lablink-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lablink_cli/__init__.py +8 -0
- lablink_cli/api.py +428 -0
- lablink_cli/app.py +938 -0
- lablink_cli/byo_detect.py +112 -0
- lablink_cli/commands/__init__.py +0 -0
- lablink_cli/commands/cleanup.py +647 -0
- lablink_cli/commands/deploy.py +863 -0
- lablink_cli/commands/deploy_compose.py +1203 -0
- lablink_cli/commands/doctor.py +549 -0
- lablink_cli/commands/export_metrics.py +244 -0
- lablink_cli/commands/launch.py +236 -0
- lablink_cli/commands/logs.py +434 -0
- lablink_cli/commands/register.py +839 -0
- lablink_cli/commands/reset_overlay.py +109 -0
- lablink_cli/commands/setup.py +347 -0
- lablink_cli/commands/stats.py +133 -0
- lablink_cli/commands/status.py +934 -0
- lablink_cli/commands/unregister.py +188 -0
- lablink_cli/commands/utils.py +552 -0
- lablink_cli/config/__init__.py +0 -0
- lablink_cli/config/schema.py +212 -0
- lablink_cli/deployment_metrics.py +94 -0
- lablink_cli/docker.py +419 -0
- lablink_cli/log_shipper.py +441 -0
- lablink_cli/templates/docker-compose.tailscale-override.yml +55 -0
- lablink_cli/templates/docker-compose.yml +67 -0
- lablink_cli/tofu_source.py +169 -0
- lablink_cli/tui/__init__.py +0 -0
- lablink_cli/tui/logs_viewer.py +413 -0
- lablink_cli/tui/wizard.py +1814 -0
- lablink_cli-0.1.0.dist-info/METADATA +76 -0
- lablink_cli-0.1.0.dist-info/RECORD +35 -0
- lablink_cli-0.1.0.dist-info/WHEEL +5 -0
- lablink_cli-0.1.0.dist-info/entry_points.txt +2 -0
- lablink_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
"""View VM logs for a LabLink deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import subprocess
|
|
7
|
+
import tempfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from urllib.error import HTTPError, URLError
|
|
10
|
+
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
from rich.markup import escape
|
|
13
|
+
|
|
14
|
+
from lablink_allocator_service.conf.structured_config import Config
|
|
15
|
+
|
|
16
|
+
from lablink_cli.api import authenticated_json_request
|
|
17
|
+
from lablink_cli.commands.utils import (
|
|
18
|
+
AwsQueryError,
|
|
19
|
+
get_allocator_url,
|
|
20
|
+
get_deploy_dir,
|
|
21
|
+
get_tofu_outputs,
|
|
22
|
+
TofuError,
|
|
23
|
+
list_all_vms,
|
|
24
|
+
print_aws_error,
|
|
25
|
+
resolve_admin_credentials,
|
|
26
|
+
)
|
|
27
|
+
from lablink_cli.docker import Docker, default_docker
|
|
28
|
+
|
|
29
|
+
console = Console()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# ------------------------------------------------------------------
|
|
33
|
+
# Log fetching — client VMs
|
|
34
|
+
# ------------------------------------------------------------------
|
|
35
|
+
def fetch_client_logs(
|
|
36
|
+
allocator_url: str,
|
|
37
|
+
hostname: str,
|
|
38
|
+
admin_user: str,
|
|
39
|
+
admin_pw: str,
|
|
40
|
+
ssl_provider: str = "none",
|
|
41
|
+
) -> dict:
|
|
42
|
+
"""Fetch logs for a client VM from the allocator API."""
|
|
43
|
+
url = f"{allocator_url}/api/vm-logs/{hostname}"
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
body = authenticated_json_request(
|
|
47
|
+
url, admin_user, admin_pw, ssl_provider=ssl_provider, timeout=30
|
|
48
|
+
)
|
|
49
|
+
return {
|
|
50
|
+
"cloud_init_logs": body.get("cloud_init_logs"),
|
|
51
|
+
"docker_logs": body.get("docker_logs"),
|
|
52
|
+
"error": None,
|
|
53
|
+
}
|
|
54
|
+
except HTTPError as e:
|
|
55
|
+
if e.code == 404:
|
|
56
|
+
return {
|
|
57
|
+
"cloud_init_logs": None,
|
|
58
|
+
"docker_logs": None,
|
|
59
|
+
"error": "VM not found in allocator database.",
|
|
60
|
+
}
|
|
61
|
+
elif e.code == 503:
|
|
62
|
+
return {
|
|
63
|
+
"cloud_init_logs": None,
|
|
64
|
+
"docker_logs": None,
|
|
65
|
+
"error": "VM is still initializing...",
|
|
66
|
+
}
|
|
67
|
+
elif e.code == 401:
|
|
68
|
+
return {
|
|
69
|
+
"cloud_init_logs": None,
|
|
70
|
+
"docker_logs": None,
|
|
71
|
+
"error": "Authentication failed. Check admin credentials.",
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
"cloud_init_logs": None,
|
|
75
|
+
"docker_logs": None,
|
|
76
|
+
"error": f"HTTP {e.code}: {e.reason}",
|
|
77
|
+
}
|
|
78
|
+
except URLError as e:
|
|
79
|
+
return {
|
|
80
|
+
"cloud_init_logs": None,
|
|
81
|
+
"docker_logs": None,
|
|
82
|
+
"error": f"Connection error: {e.reason}",
|
|
83
|
+
}
|
|
84
|
+
except Exception as e:
|
|
85
|
+
return {
|
|
86
|
+
"cloud_init_logs": None,
|
|
87
|
+
"docker_logs": None,
|
|
88
|
+
"error": f"Unexpected error: {e}",
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------------
|
|
93
|
+
# Log fetching — allocator VM (via SSH)
|
|
94
|
+
# ------------------------------------------------------------------
|
|
95
|
+
def _ssh_via_instance_connect(
|
|
96
|
+
instance_id: str,
|
|
97
|
+
region: str,
|
|
98
|
+
command: str,
|
|
99
|
+
) -> str | None:
|
|
100
|
+
"""Try SSH via ec2-instance-connect. Returns stdout or None."""
|
|
101
|
+
try:
|
|
102
|
+
result = subprocess.run(
|
|
103
|
+
[
|
|
104
|
+
"aws",
|
|
105
|
+
"ec2-instance-connect",
|
|
106
|
+
"ssh",
|
|
107
|
+
"--instance-id",
|
|
108
|
+
instance_id,
|
|
109
|
+
"--os-user",
|
|
110
|
+
"ubuntu",
|
|
111
|
+
"--connection-type",
|
|
112
|
+
"eice",
|
|
113
|
+
"--region",
|
|
114
|
+
region,
|
|
115
|
+
"--",
|
|
116
|
+
command,
|
|
117
|
+
],
|
|
118
|
+
capture_output=True,
|
|
119
|
+
text=True,
|
|
120
|
+
timeout=30,
|
|
121
|
+
)
|
|
122
|
+
if result.returncode == 0:
|
|
123
|
+
return result.stdout
|
|
124
|
+
except (subprocess.TimeoutExpired, FileNotFoundError):
|
|
125
|
+
pass
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _ssh_via_private_key(
|
|
130
|
+
public_ip: str,
|
|
131
|
+
command: str,
|
|
132
|
+
deploy_dir: Path,
|
|
133
|
+
) -> str | None:
|
|
134
|
+
"""Try SSH with the OpenTofu-provisioned private key.
|
|
135
|
+
|
|
136
|
+
Returns stdout/stderr or None.
|
|
137
|
+
"""
|
|
138
|
+
ip = public_ip if public_ip != "—" else None
|
|
139
|
+
if not ip:
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
outputs = get_tofu_outputs(deploy_dir)
|
|
144
|
+
except TofuError as e:
|
|
145
|
+
console.print(
|
|
146
|
+
f" [yellow]Could not read the SSH key:[/yellow] {escape(str(e))}"
|
|
147
|
+
)
|
|
148
|
+
return None
|
|
149
|
+
private_key_pem = outputs.get("private_key_pem", "")
|
|
150
|
+
if not private_key_pem:
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
key_file = None
|
|
154
|
+
try:
|
|
155
|
+
key_file = tempfile.NamedTemporaryFile(
|
|
156
|
+
mode="w", suffix=".pem", delete=False
|
|
157
|
+
)
|
|
158
|
+
key_file.write(private_key_pem)
|
|
159
|
+
key_file.close()
|
|
160
|
+
os.chmod(key_file.name, 0o600)
|
|
161
|
+
|
|
162
|
+
result = subprocess.run(
|
|
163
|
+
[
|
|
164
|
+
"ssh",
|
|
165
|
+
"-i",
|
|
166
|
+
key_file.name,
|
|
167
|
+
"-o",
|
|
168
|
+
"StrictHostKeyChecking=no",
|
|
169
|
+
"-o",
|
|
170
|
+
"UserKnownHostsFile=/dev/null",
|
|
171
|
+
"-o",
|
|
172
|
+
"ConnectTimeout=10",
|
|
173
|
+
f"ubuntu@{ip}",
|
|
174
|
+
command,
|
|
175
|
+
],
|
|
176
|
+
capture_output=True,
|
|
177
|
+
text=True,
|
|
178
|
+
timeout=30,
|
|
179
|
+
)
|
|
180
|
+
if result.returncode == 0:
|
|
181
|
+
return result.stdout
|
|
182
|
+
return (
|
|
183
|
+
result.stderr
|
|
184
|
+
or f"SSH exited with code {result.returncode}"
|
|
185
|
+
)
|
|
186
|
+
except (subprocess.TimeoutExpired, FileNotFoundError):
|
|
187
|
+
return None
|
|
188
|
+
finally:
|
|
189
|
+
if key_file and os.path.exists(key_file.name):
|
|
190
|
+
os.unlink(key_file.name)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _run_ssh_command(
|
|
194
|
+
instance_id: str,
|
|
195
|
+
public_ip: str,
|
|
196
|
+
region: str,
|
|
197
|
+
command: str,
|
|
198
|
+
deploy_dir: Path,
|
|
199
|
+
) -> str | None:
|
|
200
|
+
"""Run a command on the allocator via SSH.
|
|
201
|
+
|
|
202
|
+
Tries ec2-instance-connect first, then falls back to direct SSH
|
|
203
|
+
using the OpenTofu-provisioned private key.
|
|
204
|
+
"""
|
|
205
|
+
return _ssh_via_instance_connect(
|
|
206
|
+
instance_id, region, command
|
|
207
|
+
) or _ssh_via_private_key(public_ip, command, deploy_dir)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
_LOG_DELIMITER = "===LABLINK_LOG_SEPARATOR==="
|
|
211
|
+
|
|
212
|
+
_COMBINED_LOG_CMD = (
|
|
213
|
+
"cat /var/log/cloud-init-output.log 2>/dev/null;"
|
|
214
|
+
f" echo '{_LOG_DELIMITER}';"
|
|
215
|
+
" sudo docker logs $(sudo docker ps -q | head -1)"
|
|
216
|
+
" --tail 2000 2>&1"
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def fetch_allocator_logs(
|
|
221
|
+
instance_id: str,
|
|
222
|
+
public_ip: str,
|
|
223
|
+
region: str,
|
|
224
|
+
deploy_dir: Path,
|
|
225
|
+
) -> dict:
|
|
226
|
+
"""Fetch cloud-init and docker logs from the allocator EC2 instance."""
|
|
227
|
+
output = _run_ssh_command(
|
|
228
|
+
instance_id,
|
|
229
|
+
public_ip,
|
|
230
|
+
region,
|
|
231
|
+
_COMBINED_LOG_CMD,
|
|
232
|
+
deploy_dir,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
if output is None:
|
|
236
|
+
return {
|
|
237
|
+
"cloud_init_logs": None,
|
|
238
|
+
"docker_logs": None,
|
|
239
|
+
"error": (
|
|
240
|
+
"Could not SSH into allocator. "
|
|
241
|
+
"Ensure ec2-instance-connect is available or "
|
|
242
|
+
"port 22 is open."
|
|
243
|
+
),
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
parts = output.split(_LOG_DELIMITER, 1)
|
|
247
|
+
cloud_init = parts[0].strip() or None
|
|
248
|
+
docker = parts[1].strip() if len(parts) > 1 else None
|
|
249
|
+
|
|
250
|
+
return {
|
|
251
|
+
"cloud_init_logs": cloud_init,
|
|
252
|
+
"docker_logs": docker,
|
|
253
|
+
"error": None,
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
# ------------------------------------------------------------------
|
|
258
|
+
# Log fetching — manual-provider allocator (local docker container)
|
|
259
|
+
# ------------------------------------------------------------------
|
|
260
|
+
_MANUAL_ALLOCATOR_TAIL = 2000
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def fetch_manual_allocator_logs(*, docker: Docker | None = None) -> dict:
|
|
264
|
+
"""Snapshot the local lablink-allocator container's logs.
|
|
265
|
+
|
|
266
|
+
Mirrors :func:`fetch_allocator_logs` / :func:`fetch_client_logs`
|
|
267
|
+
contract (cloud_init_logs, docker_logs, error keys) so the TUI can
|
|
268
|
+
treat manual + AWS uniformly.
|
|
269
|
+
"""
|
|
270
|
+
docker = docker or default_docker()
|
|
271
|
+
result = docker.logs(
|
|
272
|
+
"lablink-allocator", tail=_MANUAL_ALLOCATOR_TAIL, timeout=30
|
|
273
|
+
)
|
|
274
|
+
if not result.ok:
|
|
275
|
+
stderr = result.stderr.strip()
|
|
276
|
+
if "No such container" in stderr:
|
|
277
|
+
err = (
|
|
278
|
+
"lablink-allocator container is not running. "
|
|
279
|
+
"Run `lablink deploy` to start it."
|
|
280
|
+
)
|
|
281
|
+
else:
|
|
282
|
+
err = stderr or f"docker logs exited {result.returncode}"
|
|
283
|
+
return {"cloud_init_logs": None, "docker_logs": None, "error": err}
|
|
284
|
+
|
|
285
|
+
# docker writes the container's own stdout to its stdout, stderr to its
|
|
286
|
+
# stderr — merge them so the TUI shows everything chronologically.
|
|
287
|
+
combined = (result.stdout or "") + (result.stderr or "")
|
|
288
|
+
return {
|
|
289
|
+
"cloud_init_logs": None,
|
|
290
|
+
"docker_logs": combined.strip() or None,
|
|
291
|
+
"error": None,
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# ------------------------------------------------------------------
|
|
296
|
+
# Manual-provider TUI launcher
|
|
297
|
+
# ------------------------------------------------------------------
|
|
298
|
+
def _run_logs_manual(cfg: Config) -> None:
|
|
299
|
+
"""Discover BYO clients via /api/v1/clients and launch the TUI.
|
|
300
|
+
|
|
301
|
+
The TUI shows the local allocator container plus every registered
|
|
302
|
+
BYO client. Client logs come from /api/vm-logs/<hostname> (populated
|
|
303
|
+
by the manual-client log shipper). The allocator entry is fetched
|
|
304
|
+
via local ``docker logs lablink-allocator`` instead of SSH.
|
|
305
|
+
"""
|
|
306
|
+
from lablink_cli.commands.deploy_compose import DEFAULT_HTTP_PORT
|
|
307
|
+
from lablink_cli.commands.status import (
|
|
308
|
+
_fetch_registered_clients,
|
|
309
|
+
_resolve_manual_admin_credentials,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
workdir = Path.home() / ".lablink" / "compose" / (
|
|
313
|
+
cfg.deployment_name or "lablink"
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
creds = _resolve_manual_admin_credentials(cfg, workdir)
|
|
317
|
+
if not creds:
|
|
318
|
+
console.print(
|
|
319
|
+
"[red]Could not resolve allocator admin credentials.[/red]\n"
|
|
320
|
+
f"Run `lablink deploy` first (expected workdir: {workdir})."
|
|
321
|
+
)
|
|
322
|
+
raise SystemExit(1)
|
|
323
|
+
admin_user, admin_pw = creds
|
|
324
|
+
|
|
325
|
+
allocator_url = f"http://localhost:{DEFAULT_HTTP_PORT}"
|
|
326
|
+
|
|
327
|
+
console.print(
|
|
328
|
+
"[dim]Fetching registered BYO clients from the allocator...[/dim]"
|
|
329
|
+
)
|
|
330
|
+
clients, err = _fetch_registered_clients(
|
|
331
|
+
allocator_url, admin_user, admin_pw
|
|
332
|
+
)
|
|
333
|
+
if clients is None:
|
|
334
|
+
console.print(
|
|
335
|
+
f"[red]Failed to list clients:[/red] {err}\n"
|
|
336
|
+
"Is the allocator running? Try `lablink status`."
|
|
337
|
+
)
|
|
338
|
+
raise SystemExit(1)
|
|
339
|
+
|
|
340
|
+
# Synthetic VM list: allocator first, then clients. Shapes match the
|
|
341
|
+
# AWS-mode dicts (name, type, vm_type, public_ip, state) so LogsApp +
|
|
342
|
+
# VMListItem work uniformly. The allocator is implicitly "running"
|
|
343
|
+
# here — _fetch_registered_clients just succeeded against it.
|
|
344
|
+
vms: list[dict] = [
|
|
345
|
+
{
|
|
346
|
+
"name": "lablink-allocator",
|
|
347
|
+
"type": "compose",
|
|
348
|
+
"vm_type": "allocator",
|
|
349
|
+
"public_ip": "localhost",
|
|
350
|
+
"state": "running",
|
|
351
|
+
}
|
|
352
|
+
]
|
|
353
|
+
for c in clients:
|
|
354
|
+
hostname = c.get("hostname") or "-"
|
|
355
|
+
vms.append({
|
|
356
|
+
"name": hostname,
|
|
357
|
+
"type": "byo",
|
|
358
|
+
"vm_type": "client",
|
|
359
|
+
"public_ip": c.get("lan_ip") or "—",
|
|
360
|
+
"state": c.get("status") or "unknown",
|
|
361
|
+
})
|
|
362
|
+
|
|
363
|
+
from lablink_cli.tui.logs_viewer import LogsApp
|
|
364
|
+
|
|
365
|
+
app = LogsApp(
|
|
366
|
+
cfg=cfg,
|
|
367
|
+
vms=vms,
|
|
368
|
+
allocator_url=allocator_url,
|
|
369
|
+
admin_user=admin_user,
|
|
370
|
+
admin_pw=admin_pw,
|
|
371
|
+
deploy_dir=workdir,
|
|
372
|
+
manual=True,
|
|
373
|
+
)
|
|
374
|
+
app.run()
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# ------------------------------------------------------------------
|
|
378
|
+
# Entry point
|
|
379
|
+
# ------------------------------------------------------------------
|
|
380
|
+
def run_logs(cfg: Config) -> None:
|
|
381
|
+
"""Launch the log viewer TUI."""
|
|
382
|
+
if getattr(cfg, "provider", "aws") == "manual":
|
|
383
|
+
_run_logs_manual(cfg)
|
|
384
|
+
return
|
|
385
|
+
|
|
386
|
+
deploy_dir = get_deploy_dir(cfg)
|
|
387
|
+
|
|
388
|
+
if not deploy_dir.exists():
|
|
389
|
+
console.print(
|
|
390
|
+
f"[red]No deployment found for "
|
|
391
|
+
f"'{cfg.deployment_name}'.[/red]\n"
|
|
392
|
+
"Run 'lablink deploy' first."
|
|
393
|
+
)
|
|
394
|
+
raise SystemExit(1)
|
|
395
|
+
|
|
396
|
+
console.print(
|
|
397
|
+
f"[dim]Discovering VMs for deployment "
|
|
398
|
+
f"'{cfg.deployment_name}' ({cfg.environment})...[/dim]"
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
try:
|
|
402
|
+
vms = list_all_vms(cfg)
|
|
403
|
+
except AwsQueryError as e:
|
|
404
|
+
print_aws_error(e, prefix="Could not list VMs")
|
|
405
|
+
raise SystemExit(1) from e
|
|
406
|
+
|
|
407
|
+
if not vms:
|
|
408
|
+
console.print(
|
|
409
|
+
f"[red]No running VMs found for deployment "
|
|
410
|
+
f"'{cfg.deployment_name}'.[/red]\n"
|
|
411
|
+
"Run 'lablink deploy' and 'lablink client launch' first."
|
|
412
|
+
)
|
|
413
|
+
raise SystemExit(1)
|
|
414
|
+
|
|
415
|
+
allocator_url = get_allocator_url(cfg)
|
|
416
|
+
if not allocator_url:
|
|
417
|
+
console.print(
|
|
418
|
+
"[yellow]Warning: Could not determine allocator URL. "
|
|
419
|
+
"Client VM logs will not be available.[/yellow]"
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
admin_user, admin_pw = resolve_admin_credentials(cfg)
|
|
423
|
+
|
|
424
|
+
from lablink_cli.tui.logs_viewer import LogsApp
|
|
425
|
+
|
|
426
|
+
app = LogsApp(
|
|
427
|
+
cfg=cfg,
|
|
428
|
+
vms=vms,
|
|
429
|
+
allocator_url=allocator_url,
|
|
430
|
+
admin_user=admin_user,
|
|
431
|
+
admin_pw=admin_pw,
|
|
432
|
+
deploy_dir=deploy_dir,
|
|
433
|
+
)
|
|
434
|
+
app.run()
|