fadenstack 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- faden_cli/__init__.py +16 -0
- faden_cli/__main__.py +9 -0
- faden_cli/bundle/shared/.gitignore +25 -0
- faden_cli/bundle/shared/LICENSE +187 -0
- faden_cli/bundle/shared/NOTICE +15 -0
- faden_cli/bundle/shared/OFFICIAL.md +9 -0
- faden_cli/bundle/shared/README.md +16 -0
- faden_cli/bundle/shared/agent-binaries/.gitignore +5 -0
- faden_cli/bundle/shared/alloy/config.alloy +40 -0
- faden_cli/bundle/shared/clickhouse/config.d/system-logs.xml +26 -0
- faden_cli/bundle/shared/docker-compose.gpu.yaml +34 -0
- faden_cli/bundle/shared/docker-compose.yaml +998 -0
- faden_cli/bundle/shared/grafana/dashboards/templates/vllm-runtime.json +2099 -0
- faden_cli/bundle/shared/grafana/provisioning/dashboards/dashboard.yml +10 -0
- faden_cli/bundle/shared/grafana/provisioning/dashboards/json/faden-overview.json +211 -0
- faden_cli/bundle/shared/grafana/provisioning/datasources/ds.yaml +9 -0
- faden_cli/bundle/shared/grafana/provisioning/datasources/prometheus.yaml +15 -0
- faden_cli/bundle/shared/initdb/01-init.sql +32 -0
- faden_cli/bundle/shared/loki/local-config.yaml +32 -0
- faden_cli/bundle/shared/nginx/locations.conf +245 -0
- faden_cli/bundle/shared/nginx/maintenance.html +239 -0
- faden_cli/bundle/shared/nginx/nginx-tls.conf +79 -0
- faden_cli/bundle/shared/nginx/nginx.conf +44 -0
- faden_cli/bundle/shared/nginx/rate-limits.conf +42 -0
- faden_cli/bundle/shared/nginx/security-headers.conf +22 -0
- faden_cli/bundle/shared/nginx/select-config.sh +37 -0
- faden_cli/bundle/shared/prometheus/prometheus.yml +40 -0
- faden_cli/bundle/shared/rabbitmq/definitions.json.example +37 -0
- faden_cli/bundle/shared/tls/README.md +30 -0
- faden_cli/cli.py +158 -0
- faden_cli/commands/__init__.py +1 -0
- faden_cli/commands/admin.py +161 -0
- faden_cli/commands/backup.py +159 -0
- faden_cli/commands/config.py +146 -0
- faden_cli/commands/deploy.py +949 -0
- faden_cli/commands/dev/__init__.py +1 -0
- faden_cli/commands/dev/dev_doctor.py +46 -0
- faden_cli/commands/dev/dev_down.py +241 -0
- faden_cli/commands/dev/dev_group.py +24 -0
- faden_cli/commands/dev/dev_init.py +710 -0
- faden_cli/commands/dev/dev_pull.py +297 -0
- faden_cli/commands/dev/dev_status.py +151 -0
- faden_cli/commands/dev/dev_up.py +1392 -0
- faden_cli/commands/doctor.py +225 -0
- faden_cli/commands/down.py +75 -0
- faden_cli/commands/logs_cmd.py +41 -0
- faden_cli/commands/maintenance.py +74 -0
- faden_cli/commands/module.py +113 -0
- faden_cli/commands/observe.py +95 -0
- faden_cli/commands/ports.py +91 -0
- faden_cli/commands/rebuild.py +158 -0
- faden_cli/commands/restore.py +139 -0
- faden_cli/commands/runtime_images.py +174 -0
- faden_cli/commands/status.py +71 -0
- faden_cli/commands/tls.py +722 -0
- faden_cli/commands/tune.py +37 -0
- faden_cli/commands/up.py +64 -0
- faden_cli/commands/upgrade.py +470 -0
- faden_cli/commands/version.py +78 -0
- faden_cli/core/__init__.py +1 -0
- faden_cli/core/api_client.py +77 -0
- faden_cli/core/backup.py +717 -0
- faden_cli/core/bootstrap.py +291 -0
- faden_cli/core/bundle.py +110 -0
- faden_cli/core/clickhouse.py +61 -0
- faden_cli/core/compose.py +484 -0
- faden_cli/core/console.py +106 -0
- faden_cli/core/detect.py +580 -0
- faden_cli/core/dev_modules.py +149 -0
- faden_cli/core/docker_env.py +147 -0
- faden_cli/core/edge.py +94 -0
- faden_cli/core/env_gen.py +598 -0
- faden_cli/core/git.py +219 -0
- faden_cli/core/install.py +288 -0
- faden_cli/core/installation.py +186 -0
- faden_cli/core/local_node.py +719 -0
- faden_cli/core/maintenance.py +82 -0
- faden_cli/core/private_files.py +133 -0
- faden_cli/core/registry.py +574 -0
- faden_cli/core/release_feed.py +342 -0
- faden_cli/core/rmq.py +135 -0
- faden_cli/core/runtime_images.py +309 -0
- faden_cli/core/schedule.py +278 -0
- faden_cli/core/settings.py +159 -0
- faden_cli/core/sysinfo.py +303 -0
- faden_cli/core/tls.py +790 -0
- faden_cli/core/tune.py +154 -0
- faden_cli/core/update_check.py +254 -0
- faden_cli/core/workspace.py +155 -0
- faden_cli/templates/env.j2 +33 -0
- faden_cli/tui/__init__.py +1 -0
- faden_cli/tui/widgets/__init__.py +1 -0
- faden_cli/tui/wizard/__init__.py +1 -0
- fadenstack-0.4.2.dist-info/METADATA +304 -0
- fadenstack-0.4.2.dist-info/RECORD +99 -0
- fadenstack-0.4.2.dist-info/WHEEL +4 -0
- fadenstack-0.4.2.dist-info/entry_points.txt +3 -0
- fadenstack-0.4.2.dist-info/licenses/LICENSE +187 -0
- fadenstack-0.4.2.dist-info/licenses/NOTICE +15 -0
|
@@ -0,0 +1,722 @@
|
|
|
1
|
+
"""``faden tls`` — serve the console, the API and /v1 over HTTPS, or stop.
|
|
2
|
+
|
|
3
|
+
nginx picks HTTPS or plain HTTP when it starts, from what ``<install>/tls``
|
|
4
|
+
holds (shared/nginx/select-config.sh), so every change here ends with
|
|
5
|
+
a restart of the proxy -- the only container it touches -- or, when HTTPS stays
|
|
6
|
+
on and only the certificate is new, a reload that keeps open connections.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import shutil
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
from datetime import UTC, datetime
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from urllib.parse import urlsplit
|
|
18
|
+
|
|
19
|
+
import click
|
|
20
|
+
|
|
21
|
+
from faden_cli.core import tls
|
|
22
|
+
from faden_cli.core.console import console, error, info, success, warning
|
|
23
|
+
from faden_cli.core.edge import console_url, url_for
|
|
24
|
+
from faden_cli.core.env_gen import read_env_file
|
|
25
|
+
from faden_cli.core.settings import load_config
|
|
26
|
+
|
|
27
|
+
PROXY_SERVICE = "nginx"
|
|
28
|
+
#: A connected agent asks for the authority every minute (agent's TRUST_REFRESH_SEC).
|
|
29
|
+
MACHINE_TRUST_WAIT_SEC = 70
|
|
30
|
+
PROXY_CONTAINER = "faden-nginx"
|
|
31
|
+
RELOAD_COMMAND = "sh /etc/nginx/faden/select-config.sh && nginx -s reload"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _install_dir() -> Path:
|
|
35
|
+
cfg = load_config()
|
|
36
|
+
if not cfg.install_dir:
|
|
37
|
+
error("No install found. Run faden deploy first.")
|
|
38
|
+
sys.exit(1)
|
|
39
|
+
return Path(cfg.install_dir)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _proxy_running() -> bool:
|
|
43
|
+
docker = shutil.which("docker")
|
|
44
|
+
if not docker:
|
|
45
|
+
return False
|
|
46
|
+
result = subprocess.run( # noqa: S603
|
|
47
|
+
[docker, "inspect", "-f", "{{.State.Running}}", PROXY_CONTAINER],
|
|
48
|
+
capture_output=True,
|
|
49
|
+
text=True,
|
|
50
|
+
check=False,
|
|
51
|
+
)
|
|
52
|
+
return result.returncode == 0 and result.stdout.strip() == "true"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _reload_proxy() -> bool:
|
|
56
|
+
"""Swap the certificate without dropping a connection: select-config.sh, then ``nginx -s reload``."""
|
|
57
|
+
docker = shutil.which("docker")
|
|
58
|
+
if not docker:
|
|
59
|
+
return False
|
|
60
|
+
result = subprocess.run( # noqa: S603
|
|
61
|
+
[docker, "exec", PROXY_CONTAINER, "sh", "-c", RELOAD_COMMAND],
|
|
62
|
+
capture_output=True,
|
|
63
|
+
text=True,
|
|
64
|
+
check=False,
|
|
65
|
+
)
|
|
66
|
+
if result.returncode != 0:
|
|
67
|
+
why = (result.stderr or result.stdout).strip()[:200]
|
|
68
|
+
warning(f"The proxy could not reload ({why}); restarting it.")
|
|
69
|
+
return result.returncode == 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def apply_to_proxy(install_dir: Path, *, recreate: bool = False, reload: bool = False) -> bool:
|
|
73
|
+
"""Have nginx pick up the install's material and ports; ``True`` once the console answers.
|
|
74
|
+
|
|
75
|
+
A restart re-runs select-config.sh; new ports need the container made
|
|
76
|
+
again (*recreate*), which ``docker compose up`` does when they changed.
|
|
77
|
+
With HTTPS staying on and only the certificate new (*reload*: a renewal,
|
|
78
|
+
a replacement), nginx reloads instead: new connections get the new
|
|
79
|
+
certificate and the open ones -- the machines' streams -- stay up. A
|
|
80
|
+
restart cut them, and the machines noticed only when their keepalive
|
|
81
|
+
timed out, a minute later. Switching between HTTP and HTTPS restarts, so
|
|
82
|
+
every open plain-HTTP connection meets the redirect at once.
|
|
83
|
+
"""
|
|
84
|
+
from faden_cli.core.bootstrap import wait_for_backend # noqa: PLC0415
|
|
85
|
+
from faden_cli.core.compose import build_context_from_config, restart, up # noqa: PLC0415
|
|
86
|
+
|
|
87
|
+
if not _proxy_running():
|
|
88
|
+
info("The proxy is not running; it serves this the next time the install starts.")
|
|
89
|
+
return True
|
|
90
|
+
ctx = build_context_from_config(load_config())
|
|
91
|
+
if recreate:
|
|
92
|
+
rc = up(ctx, services=[PROXY_SERVICE], detach=True, wait=True, timeout=120)
|
|
93
|
+
elif reload and _reload_proxy():
|
|
94
|
+
rc = 0
|
|
95
|
+
else:
|
|
96
|
+
rc = restart(ctx, services=[PROXY_SERVICE])
|
|
97
|
+
if rc != 0:
|
|
98
|
+
error(f"Restarting the proxy failed (exit code {rc}); see: faden logs {PROXY_SERVICE}")
|
|
99
|
+
return False
|
|
100
|
+
url = console_url(install_dir, read_env_file(install_dir / ".env"))
|
|
101
|
+
if wait_for_backend(url, timeout=90):
|
|
102
|
+
success(f"The console answers at {url}")
|
|
103
|
+
return True
|
|
104
|
+
error(f"The console does not answer at {url}; see: faden logs {PROXY_SERVICE}")
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def addresses(install_dir: Path) -> list[str]:
|
|
109
|
+
"""The URLs other machines reach the console by, the way it is served now."""
|
|
110
|
+
env = read_env_file(install_dir / ".env")
|
|
111
|
+
https = tls.material_present(install_dir)
|
|
112
|
+
loopback = {"localhost", "127.0.0.1", "::1"}
|
|
113
|
+
return [url_for(name, https=https, env=env) for name in tls.machine_names() if name not in loopback]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def print_summary(summary: tls.CertSummary) -> None:
|
|
117
|
+
"""The certificate as an operator reads it, with what to watch out for."""
|
|
118
|
+
console.print(f" Certificate: [bold]{summary.subject}[/bold]")
|
|
119
|
+
console.print(f" Issued by: {summary.issuer}{' (this install)' if summary.made_here else ''}")
|
|
120
|
+
console.print(f" Valid until: {summary.not_after:%Y-%m-%d}")
|
|
121
|
+
console.print(f" Names: {', '.join(summary.names) or '(none)'}")
|
|
122
|
+
for note in summary.warnings:
|
|
123
|
+
warning(note)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def reachable_example(urls: list[str]) -> str:
|
|
127
|
+
"""The address to show other machines: an IP or a dotted name before a bare host name.
|
|
128
|
+
|
|
129
|
+
A bare name ("fadenstack") resolves on the server itself, rarely anywhere else.
|
|
130
|
+
"""
|
|
131
|
+
for url in urls:
|
|
132
|
+
host = urlsplit(url).hostname or ""
|
|
133
|
+
if tls._is_ip(host) or "." in host: # noqa: SLF001
|
|
134
|
+
return url
|
|
135
|
+
return urls[0] if urls else "https://<this server>"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def print_trust_hints(install_dir: Path, ca_public: Path) -> None:
|
|
139
|
+
"""How browsers, API clients and machines come to trust a certificate made here."""
|
|
140
|
+
example = reachable_example(addresses(install_dir))
|
|
141
|
+
console.print(
|
|
142
|
+
"\n [bold]Trust this install's certificate authority once[/bold],"
|
|
143
|
+
" wherever the console is used from:\n"
|
|
144
|
+
f" {ca_public}\n"
|
|
145
|
+
" [dim]Browsers: import it as a trusted certificate authority (or accept the warning).\n"
|
|
146
|
+
f" API clients: curl --cacert ca.crt {example}/v1/models · Python: SSL_CERT_FILE=ca.crt\n"
|
|
147
|
+
" Machines with agent 0.1.14 or later learnt it from this server and follow it to HTTPS\n"
|
|
148
|
+
" by themselves. Older agents stay on plain HTTP, which now only redirects: point each at\n"
|
|
149
|
+
f" {example} and give it the file (faden-agent configure --set BACKEND_URL=...\n"
|
|
150
|
+
' --set TLS_CA_BUNDLE=...; Help: Operate, "HTTPS and trust", "The machines").[/dim]'
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def print_addresses(install_dir: Path) -> None:
|
|
155
|
+
"""Where other machines reach the console now."""
|
|
156
|
+
urls = addresses(install_dir)
|
|
157
|
+
if urls:
|
|
158
|
+
console.print("\n Reach the console at: " + " ".join(urls))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
TLS_MODES = ("off", "generate", "own")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _free_port(label: str, port: int, *, yes: bool, current: tuple[int, int], other: int) -> int:
|
|
165
|
+
"""*port*, or another one the operator picks when something else holds it."""
|
|
166
|
+
from faden_cli.commands.ports import port_conflicts # noqa: PLC0415
|
|
167
|
+
|
|
168
|
+
while True:
|
|
169
|
+
pair = (port, other) if label == "HTTP" else (other, port)
|
|
170
|
+
clash = [c for c in port_conflicts(*pair, current=current) if c.startswith(f"{label} port")]
|
|
171
|
+
if not clash:
|
|
172
|
+
return port
|
|
173
|
+
if yes:
|
|
174
|
+
error(f"{clash[0]} Free it, or pass --{label.lower()}-port.")
|
|
175
|
+
sys.exit(1)
|
|
176
|
+
warning(clash[0])
|
|
177
|
+
port = click.prompt(f" {label} port to use instead", type=click.IntRange(1, 65535))
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _ports_step(env_path: Path, *, yes: bool, http_port: int | None, https_port: int | None) -> None:
|
|
181
|
+
from faden_cli.commands.ports import set_ports # noqa: PLC0415
|
|
182
|
+
from faden_cli.core.edge import check_ports, ports # noqa: PLC0415
|
|
183
|
+
|
|
184
|
+
current = ports(read_env_file(env_path))
|
|
185
|
+
http = http_port if http_port is not None else current[0]
|
|
186
|
+
https = https_port if https_port is not None else current[1]
|
|
187
|
+
problem = check_ports(http, https)
|
|
188
|
+
if problem:
|
|
189
|
+
error(problem)
|
|
190
|
+
sys.exit(2)
|
|
191
|
+
http = _free_port("HTTP", http, yes=yes, current=current, other=https)
|
|
192
|
+
https = _free_port("HTTPS", https, yes=yes, current=current, other=http)
|
|
193
|
+
set_ports(env_path, http=http, https=https)
|
|
194
|
+
console.print(
|
|
195
|
+
f" HTTP on port {http}, HTTPS on port {https} "
|
|
196
|
+
"[dim](change: --http-port/--https-port, or later faden ports)[/dim]"
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _ask_mode(install_dir: Path, *, yes: bool) -> str | None:
|
|
201
|
+
"""What to do about HTTPS when deploy was not told; ``None`` to leave it as it is."""
|
|
202
|
+
present = tls.current(install_dir)
|
|
203
|
+
if present is not None:
|
|
204
|
+
info(
|
|
205
|
+
f"HTTPS is on, with {present.subject} until {present.not_after:%Y-%m-%d}"
|
|
206
|
+
" (change: faden tls)."
|
|
207
|
+
)
|
|
208
|
+
return None
|
|
209
|
+
off_by_choice = tls.chose_off(install_dir)
|
|
210
|
+
if yes:
|
|
211
|
+
if off_by_choice:
|
|
212
|
+
info(
|
|
213
|
+
"HTTPS is off, as chosen. Switch it on any time: faden tls generate,"
|
|
214
|
+
" or faden tls use CERT KEY.",
|
|
215
|
+
)
|
|
216
|
+
return None
|
|
217
|
+
# HTTPS is the default: without it sign-ins, API keys, prompts and the
|
|
218
|
+
# machines' credentials cross the network in the clear.
|
|
219
|
+
info("HTTPS by default: a certificate made here (plain HTTP instead: faden deploy --tls off).")
|
|
220
|
+
return "generate"
|
|
221
|
+
console.print(
|
|
222
|
+
" Serve the console, the API and /v1 over HTTPS?\n"
|
|
223
|
+
" [bold]generate[/bold] a certificate made here for this machine's names and addresses"
|
|
224
|
+
" (default)\n"
|
|
225
|
+
" [bold]own[/bold] your own certificate and key (PEM files)\n"
|
|
226
|
+
" [bold]off[/bold] plain HTTP (switch HTTPS on later with faden tls)"
|
|
227
|
+
)
|
|
228
|
+
default = "off" if off_by_choice else "generate"
|
|
229
|
+
return click.prompt(" HTTPS", type=click.Choice(TLS_MODES), default=default)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def renewal_scheduled() -> bool:
|
|
233
|
+
"""Whether ``faden tls renew`` runs by itself every day."""
|
|
234
|
+
from faden_cli.core import schedule # noqa: PLC0415
|
|
235
|
+
|
|
236
|
+
try:
|
|
237
|
+
return schedule.available() and schedule.job_current(schedule.TLS_UNIT) is not None
|
|
238
|
+
except OSError:
|
|
239
|
+
return False
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def schedule_renewal(install_dir: Path) -> None:
|
|
243
|
+
"""Have ``faden tls renew`` run every morning, once HTTPS is on; say how."""
|
|
244
|
+
from faden_cli.core import schedule # noqa: PLC0415
|
|
245
|
+
|
|
246
|
+
if renewal_scheduled():
|
|
247
|
+
return
|
|
248
|
+
if not schedule.available():
|
|
249
|
+
warning(
|
|
250
|
+
"No systemd user manager or cron here to renew the certificate by itself: "
|
|
251
|
+
"run faden tls renew now and then (it only acts when something is due)."
|
|
252
|
+
)
|
|
253
|
+
return
|
|
254
|
+
try:
|
|
255
|
+
how, notes = schedule.install_job(schedule.tls_renewal(install_dir / "logs" / "tls-renew.log"))
|
|
256
|
+
except (RuntimeError, OSError) as exc:
|
|
257
|
+
warning(f"Could not schedule the renewal: {exc}. Run faden tls renew now and then.")
|
|
258
|
+
return
|
|
259
|
+
success(f"Kept current by itself: faden tls renew runs every morning ({how}).")
|
|
260
|
+
for note in notes:
|
|
261
|
+
warning(note)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def unschedule_renewal() -> None:
|
|
265
|
+
"""Stop the daily renewal: with HTTPS off there is nothing to renew."""
|
|
266
|
+
from faden_cli.core import schedule # noqa: PLC0415
|
|
267
|
+
|
|
268
|
+
if schedule.available():
|
|
269
|
+
schedule.remove_job(schedule.TLS_UNIT)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def refresh_edge_info(install_dir: Path) -> None:
|
|
273
|
+
"""Tell the backend this host's addresses and ports (``tls/edge.json``); best effort."""
|
|
274
|
+
from faden_cli.core.edge import ports # noqa: PLC0415
|
|
275
|
+
|
|
276
|
+
http_port, https_port = ports(read_env_file(install_dir / ".env"))
|
|
277
|
+
try:
|
|
278
|
+
tls.write_edge_info(
|
|
279
|
+
install_dir, http_port=http_port, https_port=https_port,
|
|
280
|
+
renews="daily" if renewal_scheduled() else "by hand",
|
|
281
|
+
)
|
|
282
|
+
except OSError as exc:
|
|
283
|
+
warning(f"Could not write {install_dir / tls.TLS_DIR / 'edge.json'}: {exc}")
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _wait_for_machines(trust_changed: bool, *, no_restart: bool, no_wait: bool) -> None:
|
|
287
|
+
"""Give connected machines time to fetch a new issuer before the proxy serves what it signed."""
|
|
288
|
+
if not trust_changed or no_restart or no_wait:
|
|
289
|
+
return
|
|
290
|
+
info(
|
|
291
|
+
f"The machines have a new issuer to trust: giving them {MACHINE_TRUST_WAIT_SEC} s to fetch it"
|
|
292
|
+
" (--no-wait skips)."
|
|
293
|
+
)
|
|
294
|
+
time.sleep(MACHINE_TRUST_WAIT_SEC)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _generate_step(install_dir: Path, *, yes: bool, names: tuple[str, ...]) -> Path:
|
|
298
|
+
if not names and not yes:
|
|
299
|
+
extra = click.prompt(
|
|
300
|
+
" Other names or addresses the console is reached by (comma-separated, blank for none)",
|
|
301
|
+
default="",
|
|
302
|
+
show_default=False,
|
|
303
|
+
)
|
|
304
|
+
names = tuple(n.strip() for n in extra.split(",") if n.strip())
|
|
305
|
+
try:
|
|
306
|
+
made = tls.generate(install_dir, names)
|
|
307
|
+
except tls.TlsError as exc:
|
|
308
|
+
error(str(exc))
|
|
309
|
+
sys.exit(1)
|
|
310
|
+
success("HTTPS on, with a certificate made here.")
|
|
311
|
+
print_summary(made.summary)
|
|
312
|
+
return made.ca_public
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _own_step(
|
|
316
|
+
install_dir: Path,
|
|
317
|
+
*,
|
|
318
|
+
yes: bool,
|
|
319
|
+
cert: Path | None,
|
|
320
|
+
key: Path | None,
|
|
321
|
+
chain: Path | None,
|
|
322
|
+
ca: Path | None,
|
|
323
|
+
) -> None:
|
|
324
|
+
attempts = 1 if yes else 3
|
|
325
|
+
for attempt in range(attempts):
|
|
326
|
+
if cert is None or key is None:
|
|
327
|
+
if yes:
|
|
328
|
+
error("--tls own needs --tls-cert and --tls-key.")
|
|
329
|
+
sys.exit(2)
|
|
330
|
+
cert = Path(click.prompt(" Certificate file (PEM; may carry its chain)")).expanduser()
|
|
331
|
+
key = Path(click.prompt(" Its private key (PEM, without a passphrase)")).expanduser()
|
|
332
|
+
raw_chain = click.prompt(
|
|
333
|
+
" Intermediate certificates (PEM, blank for none)", default="", show_default=False
|
|
334
|
+
)
|
|
335
|
+
chain = Path(raw_chain).expanduser() if raw_chain.strip() else None
|
|
336
|
+
raw_ca = click.prompt(
|
|
337
|
+
" Its issuer, when it is a private one (PEM, blank for a public issuer)",
|
|
338
|
+
default="",
|
|
339
|
+
show_default=False,
|
|
340
|
+
)
|
|
341
|
+
ca = Path(raw_ca).expanduser() if raw_ca.strip() else None
|
|
342
|
+
try:
|
|
343
|
+
summary = tls.use_own(install_dir, cert, key, chain, ca_path=ca)
|
|
344
|
+
except tls.TlsError as exc:
|
|
345
|
+
error(str(exc))
|
|
346
|
+
if attempt + 1 == attempts:
|
|
347
|
+
sys.exit(1)
|
|
348
|
+
cert = key = chain = ca = None
|
|
349
|
+
continue
|
|
350
|
+
success("HTTPS on, with your certificate.")
|
|
351
|
+
print_summary(summary)
|
|
352
|
+
return
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def deploy_step(
|
|
356
|
+
install_dir: Path,
|
|
357
|
+
env_path: Path,
|
|
358
|
+
*,
|
|
359
|
+
yes: bool,
|
|
360
|
+
http_port: int | None,
|
|
361
|
+
https_port: int | None,
|
|
362
|
+
mode: str | None,
|
|
363
|
+
cert: Path | None,
|
|
364
|
+
key: Path | None,
|
|
365
|
+
chain: Path | None,
|
|
366
|
+
names: tuple[str, ...],
|
|
367
|
+
ca: Path | None = None,
|
|
368
|
+
) -> Path | None:
|
|
369
|
+
"""``faden deploy``'s ports-and-HTTPS step, before anything starts.
|
|
370
|
+
|
|
371
|
+
Returns the authority to trust when a certificate was made here. With
|
|
372
|
+
``--yes`` and no ``--tls``, what the install has stays as it is.
|
|
373
|
+
"""
|
|
374
|
+
_ports_step(env_path, yes=yes, http_port=http_port, https_port=https_port)
|
|
375
|
+
# Made even while HTTPS stays off: the machines fetch it from the backend
|
|
376
|
+
# now, so they can follow once a certificate it signed is served.
|
|
377
|
+
try:
|
|
378
|
+
ca_public, changed = tls.ensure_ca(install_dir)
|
|
379
|
+
except tls.TlsError as exc:
|
|
380
|
+
warning(str(exc))
|
|
381
|
+
else:
|
|
382
|
+
if changed:
|
|
383
|
+
info(f"Certificate authority for the machines: {ca_public}")
|
|
384
|
+
|
|
385
|
+
result = _https_step(install_dir, yes=yes, mode=mode, cert=cert, key=key, chain=chain, names=names, ca=ca)
|
|
386
|
+
if tls.material_present(install_dir):
|
|
387
|
+
schedule_renewal(install_dir)
|
|
388
|
+
refresh_edge_info(install_dir)
|
|
389
|
+
return result
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _https_step(
|
|
393
|
+
install_dir: Path,
|
|
394
|
+
*,
|
|
395
|
+
yes: bool,
|
|
396
|
+
mode: str | None,
|
|
397
|
+
cert: Path | None,
|
|
398
|
+
key: Path | None,
|
|
399
|
+
chain: Path | None,
|
|
400
|
+
names: tuple[str, ...],
|
|
401
|
+
ca: Path | None,
|
|
402
|
+
) -> Path | None:
|
|
403
|
+
if mode is None and (cert or key):
|
|
404
|
+
mode = "own"
|
|
405
|
+
if mode is None:
|
|
406
|
+
mode = _ask_mode(install_dir, yes=yes)
|
|
407
|
+
if mode is None:
|
|
408
|
+
return None
|
|
409
|
+
if mode == "off":
|
|
410
|
+
tls.remember_off(install_dir)
|
|
411
|
+
if tls.remove(install_dir):
|
|
412
|
+
unschedule_renewal()
|
|
413
|
+
info("HTTPS switched off: the certificate was removed (the install's authority is kept).")
|
|
414
|
+
else:
|
|
415
|
+
info("HTTPS is off.")
|
|
416
|
+
return None
|
|
417
|
+
if mode == "generate":
|
|
418
|
+
return _generate_step(install_dir, yes=yes, names=names)
|
|
419
|
+
_own_step(install_dir, yes=yes, cert=cert, key=key, chain=chain, ca=ca)
|
|
420
|
+
return None
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
@click.group("tls")
|
|
424
|
+
def tls_group() -> None:
|
|
425
|
+
"""HTTPS for the console, the API and the gateway (/v1).
|
|
426
|
+
|
|
427
|
+
\b
|
|
428
|
+
faden tls status what is served, until when, and how it is renewed
|
|
429
|
+
faden tls generate a certificate made here for this machine
|
|
430
|
+
faden tls use CERT KEY your own certificate (--chain, --ca for a private issuer)
|
|
431
|
+
faden tls renew renew what is due (runs every morning by itself)
|
|
432
|
+
faden tls off back to plain HTTP
|
|
433
|
+
faden tls machine-ca the machine CA behind model traffic TLS (--replace)
|
|
434
|
+
"""
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _renewal_line(install_dir: Path, summary: tls.CertSummary) -> str:
|
|
438
|
+
left = (summary.not_after - datetime.now(UTC)).days
|
|
439
|
+
auto = "every morning by faden tls renew" if renewal_scheduled() else "when you run faden tls renew"
|
|
440
|
+
if summary.made_here:
|
|
441
|
+
return f"issued again {tls.RENEW_BEFORE_DAYS} days before it runs out, {auto} ({left} days left)"
|
|
442
|
+
origin = tls.source(install_dir)
|
|
443
|
+
if origin is not None:
|
|
444
|
+
return f"taken again from {origin['cert']} when it changes there, {auto} ({left} days left)"
|
|
445
|
+
return f"by hand: faden tls use CERT KEY with the renewed files ({left} days left)"
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
@tls_group.command("status")
|
|
449
|
+
def status_cmd() -> None:
|
|
450
|
+
"""Show whether HTTPS is on, the certificate, until when, and how it is renewed."""
|
|
451
|
+
install_dir = _install_dir()
|
|
452
|
+
try:
|
|
453
|
+
summary = tls.current(install_dir)
|
|
454
|
+
except tls.TlsError as exc:
|
|
455
|
+
error(str(exc))
|
|
456
|
+
sys.exit(1)
|
|
457
|
+
if summary is None:
|
|
458
|
+
console.print("HTTPS is [bold]off[/bold]: the console is served over plain HTTP.")
|
|
459
|
+
console.print("[dim] Switch it on: faden tls generate, or faden tls use CERT KEY[/dim]")
|
|
460
|
+
else:
|
|
461
|
+
console.print("HTTPS is [bold green]on[/bold green]; plain HTTP redirects to it.")
|
|
462
|
+
print_summary(summary)
|
|
463
|
+
console.print(f" Renewed: {_renewal_line(install_dir, summary)}")
|
|
464
|
+
ca_public = install_dir / tls.TLS_DIR / tls.CA_PUBLIC_FILE
|
|
465
|
+
if ca_public.is_file():
|
|
466
|
+
console.print(f" Machines trust: {ca_public}")
|
|
467
|
+
if (summary.not_after - datetime.now(UTC)).days < tls.EXPIRY_WARNING_DAYS:
|
|
468
|
+
warning(f"The certificate runs out on {summary.not_after:%Y-%m-%d}: faden tls renew")
|
|
469
|
+
print_addresses(install_dir)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
@tls_group.command("generate")
|
|
473
|
+
@click.option(
|
|
474
|
+
"--name", "-n", "names", multiple=True, metavar="NAME",
|
|
475
|
+
help="Another DNS name or address the console is reached by (repeatable). "
|
|
476
|
+
"This machine's host names and addresses are always included.",
|
|
477
|
+
)
|
|
478
|
+
@click.option(
|
|
479
|
+
"--no-restart", is_flag=True, help="Write the files only; the proxy picks them up when it next starts."
|
|
480
|
+
)
|
|
481
|
+
@click.option(
|
|
482
|
+
"--no-wait", is_flag=True,
|
|
483
|
+
help="Switch at once even when the authority is new (machines that have not fetched it stay behind).",
|
|
484
|
+
)
|
|
485
|
+
def generate_cmd(*, names: tuple[str, ...], no_restart: bool, no_wait: bool) -> None:
|
|
486
|
+
"""Make a certificate for this machine, signed by the install's own authority."""
|
|
487
|
+
install_dir = _install_dir()
|
|
488
|
+
was_on = tls.material_present(install_dir)
|
|
489
|
+
try:
|
|
490
|
+
made = tls.generate(install_dir, names)
|
|
491
|
+
except tls.TlsError as exc:
|
|
492
|
+
error(str(exc))
|
|
493
|
+
sys.exit(1)
|
|
494
|
+
success("Certificate made and installed.")
|
|
495
|
+
print_summary(made.summary)
|
|
496
|
+
# Deploy and upgrade make the authority ahead of time; an install that has
|
|
497
|
+
# not been through either since gets it only now.
|
|
498
|
+
_wait_for_machines(made.trust_changed, no_restart=no_restart, no_wait=no_wait)
|
|
499
|
+
if not no_restart and not apply_to_proxy(install_dir, reload=was_on):
|
|
500
|
+
sys.exit(1)
|
|
501
|
+
schedule_renewal(install_dir)
|
|
502
|
+
refresh_edge_info(install_dir)
|
|
503
|
+
print_trust_hints(install_dir, made.ca_public)
|
|
504
|
+
print_addresses(install_dir)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
@tls_group.command("use")
|
|
508
|
+
@click.argument("cert", type=click.Path(dir_okay=False, path_type=Path))
|
|
509
|
+
@click.argument("key", type=click.Path(dir_okay=False, path_type=Path))
|
|
510
|
+
@click.option(
|
|
511
|
+
"--chain", type=click.Path(dir_okay=False, path_type=Path), default=None,
|
|
512
|
+
help="The issuer's intermediate certificates (PEM), when CERT does not already carry them.",
|
|
513
|
+
)
|
|
514
|
+
@click.option(
|
|
515
|
+
"--ca", "ca", type=click.Path(dir_okay=False, path_type=Path), default=None,
|
|
516
|
+
help="A private issuer's root (PEM): the machines are given it, so they trust the certificate. "
|
|
517
|
+
"Not needed for a public issuer, nor when CERT's chain ends in the root.",
|
|
518
|
+
)
|
|
519
|
+
@click.option(
|
|
520
|
+
"--no-restart", is_flag=True, help="Write the files only; the proxy picks them up when it next starts."
|
|
521
|
+
)
|
|
522
|
+
@click.option(
|
|
523
|
+
"--no-wait", is_flag=True,
|
|
524
|
+
help="Switch at once even when the issuer is new to the machines.",
|
|
525
|
+
)
|
|
526
|
+
def use_cmd(
|
|
527
|
+
*, cert: Path, key: Path, chain: Path | None, ca: Path | None, no_restart: bool, no_wait: bool
|
|
528
|
+
) -> None:
|
|
529
|
+
"""Serve your own certificate: CERT and KEY are PEM files (CERT may be a full chain).
|
|
530
|
+
|
|
531
|
+
The files are read again from the same place by faden tls renew when they
|
|
532
|
+
change there, so a certificate renewed in place (certbot, acme.sh, your own
|
|
533
|
+
tooling) is served without anyone running this again.
|
|
534
|
+
"""
|
|
535
|
+
install_dir = _install_dir()
|
|
536
|
+
was_on = tls.material_present(install_dir)
|
|
537
|
+
try:
|
|
538
|
+
summary = tls.use_own(install_dir, cert, key, chain, ca_path=ca)
|
|
539
|
+
except tls.TlsError as exc:
|
|
540
|
+
error(str(exc))
|
|
541
|
+
sys.exit(1)
|
|
542
|
+
success("Certificate checked and installed.")
|
|
543
|
+
print_summary(summary)
|
|
544
|
+
_wait_for_machines(summary.trust_changed, no_restart=no_restart, no_wait=no_wait)
|
|
545
|
+
if not no_restart and not apply_to_proxy(install_dir, reload=was_on):
|
|
546
|
+
sys.exit(1)
|
|
547
|
+
schedule_renewal(install_dir)
|
|
548
|
+
refresh_edge_info(install_dir)
|
|
549
|
+
print_addresses(install_dir)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
@tls_group.command("renew")
|
|
553
|
+
@click.option(
|
|
554
|
+
"--days", type=click.IntRange(1, tls.CA_DAYS), default=tls.RENEW_BEFORE_DAYS, show_default=True,
|
|
555
|
+
help="Issue a certificate made here again once it has fewer days left than this.",
|
|
556
|
+
)
|
|
557
|
+
@click.option("--force", is_flag=True, help="Issue it again, or take the operator's files again, now.")
|
|
558
|
+
@click.option("--no-wait", is_flag=True, help="Do not wait for the machines when their issuer changed.")
|
|
559
|
+
def renew_cmd(*, days: int, force: bool, no_wait: bool) -> None:
|
|
560
|
+
"""Renew what is due; changes nothing otherwise. Runs every morning by itself once HTTPS is on.
|
|
561
|
+
|
|
562
|
+
Exits 1 when a certificate runs out soon and cannot be renewed from here,
|
|
563
|
+
so a failed run shows (systemctl --user --failed).
|
|
564
|
+
"""
|
|
565
|
+
install_dir = _install_dir()
|
|
566
|
+
# A rotation of the authority is published first, so machines have it
|
|
567
|
+
# before anything it signed is served.
|
|
568
|
+
try:
|
|
569
|
+
_, rotated = tls.ensure_ca(install_dir)
|
|
570
|
+
except tls.TlsError as exc:
|
|
571
|
+
warning(str(exc))
|
|
572
|
+
rotated = False
|
|
573
|
+
try:
|
|
574
|
+
result = tls.renew(install_dir, days=days, force=force)
|
|
575
|
+
except tls.TlsError as exc:
|
|
576
|
+
error(str(exc))
|
|
577
|
+
sys.exit(1)
|
|
578
|
+
stamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC")
|
|
579
|
+
if result.action in ("renewed", "reimported"):
|
|
580
|
+
success(f"{stamp} {result.message}")
|
|
581
|
+
if result.summary is not None:
|
|
582
|
+
print_summary(result.summary)
|
|
583
|
+
_wait_for_machines(result.trust_changed or rotated, no_restart=False, no_wait=no_wait)
|
|
584
|
+
if not apply_to_proxy(install_dir, reload=True):
|
|
585
|
+
sys.exit(1)
|
|
586
|
+
refresh_edge_info(install_dir)
|
|
587
|
+
return
|
|
588
|
+
if result.action == "expiring":
|
|
589
|
+
warning(f"{stamp} {result.message}")
|
|
590
|
+
sys.exit(1)
|
|
591
|
+
info(f"{stamp} {result.message}")
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
@tls_group.command("off")
|
|
595
|
+
@click.option(
|
|
596
|
+
"--no-restart", is_flag=True, help="Remove the files only; the proxy follows when it next starts."
|
|
597
|
+
)
|
|
598
|
+
def off_cmd(*, no_restart: bool) -> None:
|
|
599
|
+
"""Go back to plain HTTP. The install's authority is kept for next time."""
|
|
600
|
+
install_dir = _install_dir()
|
|
601
|
+
tls.remember_off(install_dir)
|
|
602
|
+
if not tls.remove(install_dir):
|
|
603
|
+
info("HTTPS was already off.")
|
|
604
|
+
return
|
|
605
|
+
success("Certificate removed.")
|
|
606
|
+
unschedule_renewal()
|
|
607
|
+
if not no_restart and not apply_to_proxy(install_dir):
|
|
608
|
+
sys.exit(1)
|
|
609
|
+
refresh_edge_info(install_dir)
|
|
610
|
+
urls = addresses(install_dir)
|
|
611
|
+
warning(
|
|
612
|
+
"Machines that followed this server to HTTPS stay there: an agent never falls back to plain HTTP "
|
|
613
|
+
"by itself. Point each back with: faden-agent configure --set BACKEND_URL="
|
|
614
|
+
f"{reachable_example(urls) if urls else 'http://<this server>'}, then restart it."
|
|
615
|
+
)
|
|
616
|
+
env = read_env_file(install_dir / ".env")
|
|
617
|
+
if (env.get("FADEN_HSTS_MAX_AGE") or "0").strip() not in ("", "0"):
|
|
618
|
+
warning(
|
|
619
|
+
"FADEN_HSTS_MAX_AGE is set: browsers that saw it keep insisting on HTTPS until it runs out."
|
|
620
|
+
)
|
|
621
|
+
print_addresses(install_dir)
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
#: Run in the backend container: the machine CA lives in its database, sealed with its master key.
|
|
625
|
+
_MACHINE_CA_SCRIPT = """
|
|
626
|
+
import asyncio, json, sys
|
|
627
|
+
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
|
628
|
+
from faden_backend.settings import settings
|
|
629
|
+
from faden_backend.services.machine_tls import store
|
|
630
|
+
|
|
631
|
+
async def main(replace):
|
|
632
|
+
engine = create_async_engine(str(settings.db_url))
|
|
633
|
+
try:
|
|
634
|
+
async with async_sessionmaker(engine, expire_on_commit=False)() as session:
|
|
635
|
+
if replace:
|
|
636
|
+
await store.rotate(session, settings.settings_master_key)
|
|
637
|
+
await session.commit()
|
|
638
|
+
print(json.dumps({"enabled": settings.machine_tls_enabled, **(await store.status(session))}))
|
|
639
|
+
finally:
|
|
640
|
+
await engine.dispose()
|
|
641
|
+
|
|
642
|
+
asyncio.run(main(sys.argv[1] == "replace"))
|
|
643
|
+
"""
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def _machine_ca(replace: bool) -> dict:
|
|
647
|
+
import json # noqa: PLC0415
|
|
648
|
+
|
|
649
|
+
docker = shutil.which("docker")
|
|
650
|
+
if not docker:
|
|
651
|
+
error("docker not found on PATH")
|
|
652
|
+
sys.exit(1)
|
|
653
|
+
proc = subprocess.run( # noqa: S603
|
|
654
|
+
[docker, "exec", "-i", "faden-backend", "python", "-", "replace" if replace else "status"],
|
|
655
|
+
input=_MACHINE_CA_SCRIPT, capture_output=True, text=True, check=False,
|
|
656
|
+
)
|
|
657
|
+
line = next((ln for ln in reversed(proc.stdout.strip().splitlines()) if ln.startswith("{")), "")
|
|
658
|
+
if proc.returncode != 0 or not line:
|
|
659
|
+
error("Could not read the machine CA from the backend -- is faden-backend running?")
|
|
660
|
+
detail = (proc.stderr or proc.stdout).strip().splitlines()
|
|
661
|
+
if detail:
|
|
662
|
+
error(f" {detail[-1]}")
|
|
663
|
+
sys.exit(1)
|
|
664
|
+
return json.loads(line)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
@tls_group.command("machine-ca")
|
|
668
|
+
@click.option(
|
|
669
|
+
"--replace", is_flag=True, default=False,
|
|
670
|
+
help="Make a new machine CA now and distrust every earlier one (a suspected leak).",
|
|
671
|
+
)
|
|
672
|
+
@click.option("--yes", "-y", is_flag=True, default=False, help="Do not ask before --replace.")
|
|
673
|
+
def machine_ca_cmd(*, replace: bool, yes: bool) -> None:
|
|
674
|
+
"""The machine CA: the TLS between the gateway and the models on the machines.
|
|
675
|
+
|
|
676
|
+
It looks after itself -- renewals, the yearly rotation, the gateway's
|
|
677
|
+
certificate -- in the backend. This shows it; --replace is for a suspected
|
|
678
|
+
leak: the gateway and every machine move to a new CA within a minute, and
|
|
679
|
+
model calls fail until they have.
|
|
680
|
+
"""
|
|
681
|
+
if replace and not yes:
|
|
682
|
+
click.confirm(
|
|
683
|
+
"Replace the machine CA now? Model calls fail for about a minute while the gateway"
|
|
684
|
+
" and the machines take new certificates.",
|
|
685
|
+
abort=True,
|
|
686
|
+
)
|
|
687
|
+
doc = _machine_ca(replace)
|
|
688
|
+
if not doc.get("enabled"):
|
|
689
|
+
console.print(
|
|
690
|
+
"Model traffic TLS is [bold]switched off[/bold] (FADEN_BACKEND_MACHINE_TLS_ENABLED=false).",
|
|
691
|
+
)
|
|
692
|
+
return
|
|
693
|
+
if replace:
|
|
694
|
+
success("Machine CA replaced; the gateway and the machines move to it within a minute.")
|
|
695
|
+
now = datetime.now(UTC)
|
|
696
|
+
for ca in doc.get("authorities") or []:
|
|
697
|
+
until = datetime.fromisoformat(ca["not_after"])
|
|
698
|
+
role = "signs now" if ca.get("issuing") else f"signs from {ca['issuing_from'][:10]}"
|
|
699
|
+
console.print(
|
|
700
|
+
f" Machine CA {ca['fingerprint_sha256'][:16]}… until {until:%Y-%m-%d}"
|
|
701
|
+
f" ({(until - now).days} days), {role} [dim]({ca.get('reason')})[/dim]"
|
|
702
|
+
)
|
|
703
|
+
gateway = doc.get("gateway")
|
|
704
|
+
if gateway:
|
|
705
|
+
until = datetime.fromisoformat(gateway["not_after"])
|
|
706
|
+
state = ""
|
|
707
|
+
if not gateway.get("current"):
|
|
708
|
+
state = " [yellow](from an earlier CA; renewed within the hour)[/yellow]"
|
|
709
|
+
console.print(f" Gateway until {until:%Y-%m-%d} ({(until - now).days} days){state}")
|
|
710
|
+
machines = doc.get("machines") or {}
|
|
711
|
+
for entry in sorted(machines.values(), key=lambda m: ",".join(m.get("names") or [])):
|
|
712
|
+
until = datetime.fromisoformat(entry["not_after"])
|
|
713
|
+
state = ""
|
|
714
|
+
if not entry.get("current"):
|
|
715
|
+
state = " [yellow](from an earlier CA; renews within a minute)[/yellow]"
|
|
716
|
+
console.print(
|
|
717
|
+
f" Machine {', '.join(entry.get('names') or ['?']):<16} until {until:%Y-%m-%d}"
|
|
718
|
+
f" ({(until - now).days} days){state}"
|
|
719
|
+
)
|
|
720
|
+
if not machines:
|
|
721
|
+
console.print(" [dim]No machine has asked for a certificate yet (agents 0.1.16 and later do).[/dim]")
|
|
722
|
+
|