fadenstack 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- faden_cli/__init__.py +16 -0
- faden_cli/__main__.py +9 -0
- faden_cli/bundle/shared/.gitignore +25 -0
- faden_cli/bundle/shared/LICENSE +187 -0
- faden_cli/bundle/shared/NOTICE +15 -0
- faden_cli/bundle/shared/OFFICIAL.md +9 -0
- faden_cli/bundle/shared/README.md +16 -0
- faden_cli/bundle/shared/agent-binaries/.gitignore +5 -0
- faden_cli/bundle/shared/alloy/config.alloy +40 -0
- faden_cli/bundle/shared/clickhouse/config.d/system-logs.xml +26 -0
- faden_cli/bundle/shared/docker-compose.gpu.yaml +34 -0
- faden_cli/bundle/shared/docker-compose.yaml +998 -0
- faden_cli/bundle/shared/grafana/dashboards/templates/vllm-runtime.json +2099 -0
- faden_cli/bundle/shared/grafana/provisioning/dashboards/dashboard.yml +10 -0
- faden_cli/bundle/shared/grafana/provisioning/dashboards/json/faden-overview.json +211 -0
- faden_cli/bundle/shared/grafana/provisioning/datasources/ds.yaml +9 -0
- faden_cli/bundle/shared/grafana/provisioning/datasources/prometheus.yaml +15 -0
- faden_cli/bundle/shared/initdb/01-init.sql +32 -0
- faden_cli/bundle/shared/loki/local-config.yaml +32 -0
- faden_cli/bundle/shared/nginx/locations.conf +245 -0
- faden_cli/bundle/shared/nginx/maintenance.html +239 -0
- faden_cli/bundle/shared/nginx/nginx-tls.conf +79 -0
- faden_cli/bundle/shared/nginx/nginx.conf +44 -0
- faden_cli/bundle/shared/nginx/rate-limits.conf +42 -0
- faden_cli/bundle/shared/nginx/security-headers.conf +22 -0
- faden_cli/bundle/shared/nginx/select-config.sh +37 -0
- faden_cli/bundle/shared/prometheus/prometheus.yml +40 -0
- faden_cli/bundle/shared/rabbitmq/definitions.json.example +37 -0
- faden_cli/bundle/shared/tls/README.md +30 -0
- faden_cli/cli.py +158 -0
- faden_cli/commands/__init__.py +1 -0
- faden_cli/commands/admin.py +161 -0
- faden_cli/commands/backup.py +159 -0
- faden_cli/commands/config.py +146 -0
- faden_cli/commands/deploy.py +949 -0
- faden_cli/commands/dev/__init__.py +1 -0
- faden_cli/commands/dev/dev_doctor.py +46 -0
- faden_cli/commands/dev/dev_down.py +241 -0
- faden_cli/commands/dev/dev_group.py +24 -0
- faden_cli/commands/dev/dev_init.py +710 -0
- faden_cli/commands/dev/dev_pull.py +297 -0
- faden_cli/commands/dev/dev_status.py +151 -0
- faden_cli/commands/dev/dev_up.py +1392 -0
- faden_cli/commands/doctor.py +225 -0
- faden_cli/commands/down.py +75 -0
- faden_cli/commands/logs_cmd.py +41 -0
- faden_cli/commands/maintenance.py +74 -0
- faden_cli/commands/module.py +113 -0
- faden_cli/commands/observe.py +95 -0
- faden_cli/commands/ports.py +91 -0
- faden_cli/commands/rebuild.py +158 -0
- faden_cli/commands/restore.py +139 -0
- faden_cli/commands/runtime_images.py +174 -0
- faden_cli/commands/status.py +71 -0
- faden_cli/commands/tls.py +722 -0
- faden_cli/commands/tune.py +37 -0
- faden_cli/commands/up.py +64 -0
- faden_cli/commands/upgrade.py +470 -0
- faden_cli/commands/version.py +78 -0
- faden_cli/core/__init__.py +1 -0
- faden_cli/core/api_client.py +77 -0
- faden_cli/core/backup.py +717 -0
- faden_cli/core/bootstrap.py +291 -0
- faden_cli/core/bundle.py +110 -0
- faden_cli/core/clickhouse.py +61 -0
- faden_cli/core/compose.py +484 -0
- faden_cli/core/console.py +106 -0
- faden_cli/core/detect.py +580 -0
- faden_cli/core/dev_modules.py +149 -0
- faden_cli/core/docker_env.py +147 -0
- faden_cli/core/edge.py +94 -0
- faden_cli/core/env_gen.py +598 -0
- faden_cli/core/git.py +219 -0
- faden_cli/core/install.py +288 -0
- faden_cli/core/installation.py +186 -0
- faden_cli/core/local_node.py +719 -0
- faden_cli/core/maintenance.py +82 -0
- faden_cli/core/private_files.py +133 -0
- faden_cli/core/registry.py +574 -0
- faden_cli/core/release_feed.py +342 -0
- faden_cli/core/rmq.py +135 -0
- faden_cli/core/runtime_images.py +309 -0
- faden_cli/core/schedule.py +278 -0
- faden_cli/core/settings.py +159 -0
- faden_cli/core/sysinfo.py +303 -0
- faden_cli/core/tls.py +790 -0
- faden_cli/core/tune.py +154 -0
- faden_cli/core/update_check.py +254 -0
- faden_cli/core/workspace.py +155 -0
- faden_cli/templates/env.j2 +33 -0
- faden_cli/tui/__init__.py +1 -0
- faden_cli/tui/widgets/__init__.py +1 -0
- faden_cli/tui/wizard/__init__.py +1 -0
- fadenstack-0.4.2.dist-info/METADATA +304 -0
- fadenstack-0.4.2.dist-info/RECORD +99 -0
- fadenstack-0.4.2.dist-info/WHEEL +4 -0
- fadenstack-0.4.2.dist-info/entry_points.txt +3 -0
- fadenstack-0.4.2.dist-info/licenses/LICENSE +187 -0
- fadenstack-0.4.2.dist-info/licenses/NOTICE +15 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""``faden tune`` — auto-detect system resources and generate scalability settings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from faden_cli.core.settings import load_config
|
|
10
|
+
from faden_cli.core.tune import tune_env
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.command("tune")
|
|
14
|
+
@click.option(
|
|
15
|
+
"--profile",
|
|
16
|
+
type=click.Choice(["dev", "prod"]),
|
|
17
|
+
default=None,
|
|
18
|
+
help="Tuning profile: dev (conservative) or prod (production-grade). "
|
|
19
|
+
"Default: the one the install was last tuned with, else its environment's.",
|
|
20
|
+
)
|
|
21
|
+
@click.option(
|
|
22
|
+
"--resource-pct",
|
|
23
|
+
type=click.FloatRange(0.1, 1.0),
|
|
24
|
+
default=None,
|
|
25
|
+
help="Fraction of host CPU to allocate to the control plane (prod only). "
|
|
26
|
+
"Default: the last one used, else 0.25.",
|
|
27
|
+
)
|
|
28
|
+
@click.option(
|
|
29
|
+
"--dry-run",
|
|
30
|
+
is_flag=True,
|
|
31
|
+
default=False,
|
|
32
|
+
help="Show computed values without writing to .env.",
|
|
33
|
+
)
|
|
34
|
+
def tune_cmd(*, profile: str | None, resource_pct: float | None, dry_run: bool) -> None:
|
|
35
|
+
"""Detect host resources and write optimal scalability settings to .env."""
|
|
36
|
+
cfg = load_config()
|
|
37
|
+
tune_env(Path(cfg.install_dir) / ".env", profile=profile, resource_pct=resource_pct, dry_run=dry_run)
|
faden_cli/commands/up.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""``faden up`` — start Fadenstack services."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from faden_cli.core.compose import build_context_from_config, up as compose_up
|
|
10
|
+
from faden_cli.core.console import console, success, error
|
|
11
|
+
from faden_cli.core.env_gen import write_env_file, default_env_vars
|
|
12
|
+
from faden_cli.core.settings import load_config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@click.command("up")
|
|
16
|
+
@click.option("-d", "--detach/--no-detach", default=True, help="Run in detached mode (default).")
|
|
17
|
+
@click.option("--build", "do_build", is_flag=True, help="Build images before starting.")
|
|
18
|
+
@click.option("--pull/--no-pull", default=False, help="Pull images before starting.")
|
|
19
|
+
@click.option("--env-gen/--no-env-gen", "gen_env", default=False, help="Regenerate .env file before starting.")
|
|
20
|
+
@click.argument("services", nargs=-1)
|
|
21
|
+
def up_cmd(
|
|
22
|
+
*,
|
|
23
|
+
detach: bool,
|
|
24
|
+
do_build: bool,
|
|
25
|
+
pull: bool,
|
|
26
|
+
gen_env: bool,
|
|
27
|
+
services: tuple[str, ...],
|
|
28
|
+
) -> None:
|
|
29
|
+
"""Start Fadenstack infrastructure and services.
|
|
30
|
+
|
|
31
|
+
Optionally pass SERVICE names to start only specific containers.
|
|
32
|
+
"""
|
|
33
|
+
cfg = load_config()
|
|
34
|
+
|
|
35
|
+
# Generate .env if requested or if it doesn't exist
|
|
36
|
+
if gen_env:
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
|
|
39
|
+
env_path = Path(cfg.install_dir) / ".env"
|
|
40
|
+
console.print("[cyan]Generating .env file…[/cyan]")
|
|
41
|
+
env_vars = default_env_vars(profiles=cfg.profiles)
|
|
42
|
+
write_env_file(env_path, env_vars, preserve_secrets=True)
|
|
43
|
+
success(f"Environment file written to {env_path}")
|
|
44
|
+
from faden_cli.core.tune import tune_env # noqa: PLC0415
|
|
45
|
+
|
|
46
|
+
tune_env(env_path)
|
|
47
|
+
|
|
48
|
+
ctx = build_context_from_config(cfg)
|
|
49
|
+
|
|
50
|
+
console.print("[bold cyan]Starting Fadenstack services…[/bold cyan]")
|
|
51
|
+
|
|
52
|
+
returncode = compose_up(
|
|
53
|
+
ctx,
|
|
54
|
+
detach=detach,
|
|
55
|
+
build=do_build,
|
|
56
|
+
pull=pull,
|
|
57
|
+
services=list(services) if services else None,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
if returncode == 0:
|
|
61
|
+
success("Services started successfully.")
|
|
62
|
+
else:
|
|
63
|
+
error(f"docker compose up exited with code {returncode}.")
|
|
64
|
+
sys.exit(returncode)
|
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
"""``faden upgrade`` — upgrade the running Fadenstack installation.
|
|
2
|
+
|
|
3
|
+
Performs a safe in-place upgrade:
|
|
4
|
+
1. Pre-flight checks (Docker, disk)
|
|
5
|
+
2. Auto-backup (unless --no-backup)
|
|
6
|
+
3. Refresh the deployment files and .env (existing values kept)
|
|
7
|
+
4. Pull this release's images (an install made by ``faden deploy``
|
|
8
|
+
from the published images), or rebuild them (a source checkout)
|
|
9
|
+
5. Rolling restart: infra → migrators → app → modules
|
|
10
|
+
6. Health gate: poll backend /api/health
|
|
11
|
+
7. Print summary
|
|
12
|
+
|
|
13
|
+
Examples:
|
|
14
|
+
faden upgrade # standard upgrade
|
|
15
|
+
faden upgrade --no-backup # skip pre-upgrade backup
|
|
16
|
+
faden upgrade --no-build # skip image rebuild
|
|
17
|
+
faden upgrade --dry-run # show what would happen
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import sys
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import click
|
|
26
|
+
|
|
27
|
+
from faden_cli import __version__
|
|
28
|
+
from faden_cli.core.backup import create_backup
|
|
29
|
+
from faden_cli.core.bundle import installed_release, unpack
|
|
30
|
+
from faden_cli.core.compose import (
|
|
31
|
+
ComposeContext,
|
|
32
|
+
build as compose_build,
|
|
33
|
+
build_base_image,
|
|
34
|
+
build_context_from_config,
|
|
35
|
+
pull_base_images,
|
|
36
|
+
up as compose_up,
|
|
37
|
+
)
|
|
38
|
+
from faden_cli.core import installation
|
|
39
|
+
from faden_cli.core.console import console, error, info, success, warning
|
|
40
|
+
from faden_cli.core.detect import detect_docker
|
|
41
|
+
from faden_cli.core.env_gen import default_env_vars, read_env_file, write_env_file
|
|
42
|
+
from faden_cli.core.settings import load_config, save_config
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@click.command("upgrade")
|
|
46
|
+
@click.option("--no-backup", is_flag=True, default=False, help="Skip pre-upgrade backup.")
|
|
47
|
+
@click.option(
|
|
48
|
+
"--backup-dir",
|
|
49
|
+
type=click.Path(file_okay=False),
|
|
50
|
+
default="backups",
|
|
51
|
+
show_default=True,
|
|
52
|
+
help="Directory for the pre-upgrade backup.",
|
|
53
|
+
)
|
|
54
|
+
@click.option("--no-build", is_flag=True, default=False, help="Skip image rebuild.")
|
|
55
|
+
@click.option("--no-cache", is_flag=True, default=False, help="Build images without cache.")
|
|
56
|
+
@click.option("--skip-doctor", is_flag=True, default=False, help="Skip pre-flight checks.")
|
|
57
|
+
@click.option(
|
|
58
|
+
"--runtime-images",
|
|
59
|
+
"runtime_images",
|
|
60
|
+
default=None,
|
|
61
|
+
metavar="ARCHS",
|
|
62
|
+
help=(
|
|
63
|
+
"Runtime images to fetch for machines: all, none, or architectures such as "
|
|
64
|
+
"x86_64,aarch64. Default: what deploy chose (all)."
|
|
65
|
+
),
|
|
66
|
+
)
|
|
67
|
+
@click.option(
|
|
68
|
+
"--dry-run",
|
|
69
|
+
is_flag=True,
|
|
70
|
+
default=False,
|
|
71
|
+
help="Show what would happen without executing.",
|
|
72
|
+
)
|
|
73
|
+
@click.option("-y", "--yes", is_flag=True, default=False, help="Skip confirmation prompt.")
|
|
74
|
+
def upgrade_cmd(
|
|
75
|
+
*,
|
|
76
|
+
no_backup: bool,
|
|
77
|
+
backup_dir: str,
|
|
78
|
+
no_build: bool,
|
|
79
|
+
no_cache: bool,
|
|
80
|
+
skip_doctor: bool,
|
|
81
|
+
runtime_images: str | None,
|
|
82
|
+
dry_run: bool,
|
|
83
|
+
yes: bool,
|
|
84
|
+
) -> None:
|
|
85
|
+
"""Upgrade the running Fadenstack installation in place.
|
|
86
|
+
|
|
87
|
+
Creates a backup, refreshes configuration (preserving secrets),
|
|
88
|
+
rebuilds container images, and performs a rolling restart with
|
|
89
|
+
a health gate. Then fetches the runtime images this release pins, which
|
|
90
|
+
machines download from this server.
|
|
91
|
+
"""
|
|
92
|
+
console.print("\n[bold magenta]fadenstack — Upgrade[/bold magenta]\n")
|
|
93
|
+
|
|
94
|
+
cfg = load_config()
|
|
95
|
+
|
|
96
|
+
if not cfg.install_dir:
|
|
97
|
+
error(
|
|
98
|
+
"No install directory configured.\n"
|
|
99
|
+
" Run 'faden deploy' first, or set install_dir in faden.yaml."
|
|
100
|
+
)
|
|
101
|
+
sys.exit(1)
|
|
102
|
+
|
|
103
|
+
shared_dir = cfg.install_path
|
|
104
|
+
compose_file = shared_dir / cfg.compose_file
|
|
105
|
+
if not compose_file.exists():
|
|
106
|
+
error(f"Compose file not found: {compose_file}")
|
|
107
|
+
sys.exit(1)
|
|
108
|
+
|
|
109
|
+
env_path = shared_dir / ".env"
|
|
110
|
+
|
|
111
|
+
# An install made from the published images moves to this CLI's release;
|
|
112
|
+
# a source checkout rebuilds from its own code.
|
|
113
|
+
release = installed_release(shared_dir)
|
|
114
|
+
if release is not None and _older(__version__, release):
|
|
115
|
+
error(
|
|
116
|
+
f"This CLI ({__version__}) is older than the install ({release}).\n"
|
|
117
|
+
" Its database has already been migrated past this release; going back is a restore:\n"
|
|
118
|
+
" faden restore <install_dir>/backups/<time>\n"
|
|
119
|
+
" To upgrade, install the newer CLI first: pip install -U fadenstack"
|
|
120
|
+
)
|
|
121
|
+
sys.exit(1)
|
|
122
|
+
# Some releases cannot be skipped (a data migration the next one builds on): the feed says which.
|
|
123
|
+
if release is not None and _older(release, __version__):
|
|
124
|
+
first = _required_first(release)
|
|
125
|
+
if first is not None:
|
|
126
|
+
from faden_cli.core.release_feed import cli_install_command # noqa: PLC0415
|
|
127
|
+
from faden_cli.core.update_check import installer # noqa: PLC0415
|
|
128
|
+
|
|
129
|
+
error(
|
|
130
|
+
f"{release} cannot upgrade straight to {__version__}: {first} has to come first.\n"
|
|
131
|
+
f" {cli_install_command(installer(), first, latest=False)}\n"
|
|
132
|
+
" faden upgrade\n"
|
|
133
|
+
f" Then install {__version__} again and run faden upgrade once more."
|
|
134
|
+
)
|
|
135
|
+
sys.exit(1)
|
|
136
|
+
|
|
137
|
+
# Build compose file list (include GPU overlay if present)
|
|
138
|
+
from faden_cli.core.compose import has_nvidia_gpu # noqa: PLC0415
|
|
139
|
+
|
|
140
|
+
compose_files: list[Path] = [compose_file]
|
|
141
|
+
gpu_overlay = shared_dir / "docker-compose.gpu.yaml"
|
|
142
|
+
if has_nvidia_gpu() and gpu_overlay.exists():
|
|
143
|
+
compose_files.append(gpu_overlay)
|
|
144
|
+
|
|
145
|
+
ctx = ComposeContext(
|
|
146
|
+
compose_files=compose_files,
|
|
147
|
+
env_file=env_path if env_path.exists() else None,
|
|
148
|
+
project_dir=shared_dir,
|
|
149
|
+
profiles=list(cfg.profiles),
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# ── Dry-run summary ───────────────────────────────────────
|
|
153
|
+
if dry_run:
|
|
154
|
+
console.print("[bold]Dry-run — no changes will be made.[/bold]\n")
|
|
155
|
+
console.print(f" Install directory: {shared_dir}")
|
|
156
|
+
console.print(f" Compose file: {compose_file.name}")
|
|
157
|
+
console.print(f" Profiles: {', '.join(cfg.profiles) or '(none)'}")
|
|
158
|
+
console.print(f" Backup: {'skip' if no_backup else backup_dir}")
|
|
159
|
+
if release is not None:
|
|
160
|
+
console.print(f" Release: {release} -> {__version__} (pull published images)")
|
|
161
|
+
newer = _newer_than_cli()
|
|
162
|
+
if newer:
|
|
163
|
+
console.print(f" [yellow]Newer release: {newer}[/yellow]")
|
|
164
|
+
else:
|
|
165
|
+
console.print(f" Build images: {'skip' if no_build else 'yes'}")
|
|
166
|
+
console.print(f" Build cache: {'no' if no_cache else 'yes'}")
|
|
167
|
+
console.print(f" Runtime images: {runtime_images or cfg.runtime_images} (for machines)")
|
|
168
|
+
return
|
|
169
|
+
|
|
170
|
+
# ── Confirm ───────────────────────────────────────────────
|
|
171
|
+
if not yes:
|
|
172
|
+
click.confirm("Proceed with upgrade?", default=True, abort=True)
|
|
173
|
+
|
|
174
|
+
step = 0
|
|
175
|
+
|
|
176
|
+
# ── 1. Pre-flight checks ─────────────────────────────────
|
|
177
|
+
if not skip_doctor:
|
|
178
|
+
step += 1
|
|
179
|
+
console.print(f"\n[bold cyan]Step {step}: Pre-flight checks…[/bold cyan]")
|
|
180
|
+
|
|
181
|
+
docker = detect_docker()
|
|
182
|
+
if not docker.ok:
|
|
183
|
+
error("Docker is not available or the daemon is not running.")
|
|
184
|
+
if docker.error:
|
|
185
|
+
error(f" {docker.error}")
|
|
186
|
+
sys.exit(1)
|
|
187
|
+
success(
|
|
188
|
+
f"Docker {docker.version}, "
|
|
189
|
+
f"Compose {docker.compose_version}, "
|
|
190
|
+
f"daemon running"
|
|
191
|
+
)
|
|
192
|
+
else:
|
|
193
|
+
console.print("\n[dim]Skipping pre-flight checks (--skip-doctor).[/dim]")
|
|
194
|
+
|
|
195
|
+
# The compose file has no fallback for its passwords, keys and tokens any
|
|
196
|
+
# more, and every compose command stops on one missing -- the checks and
|
|
197
|
+
# the backup's too. So an older .env gets them first. A missing key that
|
|
198
|
+
# seals stored data keeps the value this install ran on
|
|
199
|
+
# (env_gen.fill_required_env).
|
|
200
|
+
if env_path.exists():
|
|
201
|
+
from faden_cli.core.env_gen import fill_required_env # noqa: PLC0415
|
|
202
|
+
|
|
203
|
+
for note in fill_required_env(env_path, profiles=list(cfg.profiles)):
|
|
204
|
+
warning(note)
|
|
205
|
+
|
|
206
|
+
# Before the backup and the builds, not after: compose cannot replace a
|
|
207
|
+
# container it did not create, and the restart would stop on it.
|
|
208
|
+
from faden_cli.core.compose import foreign_containers # noqa: PLC0415
|
|
209
|
+
|
|
210
|
+
foreign = foreign_containers(ctx)
|
|
211
|
+
if foreign:
|
|
212
|
+
error(
|
|
213
|
+
"These containers have names this installation uses, but it did not create them,\n"
|
|
214
|
+
" so it cannot replace them:\n"
|
|
215
|
+
+ "".join(
|
|
216
|
+
f" {name} (made by {f'compose project {owner}' if owner else 'docker run'})\n"
|
|
217
|
+
for name, owner in foreign
|
|
218
|
+
)
|
|
219
|
+
+ " Remove them -- their data volumes stay -- and run the upgrade again:\n"
|
|
220
|
+
f" docker rm -f {' '.join(name for name, _ in foreign)}"
|
|
221
|
+
)
|
|
222
|
+
sys.exit(1)
|
|
223
|
+
|
|
224
|
+
# ── 2. Pre-upgrade backup ─────────────────────────────────
|
|
225
|
+
if not no_backup:
|
|
226
|
+
step += 1
|
|
227
|
+
console.print(f"\n[bold cyan]Step {step}: Pre-upgrade backup…[/bold cyan]")
|
|
228
|
+
|
|
229
|
+
backup_path = Path(backup_dir)
|
|
230
|
+
if not backup_path.is_absolute():
|
|
231
|
+
backup_path = shared_dir / backup_path
|
|
232
|
+
|
|
233
|
+
result = create_backup(ctx, output_dir=backup_path)
|
|
234
|
+
if not result.ok:
|
|
235
|
+
for msg in result.errors:
|
|
236
|
+
error(msg)
|
|
237
|
+
error("Backup failed — aborting upgrade. Fix the issue or use --no-backup.")
|
|
238
|
+
sys.exit(1)
|
|
239
|
+
success(f"Backup created: {result.backup_dir}")
|
|
240
|
+
else:
|
|
241
|
+
console.print("\n[dim]Skipping backup (--no-backup).[/dim]")
|
|
242
|
+
|
|
243
|
+
# ── 3. Refresh .env (preserve secrets) ────────────────────
|
|
244
|
+
step += 1
|
|
245
|
+
console.print(f"\n[bold cyan]Step {step}: Refreshing environment…[/bold cyan]")
|
|
246
|
+
|
|
247
|
+
if release is not None:
|
|
248
|
+
changed = unpack(shared_dir)
|
|
249
|
+
success(f"Deployment files for release {__version__} ({len(changed)} changed).")
|
|
250
|
+
|
|
251
|
+
if env_path.exists():
|
|
252
|
+
# What the install has wins; a new release only adds what it
|
|
253
|
+
# introduced. Rewritten from the defaults with just the secrets kept,
|
|
254
|
+
# every upgrade dropped what deploy and the operator had set --
|
|
255
|
+
# HF_CACHE_DIR (the host's model cache), the admin name synced to
|
|
256
|
+
# Grafana, ports -- and the services came back without them.
|
|
257
|
+
env_vars = default_env_vars(profiles=list(cfg.profiles))
|
|
258
|
+
env_vars.update(read_env_file(env_path))
|
|
259
|
+
write_env_file(env_path, env_vars)
|
|
260
|
+
from faden_cli.commands.deploy import pin_release, record_installer # noqa: PLC0415
|
|
261
|
+
|
|
262
|
+
if release is not None:
|
|
263
|
+
pin_release(env_path)
|
|
264
|
+
else:
|
|
265
|
+
record_installer(env_path, shared_dir)
|
|
266
|
+
success(".env refreshed (existing settings kept).")
|
|
267
|
+
# Sized again, as deploy does: what ``faden tune`` last chose.
|
|
268
|
+
from faden_cli.core.tune import tune_env # noqa: PLC0415
|
|
269
|
+
|
|
270
|
+
tune_env(env_path)
|
|
271
|
+
# RabbitMQ makes its users from definitions.json on every start; the
|
|
272
|
+
# services log in with the passwords in .env. Written only by deploy,
|
|
273
|
+
# the file kept an older install's passwords, and after an upgrade
|
|
274
|
+
# the gateway and backend were refused ("invalid credentials").
|
|
275
|
+
from faden_cli.commands.deploy import _regenerate_rmq_definitions # noqa: PLC0415
|
|
276
|
+
|
|
277
|
+
_regenerate_rmq_definitions(shared_dir, read_env_file(env_path), cfg.profiles)
|
|
278
|
+
success("RabbitMQ users written from .env.")
|
|
279
|
+
else:
|
|
280
|
+
warning("No .env file found — skipping env refresh.")
|
|
281
|
+
|
|
282
|
+
# .env, the saved credentials and the install's backups, readable by this user only -- an install
|
|
283
|
+
# made before kept them readable by every local user. (A --backup-dir of the operator's keeps its
|
|
284
|
+
# mode; the backup written into it is private.)
|
|
285
|
+
from faden_cli.core.private_files import secure_install # noqa: PLC0415
|
|
286
|
+
|
|
287
|
+
secure_install(shared_dir)
|
|
288
|
+
|
|
289
|
+
# The authority machines keep for when HTTPS is switched on (faden tls):
|
|
290
|
+
# made while they can still fetch it over plain HTTP.
|
|
291
|
+
from faden_cli.core import tls as edge_tls # noqa: PLC0415
|
|
292
|
+
|
|
293
|
+
try:
|
|
294
|
+
ca_public, ca_created = edge_tls.ensure_ca(shared_dir)
|
|
295
|
+
except edge_tls.TlsError as exc:
|
|
296
|
+
warning(str(exc))
|
|
297
|
+
else:
|
|
298
|
+
if ca_created:
|
|
299
|
+
success(f"Certificate authority for the machines made: {ca_public}")
|
|
300
|
+
from faden_cli.commands.tls import refresh_edge_info # noqa: PLC0415
|
|
301
|
+
|
|
302
|
+
refresh_edge_info(shared_dir)
|
|
303
|
+
|
|
304
|
+
# ── 4. Images ─────────────────────────────────────────────
|
|
305
|
+
if release is not None:
|
|
306
|
+
step += 1
|
|
307
|
+
console.print(f"\n[bold cyan]Step {step}: Pulling release {__version__}…[/bold cyan]")
|
|
308
|
+
from faden_cli.core.compose import pull as compose_pull # noqa: PLC0415
|
|
309
|
+
|
|
310
|
+
rc = compose_pull(ctx)
|
|
311
|
+
if rc != 0:
|
|
312
|
+
error("Could not pull this release's images; nothing was restarted.")
|
|
313
|
+
sys.exit(rc)
|
|
314
|
+
success("Images pulled.")
|
|
315
|
+
elif not no_build:
|
|
316
|
+
step += 1
|
|
317
|
+
console.print(f"\n[bold cyan]Step {step}: Building container images…[/bold cyan]")
|
|
318
|
+
console.print("[dim]This may take several minutes.[/dim]")
|
|
319
|
+
|
|
320
|
+
pull_base_images()
|
|
321
|
+
|
|
322
|
+
info("Building platform base image…")
|
|
323
|
+
rc = build_base_image(ctx, no_cache=no_cache)
|
|
324
|
+
if rc != 0:
|
|
325
|
+
warning("Base image build failed — service builds may fail.")
|
|
326
|
+
else:
|
|
327
|
+
success("Base image built.")
|
|
328
|
+
|
|
329
|
+
rc = compose_build(ctx, no_cache=no_cache)
|
|
330
|
+
installation.record_build(shared_dir, complete=rc == 0)
|
|
331
|
+
if rc != 0:
|
|
332
|
+
warning("Some images failed to build. Continuing with available images.")
|
|
333
|
+
else:
|
|
334
|
+
success("All images built successfully.")
|
|
335
|
+
else:
|
|
336
|
+
console.print("\n[dim]Skipping image build (--no-build).[/dim]")
|
|
337
|
+
|
|
338
|
+
# ── 5. Rolling restart ────────────────────────────────────
|
|
339
|
+
# Behind the maintenance page, from here until the backend is healthy:
|
|
340
|
+
# users see "being updated" rather than a gateway error, and an open
|
|
341
|
+
# console reloads into that page and comes back by itself afterwards.
|
|
342
|
+
# Lifted however this ends -- a failed upgrade is better reached.
|
|
343
|
+
from faden_cli.core import maintenance # noqa: PLC0415
|
|
344
|
+
|
|
345
|
+
with maintenance.during(shared_dir, reason="upgrade"):
|
|
346
|
+
step = _restart_and_check(ctx, shared_dir, env_path, step)
|
|
347
|
+
|
|
348
|
+
# ── 7. Runtime images ─────────────────────────────────────
|
|
349
|
+
# After the restart, not before it: the service images are what the
|
|
350
|
+
# upgrade waits on, and a runtime image is several gigabytes that only
|
|
351
|
+
# matters once a machine starts a cluster.
|
|
352
|
+
step += 1
|
|
353
|
+
console.print(f"\n[bold cyan]Step {step}: Runtime images for machines…[/bold cyan]")
|
|
354
|
+
from faden_cli.commands.runtime_images import run_step # noqa: PLC0415
|
|
355
|
+
|
|
356
|
+
run_step(cfg, value=runtime_images)
|
|
357
|
+
|
|
358
|
+
# ── Done ──────────────────────────────────────────────────
|
|
359
|
+
console.print()
|
|
360
|
+
success("Upgrade complete.")
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _restart_and_check(ctx: ComposeContext, shared_dir: Path, env_path: Path, step: int) -> int:
|
|
364
|
+
"""Steps 5 and 6: restart on the new images, then wait for the backend; returns the last step's number."""
|
|
365
|
+
step += 1
|
|
366
|
+
console.print(f"\n[bold cyan]Step {step}: Rolling restart…[/bold cyan]")
|
|
367
|
+
|
|
368
|
+
# Sync postgres password before restarting (same as deploy)
|
|
369
|
+
from faden_cli.commands.deploy import _sync_postgres_password, retire_fixed_logins # noqa: PLC0415
|
|
370
|
+
|
|
371
|
+
_sync_postgres_password(ctx, env_path)
|
|
372
|
+
# Earlier releases made two logins with fixed passwords on every install.
|
|
373
|
+
retire_fixed_logins(env_path)
|
|
374
|
+
|
|
375
|
+
# The front door first, on its own: recreated with this release's config
|
|
376
|
+
# it serves the maintenance page while the rest restarts -- even on the
|
|
377
|
+
# first upgrade to a release that has one. Then everything else, so it
|
|
378
|
+
# is not recreated a second time (a few seconds of nothing answering).
|
|
379
|
+
from faden_cli.core.compose import service_names # noqa: PLC0415
|
|
380
|
+
|
|
381
|
+
front = "nginx"
|
|
382
|
+
names = service_names(ctx)
|
|
383
|
+
if front in names:
|
|
384
|
+
rc = compose_up(ctx, services=[front], no_deps=True, pull="missing", force_recreate=True)
|
|
385
|
+
if rc != 0:
|
|
386
|
+
warning("The front door did not restart first; it restarts with the rest.")
|
|
387
|
+
names = []
|
|
388
|
+
rest = [name for name in names if name != front] if front in names and names else None
|
|
389
|
+
|
|
390
|
+
# Bring everything up (force-recreate to pick up new images)
|
|
391
|
+
rc = compose_up(
|
|
392
|
+
ctx, services=rest, detach=True, build=False, pull="missing",
|
|
393
|
+
force_recreate=True, wait=True, timeout=180,
|
|
394
|
+
)
|
|
395
|
+
if rc != 0:
|
|
396
|
+
error(f"docker compose up failed (exit code {rc}).")
|
|
397
|
+
sys.exit(rc)
|
|
398
|
+
success("All services restarted.")
|
|
399
|
+
|
|
400
|
+
# ClickHouse's diagnostic logs are off from this release; what they hold
|
|
401
|
+
# stays until dropped (41 GB of trace_log filled one install's disk).
|
|
402
|
+
from faden_cli.core.clickhouse import drop_stale_logs # noqa: PLC0415
|
|
403
|
+
|
|
404
|
+
try:
|
|
405
|
+
dropped = drop_stale_logs(ctx)
|
|
406
|
+
except RuntimeError as exc:
|
|
407
|
+
warning(f"Could not clear ClickHouse's old diagnostic logs: {exc}")
|
|
408
|
+
else:
|
|
409
|
+
if dropped:
|
|
410
|
+
success(f"Dropped ClickHouse's old diagnostic logs: {', '.join(dropped)}.")
|
|
411
|
+
|
|
412
|
+
# ── 6. Health gate ────────────────────────────────────────
|
|
413
|
+
step += 1
|
|
414
|
+
console.print(f"\n[bold cyan]Step {step}: Health check…[/bold cyan]")
|
|
415
|
+
|
|
416
|
+
from faden_cli.core.bootstrap import wait_for_backend # noqa: PLC0415
|
|
417
|
+
from faden_cli.core.edge import console_url # noqa: PLC0415
|
|
418
|
+
|
|
419
|
+
backend_url = console_url(shared_dir, read_env_file(env_path) if env_path.exists() else {})
|
|
420
|
+
|
|
421
|
+
console.print(" [dim]Waiting for backend to become healthy…[/dim]")
|
|
422
|
+
if wait_for_backend(backend_url, timeout=120):
|
|
423
|
+
success("Backend is healthy.")
|
|
424
|
+
else:
|
|
425
|
+
warning(
|
|
426
|
+
"Backend did not become healthy within 120 s.\n"
|
|
427
|
+
" Check logs: faden logs backend"
|
|
428
|
+
)
|
|
429
|
+
return step
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _older(a: str, b: str) -> bool:
|
|
433
|
+
"""Whether release *a* comes before release *b* (``0.3.0`` < ``0.10.0``, ``0.5.0rc1`` < ``0.5.0``)."""
|
|
434
|
+
from faden_cli.core.release_feed import is_newer # noqa: PLC0415
|
|
435
|
+
|
|
436
|
+
return is_newer(b, a)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _newer_than_cli() -> str | None:
|
|
440
|
+
"""A release newer than this CLI's, from the last update check: the upgrade would stop short of it."""
|
|
441
|
+
from faden_cli.core import release_feed, update_check # noqa: PLC0415
|
|
442
|
+
|
|
443
|
+
cfg = load_config()
|
|
444
|
+
feed = update_check.cached_feed() if update_check.enabled(cfg) else None
|
|
445
|
+
if feed is None:
|
|
446
|
+
return None
|
|
447
|
+
found = release_feed.assess(feed, __version__, cfg.update_channel)
|
|
448
|
+
commands = release_feed.upgrade_commands(update_check.installer(), found)
|
|
449
|
+
if not found.update_available or found.latest is None or not commands:
|
|
450
|
+
return None
|
|
451
|
+
return (
|
|
452
|
+
f"{found.latest.version} is out; this CLI installs {__version__}. "
|
|
453
|
+
f"For the newer one first: {commands[0]}"
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _required_first(installed: str) -> str | None:
|
|
458
|
+
"""The release *installed* has to pass through before this CLI's, when the release feed says one."""
|
|
459
|
+
from faden_cli.core import release_feed, update_check # noqa: PLC0415
|
|
460
|
+
|
|
461
|
+
cfg = load_config()
|
|
462
|
+
feed = None
|
|
463
|
+
if update_check.enabled(cfg):
|
|
464
|
+
try:
|
|
465
|
+
feed = update_check.refresh(cfg)
|
|
466
|
+
except Exception: # offline: the last check, if any
|
|
467
|
+
feed = update_check.cached_feed()
|
|
468
|
+
else:
|
|
469
|
+
feed = update_check.cached_feed()
|
|
470
|
+
return release_feed.required_first(feed, installed, __version__) if feed is not None else None
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""``faden version`` — print version and runtime information."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import platform
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
|
|
10
|
+
from faden_cli import __version__
|
|
11
|
+
from faden_cli.core import installation, release_feed, update_check
|
|
12
|
+
from faden_cli.core.console import console, kv_table
|
|
13
|
+
from faden_cli.core.detect import detect_docker
|
|
14
|
+
from faden_cli.core.settings import FadenConfig, load_config
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@click.command("version")
|
|
18
|
+
@click.option("--short", is_flag=True, help="Print only the CLI version string.")
|
|
19
|
+
def version_cmd(*, short: bool) -> None:
|
|
20
|
+
"""Display version and runtime information."""
|
|
21
|
+
if short:
|
|
22
|
+
click.echo(__version__)
|
|
23
|
+
return
|
|
24
|
+
|
|
25
|
+
docker = detect_docker()
|
|
26
|
+
|
|
27
|
+
rows: list[tuple[str, str]] = [
|
|
28
|
+
("faden CLI", __version__),
|
|
29
|
+
("Python", f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"),
|
|
30
|
+
("Platform", platform.platform()),
|
|
31
|
+
("Docker", docker.version or "[red]not found[/red]"),
|
|
32
|
+
("Compose", docker.compose_version or "[red]not found[/red]"),
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
kv_table("Fadenstack CLI", rows)
|
|
36
|
+
console.print()
|
|
37
|
+
|
|
38
|
+
# What the installation this CLI looks after runs: the CLI's own version says nothing about that.
|
|
39
|
+
cfg = load_config()
|
|
40
|
+
if cfg.install_dir:
|
|
41
|
+
kv_table("Fadenstack installation", installation.describe(cfg.install_path))
|
|
42
|
+
console.print()
|
|
43
|
+
|
|
44
|
+
kv_table("Releases", release_rows(cfg))
|
|
45
|
+
console.print()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def release_rows(cfg: FadenConfig) -> list[tuple[str, str]]:
|
|
49
|
+
"""What the release feed says about this CLI's release, checked now (cached when offline)."""
|
|
50
|
+
if not update_check.enabled(cfg):
|
|
51
|
+
return [("Update check", "off (update_check in faden.yaml, or FADEN_NO_UPDATE_CHECK)")]
|
|
52
|
+
rows: list[tuple[str, str]] = []
|
|
53
|
+
try:
|
|
54
|
+
feed: release_feed.Feed | None = update_check.refresh(cfg)
|
|
55
|
+
except Exception as exc: # offline is an answer, not a failure
|
|
56
|
+
feed = update_check.cached_feed()
|
|
57
|
+
rows.append(("Update check", f"[yellow]could not read {update_check.feed_url(cfg)}: {exc}[/yellow]"))
|
|
58
|
+
if feed is None:
|
|
59
|
+
return rows
|
|
60
|
+
found = release_feed.assess(feed, __version__, cfg.update_channel)
|
|
61
|
+
latest = found.latest.version if found.latest else "-"
|
|
62
|
+
rows.append((f"Newest ({found.channel})", latest))
|
|
63
|
+
if not found.update_available:
|
|
64
|
+
rows.append(("Status", "[green]up to date[/green]"))
|
|
65
|
+
else:
|
|
66
|
+
colour = {"critical": "red", "security": "yellow"}.get(found.severity, "cyan")
|
|
67
|
+
rows.append(("Status", f"[{colour}]update available ({found.severity})[/{colour}]"))
|
|
68
|
+
if len(found.path) > 1:
|
|
69
|
+
rows.append(("Path", " -> ".join(found.path)))
|
|
70
|
+
commands = release_feed.upgrade_commands(update_check.upgrade_kind(cfg), found)
|
|
71
|
+
if commands:
|
|
72
|
+
rows.append(("Upgrade", "\n".join(commands)))
|
|
73
|
+
if found.latest and found.latest.notes_url:
|
|
74
|
+
rows.append(("Notes", found.latest.notes_url))
|
|
75
|
+
if found.unsupported:
|
|
76
|
+
until = found.supported_until
|
|
77
|
+
rows.append(("Support", f"[red]{found.current} no longer gets fixes (since {until})[/red]"))
|
|
78
|
+
return rows
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Core utilities shared across all commands."""
|