loft-cli 0.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loft_cli/__init__.py +3 -0
- loft_cli/_builtins.py +269 -0
- loft_cli/addons/__init__.py +12 -0
- loft_cli/addons/goss/__init__.py +1 -0
- loft_cli/addons/goss/generator.py +202 -0
- loft_cli/addons/goss/renderer.py +207 -0
- loft_cli/addons/goss/shipper.py +174 -0
- loft_cli/agent_installer.py +65 -0
- loft_cli/checks/__init__.py +0 -0
- loft_cli/checks/compose.py +147 -0
- loft_cli/checks/container.py +24 -0
- loft_cli/checks/http.py +30 -0
- loft_cli/checks/nginx.py +21 -0
- loft_cli/checks/ports.py +24 -0
- loft_cli/checks/postgres.py +21 -0
- loft_cli/checks/ssh.py +66 -0
- loft_cli/checks/wireguard.py +21 -0
- loft_cli/cli.py +1161 -0
- loft_cli/compiler/__init__.py +0 -0
- loft_cli/compiler/normalizer.py +420 -0
- loft_cli/compiler/parser.py +42 -0
- loft_cli/compiler/planner.py +2543 -0
- loft_cli/local/__init__.py +0 -0
- loft_cli/local/ddl/__init__.py +0 -0
- loft_cli/local/ddl/bootstrap_tables.py +75 -0
- loft_cli/local/ddl/versionize_system.py +374 -0
- loft_cli/local/inventory.py +449 -0
- loft_cli/local/inventory_db.py +255 -0
- loft_cli/local/keys.py +45 -0
- loft_cli/local/remove.py +135 -0
- loft_cli/local/ssh_config.py +118 -0
- loft_cli/local/tunnel.py +228 -0
- loft_cli/local/wireguard_store.py +138 -0
- loft_cli/logs/__init__.py +0 -0
- loft_cli/logs/reader.py +55 -0
- loft_cli/logs/writer.py +58 -0
- loft_cli/runtime/__init__.py +0 -0
- loft_cli/runtime/agent_transport.py +115 -0
- loft_cli/runtime/executor.py +798 -0
- loft_cli/runtime/fabric_transport.py +68 -0
- loft_cli/runtime/secret_rotation.py +99 -0
- loft_cli/runtime/ssh.py +147 -0
- loft_cli/runtime/steps/__init__.py +0 -0
- loft_cli/runtime/steps/backup.py +72 -0
- loft_cli/runtime/steps/bootstrap.py +227 -0
- loft_cli/runtime/steps/compose.py +43 -0
- loft_cli/runtime/steps/container.py +40 -0
- loft_cli/runtime/steps/docker.py +15 -0
- loft_cli/runtime/steps/file_template.py +25 -0
- loft_cli/runtime/steps/nginx.py +126 -0
- loft_cli/runtime/steps/postgres.py +101 -0
- loft_cli/runtime/steps/postgres_ensure.py +141 -0
- loft_cli/runtime/steps/systemd.py +131 -0
- loft_cli/runtime/steps/wireguard.py +76 -0
- loft_cli/runtime/transport.py +43 -0
- loft_cli/updater.py +208 -0
- loft_cli-0.6.3.dist-info/METADATA +19 -0
- loft_cli-0.6.3.dist-info/RECORD +61 -0
- loft_cli-0.6.3.dist-info/WHEEL +5 -0
- loft_cli-0.6.3.dist-info/entry_points.txt +5 -0
- loft_cli-0.6.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2543 @@
|
|
|
1
|
+
"""Phase 3: Convert normalized spec into an ordered Plan.
|
|
2
|
+
|
|
3
|
+
CRITICAL INVARIANT (SSH lockout prevention):
|
|
4
|
+
Steps 'disable_root_login' and 'disable_password_auth' MUST have
|
|
5
|
+
depends_on referencing the index of 'verify_admin_login_on_new_port'.
|
|
6
|
+
'verify_admin_login_on_new_port' MUST be gate=True.
|
|
7
|
+
Local steps MUST only be generated after all critical remote steps.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import contextlib
|
|
13
|
+
from datetime import UTC, datetime
|
|
14
|
+
|
|
15
|
+
from loft_cli.compiler.normalizer import NormalizedContext
|
|
16
|
+
from loft_cli_core.plan.models import Plan, Step, StepKind, StepScope
|
|
17
|
+
from loft_cli_core.specs.backup_job_schema import BackupJobSpec
|
|
18
|
+
from loft_cli_core.specs.bootstrap_schema import BootstrapSpec
|
|
19
|
+
from loft_cli_core.specs.compose_project_schema import ComposeProjectSpec
|
|
20
|
+
from loft_cli_core.specs.file_template_schema import FileTemplateSpec
|
|
21
|
+
from loft_cli_core.specs.http_check_schema import HttpCheckSpec
|
|
22
|
+
from loft_cli_core.specs.postgres_ensure_schema import PostgresEnsureSpec
|
|
23
|
+
from loft_cli_core.specs.service_schema import ServiceSpec
|
|
24
|
+
from loft_cli_core.specs.stack_schema import StackSpec
|
|
25
|
+
from loft_cli_core.specs.systemd_timer_schema import SystemdTimerSpec
|
|
26
|
+
from loft_cli_core.specs.systemd_unit_schema import SystemdUnitSpec
|
|
27
|
+
from loft_cli_core.utils.hashing import sha256_string
|
|
28
|
+
|
|
29
|
+
AnySpec = (
|
|
30
|
+
BootstrapSpec
|
|
31
|
+
| ServiceSpec
|
|
32
|
+
| FileTemplateSpec
|
|
33
|
+
| ComposeProjectSpec
|
|
34
|
+
| StackSpec
|
|
35
|
+
| HttpCheckSpec
|
|
36
|
+
| SystemdUnitSpec
|
|
37
|
+
| SystemdTimerSpec
|
|
38
|
+
| BackupJobSpec
|
|
39
|
+
| PostgresEnsureSpec
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _encode_check_command(check, spec) -> str:
|
|
44
|
+
"""Encode a CheckBlock into a command string for the executor to dispatch."""
|
|
45
|
+
ctype = check.type
|
|
46
|
+
host = spec.host.address
|
|
47
|
+
if ctype == "ssh_reachable":
|
|
48
|
+
port = check.port or 22
|
|
49
|
+
user = check.user or "root"
|
|
50
|
+
return f"check:ssh_reachable:{host}:{port}:{user}"
|
|
51
|
+
elif ctype == "port_open":
|
|
52
|
+
port = check.port or 22
|
|
53
|
+
return f"check:port_open:{host}:{port}"
|
|
54
|
+
elif ctype == "wireguard_up":
|
|
55
|
+
iface = check.interface or "wg0"
|
|
56
|
+
return f"check:wireguard_up:{iface}"
|
|
57
|
+
elif ctype == "container_running":
|
|
58
|
+
name = check.name or "unknown"
|
|
59
|
+
return f"check:container_running:{name}"
|
|
60
|
+
elif ctype == "http":
|
|
61
|
+
url = check.url or ""
|
|
62
|
+
status = check.expect_status or 200
|
|
63
|
+
return f"check:http:{url}:{status}"
|
|
64
|
+
elif ctype == "postgres_ready":
|
|
65
|
+
return "check:postgres_ready"
|
|
66
|
+
elif ctype == "nginx_ready":
|
|
67
|
+
return "check:nginx_ready"
|
|
68
|
+
return f"check:{ctype}"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def plan(ctx: NormalizedContext) -> Plan:
|
|
72
|
+
"""Convert a NormalizedContext into an executable Plan."""
|
|
73
|
+
from loft_cli_core.registry import get_planner, load_addons
|
|
74
|
+
|
|
75
|
+
load_addons()
|
|
76
|
+
|
|
77
|
+
spec = ctx.spec
|
|
78
|
+
planner_fn = get_planner(spec.kind)
|
|
79
|
+
if planner_fn is None:
|
|
80
|
+
raise RuntimeError(f"No planner registered for spec kind '{spec.kind}'")
|
|
81
|
+
steps = planner_fn(spec, ctx)
|
|
82
|
+
|
|
83
|
+
# Re-index all steps
|
|
84
|
+
for i, step in enumerate(steps):
|
|
85
|
+
step.index = i
|
|
86
|
+
|
|
87
|
+
spec_hash = sha256_string(spec.model_dump_json())
|
|
88
|
+
plan_obj = Plan(
|
|
89
|
+
spec_name=spec.meta.name,
|
|
90
|
+
spec_kind=spec.kind,
|
|
91
|
+
target_host=spec.host.address,
|
|
92
|
+
spec_hash=spec_hash,
|
|
93
|
+
plan_hash="",
|
|
94
|
+
steps=steps,
|
|
95
|
+
created_at=datetime.now(UTC).isoformat(),
|
|
96
|
+
)
|
|
97
|
+
plan_obj.plan_hash = sha256_string("".join(s.id + (s.command or "") for s in steps))
|
|
98
|
+
return plan_obj
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _s(
|
|
102
|
+
id: str,
|
|
103
|
+
description: str,
|
|
104
|
+
scope: StepScope,
|
|
105
|
+
kind: StepKind,
|
|
106
|
+
*,
|
|
107
|
+
command: str | None = None,
|
|
108
|
+
file_content: str | None = None,
|
|
109
|
+
target_path: str | None = None,
|
|
110
|
+
sudo: bool = False,
|
|
111
|
+
check_command: str | None = None,
|
|
112
|
+
rollback_hint: str | None = None,
|
|
113
|
+
depends_on: list[int] | None = None,
|
|
114
|
+
gate: bool = False,
|
|
115
|
+
tags: list[str] | None = None,
|
|
116
|
+
) -> Step:
|
|
117
|
+
"""Helper to construct a Step with index placeholder (set later)."""
|
|
118
|
+
return Step(
|
|
119
|
+
id=id,
|
|
120
|
+
index=0, # filled in by plan()
|
|
121
|
+
description=description,
|
|
122
|
+
scope=scope,
|
|
123
|
+
kind=kind,
|
|
124
|
+
command=command,
|
|
125
|
+
file_content=file_content,
|
|
126
|
+
target_path=target_path,
|
|
127
|
+
sudo=sudo,
|
|
128
|
+
check_command=check_command,
|
|
129
|
+
rollback_hint=rollback_hint,
|
|
130
|
+
depends_on=depends_on or [],
|
|
131
|
+
gate=gate,
|
|
132
|
+
tags=tags or [],
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _plan_bootstrap(spec: BootstrapSpec, ctx: NormalizedContext) -> list[Step]:
|
|
137
|
+
from loft_cli.runtime.steps import bootstrap as bs
|
|
138
|
+
from loft_cli.runtime.steps import wireguard as wg
|
|
139
|
+
|
|
140
|
+
steps: list[Step] = []
|
|
141
|
+
R = StepScope.REMOTE
|
|
142
|
+
L = StepScope.LOCAL
|
|
143
|
+
V = StepScope.VERIFY
|
|
144
|
+
|
|
145
|
+
pubkey_content = "\n".join(ctx.pubkey_contents) if ctx.pubkey_contents else ""
|
|
146
|
+
|
|
147
|
+
# ------------------------------------------------------------------ #
|
|
148
|
+
# REMOTE: critical bootstrap path
|
|
149
|
+
# ------------------------------------------------------------------ #
|
|
150
|
+
|
|
151
|
+
# 0: preflight — verify root SSH access
|
|
152
|
+
steps.append(
|
|
153
|
+
_s(
|
|
154
|
+
"preflight_connect_root",
|
|
155
|
+
f"Verify root SSH access to {spec.host.address}:{spec.login.port}",
|
|
156
|
+
R,
|
|
157
|
+
StepKind.SSH_COMMAND,
|
|
158
|
+
command="echo 'preflight ok'",
|
|
159
|
+
rollback_hint="Check SSH credentials and network connectivity.",
|
|
160
|
+
tags=["preflight"],
|
|
161
|
+
)
|
|
162
|
+
)
|
|
163
|
+
idx_preflight = len(steps) - 1
|
|
164
|
+
|
|
165
|
+
# 1: detect OS
|
|
166
|
+
steps.append(
|
|
167
|
+
_s(
|
|
168
|
+
"detect_os",
|
|
169
|
+
"Detect remote OS (assert Debian/Ubuntu)",
|
|
170
|
+
R,
|
|
171
|
+
StepKind.SSH_COMMAND,
|
|
172
|
+
command="cat /etc/os-release",
|
|
173
|
+
depends_on=[idx_preflight],
|
|
174
|
+
tags=["os"],
|
|
175
|
+
)
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
# 2: update apt package index
|
|
179
|
+
steps.append(
|
|
180
|
+
_s(
|
|
181
|
+
"apt_update",
|
|
182
|
+
"Update apt package index",
|
|
183
|
+
R,
|
|
184
|
+
StepKind.SSH_COMMAND,
|
|
185
|
+
command=bs.apt_update(),
|
|
186
|
+
sudo=True,
|
|
187
|
+
rollback_hint="Check apt sources and network connectivity.",
|
|
188
|
+
tags=["packages"],
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
# 3: install base packages
|
|
193
|
+
base_packages = ["ufw"]
|
|
194
|
+
if spec.wireguard.enabled:
|
|
195
|
+
base_packages.append("wireguard")
|
|
196
|
+
pkg_list = " ".join(base_packages)
|
|
197
|
+
steps.append(
|
|
198
|
+
_s(
|
|
199
|
+
"install_base_packages",
|
|
200
|
+
f"Install base packages: {pkg_list}",
|
|
201
|
+
R,
|
|
202
|
+
StepKind.SSH_COMMAND,
|
|
203
|
+
command=bs.install_packages(base_packages),
|
|
204
|
+
sudo=True,
|
|
205
|
+
rollback_hint="Check apt sources and network connectivity.",
|
|
206
|
+
tags=["packages"],
|
|
207
|
+
)
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# 3: create admin user (idempotent)
|
|
211
|
+
steps.append(
|
|
212
|
+
_s(
|
|
213
|
+
"create_admin_user",
|
|
214
|
+
f"Create admin user '{spec.admin_user.name}' with sudo",
|
|
215
|
+
R,
|
|
216
|
+
StepKind.SSH_COMMAND,
|
|
217
|
+
command=bs.create_admin_user(
|
|
218
|
+
spec.admin_user.name,
|
|
219
|
+
spec.admin_user.groups,
|
|
220
|
+
),
|
|
221
|
+
sudo=True,
|
|
222
|
+
tags=["user"],
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# 3b: grant passwordless sudo so the admin can run sudo without a password
|
|
227
|
+
steps.append(
|
|
228
|
+
_s(
|
|
229
|
+
"configure_nopasswd_sudo",
|
|
230
|
+
f"Grant {spec.admin_user.name} passwordless sudo",
|
|
231
|
+
R,
|
|
232
|
+
StepKind.SSH_COMMAND,
|
|
233
|
+
command=bs.nopasswd_sudoers(spec.admin_user.name),
|
|
234
|
+
sudo=True,
|
|
235
|
+
tags=["user"],
|
|
236
|
+
)
|
|
237
|
+
)
|
|
238
|
+
steps.append(
|
|
239
|
+
_s(
|
|
240
|
+
"secure_sudoers_file",
|
|
241
|
+
f"Secure /etc/sudoers.d/{spec.admin_user.name} (chmod 440)",
|
|
242
|
+
R,
|
|
243
|
+
StepKind.SSH_COMMAND,
|
|
244
|
+
command=bs.secure_sudoers(spec.admin_user.name),
|
|
245
|
+
sudo=True,
|
|
246
|
+
tags=["user"],
|
|
247
|
+
)
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
# 4: install authorized keys
|
|
251
|
+
if pubkey_content:
|
|
252
|
+
steps.append(
|
|
253
|
+
_s(
|
|
254
|
+
"install_authorized_keys",
|
|
255
|
+
f"Install SSH authorized keys for {spec.admin_user.name}",
|
|
256
|
+
R,
|
|
257
|
+
StepKind.SSH_COMMAND,
|
|
258
|
+
command=bs.install_authorized_keys(spec.admin_user.name, pubkey_content),
|
|
259
|
+
sudo=True,
|
|
260
|
+
tags=["ssh", "keys"],
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
else:
|
|
264
|
+
steps.append(
|
|
265
|
+
_s(
|
|
266
|
+
"install_authorized_keys",
|
|
267
|
+
"No pubkeys configured — skipping authorized_keys install",
|
|
268
|
+
R,
|
|
269
|
+
StepKind.VERIFY,
|
|
270
|
+
command="echo 'no pubkeys configured'",
|
|
271
|
+
tags=["ssh", "keys"],
|
|
272
|
+
)
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
# 4b: ensure PubkeyAuthentication yes + reload sshd
|
|
276
|
+
# Some images (e.g. this VirtualBox Ubuntu) ship with PubkeyAuthentication no.
|
|
277
|
+
# The gate below requires key auth to work, so we must enable it first.
|
|
278
|
+
steps.append(
|
|
279
|
+
_s(
|
|
280
|
+
"enable_pubkey_auth",
|
|
281
|
+
"Enable PubkeyAuthentication in sshd and reload",
|
|
282
|
+
R,
|
|
283
|
+
StepKind.SSH_COMMAND,
|
|
284
|
+
command=bs.enable_pubkey_auth(),
|
|
285
|
+
sudo=True,
|
|
286
|
+
tags=["ssh", "sshd"],
|
|
287
|
+
)
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
# 4c: GATE — verify admin login on current port BEFORE touching sshd
|
|
291
|
+
# This is the critical safety gate: if admin key login doesn't work yet,
|
|
292
|
+
# we must NOT change the SSH port — the server would become unrecoverable.
|
|
293
|
+
if pubkey_content:
|
|
294
|
+
steps.append(
|
|
295
|
+
_s(
|
|
296
|
+
"verify_admin_login_before_port_change",
|
|
297
|
+
f"[GATE] Verify admin SSH login before port change: "
|
|
298
|
+
f"{spec.admin_user.name}@{spec.host.address}:{spec.login.port}",
|
|
299
|
+
V,
|
|
300
|
+
StepKind.GATE,
|
|
301
|
+
command=f"ssh_check:{spec.host.address}:{spec.login.port}:{spec.admin_user.name}",
|
|
302
|
+
rollback_hint=(
|
|
303
|
+
"Admin key login failed — SSH port has NOT been changed. "
|
|
304
|
+
"Safe to re-run. Check that admin user and authorized_keys were created correctly."
|
|
305
|
+
),
|
|
306
|
+
gate=True,
|
|
307
|
+
tags=["gate", "ssh", "lockout-prevention"],
|
|
308
|
+
)
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
# 5: write sshd config candidate (port change, defer root/password disable)
|
|
312
|
+
steps.append(
|
|
313
|
+
_s(
|
|
314
|
+
"write_sshd_config_candidate",
|
|
315
|
+
f"Configure SSH daemon: port={spec.ssh.port} (root/password hardening deferred)",
|
|
316
|
+
R,
|
|
317
|
+
StepKind.SSH_COMMAND,
|
|
318
|
+
command=bs.write_sshd_config_candidate(spec.ssh.port),
|
|
319
|
+
sudo=True,
|
|
320
|
+
rollback_hint="Restore /etc/ssh/sshd_config from backup: "
|
|
321
|
+
"cp /etc/ssh/sshd_config.bak /etc/ssh/sshd_config",
|
|
322
|
+
tags=["ssh", "sshd"],
|
|
323
|
+
)
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
# 6: open new SSH port in firewall
|
|
327
|
+
steps.append(
|
|
328
|
+
_s(
|
|
329
|
+
"open_new_ssh_port_in_firewall",
|
|
330
|
+
f"Open firewall for new SSH port {spec.ssh.port}/tcp",
|
|
331
|
+
R,
|
|
332
|
+
StepKind.SSH_COMMAND,
|
|
333
|
+
command=bs.open_firewall_port(spec.ssh.port),
|
|
334
|
+
sudo=True,
|
|
335
|
+
tags=["firewall"],
|
|
336
|
+
)
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
# 7: validate sshd config
|
|
340
|
+
steps.append(
|
|
341
|
+
_s(
|
|
342
|
+
"validate_sshd_config",
|
|
343
|
+
"Validate sshd configuration (sshd -t)",
|
|
344
|
+
R,
|
|
345
|
+
StepKind.SSH_COMMAND,
|
|
346
|
+
command="sshd -t",
|
|
347
|
+
sudo=True,
|
|
348
|
+
rollback_hint="Fix /etc/ssh/sshd_config errors before reloading.",
|
|
349
|
+
tags=["ssh", "sshd"],
|
|
350
|
+
)
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
# 8: reload sshd
|
|
354
|
+
steps.append(
|
|
355
|
+
_s(
|
|
356
|
+
"reload_sshd",
|
|
357
|
+
"Reload SSH daemon to apply config",
|
|
358
|
+
R,
|
|
359
|
+
StepKind.SSH_COMMAND,
|
|
360
|
+
command=bs.reload_sshd(),
|
|
361
|
+
sudo=True,
|
|
362
|
+
rollback_hint="If reload fails, check sshd_config syntax with 'sshd -t'.",
|
|
363
|
+
tags=["ssh", "sshd"],
|
|
364
|
+
)
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
# 9: GATE — verify admin login on new port
|
|
368
|
+
# This is the SSH lockout prevention gate.
|
|
369
|
+
# Steps that disable root login and password auth MUST depend on this index.
|
|
370
|
+
steps.append(
|
|
371
|
+
_s(
|
|
372
|
+
"verify_admin_login_on_new_port",
|
|
373
|
+
f"[GATE] Verify admin SSH login: {spec.admin_user.name}@{spec.host.address}:{spec.ssh.port}",
|
|
374
|
+
V,
|
|
375
|
+
StepKind.GATE,
|
|
376
|
+
command=f"ssh_check:{spec.host.address}:{spec.ssh.port}:{spec.admin_user.name}",
|
|
377
|
+
rollback_hint=(
|
|
378
|
+
"Admin login failed. Do NOT disable root login or password auth. "
|
|
379
|
+
"Restore sshd_config: cp /etc/ssh/sshd_config.bak /etc/ssh/sshd_config && "
|
|
380
|
+
"systemctl reload ssh"
|
|
381
|
+
),
|
|
382
|
+
gate=True,
|
|
383
|
+
tags=["gate", "ssh", "lockout-prevention"],
|
|
384
|
+
)
|
|
385
|
+
)
|
|
386
|
+
idx_gate = len(steps) - 1
|
|
387
|
+
|
|
388
|
+
# 10: disable root login — MUST depend on gate
|
|
389
|
+
steps.append(
|
|
390
|
+
_s(
|
|
391
|
+
"disable_root_login",
|
|
392
|
+
"Disable root SSH login (PermitRootLogin no)",
|
|
393
|
+
R,
|
|
394
|
+
StepKind.SSH_COMMAND,
|
|
395
|
+
command=bs.disable_root_login(),
|
|
396
|
+
sudo=True,
|
|
397
|
+
depends_on=[idx_gate],
|
|
398
|
+
rollback_hint="sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config && systemctl reload ssh",
|
|
399
|
+
tags=["ssh", "hardening", "lockout-prevention"],
|
|
400
|
+
)
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
# 11: disable password auth — MUST depend on gate
|
|
404
|
+
if spec.ssh.disable_password_auth and pubkey_content:
|
|
405
|
+
steps.append(
|
|
406
|
+
_s(
|
|
407
|
+
"disable_password_auth",
|
|
408
|
+
"Disable SSH password authentication",
|
|
409
|
+
R,
|
|
410
|
+
StepKind.SSH_COMMAND,
|
|
411
|
+
command=bs.disable_password_auth(),
|
|
412
|
+
sudo=True,
|
|
413
|
+
depends_on=[idx_gate],
|
|
414
|
+
rollback_hint="sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config && systemctl reload ssh",
|
|
415
|
+
tags=["ssh", "hardening", "lockout-prevention"],
|
|
416
|
+
)
|
|
417
|
+
)
|
|
418
|
+
else:
|
|
419
|
+
steps.append(
|
|
420
|
+
_s(
|
|
421
|
+
"disable_password_auth",
|
|
422
|
+
"Password auth not disabled (no pubkeys or config not set)",
|
|
423
|
+
R,
|
|
424
|
+
StepKind.VERIFY,
|
|
425
|
+
command="echo 'password auth left enabled'",
|
|
426
|
+
depends_on=[idx_gate],
|
|
427
|
+
tags=["ssh", "hardening"],
|
|
428
|
+
)
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
# 12: finalize firewall (three separate steps — no shell chaining)
|
|
432
|
+
steps.append(
|
|
433
|
+
_s(
|
|
434
|
+
"ufw_default_deny_incoming",
|
|
435
|
+
"Firewall: default deny incoming",
|
|
436
|
+
R,
|
|
437
|
+
StepKind.SSH_COMMAND,
|
|
438
|
+
command=bs.ufw_default_deny_incoming(),
|
|
439
|
+
sudo=True,
|
|
440
|
+
tags=["firewall"],
|
|
441
|
+
)
|
|
442
|
+
)
|
|
443
|
+
steps.append(
|
|
444
|
+
_s(
|
|
445
|
+
"ufw_default_allow_outgoing",
|
|
446
|
+
"Firewall: default allow outgoing",
|
|
447
|
+
R,
|
|
448
|
+
StepKind.SSH_COMMAND,
|
|
449
|
+
command=bs.ufw_default_allow_outgoing(),
|
|
450
|
+
sudo=True,
|
|
451
|
+
tags=["firewall"],
|
|
452
|
+
)
|
|
453
|
+
)
|
|
454
|
+
steps.append(
|
|
455
|
+
_s(
|
|
456
|
+
"ufw_force_enable",
|
|
457
|
+
"Enable firewall (ufw --force enable)",
|
|
458
|
+
R,
|
|
459
|
+
StepKind.SSH_COMMAND,
|
|
460
|
+
command=bs.ufw_force_enable(),
|
|
461
|
+
sudo=True,
|
|
462
|
+
tags=["firewall"],
|
|
463
|
+
)
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
# 13: reload sshd again after hardening changes
|
|
467
|
+
steps.append(
|
|
468
|
+
_s(
|
|
469
|
+
"reload_sshd_final",
|
|
470
|
+
"Reload SSH daemon to apply hardening changes",
|
|
471
|
+
R,
|
|
472
|
+
StepKind.SSH_COMMAND,
|
|
473
|
+
command=bs.reload_sshd(),
|
|
474
|
+
sudo=True,
|
|
475
|
+
depends_on=[idx_gate],
|
|
476
|
+
tags=["ssh", "sshd"],
|
|
477
|
+
)
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
# 14-18: WireGuard (remote steps + local state save)
|
|
481
|
+
if spec.wireguard.enabled:
|
|
482
|
+
import ipaddress as _ip
|
|
483
|
+
|
|
484
|
+
wg_port = (
|
|
485
|
+
int(spec.wireguard.endpoint.split(":")[-1]) if ":" in spec.wireguard.endpoint else 51820
|
|
486
|
+
)
|
|
487
|
+
# VPN subnet derived from the server's interface address (e.g. 10.10.0.0/24)
|
|
488
|
+
vpn_subnet = str(_ip.ip_interface(spec.wireguard.address).network)
|
|
489
|
+
|
|
490
|
+
# Server config: [Interface] with ListenPort + [Peer] with client public key
|
|
491
|
+
server_conf = wg.generate_server_config(
|
|
492
|
+
interface=spec.wireguard.interface,
|
|
493
|
+
address=spec.wireguard.address,
|
|
494
|
+
private_key=ctx.wireguard_private_key,
|
|
495
|
+
listen_port=wg_port,
|
|
496
|
+
client_public_key=ctx.wg_client_public_key,
|
|
497
|
+
peer_address=spec.wireguard.peer_address,
|
|
498
|
+
)
|
|
499
|
+
# Client config: saved locally to ~/.wg/loft-cli/{host}/client.conf
|
|
500
|
+
client_conf = wg.generate_client_config(
|
|
501
|
+
client_private_key=ctx.wg_client_private_key,
|
|
502
|
+
peer_address=spec.wireguard.peer_address,
|
|
503
|
+
server_public_key=ctx.wireguard_public_key,
|
|
504
|
+
endpoint=spec.wireguard.endpoint,
|
|
505
|
+
vpn_subnet=vpn_subnet,
|
|
506
|
+
persistent_keepalive=spec.wireguard.persistent_keepalive,
|
|
507
|
+
)
|
|
508
|
+
# Store both on ctx so the executor can persist them after apply
|
|
509
|
+
ctx.wireguard_conf_content = server_conf
|
|
510
|
+
ctx.wg_client_conf_content = client_conf
|
|
511
|
+
|
|
512
|
+
steps.append(
|
|
513
|
+
_s(
|
|
514
|
+
"set_wireguard_dir_permissions",
|
|
515
|
+
"Secure /etc/wireguard directory (chmod 700)",
|
|
516
|
+
R,
|
|
517
|
+
StepKind.SSH_COMMAND,
|
|
518
|
+
command=wg.set_wireguard_dir_permissions(),
|
|
519
|
+
sudo=True,
|
|
520
|
+
tags=["wireguard"],
|
|
521
|
+
)
|
|
522
|
+
)
|
|
523
|
+
steps.append(
|
|
524
|
+
_s(
|
|
525
|
+
"write_wireguard_config",
|
|
526
|
+
f"Write WireGuard server config: /etc/wireguard/{spec.wireguard.interface}.conf",
|
|
527
|
+
R,
|
|
528
|
+
StepKind.SSH_UPLOAD,
|
|
529
|
+
file_content=server_conf,
|
|
530
|
+
target_path=f"/etc/wireguard/{spec.wireguard.interface}.conf",
|
|
531
|
+
sudo=True,
|
|
532
|
+
tags=["wireguard"],
|
|
533
|
+
)
|
|
534
|
+
)
|
|
535
|
+
# Open the WireGuard UDP port in the firewall before starting the interface
|
|
536
|
+
steps.append(
|
|
537
|
+
_s(
|
|
538
|
+
"open_wireguard_port_in_firewall",
|
|
539
|
+
f"Open firewall for WireGuard UDP port {wg_port}",
|
|
540
|
+
R,
|
|
541
|
+
StepKind.SSH_COMMAND,
|
|
542
|
+
command=f"ufw allow {wg_port}/udp",
|
|
543
|
+
sudo=True,
|
|
544
|
+
tags=["wireguard", "firewall"],
|
|
545
|
+
)
|
|
546
|
+
)
|
|
547
|
+
steps.append(
|
|
548
|
+
_s(
|
|
549
|
+
"load_wireguard_module",
|
|
550
|
+
"Load WireGuard kernel module",
|
|
551
|
+
R,
|
|
552
|
+
StepKind.SSH_COMMAND,
|
|
553
|
+
command=wg.load_wireguard_module(),
|
|
554
|
+
sudo=True,
|
|
555
|
+
tags=["wireguard"],
|
|
556
|
+
)
|
|
557
|
+
)
|
|
558
|
+
steps.append(
|
|
559
|
+
_s(
|
|
560
|
+
"enable_wireguard",
|
|
561
|
+
f"Enable and start WireGuard: wg-quick@{spec.wireguard.interface}",
|
|
562
|
+
R,
|
|
563
|
+
StepKind.SSH_COMMAND,
|
|
564
|
+
command=wg.enable_wireguard(spec.wireguard.interface),
|
|
565
|
+
sudo=True,
|
|
566
|
+
tags=["wireguard"],
|
|
567
|
+
)
|
|
568
|
+
)
|
|
569
|
+
steps.append(
|
|
570
|
+
_s(
|
|
571
|
+
"verify_wireguard",
|
|
572
|
+
f"Verify WireGuard interface {spec.wireguard.interface} is up",
|
|
573
|
+
V,
|
|
574
|
+
StepKind.VERIFY,
|
|
575
|
+
command=f"wg show {spec.wireguard.interface}",
|
|
576
|
+
sudo=True,
|
|
577
|
+
tags=["wireguard"],
|
|
578
|
+
)
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
# 17: postflight checks (from spec)
|
|
582
|
+
for check in spec.checks:
|
|
583
|
+
steps.append(
|
|
584
|
+
_s(
|
|
585
|
+
f"postflight_{check.type}",
|
|
586
|
+
f"Postflight check: {check.type}",
|
|
587
|
+
V,
|
|
588
|
+
StepKind.VERIFY,
|
|
589
|
+
command=_encode_check_command(check, spec),
|
|
590
|
+
tags=["postflight", check.type],
|
|
591
|
+
)
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
# ------------------------------------------------------------------ #
|
|
595
|
+
# 18-19: Goss — generate, ship, and run server-spec verification.
|
|
596
|
+
# This is optional but strongly recommended. If the goss generator
|
|
597
|
+
# raises for any reason the steps are emitted as no-ops so the plan
|
|
598
|
+
# never breaks because of test-tooling issues.
|
|
599
|
+
# ------------------------------------------------------------------ #
|
|
600
|
+
goss_content: str | None = None
|
|
601
|
+
try:
|
|
602
|
+
from loft_cli.addons.goss.generator import generate_goss_yaml
|
|
603
|
+
|
|
604
|
+
goss_content = generate_goss_yaml(spec)
|
|
605
|
+
except Exception:
|
|
606
|
+
goss_content = None
|
|
607
|
+
|
|
608
|
+
if goss_content is not None:
|
|
609
|
+
idx_ship = len(steps)
|
|
610
|
+
steps.append(
|
|
611
|
+
_s(
|
|
612
|
+
"ship_goss_file",
|
|
613
|
+
f"Ship goss spec to ~/.goss/{spec.meta.name}.yaml on remote",
|
|
614
|
+
R,
|
|
615
|
+
StepKind.SSH_UPLOAD,
|
|
616
|
+
file_content=goss_content,
|
|
617
|
+
target_path=f"~/.goss/{spec.meta.name}.yaml",
|
|
618
|
+
sudo=False,
|
|
619
|
+
tags=["goss"],
|
|
620
|
+
)
|
|
621
|
+
)
|
|
622
|
+
steps.append(
|
|
623
|
+
_s(
|
|
624
|
+
"run_goss_validate",
|
|
625
|
+
"Run goss validate and display verification results",
|
|
626
|
+
V,
|
|
627
|
+
StepKind.VERIFY,
|
|
628
|
+
command="goss_validate",
|
|
629
|
+
depends_on=[idx_ship],
|
|
630
|
+
tags=["goss", "verify"],
|
|
631
|
+
)
|
|
632
|
+
)
|
|
633
|
+
else:
|
|
634
|
+
# Emit a visible warning step so the operator sees it in the plan output
|
|
635
|
+
steps.append(
|
|
636
|
+
_s(
|
|
637
|
+
"goss_unavailable",
|
|
638
|
+
"[WARNING] No goss spec available — server state will NOT be verified",
|
|
639
|
+
V,
|
|
640
|
+
StepKind.VERIFY,
|
|
641
|
+
command="goss_unavailable",
|
|
642
|
+
tags=["goss", "warning"],
|
|
643
|
+
)
|
|
644
|
+
)
|
|
645
|
+
|
|
646
|
+
# ------------------------------------------------------------------ #
|
|
647
|
+
# WireGuard SSH restriction — MUST be the absolute last remote steps.
|
|
648
|
+
# Adds a WireGuard-restricted SSH rule, verifies SSH through the tunnel
|
|
649
|
+
# from the client side, then removes the open-to-all rule.
|
|
650
|
+
# After delete_open_ssh_rule executes, direct SSH to spec.host.address
|
|
651
|
+
# stops working.
|
|
652
|
+
# ------------------------------------------------------------------ #
|
|
653
|
+
if spec.wireguard.enabled:
|
|
654
|
+
import ipaddress as _ip_wg
|
|
655
|
+
|
|
656
|
+
peer_ip = (
|
|
657
|
+
spec.wireguard.peer_address.split("/")[0]
|
|
658
|
+
if spec.firewall.registered_peers_only
|
|
659
|
+
else None
|
|
660
|
+
)
|
|
661
|
+
label_allow = (
|
|
662
|
+
f"Allow SSH port {spec.ssh.port} on WireGuard peer {peer_ip}"
|
|
663
|
+
if peer_ip
|
|
664
|
+
else f"Allow SSH port {spec.ssh.port} on WireGuard interface {spec.wireguard.interface}"
|
|
665
|
+
)
|
|
666
|
+
steps.append(
|
|
667
|
+
_s(
|
|
668
|
+
"allow_ssh_on_wireguard",
|
|
669
|
+
label_allow,
|
|
670
|
+
R,
|
|
671
|
+
StepKind.SSH_COMMAND,
|
|
672
|
+
command=bs.allow_ssh_on_wireguard(
|
|
673
|
+
spec.ssh.port,
|
|
674
|
+
spec.wireguard.interface,
|
|
675
|
+
peer_ip,
|
|
676
|
+
),
|
|
677
|
+
sudo=True,
|
|
678
|
+
tags=["ssh", "firewall", "wireguard"],
|
|
679
|
+
)
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
# Tunnel safety gate: bring up WireGuard tunnel locally, verify SSH
|
|
683
|
+
# through the VPN IP, and only then proceed to delete the open SSH rule.
|
|
684
|
+
# If this gate fails, the open SSH rule is NOT deleted — the server
|
|
685
|
+
# remains accessible via public IP.
|
|
686
|
+
wg_server_vpn_ip = str(_ip_wg.ip_interface(spec.wireguard.address).ip)
|
|
687
|
+
steps.append(
|
|
688
|
+
_s(
|
|
689
|
+
"verify_ssh_over_wireguard_tunnel",
|
|
690
|
+
(
|
|
691
|
+
f"Verify SSH through WireGuard tunnel "
|
|
692
|
+
f"({spec.admin_user.name}@{wg_server_vpn_ip}:{spec.ssh.port})"
|
|
693
|
+
),
|
|
694
|
+
L,
|
|
695
|
+
StepKind.GATE,
|
|
696
|
+
command=(
|
|
697
|
+
f"tunnel_ssh_gate:{spec.host.name}:{wg_server_vpn_ip}"
|
|
698
|
+
f":{spec.ssh.port}:{spec.admin_user.name}"
|
|
699
|
+
),
|
|
700
|
+
gate=True,
|
|
701
|
+
rollback_hint=(
|
|
702
|
+
"WireGuard tunnel SSH verification failed. The open SSH rule was NOT deleted — "
|
|
703
|
+
"the server is still accessible via its public IP. "
|
|
704
|
+
"Check WireGuard configuration, ensure wg-quick is installed locally, "
|
|
705
|
+
"and verify the tunnel can be established."
|
|
706
|
+
),
|
|
707
|
+
tags=["wireguard", "ssh", "gate", "tunnel"],
|
|
708
|
+
)
|
|
709
|
+
)
|
|
710
|
+
|
|
711
|
+
steps.append(
|
|
712
|
+
_s(
|
|
713
|
+
"delete_open_ssh_rule",
|
|
714
|
+
f"Remove open-to-all SSH rule for port {spec.ssh.port}/tcp",
|
|
715
|
+
R,
|
|
716
|
+
StepKind.SSH_COMMAND,
|
|
717
|
+
command=bs.delete_open_ssh_rule(spec.ssh.port),
|
|
718
|
+
sudo=True,
|
|
719
|
+
tags=["ssh", "firewall", "wireguard"],
|
|
720
|
+
)
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
# ------------------------------------------------------------------ #
|
|
724
|
+
# LOCAL: only after remote success
|
|
725
|
+
# ------------------------------------------------------------------ #
|
|
726
|
+
|
|
727
|
+
# SSH conf.d entry — only when local.ssh_config.enabled (default: true)
|
|
728
|
+
if spec.local.ssh_config.enabled:
|
|
729
|
+
steps.append(
|
|
730
|
+
_s(
|
|
731
|
+
"backup_local_ssh_config",
|
|
732
|
+
"Backup local ~/.ssh/config",
|
|
733
|
+
L,
|
|
734
|
+
StepKind.LOCAL_COMMAND,
|
|
735
|
+
command="backup_ssh_config",
|
|
736
|
+
tags=["local", "ssh-config"],
|
|
737
|
+
)
|
|
738
|
+
)
|
|
739
|
+
steps.append(
|
|
740
|
+
_s(
|
|
741
|
+
"write_local_ssh_conf_d",
|
|
742
|
+
f"Write local SSH conf.d entry: {ctx.ssh_conf_d_path}",
|
|
743
|
+
L,
|
|
744
|
+
StepKind.LOCAL_FILE_WRITE,
|
|
745
|
+
target_path=str(ctx.ssh_conf_d_path) if ctx.ssh_conf_d_path else "",
|
|
746
|
+
tags=["local", "ssh-config"],
|
|
747
|
+
)
|
|
748
|
+
)
|
|
749
|
+
steps.append(
|
|
750
|
+
_s(
|
|
751
|
+
"ensure_include_directive",
|
|
752
|
+
"Ensure Include directive in ~/.ssh/config",
|
|
753
|
+
L,
|
|
754
|
+
StepKind.LOCAL_COMMAND,
|
|
755
|
+
command="ensure_include",
|
|
756
|
+
tags=["local", "ssh-config"],
|
|
757
|
+
)
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
# WireGuard local state — save key material + metadata after remote success
|
|
761
|
+
if spec.wireguard.enabled:
|
|
762
|
+
steps.append(
|
|
763
|
+
_s(
|
|
764
|
+
"save_local_wireguard_state",
|
|
765
|
+
f"Save WireGuard state to ~/.wg/loft-cli/{spec.host.name}/",
|
|
766
|
+
L,
|
|
767
|
+
StepKind.LOCAL_COMMAND,
|
|
768
|
+
command="save_wireguard_state",
|
|
769
|
+
tags=["local", "wireguard"],
|
|
770
|
+
)
|
|
771
|
+
)
|
|
772
|
+
|
|
773
|
+
# Inventory DB
|
|
774
|
+
if spec.local.inventory.enabled:
|
|
775
|
+
steps.append(
|
|
776
|
+
_s(
|
|
777
|
+
"open_or_init_local_inventory",
|
|
778
|
+
"Open or initialize local SQLCipher inventory database",
|
|
779
|
+
L,
|
|
780
|
+
StepKind.LOCAL_DB_WRITE,
|
|
781
|
+
command="init_inventory",
|
|
782
|
+
tags=["local", "inventory"],
|
|
783
|
+
)
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
# 22: upsert server record
|
|
787
|
+
steps.append(
|
|
788
|
+
_s(
|
|
789
|
+
"upsert_server_inventory",
|
|
790
|
+
f"Upsert server record in inventory: {spec.host.name}",
|
|
791
|
+
L,
|
|
792
|
+
StepKind.LOCAL_DB_WRITE,
|
|
793
|
+
command="upsert_server",
|
|
794
|
+
tags=["local", "inventory"],
|
|
795
|
+
)
|
|
796
|
+
)
|
|
797
|
+
|
|
798
|
+
# 23: record run metadata
|
|
799
|
+
steps.append(
|
|
800
|
+
_s(
|
|
801
|
+
"record_run_metadata",
|
|
802
|
+
"Record bootstrap run metadata in inventory",
|
|
803
|
+
L,
|
|
804
|
+
StepKind.LOCAL_DB_WRITE,
|
|
805
|
+
command="record_run",
|
|
806
|
+
tags=["local", "inventory"],
|
|
807
|
+
)
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
return steps
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def _plan_service(spec: ServiceSpec, ctx: NormalizedContext) -> list[Step]:
|
|
814
|
+
from loft_cli.runtime.steps import container as ct
|
|
815
|
+
from loft_cli.runtime.steps import docker as dk
|
|
816
|
+
from loft_cli.runtime.steps import nginx as nx
|
|
817
|
+
from loft_cli.runtime.steps import postgres as pg
|
|
818
|
+
|
|
819
|
+
steps: list[Step] = []
|
|
820
|
+
R = StepScope.REMOTE
|
|
821
|
+
L = StepScope.LOCAL
|
|
822
|
+
V = StepScope.VERIFY
|
|
823
|
+
|
|
824
|
+
# preflight
|
|
825
|
+
steps.append(
|
|
826
|
+
_s(
|
|
827
|
+
"preflight_connect_admin",
|
|
828
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
829
|
+
R,
|
|
830
|
+
StepKind.SSH_COMMAND,
|
|
831
|
+
command="echo 'preflight ok'",
|
|
832
|
+
tags=["preflight"],
|
|
833
|
+
)
|
|
834
|
+
)
|
|
835
|
+
|
|
836
|
+
steps.append(
|
|
837
|
+
_s(
|
|
838
|
+
"detect_os",
|
|
839
|
+
"Detect remote OS",
|
|
840
|
+
R,
|
|
841
|
+
StepKind.SSH_COMMAND,
|
|
842
|
+
command="cat /etc/os-release",
|
|
843
|
+
tags=["os"],
|
|
844
|
+
)
|
|
845
|
+
)
|
|
846
|
+
|
|
847
|
+
# apt update — shared by all service installations that need packages
|
|
848
|
+
needs_apt = (
|
|
849
|
+
(spec.postgres and spec.postgres.enabled)
|
|
850
|
+
or (spec.nginx and spec.nginx.enabled)
|
|
851
|
+
or (spec.docker and spec.docker.enabled)
|
|
852
|
+
or bool(spec.containers)
|
|
853
|
+
)
|
|
854
|
+
if needs_apt:
|
|
855
|
+
steps.append(
|
|
856
|
+
_s(
|
|
857
|
+
"apt_update",
|
|
858
|
+
"Update apt package index",
|
|
859
|
+
R,
|
|
860
|
+
StepKind.SSH_COMMAND,
|
|
861
|
+
command="apt-get update -y",
|
|
862
|
+
sudo=True,
|
|
863
|
+
tags=["packages"],
|
|
864
|
+
)
|
|
865
|
+
)
|
|
866
|
+
|
|
867
|
+
# Postgres
|
|
868
|
+
if spec.postgres and spec.postgres.enabled:
|
|
869
|
+
# PGDG repository — ensures the requested PostgreSQL version is
|
|
870
|
+
# available regardless of the distro's default packages.
|
|
871
|
+
steps.append(
|
|
872
|
+
_s(
|
|
873
|
+
"install_pgdg_prerequisites",
|
|
874
|
+
"Install PGDG repository prerequisites",
|
|
875
|
+
R,
|
|
876
|
+
StepKind.SSH_COMMAND,
|
|
877
|
+
command=pg.install_pgdg_prerequisites(),
|
|
878
|
+
sudo=True,
|
|
879
|
+
tags=["postgres", "pgdg"],
|
|
880
|
+
)
|
|
881
|
+
)
|
|
882
|
+
steps.append(
|
|
883
|
+
_s(
|
|
884
|
+
"add_pgdg_signing_key",
|
|
885
|
+
"Import PostgreSQL PGDG apt signing key",
|
|
886
|
+
R,
|
|
887
|
+
StepKind.SSH_COMMAND,
|
|
888
|
+
command=pg.add_pgdg_signing_key(),
|
|
889
|
+
sudo=True,
|
|
890
|
+
tags=["postgres", "pgdg"],
|
|
891
|
+
)
|
|
892
|
+
)
|
|
893
|
+
steps.append(
|
|
894
|
+
_s(
|
|
895
|
+
"add_pgdg_source_list",
|
|
896
|
+
"Add PGDG apt repository source list",
|
|
897
|
+
R,
|
|
898
|
+
StepKind.SSH_COMMAND,
|
|
899
|
+
command=pg.add_pgdg_source_list(),
|
|
900
|
+
sudo=True,
|
|
901
|
+
tags=["postgres", "pgdg"],
|
|
902
|
+
)
|
|
903
|
+
)
|
|
904
|
+
steps.append(
|
|
905
|
+
_s(
|
|
906
|
+
"apt_update_pgdg",
|
|
907
|
+
"Update apt package index (with PGDG repo)",
|
|
908
|
+
R,
|
|
909
|
+
StepKind.SSH_COMMAND,
|
|
910
|
+
command="apt-get update -y",
|
|
911
|
+
sudo=True,
|
|
912
|
+
tags=["postgres", "pgdg", "packages"],
|
|
913
|
+
)
|
|
914
|
+
)
|
|
915
|
+
steps.append(
|
|
916
|
+
_s(
|
|
917
|
+
"install_postgres",
|
|
918
|
+
f"Install PostgreSQL {spec.postgres.version}",
|
|
919
|
+
R,
|
|
920
|
+
StepKind.SSH_COMMAND,
|
|
921
|
+
command=pg.install_postgres(spec.postgres.version),
|
|
922
|
+
sudo=True,
|
|
923
|
+
tags=["postgres"],
|
|
924
|
+
)
|
|
925
|
+
)
|
|
926
|
+
steps.append(
|
|
927
|
+
_s(
|
|
928
|
+
"configure_postgres_listen",
|
|
929
|
+
"Configure PostgreSQL listen_addresses",
|
|
930
|
+
R,
|
|
931
|
+
StepKind.SSH_COMMAND,
|
|
932
|
+
command=pg.configure_listen(spec.postgres.listen_addresses),
|
|
933
|
+
sudo=True,
|
|
934
|
+
tags=["postgres"],
|
|
935
|
+
)
|
|
936
|
+
)
|
|
937
|
+
steps.append(
|
|
938
|
+
_s(
|
|
939
|
+
"enable_postgres",
|
|
940
|
+
"Enable and start PostgreSQL service",
|
|
941
|
+
R,
|
|
942
|
+
StepKind.SSH_COMMAND,
|
|
943
|
+
command=pg.enable_postgres(),
|
|
944
|
+
sudo=True,
|
|
945
|
+
tags=["postgres"],
|
|
946
|
+
)
|
|
947
|
+
)
|
|
948
|
+
if spec.postgres.create_role:
|
|
949
|
+
steps.append(
|
|
950
|
+
_s(
|
|
951
|
+
"create_db_role",
|
|
952
|
+
f"Create PostgreSQL role: {spec.postgres.create_role.name}",
|
|
953
|
+
R,
|
|
954
|
+
StepKind.SSH_COMMAND,
|
|
955
|
+
command=pg.create_role(
|
|
956
|
+
spec.postgres.create_role.name,
|
|
957
|
+
spec.postgres.create_role.password_env,
|
|
958
|
+
),
|
|
959
|
+
sudo=True,
|
|
960
|
+
tags=["postgres"],
|
|
961
|
+
)
|
|
962
|
+
)
|
|
963
|
+
if spec.postgres.create_database:
|
|
964
|
+
steps.append(
|
|
965
|
+
_s(
|
|
966
|
+
"create_database",
|
|
967
|
+
f"Create PostgreSQL database: {spec.postgres.create_database.name}",
|
|
968
|
+
R,
|
|
969
|
+
StepKind.SSH_COMMAND,
|
|
970
|
+
command=pg.create_database(
|
|
971
|
+
spec.postgres.create_database.name,
|
|
972
|
+
spec.postgres.create_database.owner,
|
|
973
|
+
),
|
|
974
|
+
sudo=True,
|
|
975
|
+
tags=["postgres"],
|
|
976
|
+
)
|
|
977
|
+
)
|
|
978
|
+
steps.append(
|
|
979
|
+
_s(
|
|
980
|
+
"postgres_ready_check",
|
|
981
|
+
"Verify PostgreSQL is ready",
|
|
982
|
+
V,
|
|
983
|
+
StepKind.VERIFY,
|
|
984
|
+
command="pg_isready",
|
|
985
|
+
sudo=True,
|
|
986
|
+
tags=["postgres", "verify"],
|
|
987
|
+
)
|
|
988
|
+
)
|
|
989
|
+
|
|
990
|
+
# Nginx
|
|
991
|
+
if spec.nginx and spec.nginx.enabled:
|
|
992
|
+
steps.append(
|
|
993
|
+
_s(
|
|
994
|
+
"install_nginx",
|
|
995
|
+
"Install nginx",
|
|
996
|
+
R,
|
|
997
|
+
StepKind.SSH_COMMAND,
|
|
998
|
+
command=nx.install_nginx(),
|
|
999
|
+
sudo=True,
|
|
1000
|
+
tags=["nginx"],
|
|
1001
|
+
)
|
|
1002
|
+
)
|
|
1003
|
+
steps.append(
|
|
1004
|
+
_s(
|
|
1005
|
+
"enable_nginx",
|
|
1006
|
+
"Enable and start nginx service",
|
|
1007
|
+
R,
|
|
1008
|
+
StepKind.SSH_COMMAND,
|
|
1009
|
+
command=nx.enable_nginx(),
|
|
1010
|
+
sudo=True,
|
|
1011
|
+
tags=["nginx"],
|
|
1012
|
+
)
|
|
1013
|
+
)
|
|
1014
|
+
steps.append(
|
|
1015
|
+
_s(
|
|
1016
|
+
"remove_nginx_default_site",
|
|
1017
|
+
"Remove default nginx site",
|
|
1018
|
+
R,
|
|
1019
|
+
StepKind.SSH_COMMAND,
|
|
1020
|
+
command=nx.remove_default_site(),
|
|
1021
|
+
sudo=True,
|
|
1022
|
+
tags=["nginx"],
|
|
1023
|
+
)
|
|
1024
|
+
)
|
|
1025
|
+
for site in spec.nginx.sites:
|
|
1026
|
+
safe_name = site.domain.replace(".", "_")
|
|
1027
|
+
steps.append(
|
|
1028
|
+
_s(
|
|
1029
|
+
f"write_nginx_site_{safe_name}",
|
|
1030
|
+
f"Write nginx site config for {site.domain}",
|
|
1031
|
+
R,
|
|
1032
|
+
StepKind.SSH_UPLOAD,
|
|
1033
|
+
file_content=nx.site_config_content(site),
|
|
1034
|
+
target_path=nx.site_config_path(site),
|
|
1035
|
+
sudo=True,
|
|
1036
|
+
tags=["nginx", site.domain],
|
|
1037
|
+
)
|
|
1038
|
+
)
|
|
1039
|
+
steps.append(
|
|
1040
|
+
_s(
|
|
1041
|
+
f"enable_nginx_site_{safe_name}",
|
|
1042
|
+
f"Enable nginx site: {site.domain}",
|
|
1043
|
+
R,
|
|
1044
|
+
StepKind.SSH_COMMAND,
|
|
1045
|
+
command=nx.enable_site(site),
|
|
1046
|
+
sudo=True,
|
|
1047
|
+
tags=["nginx", site.domain],
|
|
1048
|
+
)
|
|
1049
|
+
)
|
|
1050
|
+
steps.append(
|
|
1051
|
+
_s(
|
|
1052
|
+
"validate_nginx_config",
|
|
1053
|
+
"Validate nginx configuration",
|
|
1054
|
+
R,
|
|
1055
|
+
StepKind.SSH_COMMAND,
|
|
1056
|
+
command=nx.validate_nginx_config(),
|
|
1057
|
+
sudo=True,
|
|
1058
|
+
rollback_hint="Check nginx config files for syntax errors.",
|
|
1059
|
+
tags=["nginx"],
|
|
1060
|
+
)
|
|
1061
|
+
)
|
|
1062
|
+
steps.append(
|
|
1063
|
+
_s(
|
|
1064
|
+
"reload_nginx",
|
|
1065
|
+
"Reload nginx to apply configuration",
|
|
1066
|
+
R,
|
|
1067
|
+
StepKind.SSH_COMMAND,
|
|
1068
|
+
command=nx.reload_nginx_service(),
|
|
1069
|
+
sudo=True,
|
|
1070
|
+
tags=["nginx"],
|
|
1071
|
+
)
|
|
1072
|
+
)
|
|
1073
|
+
steps.append(
|
|
1074
|
+
_s(
|
|
1075
|
+
"nginx_config_check",
|
|
1076
|
+
"Verify nginx configuration is valid",
|
|
1077
|
+
V,
|
|
1078
|
+
StepKind.VERIFY,
|
|
1079
|
+
command="nginx -t",
|
|
1080
|
+
sudo=True,
|
|
1081
|
+
tags=["nginx", "verify"],
|
|
1082
|
+
)
|
|
1083
|
+
)
|
|
1084
|
+
|
|
1085
|
+
# Docker
|
|
1086
|
+
needs_docker = spec.docker and spec.docker.enabled or bool(spec.containers)
|
|
1087
|
+
if needs_docker:
|
|
1088
|
+
steps.append(
|
|
1089
|
+
_s(
|
|
1090
|
+
"install_docker",
|
|
1091
|
+
"Install Docker",
|
|
1092
|
+
R,
|
|
1093
|
+
StepKind.SSH_COMMAND,
|
|
1094
|
+
command=dk.install_docker(),
|
|
1095
|
+
sudo=True,
|
|
1096
|
+
tags=["docker"],
|
|
1097
|
+
)
|
|
1098
|
+
)
|
|
1099
|
+
steps.append(
|
|
1100
|
+
_s(
|
|
1101
|
+
"enable_docker",
|
|
1102
|
+
"Enable Docker service",
|
|
1103
|
+
R,
|
|
1104
|
+
StepKind.SSH_COMMAND,
|
|
1105
|
+
command=dk.enable_docker(),
|
|
1106
|
+
sudo=True,
|
|
1107
|
+
tags=["docker"],
|
|
1108
|
+
)
|
|
1109
|
+
)
|
|
1110
|
+
steps.append(
|
|
1111
|
+
_s(
|
|
1112
|
+
"docker_version_check",
|
|
1113
|
+
"Verify Docker installation",
|
|
1114
|
+
V,
|
|
1115
|
+
StepKind.VERIFY,
|
|
1116
|
+
command="docker --version",
|
|
1117
|
+
sudo=True,
|
|
1118
|
+
tags=["docker", "verify"],
|
|
1119
|
+
)
|
|
1120
|
+
)
|
|
1121
|
+
|
|
1122
|
+
# Containers
|
|
1123
|
+
for c in spec.containers:
|
|
1124
|
+
steps.append(
|
|
1125
|
+
_s(
|
|
1126
|
+
f"pull_image_{c.name}",
|
|
1127
|
+
f"Pull container image: {c.image}",
|
|
1128
|
+
R,
|
|
1129
|
+
StepKind.SSH_COMMAND,
|
|
1130
|
+
command=ct.pull_image(c.image),
|
|
1131
|
+
sudo=True,
|
|
1132
|
+
tags=["container", c.name],
|
|
1133
|
+
)
|
|
1134
|
+
)
|
|
1135
|
+
steps.append(
|
|
1136
|
+
_s(
|
|
1137
|
+
f"stop_container_{c.name}",
|
|
1138
|
+
f"Stop existing container '{c.name}' (if running)",
|
|
1139
|
+
R,
|
|
1140
|
+
StepKind.SSH_COMMAND,
|
|
1141
|
+
command=ct.stop_container(c.name),
|
|
1142
|
+
sudo=True,
|
|
1143
|
+
tags=["container", c.name],
|
|
1144
|
+
)
|
|
1145
|
+
)
|
|
1146
|
+
steps.append(
|
|
1147
|
+
_s(
|
|
1148
|
+
f"remove_container_{c.name}",
|
|
1149
|
+
f"Remove existing container '{c.name}' (if present)",
|
|
1150
|
+
R,
|
|
1151
|
+
StepKind.SSH_COMMAND,
|
|
1152
|
+
command=ct.remove_container(c.name),
|
|
1153
|
+
sudo=True,
|
|
1154
|
+
tags=["container", c.name],
|
|
1155
|
+
)
|
|
1156
|
+
)
|
|
1157
|
+
steps.append(
|
|
1158
|
+
_s(
|
|
1159
|
+
f"run_container_{c.name}",
|
|
1160
|
+
f"Start container '{c.name}' from {c.image}",
|
|
1161
|
+
R,
|
|
1162
|
+
StepKind.SSH_COMMAND,
|
|
1163
|
+
command=ct.run_container(c),
|
|
1164
|
+
sudo=True,
|
|
1165
|
+
tags=["container", c.name],
|
|
1166
|
+
)
|
|
1167
|
+
)
|
|
1168
|
+
steps.append(
|
|
1169
|
+
_s(
|
|
1170
|
+
f"container_running_check_{c.name}",
|
|
1171
|
+
f"Verify container '{c.name}' is running",
|
|
1172
|
+
V,
|
|
1173
|
+
StepKind.VERIFY,
|
|
1174
|
+
command=f"docker inspect --format='{{{{.State.Running}}}}' {c.name}",
|
|
1175
|
+
sudo=True,
|
|
1176
|
+
tags=["container", c.name, "verify"],
|
|
1177
|
+
)
|
|
1178
|
+
)
|
|
1179
|
+
if c.healthcheck:
|
|
1180
|
+
steps.append(
|
|
1181
|
+
_s(
|
|
1182
|
+
f"http_health_check_{c.name}",
|
|
1183
|
+
f"HTTP health check: {c.healthcheck.url}",
|
|
1184
|
+
V,
|
|
1185
|
+
StepKind.VERIFY,
|
|
1186
|
+
command=f"http_check:{c.healthcheck.url}:{c.healthcheck.expect_status}",
|
|
1187
|
+
tags=["container", c.name, "health"],
|
|
1188
|
+
)
|
|
1189
|
+
)
|
|
1190
|
+
|
|
1191
|
+
# Local inventory
|
|
1192
|
+
if spec.local.inventory.enabled:
|
|
1193
|
+
steps.append(
|
|
1194
|
+
_s(
|
|
1195
|
+
"open_or_init_local_inventory",
|
|
1196
|
+
"Open or initialize local SQLCipher inventory database",
|
|
1197
|
+
L,
|
|
1198
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1199
|
+
command="init_inventory",
|
|
1200
|
+
tags=["local", "inventory"],
|
|
1201
|
+
)
|
|
1202
|
+
)
|
|
1203
|
+
steps.append(
|
|
1204
|
+
_s(
|
|
1205
|
+
"update_server_services_metadata",
|
|
1206
|
+
f"Update service metadata in inventory for {spec.host.name}",
|
|
1207
|
+
L,
|
|
1208
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1209
|
+
command="upsert_services",
|
|
1210
|
+
tags=["local", "inventory"],
|
|
1211
|
+
)
|
|
1212
|
+
)
|
|
1213
|
+
steps.append(
|
|
1214
|
+
_s(
|
|
1215
|
+
"record_service_run_metadata",
|
|
1216
|
+
"Record service run metadata in inventory",
|
|
1217
|
+
L,
|
|
1218
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1219
|
+
command="record_run",
|
|
1220
|
+
tags=["local", "inventory"],
|
|
1221
|
+
)
|
|
1222
|
+
)
|
|
1223
|
+
|
|
1224
|
+
return steps
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
def _plan_file_template(spec: FileTemplateSpec, ctx: NormalizedContext) -> list[Step]:
|
|
1228
|
+
"""Generate steps for rendering and uploading managed configuration files."""
|
|
1229
|
+
from loft_cli.runtime.steps import file_template as ft
|
|
1230
|
+
|
|
1231
|
+
steps: list[Step] = []
|
|
1232
|
+
R = StepScope.REMOTE
|
|
1233
|
+
L = StepScope.LOCAL
|
|
1234
|
+
V = StepScope.VERIFY
|
|
1235
|
+
|
|
1236
|
+
# Preflight
|
|
1237
|
+
steps.append(
|
|
1238
|
+
_s(
|
|
1239
|
+
"preflight_connect_admin",
|
|
1240
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
1241
|
+
R,
|
|
1242
|
+
StepKind.SSH_COMMAND,
|
|
1243
|
+
command="echo 'preflight ok'",
|
|
1244
|
+
tags=["preflight"],
|
|
1245
|
+
)
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
# For each template: mkdir parent -> upload rendered content -> chmod -> chown
|
|
1249
|
+
for t in spec.templates:
|
|
1250
|
+
safe_id = t.dest.replace("/", "_").strip("_")
|
|
1251
|
+
rendered = ctx.rendered_templates.get(t.dest, "")
|
|
1252
|
+
|
|
1253
|
+
# mkdir parent directory
|
|
1254
|
+
steps.append(
|
|
1255
|
+
_s(
|
|
1256
|
+
f"mkdir_{safe_id}",
|
|
1257
|
+
f"Create parent directory for {t.dest}",
|
|
1258
|
+
R,
|
|
1259
|
+
StepKind.SSH_COMMAND,
|
|
1260
|
+
command=ft.mkdir_for_file(t.dest),
|
|
1261
|
+
sudo=True,
|
|
1262
|
+
tags=["file_template", "mkdir"],
|
|
1263
|
+
)
|
|
1264
|
+
)
|
|
1265
|
+
|
|
1266
|
+
# Upload rendered content
|
|
1267
|
+
steps.append(
|
|
1268
|
+
_s(
|
|
1269
|
+
f"upload_{safe_id}",
|
|
1270
|
+
f"Upload rendered template to {t.dest}",
|
|
1271
|
+
R,
|
|
1272
|
+
StepKind.SSH_UPLOAD,
|
|
1273
|
+
file_content=rendered,
|
|
1274
|
+
target_path=t.dest,
|
|
1275
|
+
sudo=True,
|
|
1276
|
+
tags=["file_template", "upload"],
|
|
1277
|
+
)
|
|
1278
|
+
)
|
|
1279
|
+
|
|
1280
|
+
# Set permissions
|
|
1281
|
+
steps.append(
|
|
1282
|
+
_s(
|
|
1283
|
+
f"chmod_{safe_id}",
|
|
1284
|
+
f"Set permissions {t.mode} on {t.dest}",
|
|
1285
|
+
R,
|
|
1286
|
+
StepKind.SSH_COMMAND,
|
|
1287
|
+
command=ft.chmod_file(t.dest, t.mode),
|
|
1288
|
+
sudo=True,
|
|
1289
|
+
tags=["file_template", "chmod"],
|
|
1290
|
+
)
|
|
1291
|
+
)
|
|
1292
|
+
|
|
1293
|
+
# Set ownership
|
|
1294
|
+
steps.append(
|
|
1295
|
+
_s(
|
|
1296
|
+
f"chown_{safe_id}",
|
|
1297
|
+
f"Set ownership {t.owner}:{t.group} on {t.dest}",
|
|
1298
|
+
R,
|
|
1299
|
+
StepKind.SSH_COMMAND,
|
|
1300
|
+
command=ft.chown_file(t.dest, t.owner, t.group),
|
|
1301
|
+
sudo=True,
|
|
1302
|
+
tags=["file_template", "chown"],
|
|
1303
|
+
)
|
|
1304
|
+
)
|
|
1305
|
+
|
|
1306
|
+
# Postflight checks
|
|
1307
|
+
for check in spec.checks:
|
|
1308
|
+
steps.append(
|
|
1309
|
+
_s(
|
|
1310
|
+
f"postflight_{check.type}",
|
|
1311
|
+
f"Postflight check: {check.type}",
|
|
1312
|
+
V,
|
|
1313
|
+
StepKind.VERIFY,
|
|
1314
|
+
command=_encode_check_command(check, spec),
|
|
1315
|
+
tags=["postflight", check.type],
|
|
1316
|
+
)
|
|
1317
|
+
)
|
|
1318
|
+
|
|
1319
|
+
# Local inventory
|
|
1320
|
+
if spec.local.inventory.enabled:
|
|
1321
|
+
steps.append(
|
|
1322
|
+
_s(
|
|
1323
|
+
"open_or_init_local_inventory",
|
|
1324
|
+
"Open or initialize local inventory database",
|
|
1325
|
+
L,
|
|
1326
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1327
|
+
command="init_inventory",
|
|
1328
|
+
tags=["local", "inventory"],
|
|
1329
|
+
)
|
|
1330
|
+
)
|
|
1331
|
+
steps.append(
|
|
1332
|
+
_s(
|
|
1333
|
+
"update_file_template_metadata",
|
|
1334
|
+
f"Update file_template metadata in inventory for {spec.host.name}",
|
|
1335
|
+
L,
|
|
1336
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1337
|
+
command="upsert_services",
|
|
1338
|
+
tags=["local", "inventory"],
|
|
1339
|
+
)
|
|
1340
|
+
)
|
|
1341
|
+
steps.append(
|
|
1342
|
+
_s(
|
|
1343
|
+
"record_file_template_run",
|
|
1344
|
+
"Record file_template run metadata in inventory",
|
|
1345
|
+
L,
|
|
1346
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1347
|
+
command="record_run",
|
|
1348
|
+
tags=["local", "inventory"],
|
|
1349
|
+
)
|
|
1350
|
+
)
|
|
1351
|
+
|
|
1352
|
+
return steps
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def _plan_compose_project(spec: ComposeProjectSpec, ctx: NormalizedContext) -> list[Step]:
|
|
1356
|
+
"""Generate steps for deploying a Docker Compose project."""
|
|
1357
|
+
from loft_cli.runtime.steps import compose as cp
|
|
1358
|
+
from loft_cli.runtime.steps import docker as dk
|
|
1359
|
+
|
|
1360
|
+
steps: list[Step] = []
|
|
1361
|
+
R = StepScope.REMOTE
|
|
1362
|
+
L = StepScope.LOCAL
|
|
1363
|
+
V = StepScope.VERIFY
|
|
1364
|
+
|
|
1365
|
+
p = spec.project
|
|
1366
|
+
|
|
1367
|
+
# Preflight
|
|
1368
|
+
steps.append(
|
|
1369
|
+
_s(
|
|
1370
|
+
"preflight_connect_admin",
|
|
1371
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
1372
|
+
R,
|
|
1373
|
+
StepKind.SSH_COMMAND,
|
|
1374
|
+
command="echo 'preflight ok'",
|
|
1375
|
+
tags=["preflight"],
|
|
1376
|
+
)
|
|
1377
|
+
)
|
|
1378
|
+
|
|
1379
|
+
# Docker — always required for compose_project
|
|
1380
|
+
steps.append(
|
|
1381
|
+
_s(
|
|
1382
|
+
"apt_update",
|
|
1383
|
+
"Update apt package index",
|
|
1384
|
+
R,
|
|
1385
|
+
StepKind.SSH_COMMAND,
|
|
1386
|
+
command="apt-get update -y",
|
|
1387
|
+
sudo=True,
|
|
1388
|
+
tags=["packages"],
|
|
1389
|
+
)
|
|
1390
|
+
)
|
|
1391
|
+
steps.append(
|
|
1392
|
+
_s(
|
|
1393
|
+
"install_docker",
|
|
1394
|
+
"Install Docker",
|
|
1395
|
+
R,
|
|
1396
|
+
StepKind.SSH_COMMAND,
|
|
1397
|
+
command=dk.install_docker(),
|
|
1398
|
+
sudo=True,
|
|
1399
|
+
tags=["docker"],
|
|
1400
|
+
)
|
|
1401
|
+
)
|
|
1402
|
+
steps.append(
|
|
1403
|
+
_s(
|
|
1404
|
+
"enable_docker",
|
|
1405
|
+
"Enable Docker service",
|
|
1406
|
+
R,
|
|
1407
|
+
StepKind.SSH_COMMAND,
|
|
1408
|
+
command=dk.enable_docker(),
|
|
1409
|
+
sudo=True,
|
|
1410
|
+
tags=["docker"],
|
|
1411
|
+
)
|
|
1412
|
+
)
|
|
1413
|
+
steps.append(
|
|
1414
|
+
_s(
|
|
1415
|
+
"docker_version_check",
|
|
1416
|
+
"Verify Docker installation",
|
|
1417
|
+
V,
|
|
1418
|
+
StepKind.VERIFY,
|
|
1419
|
+
command="docker --version",
|
|
1420
|
+
sudo=True,
|
|
1421
|
+
tags=["docker", "verify"],
|
|
1422
|
+
)
|
|
1423
|
+
)
|
|
1424
|
+
|
|
1425
|
+
# Create project directory
|
|
1426
|
+
steps.append(
|
|
1427
|
+
_s(
|
|
1428
|
+
"mkdir_project_dir",
|
|
1429
|
+
f"Create project directory: {p.directory}",
|
|
1430
|
+
R,
|
|
1431
|
+
StepKind.SSH_COMMAND,
|
|
1432
|
+
command=f"mkdir -p {p.directory}",
|
|
1433
|
+
sudo=True,
|
|
1434
|
+
tags=["compose", "mkdir"],
|
|
1435
|
+
)
|
|
1436
|
+
)
|
|
1437
|
+
|
|
1438
|
+
# Create managed directories
|
|
1439
|
+
for d in p.directories:
|
|
1440
|
+
if d.path.startswith("/"):
|
|
1441
|
+
full_path = d.path
|
|
1442
|
+
else:
|
|
1443
|
+
full_path = f"{p.directory}/{d.path}"
|
|
1444
|
+
safe_id = full_path.replace("/", "_").strip("_")
|
|
1445
|
+
steps.append(
|
|
1446
|
+
_s(
|
|
1447
|
+
f"mkdir_{safe_id}",
|
|
1448
|
+
f"Create managed directory: {full_path}",
|
|
1449
|
+
R,
|
|
1450
|
+
StepKind.SSH_COMMAND,
|
|
1451
|
+
command=cp.mkdir_with_permissions(full_path, d.mode, d.owner, d.group),
|
|
1452
|
+
sudo=True,
|
|
1453
|
+
tags=["compose", "mkdir"],
|
|
1454
|
+
)
|
|
1455
|
+
)
|
|
1456
|
+
|
|
1457
|
+
# Upload rendered templates
|
|
1458
|
+
for t in p.templates:
|
|
1459
|
+
if t.dest.startswith("/"):
|
|
1460
|
+
full_dest = t.dest
|
|
1461
|
+
else:
|
|
1462
|
+
full_dest = f"{p.directory}/{t.dest}"
|
|
1463
|
+
safe_id = full_dest.replace("/", "_").strip("_")
|
|
1464
|
+
rendered = ctx.rendered_templates.get(full_dest, "")
|
|
1465
|
+
|
|
1466
|
+
steps.append(
|
|
1467
|
+
_s(
|
|
1468
|
+
f"upload_template_{safe_id}",
|
|
1469
|
+
f"Upload rendered template: {full_dest}",
|
|
1470
|
+
R,
|
|
1471
|
+
StepKind.SSH_UPLOAD,
|
|
1472
|
+
file_content=rendered,
|
|
1473
|
+
target_path=full_dest,
|
|
1474
|
+
sudo=True,
|
|
1475
|
+
tags=["compose", "template"],
|
|
1476
|
+
)
|
|
1477
|
+
)
|
|
1478
|
+
|
|
1479
|
+
# Upload compose file
|
|
1480
|
+
compose_dest = f"{p.directory}/{p.compose_file}"
|
|
1481
|
+
steps.append(
|
|
1482
|
+
_s(
|
|
1483
|
+
"upload_compose_file",
|
|
1484
|
+
f"Upload compose file: {compose_dest}",
|
|
1485
|
+
R,
|
|
1486
|
+
StepKind.SSH_UPLOAD,
|
|
1487
|
+
file_content=ctx.compose_file_content,
|
|
1488
|
+
target_path=compose_dest,
|
|
1489
|
+
sudo=True,
|
|
1490
|
+
tags=["compose", "compose-file"],
|
|
1491
|
+
)
|
|
1492
|
+
)
|
|
1493
|
+
|
|
1494
|
+
# Validate compose config
|
|
1495
|
+
steps.append(
|
|
1496
|
+
_s(
|
|
1497
|
+
"compose_config_validate",
|
|
1498
|
+
f"Validate compose configuration for project '{p.name}'",
|
|
1499
|
+
R,
|
|
1500
|
+
StepKind.SSH_COMMAND,
|
|
1501
|
+
command=cp.compose_config(p.directory, p.compose_file, p.name),
|
|
1502
|
+
sudo=True,
|
|
1503
|
+
rollback_hint="Check compose file syntax and variable substitution.",
|
|
1504
|
+
tags=["compose", "validate"],
|
|
1505
|
+
)
|
|
1506
|
+
)
|
|
1507
|
+
|
|
1508
|
+
# Pull images (optional)
|
|
1509
|
+
if p.pull_before_up:
|
|
1510
|
+
steps.append(
|
|
1511
|
+
_s(
|
|
1512
|
+
"compose_pull",
|
|
1513
|
+
f"Pull images for project '{p.name}'",
|
|
1514
|
+
R,
|
|
1515
|
+
StepKind.SSH_COMMAND,
|
|
1516
|
+
command=cp.compose_pull(p.directory, p.compose_file, p.name),
|
|
1517
|
+
sudo=True,
|
|
1518
|
+
tags=["compose", "pull"],
|
|
1519
|
+
)
|
|
1520
|
+
)
|
|
1521
|
+
|
|
1522
|
+
# Start the stack
|
|
1523
|
+
steps.append(
|
|
1524
|
+
_s(
|
|
1525
|
+
"compose_up",
|
|
1526
|
+
f"Start compose project '{p.name}' (detached)",
|
|
1527
|
+
R,
|
|
1528
|
+
StepKind.SSH_COMMAND,
|
|
1529
|
+
command=cp.compose_up(p.directory, p.compose_file, p.name),
|
|
1530
|
+
sudo=True,
|
|
1531
|
+
tags=["compose", "up"],
|
|
1532
|
+
)
|
|
1533
|
+
)
|
|
1534
|
+
|
|
1535
|
+
# Health check
|
|
1536
|
+
if p.healthcheck.enabled:
|
|
1537
|
+
# Command encodes the parameters for the handler to parse
|
|
1538
|
+
health_cmd = (
|
|
1539
|
+
f"compose_health:{p.directory}:{p.compose_file}:{p.name}"
|
|
1540
|
+
f":{p.healthcheck.timeout}:{p.healthcheck.interval}"
|
|
1541
|
+
)
|
|
1542
|
+
steps.append(
|
|
1543
|
+
_s(
|
|
1544
|
+
"compose_health_check",
|
|
1545
|
+
f"Wait for containers healthy (timeout={p.healthcheck.timeout}s)",
|
|
1546
|
+
V,
|
|
1547
|
+
StepKind.COMPOSE_HEALTH_CHECK,
|
|
1548
|
+
command=health_cmd,
|
|
1549
|
+
tags=["compose", "health"],
|
|
1550
|
+
)
|
|
1551
|
+
)
|
|
1552
|
+
|
|
1553
|
+
# Postflight checks
|
|
1554
|
+
for check in spec.checks:
|
|
1555
|
+
steps.append(
|
|
1556
|
+
_s(
|
|
1557
|
+
f"postflight_{check.type}",
|
|
1558
|
+
f"Postflight check: {check.type}",
|
|
1559
|
+
V,
|
|
1560
|
+
StepKind.VERIFY,
|
|
1561
|
+
command=_encode_check_command(check, spec),
|
|
1562
|
+
tags=["postflight", check.type],
|
|
1563
|
+
)
|
|
1564
|
+
)
|
|
1565
|
+
|
|
1566
|
+
# Local inventory
|
|
1567
|
+
if spec.local.inventory.enabled:
|
|
1568
|
+
steps.append(
|
|
1569
|
+
_s(
|
|
1570
|
+
"open_or_init_local_inventory",
|
|
1571
|
+
"Open or initialize local inventory database",
|
|
1572
|
+
L,
|
|
1573
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1574
|
+
command="init_inventory",
|
|
1575
|
+
tags=["local", "inventory"],
|
|
1576
|
+
)
|
|
1577
|
+
)
|
|
1578
|
+
steps.append(
|
|
1579
|
+
_s(
|
|
1580
|
+
"update_compose_project_metadata",
|
|
1581
|
+
f"Update compose_project metadata in inventory for {spec.host.name}",
|
|
1582
|
+
L,
|
|
1583
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1584
|
+
command="upsert_services",
|
|
1585
|
+
tags=["local", "inventory"],
|
|
1586
|
+
)
|
|
1587
|
+
)
|
|
1588
|
+
steps.append(
|
|
1589
|
+
_s(
|
|
1590
|
+
"record_compose_project_run",
|
|
1591
|
+
"Record compose_project run metadata in inventory",
|
|
1592
|
+
L,
|
|
1593
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1594
|
+
command="record_run",
|
|
1595
|
+
tags=["local", "inventory"],
|
|
1596
|
+
)
|
|
1597
|
+
)
|
|
1598
|
+
|
|
1599
|
+
return steps
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
def _plan_stack(spec: StackSpec, ctx: NormalizedContext) -> list[Step]:
|
|
1603
|
+
"""Plan a stack by expanding resources in dependency order.
|
|
1604
|
+
|
|
1605
|
+
Stack resources are topologically sorted by ``depends_on``, then each
|
|
1606
|
+
resource is delegated to its kind's registered planner. Steps from
|
|
1607
|
+
each resource are prefixed with the resource name for traceability.
|
|
1608
|
+
"""
|
|
1609
|
+
from loft_cli_core.registry import get_normalizer, get_planner, get_spec_model
|
|
1610
|
+
|
|
1611
|
+
R = StepScope.REMOTE
|
|
1612
|
+
L = StepScope.LOCAL
|
|
1613
|
+
V = StepScope.VERIFY
|
|
1614
|
+
|
|
1615
|
+
steps: list[Step] = []
|
|
1616
|
+
|
|
1617
|
+
# Topological sort
|
|
1618
|
+
ordered = _topo_sort(spec.resources)
|
|
1619
|
+
|
|
1620
|
+
for res in ordered:
|
|
1621
|
+
planner_fn = get_planner(res.kind)
|
|
1622
|
+
if planner_fn is None:
|
|
1623
|
+
steps.append(
|
|
1624
|
+
_s(
|
|
1625
|
+
f"stack_{res.name}_error",
|
|
1626
|
+
f"No planner for resource kind '{res.kind}'",
|
|
1627
|
+
R,
|
|
1628
|
+
StepKind.AGENT_COMMAND,
|
|
1629
|
+
command=f"echo 'ERROR: no planner for {res.kind}'",
|
|
1630
|
+
tags=["stack", res.name, "error"],
|
|
1631
|
+
)
|
|
1632
|
+
)
|
|
1633
|
+
continue
|
|
1634
|
+
|
|
1635
|
+
# Build a minimal spec-like object for the resource's planner.
|
|
1636
|
+
model_class = get_spec_model(res.kind)
|
|
1637
|
+
if model_class is None:
|
|
1638
|
+
continue
|
|
1639
|
+
|
|
1640
|
+
# Construct a synthetic spec for the child resource. We merge
|
|
1641
|
+
# the stack-level host/login/local into the resource config.
|
|
1642
|
+
child_data = {
|
|
1643
|
+
"kind": res.kind,
|
|
1644
|
+
"meta": {"name": f"{spec.meta.name}/{res.name}"},
|
|
1645
|
+
"host": spec.host.model_dump(),
|
|
1646
|
+
**res.config,
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
# Inject login and local if the model expects them and they're
|
|
1650
|
+
# not already in the config.
|
|
1651
|
+
if "login" not in child_data:
|
|
1652
|
+
child_data["login"] = spec.login.model_dump()
|
|
1653
|
+
if "local" not in child_data:
|
|
1654
|
+
child_data["local"] = spec.local.model_dump()
|
|
1655
|
+
|
|
1656
|
+
try:
|
|
1657
|
+
child_spec = model_class.model_validate(child_data)
|
|
1658
|
+
except Exception:
|
|
1659
|
+
steps.append(
|
|
1660
|
+
_s(
|
|
1661
|
+
f"stack_{res.name}_validation_error",
|
|
1662
|
+
f"Failed to validate resource '{res.name}' config as {res.kind}",
|
|
1663
|
+
R,
|
|
1664
|
+
StepKind.AGENT_COMMAND,
|
|
1665
|
+
command=f"echo 'ERROR: validation failed for {res.name}'",
|
|
1666
|
+
tags=["stack", res.name, "error"],
|
|
1667
|
+
)
|
|
1668
|
+
)
|
|
1669
|
+
continue
|
|
1670
|
+
|
|
1671
|
+
# Normalize the child spec
|
|
1672
|
+
child_ctx = NormalizedContext(spec=child_spec, spec_dir=ctx.spec_dir)
|
|
1673
|
+
normalizer_fn = get_normalizer(res.kind)
|
|
1674
|
+
if normalizer_fn is not None:
|
|
1675
|
+
with contextlib.suppress(Exception):
|
|
1676
|
+
normalizer_fn(child_spec, child_ctx)
|
|
1677
|
+
|
|
1678
|
+
# Generate steps from the child planner
|
|
1679
|
+
child_steps = planner_fn(child_spec, child_ctx)
|
|
1680
|
+
|
|
1681
|
+
# Prefix step IDs with the resource name for traceability
|
|
1682
|
+
for cs in child_steps:
|
|
1683
|
+
cs.id = f"stack_{res.name}_{cs.id}"
|
|
1684
|
+
cs.tags = ["stack", res.name] + cs.tags
|
|
1685
|
+
|
|
1686
|
+
steps.extend(child_steps)
|
|
1687
|
+
|
|
1688
|
+
# Postflight checks for the stack
|
|
1689
|
+
for check in spec.checks:
|
|
1690
|
+
steps.append(
|
|
1691
|
+
_s(
|
|
1692
|
+
f"stack_postflight_{check.type}",
|
|
1693
|
+
f"Stack postflight check: {check.type}",
|
|
1694
|
+
V,
|
|
1695
|
+
StepKind.VERIFY,
|
|
1696
|
+
command=_encode_check_command(check, spec),
|
|
1697
|
+
tags=["stack", "postflight", check.type],
|
|
1698
|
+
)
|
|
1699
|
+
)
|
|
1700
|
+
|
|
1701
|
+
# Local inventory
|
|
1702
|
+
if spec.local.inventory.enabled:
|
|
1703
|
+
steps.append(
|
|
1704
|
+
_s(
|
|
1705
|
+
"stack_open_or_init_local_inventory",
|
|
1706
|
+
"Open or initialize local inventory database",
|
|
1707
|
+
L,
|
|
1708
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1709
|
+
command="init_inventory",
|
|
1710
|
+
tags=["local", "inventory", "stack"],
|
|
1711
|
+
)
|
|
1712
|
+
)
|
|
1713
|
+
|
|
1714
|
+
return steps
|
|
1715
|
+
|
|
1716
|
+
|
|
1717
|
+
def _plan_postgres_ensure(spec: PostgresEnsureSpec, ctx: NormalizedContext) -> list[Step]:
|
|
1718
|
+
"""Generate steps for ensuring PostgreSQL resources exist."""
|
|
1719
|
+
from loft_cli.runtime.steps.postgres_ensure import (
|
|
1720
|
+
ensure_database_cmd,
|
|
1721
|
+
ensure_extension_cmd,
|
|
1722
|
+
ensure_grant_cmd,
|
|
1723
|
+
ensure_user_cmd,
|
|
1724
|
+
pg_isready_cmd,
|
|
1725
|
+
)
|
|
1726
|
+
|
|
1727
|
+
steps: list[Step] = []
|
|
1728
|
+
R = StepScope.REMOTE
|
|
1729
|
+
L = StepScope.LOCAL
|
|
1730
|
+
V = StepScope.VERIFY
|
|
1731
|
+
|
|
1732
|
+
conn = spec.connection
|
|
1733
|
+
conn_kwargs = {
|
|
1734
|
+
"conn_host": conn.host,
|
|
1735
|
+
"conn_port": conn.port,
|
|
1736
|
+
"admin_user": conn.admin_user,
|
|
1737
|
+
"docker_exec": conn.docker_exec,
|
|
1738
|
+
}
|
|
1739
|
+
|
|
1740
|
+
# Preflight
|
|
1741
|
+
steps.append(
|
|
1742
|
+
_s(
|
|
1743
|
+
"preflight_connect_admin",
|
|
1744
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
1745
|
+
R,
|
|
1746
|
+
StepKind.SSH_COMMAND,
|
|
1747
|
+
command="echo 'preflight ok'",
|
|
1748
|
+
tags=["preflight"],
|
|
1749
|
+
)
|
|
1750
|
+
)
|
|
1751
|
+
|
|
1752
|
+
# Gate: verify PostgreSQL is ready
|
|
1753
|
+
steps.append(
|
|
1754
|
+
_s(
|
|
1755
|
+
"pg_isready_gate",
|
|
1756
|
+
"Verify PostgreSQL is accepting connections",
|
|
1757
|
+
R,
|
|
1758
|
+
StepKind.GATE,
|
|
1759
|
+
command=pg_isready_cmd(**conn_kwargs),
|
|
1760
|
+
gate=True,
|
|
1761
|
+
sudo=True,
|
|
1762
|
+
tags=["postgres_ensure", "always"],
|
|
1763
|
+
)
|
|
1764
|
+
)
|
|
1765
|
+
|
|
1766
|
+
# Ensure users
|
|
1767
|
+
for u in spec.users:
|
|
1768
|
+
password = u.password_env if u.password_env else None
|
|
1769
|
+
steps.append(
|
|
1770
|
+
_s(
|
|
1771
|
+
f"ensure_user_{u.name}",
|
|
1772
|
+
f"Ensure PostgreSQL user '{u.name}' exists",
|
|
1773
|
+
R,
|
|
1774
|
+
StepKind.SSH_COMMAND,
|
|
1775
|
+
command=ensure_user_cmd(u.name, password, **conn_kwargs),
|
|
1776
|
+
sudo=True,
|
|
1777
|
+
tags=["postgres_ensure", "user"],
|
|
1778
|
+
)
|
|
1779
|
+
)
|
|
1780
|
+
|
|
1781
|
+
# Ensure databases
|
|
1782
|
+
for d in spec.databases:
|
|
1783
|
+
steps.append(
|
|
1784
|
+
_s(
|
|
1785
|
+
f"ensure_database_{d.name}",
|
|
1786
|
+
f"Ensure PostgreSQL database '{d.name}' exists with owner '{d.owner}'",
|
|
1787
|
+
R,
|
|
1788
|
+
StepKind.SSH_COMMAND,
|
|
1789
|
+
command=ensure_database_cmd(d.name, d.owner, **conn_kwargs),
|
|
1790
|
+
sudo=True,
|
|
1791
|
+
tags=["postgres_ensure", "database"],
|
|
1792
|
+
)
|
|
1793
|
+
)
|
|
1794
|
+
|
|
1795
|
+
# Ensure extensions
|
|
1796
|
+
for e in spec.extensions:
|
|
1797
|
+
steps.append(
|
|
1798
|
+
_s(
|
|
1799
|
+
f"ensure_extension_{e.name}_on_{e.database}",
|
|
1800
|
+
f"Ensure extension '{e.name}' on database '{e.database}'",
|
|
1801
|
+
R,
|
|
1802
|
+
StepKind.SSH_COMMAND,
|
|
1803
|
+
command=ensure_extension_cmd(e.name, e.database, **conn_kwargs),
|
|
1804
|
+
sudo=True,
|
|
1805
|
+
tags=["postgres_ensure", "extension"],
|
|
1806
|
+
)
|
|
1807
|
+
)
|
|
1808
|
+
|
|
1809
|
+
# Ensure grants
|
|
1810
|
+
for g in spec.grants:
|
|
1811
|
+
steps.append(
|
|
1812
|
+
_s(
|
|
1813
|
+
f"grant_{g.privilege.lower()}_{g.on_database}_to_{g.to_user}",
|
|
1814
|
+
f"Grant {g.privilege} on {g.on_database} to {g.to_user}",
|
|
1815
|
+
R,
|
|
1816
|
+
StepKind.SSH_COMMAND,
|
|
1817
|
+
command=ensure_grant_cmd(g.privilege, g.on_database, g.to_user, **conn_kwargs),
|
|
1818
|
+
sudo=True,
|
|
1819
|
+
tags=["postgres_ensure", "grant"],
|
|
1820
|
+
)
|
|
1821
|
+
)
|
|
1822
|
+
|
|
1823
|
+
# Postflight checks
|
|
1824
|
+
for check in spec.checks:
|
|
1825
|
+
steps.append(
|
|
1826
|
+
_s(
|
|
1827
|
+
f"postflight_{check.type}",
|
|
1828
|
+
f"Postflight check: {check.type}",
|
|
1829
|
+
V,
|
|
1830
|
+
StepKind.VERIFY,
|
|
1831
|
+
command=_encode_check_command(check, spec),
|
|
1832
|
+
tags=["postflight", check.type],
|
|
1833
|
+
)
|
|
1834
|
+
)
|
|
1835
|
+
|
|
1836
|
+
# Local inventory
|
|
1837
|
+
if spec.local.inventory.enabled:
|
|
1838
|
+
steps.append(
|
|
1839
|
+
_s(
|
|
1840
|
+
"open_or_init_local_inventory",
|
|
1841
|
+
"Open or initialize local inventory database",
|
|
1842
|
+
L,
|
|
1843
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1844
|
+
command="init_inventory",
|
|
1845
|
+
tags=["local", "inventory"],
|
|
1846
|
+
)
|
|
1847
|
+
)
|
|
1848
|
+
steps.append(
|
|
1849
|
+
_s(
|
|
1850
|
+
"update_postgres_ensure_metadata",
|
|
1851
|
+
f"Update postgres_ensure metadata in inventory for {spec.host.name}",
|
|
1852
|
+
L,
|
|
1853
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1854
|
+
command="upsert_services",
|
|
1855
|
+
tags=["local", "inventory"],
|
|
1856
|
+
)
|
|
1857
|
+
)
|
|
1858
|
+
steps.append(
|
|
1859
|
+
_s(
|
|
1860
|
+
"record_postgres_ensure_run",
|
|
1861
|
+
"Record postgres_ensure run metadata in inventory",
|
|
1862
|
+
L,
|
|
1863
|
+
StepKind.LOCAL_DB_WRITE,
|
|
1864
|
+
command="record_run",
|
|
1865
|
+
tags=["local", "inventory"],
|
|
1866
|
+
)
|
|
1867
|
+
)
|
|
1868
|
+
|
|
1869
|
+
return steps
|
|
1870
|
+
|
|
1871
|
+
|
|
1872
|
+
def _plan_backup_job(spec: BackupJobSpec, ctx: NormalizedContext) -> list[Step]:
|
|
1873
|
+
"""Generate steps for deploying a backup job with systemd timer."""
|
|
1874
|
+
from loft_cli.runtime.steps.backup import render_backup_script
|
|
1875
|
+
from loft_cli.runtime.steps.systemd import (
|
|
1876
|
+
daemon_reload,
|
|
1877
|
+
enable_now_unit,
|
|
1878
|
+
is_active,
|
|
1879
|
+
render_service_unit,
|
|
1880
|
+
render_timer_unit,
|
|
1881
|
+
)
|
|
1882
|
+
|
|
1883
|
+
steps: list[Step] = []
|
|
1884
|
+
R = StepScope.REMOTE
|
|
1885
|
+
L = StepScope.LOCAL
|
|
1886
|
+
V = StepScope.VERIFY
|
|
1887
|
+
|
|
1888
|
+
b = spec.backup
|
|
1889
|
+
src = b.source
|
|
1890
|
+
script_name = f"loft-cli-backup-{b.name}"
|
|
1891
|
+
script_path = f"/usr/local/bin/{script_name}.sh"
|
|
1892
|
+
|
|
1893
|
+
# Preflight
|
|
1894
|
+
steps.append(
|
|
1895
|
+
_s(
|
|
1896
|
+
"preflight_connect_admin",
|
|
1897
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
1898
|
+
R,
|
|
1899
|
+
StepKind.SSH_COMMAND,
|
|
1900
|
+
command="echo 'preflight ok'",
|
|
1901
|
+
tags=["preflight"],
|
|
1902
|
+
)
|
|
1903
|
+
)
|
|
1904
|
+
|
|
1905
|
+
# Create backup destination directory
|
|
1906
|
+
steps.append(
|
|
1907
|
+
_s(
|
|
1908
|
+
f"mkdir_backup_dest_{b.name}",
|
|
1909
|
+
f"Create backup destination directory {b.destination.path}",
|
|
1910
|
+
R,
|
|
1911
|
+
StepKind.SSH_COMMAND,
|
|
1912
|
+
command=f"mkdir -p {b.destination.path}",
|
|
1913
|
+
sudo=True,
|
|
1914
|
+
tags=["backup_job", "mkdir"],
|
|
1915
|
+
)
|
|
1916
|
+
)
|
|
1917
|
+
|
|
1918
|
+
# Write backup script
|
|
1919
|
+
script_content = render_backup_script(
|
|
1920
|
+
name=b.name,
|
|
1921
|
+
source_type=src.type,
|
|
1922
|
+
destination_path=b.destination.path,
|
|
1923
|
+
retention_count=b.retention.count,
|
|
1924
|
+
database=src.database,
|
|
1925
|
+
pg_host=src.host,
|
|
1926
|
+
pg_port=src.port,
|
|
1927
|
+
pg_user=src.user,
|
|
1928
|
+
docker_exec=src.docker_exec,
|
|
1929
|
+
source_path=src.path,
|
|
1930
|
+
)
|
|
1931
|
+
steps.append(
|
|
1932
|
+
_s(
|
|
1933
|
+
f"write_backup_script_{b.name}",
|
|
1934
|
+
f"Write backup script {script_path}",
|
|
1935
|
+
R,
|
|
1936
|
+
StepKind.SSH_UPLOAD,
|
|
1937
|
+
file_content=script_content,
|
|
1938
|
+
target_path=script_path,
|
|
1939
|
+
sudo=True,
|
|
1940
|
+
tags=["backup_job", "upload"],
|
|
1941
|
+
)
|
|
1942
|
+
)
|
|
1943
|
+
|
|
1944
|
+
# Make script executable
|
|
1945
|
+
steps.append(
|
|
1946
|
+
_s(
|
|
1947
|
+
f"chmod_backup_script_{b.name}",
|
|
1948
|
+
"Make backup script executable",
|
|
1949
|
+
R,
|
|
1950
|
+
StepKind.SSH_COMMAND,
|
|
1951
|
+
command=f"chmod 0755 {script_path}",
|
|
1952
|
+
sudo=True,
|
|
1953
|
+
tags=["backup_job"],
|
|
1954
|
+
)
|
|
1955
|
+
)
|
|
1956
|
+
|
|
1957
|
+
# Write systemd service (oneshot)
|
|
1958
|
+
service_content = render_service_unit(
|
|
1959
|
+
description=f"Backup job: {b.name}",
|
|
1960
|
+
exec_start=script_path,
|
|
1961
|
+
service_type="oneshot",
|
|
1962
|
+
restart="no",
|
|
1963
|
+
restart_sec=0,
|
|
1964
|
+
wanted_by="",
|
|
1965
|
+
)
|
|
1966
|
+
service_file = f"/etc/systemd/system/{script_name}.service"
|
|
1967
|
+
steps.append(
|
|
1968
|
+
_s(
|
|
1969
|
+
f"write_backup_service_{b.name}",
|
|
1970
|
+
f"Write backup service file {service_file}",
|
|
1971
|
+
R,
|
|
1972
|
+
StepKind.SSH_UPLOAD,
|
|
1973
|
+
file_content=service_content,
|
|
1974
|
+
target_path=service_file,
|
|
1975
|
+
sudo=True,
|
|
1976
|
+
tags=["backup_job", "upload"],
|
|
1977
|
+
)
|
|
1978
|
+
)
|
|
1979
|
+
|
|
1980
|
+
# Write systemd timer
|
|
1981
|
+
timer_content = render_timer_unit(
|
|
1982
|
+
description=f"Backup timer: {b.name}",
|
|
1983
|
+
on_calendar=b.schedule,
|
|
1984
|
+
persistent=True,
|
|
1985
|
+
)
|
|
1986
|
+
timer_file = f"/etc/systemd/system/{script_name}.timer"
|
|
1987
|
+
steps.append(
|
|
1988
|
+
_s(
|
|
1989
|
+
f"write_backup_timer_{b.name}",
|
|
1990
|
+
f"Write backup timer file {timer_file}",
|
|
1991
|
+
R,
|
|
1992
|
+
StepKind.SSH_UPLOAD,
|
|
1993
|
+
file_content=timer_content,
|
|
1994
|
+
target_path=timer_file,
|
|
1995
|
+
sudo=True,
|
|
1996
|
+
tags=["backup_job", "upload"],
|
|
1997
|
+
)
|
|
1998
|
+
)
|
|
1999
|
+
|
|
2000
|
+
# Daemon reload
|
|
2001
|
+
steps.append(
|
|
2002
|
+
_s(
|
|
2003
|
+
"systemd_daemon_reload",
|
|
2004
|
+
"Reload systemd daemon",
|
|
2005
|
+
R,
|
|
2006
|
+
StepKind.SSH_COMMAND,
|
|
2007
|
+
command=daemon_reload(),
|
|
2008
|
+
sudo=True,
|
|
2009
|
+
tags=["backup_job", "always"],
|
|
2010
|
+
)
|
|
2011
|
+
)
|
|
2012
|
+
|
|
2013
|
+
# Enable and start timer
|
|
2014
|
+
steps.append(
|
|
2015
|
+
_s(
|
|
2016
|
+
f"enable_start_backup_timer_{b.name}",
|
|
2017
|
+
f"Enable and start {script_name}.timer",
|
|
2018
|
+
R,
|
|
2019
|
+
StepKind.SSH_COMMAND,
|
|
2020
|
+
command=enable_now_unit(f"{script_name}.timer"),
|
|
2021
|
+
sudo=True,
|
|
2022
|
+
tags=["backup_job", "always"],
|
|
2023
|
+
)
|
|
2024
|
+
)
|
|
2025
|
+
|
|
2026
|
+
# Verify timer active
|
|
2027
|
+
steps.append(
|
|
2028
|
+
_s(
|
|
2029
|
+
f"verify_backup_timer_{b.name}_active",
|
|
2030
|
+
f"Verify {script_name}.timer is active",
|
|
2031
|
+
V,
|
|
2032
|
+
StepKind.VERIFY,
|
|
2033
|
+
command=is_active(f"{script_name}.timer"),
|
|
2034
|
+
tags=["backup_job", "verify"],
|
|
2035
|
+
)
|
|
2036
|
+
)
|
|
2037
|
+
|
|
2038
|
+
# Postflight checks
|
|
2039
|
+
for check in spec.checks:
|
|
2040
|
+
steps.append(
|
|
2041
|
+
_s(
|
|
2042
|
+
f"postflight_{check.type}",
|
|
2043
|
+
f"Postflight check: {check.type}",
|
|
2044
|
+
V,
|
|
2045
|
+
StepKind.VERIFY,
|
|
2046
|
+
command=_encode_check_command(check, spec),
|
|
2047
|
+
tags=["postflight", check.type],
|
|
2048
|
+
)
|
|
2049
|
+
)
|
|
2050
|
+
|
|
2051
|
+
# Local inventory
|
|
2052
|
+
if spec.local.inventory.enabled:
|
|
2053
|
+
steps.append(
|
|
2054
|
+
_s(
|
|
2055
|
+
"open_or_init_local_inventory",
|
|
2056
|
+
"Open or initialize local inventory database",
|
|
2057
|
+
L,
|
|
2058
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2059
|
+
command="init_inventory",
|
|
2060
|
+
tags=["local", "inventory"],
|
|
2061
|
+
)
|
|
2062
|
+
)
|
|
2063
|
+
steps.append(
|
|
2064
|
+
_s(
|
|
2065
|
+
"update_backup_job_metadata",
|
|
2066
|
+
f"Update backup_job metadata in inventory for {spec.host.name}",
|
|
2067
|
+
L,
|
|
2068
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2069
|
+
command="upsert_services",
|
|
2070
|
+
tags=["local", "inventory"],
|
|
2071
|
+
)
|
|
2072
|
+
)
|
|
2073
|
+
steps.append(
|
|
2074
|
+
_s(
|
|
2075
|
+
"record_backup_job_run",
|
|
2076
|
+
"Record backup_job run metadata in inventory",
|
|
2077
|
+
L,
|
|
2078
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2079
|
+
command="record_run",
|
|
2080
|
+
tags=["local", "inventory"],
|
|
2081
|
+
)
|
|
2082
|
+
)
|
|
2083
|
+
|
|
2084
|
+
return steps
|
|
2085
|
+
|
|
2086
|
+
|
|
2087
|
+
def _plan_systemd_unit(spec: SystemdUnitSpec, ctx: NormalizedContext) -> list[Step]:
|
|
2088
|
+
"""Generate steps for deploying a systemd service unit."""
|
|
2089
|
+
from loft_cli.runtime.steps.systemd import (
|
|
2090
|
+
daemon_reload,
|
|
2091
|
+
enable_unit,
|
|
2092
|
+
is_active,
|
|
2093
|
+
render_logrotate_config,
|
|
2094
|
+
render_service_unit,
|
|
2095
|
+
restart_unit,
|
|
2096
|
+
)
|
|
2097
|
+
|
|
2098
|
+
steps: list[Step] = []
|
|
2099
|
+
R = StepScope.REMOTE
|
|
2100
|
+
L = StepScope.LOCAL
|
|
2101
|
+
V = StepScope.VERIFY
|
|
2102
|
+
|
|
2103
|
+
u = spec.unit
|
|
2104
|
+
unit_file = f"/etc/systemd/system/{u.unit_name}.service"
|
|
2105
|
+
|
|
2106
|
+
# Preflight
|
|
2107
|
+
steps.append(
|
|
2108
|
+
_s(
|
|
2109
|
+
"preflight_connect_admin",
|
|
2110
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
2111
|
+
R,
|
|
2112
|
+
StepKind.SSH_COMMAND,
|
|
2113
|
+
command="echo 'preflight ok'",
|
|
2114
|
+
tags=["preflight"],
|
|
2115
|
+
)
|
|
2116
|
+
)
|
|
2117
|
+
|
|
2118
|
+
# Write unit file
|
|
2119
|
+
unit_content = render_service_unit(
|
|
2120
|
+
description=u.description or spec.meta.description or u.unit_name,
|
|
2121
|
+
exec_start=u.exec_start,
|
|
2122
|
+
exec_stop=u.exec_stop,
|
|
2123
|
+
working_directory=u.working_directory,
|
|
2124
|
+
user=u.user,
|
|
2125
|
+
group=u.group,
|
|
2126
|
+
restart=u.restart,
|
|
2127
|
+
restart_sec=u.restart_sec,
|
|
2128
|
+
after=u.after,
|
|
2129
|
+
environment=u.environment,
|
|
2130
|
+
environment_file=u.environment_file,
|
|
2131
|
+
service_type=u.type,
|
|
2132
|
+
wanted_by=u.wanted_by,
|
|
2133
|
+
)
|
|
2134
|
+
|
|
2135
|
+
steps.append(
|
|
2136
|
+
_s(
|
|
2137
|
+
f"write_unit_{u.unit_name}",
|
|
2138
|
+
f"Write systemd unit file {unit_file}",
|
|
2139
|
+
R,
|
|
2140
|
+
StepKind.SSH_UPLOAD,
|
|
2141
|
+
file_content=unit_content,
|
|
2142
|
+
target_path=unit_file,
|
|
2143
|
+
sudo=True,
|
|
2144
|
+
tags=["systemd_unit", "upload"],
|
|
2145
|
+
)
|
|
2146
|
+
)
|
|
2147
|
+
|
|
2148
|
+
# Daemon reload
|
|
2149
|
+
steps.append(
|
|
2150
|
+
_s(
|
|
2151
|
+
"systemd_daemon_reload",
|
|
2152
|
+
"Reload systemd daemon",
|
|
2153
|
+
R,
|
|
2154
|
+
StepKind.SSH_COMMAND,
|
|
2155
|
+
command=daemon_reload(),
|
|
2156
|
+
sudo=True,
|
|
2157
|
+
tags=["systemd_unit", "always"],
|
|
2158
|
+
)
|
|
2159
|
+
)
|
|
2160
|
+
|
|
2161
|
+
# Enable
|
|
2162
|
+
steps.append(
|
|
2163
|
+
_s(
|
|
2164
|
+
f"enable_{u.unit_name}",
|
|
2165
|
+
f"Enable {u.unit_name}.service",
|
|
2166
|
+
R,
|
|
2167
|
+
StepKind.SSH_COMMAND,
|
|
2168
|
+
command=enable_unit(u.unit_name),
|
|
2169
|
+
sudo=True,
|
|
2170
|
+
tags=["systemd_unit"],
|
|
2171
|
+
)
|
|
2172
|
+
)
|
|
2173
|
+
|
|
2174
|
+
# Restart
|
|
2175
|
+
steps.append(
|
|
2176
|
+
_s(
|
|
2177
|
+
f"restart_{u.unit_name}",
|
|
2178
|
+
f"Restart {u.unit_name}.service",
|
|
2179
|
+
R,
|
|
2180
|
+
StepKind.SSH_COMMAND,
|
|
2181
|
+
command=restart_unit(u.unit_name),
|
|
2182
|
+
sudo=True,
|
|
2183
|
+
tags=["systemd_unit", "always"],
|
|
2184
|
+
)
|
|
2185
|
+
)
|
|
2186
|
+
|
|
2187
|
+
# Verify active
|
|
2188
|
+
steps.append(
|
|
2189
|
+
_s(
|
|
2190
|
+
f"verify_{u.unit_name}_active",
|
|
2191
|
+
f"Verify {u.unit_name}.service is active",
|
|
2192
|
+
V,
|
|
2193
|
+
StepKind.VERIFY,
|
|
2194
|
+
command=is_active(u.unit_name),
|
|
2195
|
+
tags=["systemd_unit", "verify"],
|
|
2196
|
+
)
|
|
2197
|
+
)
|
|
2198
|
+
|
|
2199
|
+
# Optional logrotate
|
|
2200
|
+
lr = spec.logrotate
|
|
2201
|
+
if lr and lr.enabled:
|
|
2202
|
+
logrotate_content = render_logrotate_config(
|
|
2203
|
+
name=u.unit_name,
|
|
2204
|
+
path=lr.path,
|
|
2205
|
+
rotate=lr.rotate,
|
|
2206
|
+
frequency=lr.frequency,
|
|
2207
|
+
compress=lr.compress,
|
|
2208
|
+
max_size=lr.max_size,
|
|
2209
|
+
)
|
|
2210
|
+
steps.append(
|
|
2211
|
+
_s(
|
|
2212
|
+
f"write_logrotate_{u.unit_name}",
|
|
2213
|
+
f"Write logrotate config for {u.unit_name}",
|
|
2214
|
+
R,
|
|
2215
|
+
StepKind.SSH_UPLOAD,
|
|
2216
|
+
file_content=logrotate_content,
|
|
2217
|
+
target_path=f"/etc/logrotate.d/loft-cli-{u.unit_name}",
|
|
2218
|
+
sudo=True,
|
|
2219
|
+
tags=["systemd_unit", "logrotate"],
|
|
2220
|
+
)
|
|
2221
|
+
)
|
|
2222
|
+
|
|
2223
|
+
# Postflight checks
|
|
2224
|
+
for check in spec.checks:
|
|
2225
|
+
steps.append(
|
|
2226
|
+
_s(
|
|
2227
|
+
f"postflight_{check.type}",
|
|
2228
|
+
f"Postflight check: {check.type}",
|
|
2229
|
+
V,
|
|
2230
|
+
StepKind.VERIFY,
|
|
2231
|
+
command=_encode_check_command(check, spec),
|
|
2232
|
+
tags=["postflight", check.type],
|
|
2233
|
+
)
|
|
2234
|
+
)
|
|
2235
|
+
|
|
2236
|
+
# Local inventory
|
|
2237
|
+
if spec.local.inventory.enabled:
|
|
2238
|
+
steps.append(
|
|
2239
|
+
_s(
|
|
2240
|
+
"open_or_init_local_inventory",
|
|
2241
|
+
"Open or initialize local inventory database",
|
|
2242
|
+
L,
|
|
2243
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2244
|
+
command="init_inventory",
|
|
2245
|
+
tags=["local", "inventory"],
|
|
2246
|
+
)
|
|
2247
|
+
)
|
|
2248
|
+
steps.append(
|
|
2249
|
+
_s(
|
|
2250
|
+
"update_systemd_unit_metadata",
|
|
2251
|
+
f"Update systemd_unit metadata in inventory for {spec.host.name}",
|
|
2252
|
+
L,
|
|
2253
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2254
|
+
command="upsert_services",
|
|
2255
|
+
tags=["local", "inventory"],
|
|
2256
|
+
)
|
|
2257
|
+
)
|
|
2258
|
+
steps.append(
|
|
2259
|
+
_s(
|
|
2260
|
+
"record_systemd_unit_run",
|
|
2261
|
+
"Record systemd_unit run metadata in inventory",
|
|
2262
|
+
L,
|
|
2263
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2264
|
+
command="record_run",
|
|
2265
|
+
tags=["local", "inventory"],
|
|
2266
|
+
)
|
|
2267
|
+
)
|
|
2268
|
+
|
|
2269
|
+
return steps
|
|
2270
|
+
|
|
2271
|
+
|
|
2272
|
+
def _plan_systemd_timer(spec: SystemdTimerSpec, ctx: NormalizedContext) -> list[Step]:
|
|
2273
|
+
"""Generate steps for deploying a systemd timer with companion oneshot service."""
|
|
2274
|
+
from loft_cli.runtime.steps.systemd import (
|
|
2275
|
+
daemon_reload,
|
|
2276
|
+
enable_now_unit,
|
|
2277
|
+
is_active,
|
|
2278
|
+
render_service_unit,
|
|
2279
|
+
render_timer_unit,
|
|
2280
|
+
)
|
|
2281
|
+
|
|
2282
|
+
steps: list[Step] = []
|
|
2283
|
+
R = StepScope.REMOTE
|
|
2284
|
+
L = StepScope.LOCAL
|
|
2285
|
+
V = StepScope.VERIFY
|
|
2286
|
+
|
|
2287
|
+
t = spec.timer
|
|
2288
|
+
s = spec.service
|
|
2289
|
+
|
|
2290
|
+
# Preflight
|
|
2291
|
+
steps.append(
|
|
2292
|
+
_s(
|
|
2293
|
+
"preflight_connect_admin",
|
|
2294
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
2295
|
+
R,
|
|
2296
|
+
StepKind.SSH_COMMAND,
|
|
2297
|
+
command="echo 'preflight ok'",
|
|
2298
|
+
tags=["preflight"],
|
|
2299
|
+
)
|
|
2300
|
+
)
|
|
2301
|
+
|
|
2302
|
+
# Write companion .service (Type=oneshot)
|
|
2303
|
+
service_content = render_service_unit(
|
|
2304
|
+
description=t.description or f"Service for timer {t.timer_name}",
|
|
2305
|
+
exec_start=s.exec_start,
|
|
2306
|
+
working_directory=s.working_directory,
|
|
2307
|
+
user=s.user,
|
|
2308
|
+
group=s.group,
|
|
2309
|
+
restart="no",
|
|
2310
|
+
restart_sec=0,
|
|
2311
|
+
environment=s.environment,
|
|
2312
|
+
service_type="oneshot",
|
|
2313
|
+
wanted_by="",
|
|
2314
|
+
)
|
|
2315
|
+
service_file = f"/etc/systemd/system/{t.timer_name}.service"
|
|
2316
|
+
steps.append(
|
|
2317
|
+
_s(
|
|
2318
|
+
f"write_service_{t.timer_name}",
|
|
2319
|
+
f"Write companion service file {service_file}",
|
|
2320
|
+
R,
|
|
2321
|
+
StepKind.SSH_UPLOAD,
|
|
2322
|
+
file_content=service_content,
|
|
2323
|
+
target_path=service_file,
|
|
2324
|
+
sudo=True,
|
|
2325
|
+
tags=["systemd_timer", "upload"],
|
|
2326
|
+
)
|
|
2327
|
+
)
|
|
2328
|
+
|
|
2329
|
+
# Write .timer
|
|
2330
|
+
timer_content = render_timer_unit(
|
|
2331
|
+
description=t.description or f"Timer for {t.timer_name}",
|
|
2332
|
+
on_calendar=t.on_calendar,
|
|
2333
|
+
persistent=t.persistent,
|
|
2334
|
+
accuracy_sec=t.accuracy_sec,
|
|
2335
|
+
)
|
|
2336
|
+
timer_file = f"/etc/systemd/system/{t.timer_name}.timer"
|
|
2337
|
+
steps.append(
|
|
2338
|
+
_s(
|
|
2339
|
+
f"write_timer_{t.timer_name}",
|
|
2340
|
+
f"Write timer unit file {timer_file}",
|
|
2341
|
+
R,
|
|
2342
|
+
StepKind.SSH_UPLOAD,
|
|
2343
|
+
file_content=timer_content,
|
|
2344
|
+
target_path=timer_file,
|
|
2345
|
+
sudo=True,
|
|
2346
|
+
tags=["systemd_timer", "upload"],
|
|
2347
|
+
)
|
|
2348
|
+
)
|
|
2349
|
+
|
|
2350
|
+
# Daemon reload
|
|
2351
|
+
steps.append(
|
|
2352
|
+
_s(
|
|
2353
|
+
"systemd_daemon_reload",
|
|
2354
|
+
"Reload systemd daemon",
|
|
2355
|
+
R,
|
|
2356
|
+
StepKind.SSH_COMMAND,
|
|
2357
|
+
command=daemon_reload(),
|
|
2358
|
+
sudo=True,
|
|
2359
|
+
tags=["systemd_timer", "always"],
|
|
2360
|
+
)
|
|
2361
|
+
)
|
|
2362
|
+
|
|
2363
|
+
# Enable and start timer
|
|
2364
|
+
steps.append(
|
|
2365
|
+
_s(
|
|
2366
|
+
f"enable_start_{t.timer_name}_timer",
|
|
2367
|
+
f"Enable and start {t.timer_name}.timer",
|
|
2368
|
+
R,
|
|
2369
|
+
StepKind.SSH_COMMAND,
|
|
2370
|
+
command=enable_now_unit(f"{t.timer_name}.timer"),
|
|
2371
|
+
sudo=True,
|
|
2372
|
+
tags=["systemd_timer", "always"],
|
|
2373
|
+
)
|
|
2374
|
+
)
|
|
2375
|
+
|
|
2376
|
+
# Verify timer active
|
|
2377
|
+
steps.append(
|
|
2378
|
+
_s(
|
|
2379
|
+
f"verify_{t.timer_name}_timer_active",
|
|
2380
|
+
f"Verify {t.timer_name}.timer is active",
|
|
2381
|
+
V,
|
|
2382
|
+
StepKind.VERIFY,
|
|
2383
|
+
command=is_active(f"{t.timer_name}.timer"),
|
|
2384
|
+
tags=["systemd_timer", "verify"],
|
|
2385
|
+
)
|
|
2386
|
+
)
|
|
2387
|
+
|
|
2388
|
+
# Postflight checks
|
|
2389
|
+
for check in spec.checks:
|
|
2390
|
+
steps.append(
|
|
2391
|
+
_s(
|
|
2392
|
+
f"postflight_{check.type}",
|
|
2393
|
+
f"Postflight check: {check.type}",
|
|
2394
|
+
V,
|
|
2395
|
+
StepKind.VERIFY,
|
|
2396
|
+
command=_encode_check_command(check, spec),
|
|
2397
|
+
tags=["postflight", check.type],
|
|
2398
|
+
)
|
|
2399
|
+
)
|
|
2400
|
+
|
|
2401
|
+
# Local inventory
|
|
2402
|
+
if spec.local.inventory.enabled:
|
|
2403
|
+
steps.append(
|
|
2404
|
+
_s(
|
|
2405
|
+
"open_or_init_local_inventory",
|
|
2406
|
+
"Open or initialize local inventory database",
|
|
2407
|
+
L,
|
|
2408
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2409
|
+
command="init_inventory",
|
|
2410
|
+
tags=["local", "inventory"],
|
|
2411
|
+
)
|
|
2412
|
+
)
|
|
2413
|
+
steps.append(
|
|
2414
|
+
_s(
|
|
2415
|
+
"update_systemd_timer_metadata",
|
|
2416
|
+
f"Update systemd_timer metadata in inventory for {spec.host.name}",
|
|
2417
|
+
L,
|
|
2418
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2419
|
+
command="upsert_services",
|
|
2420
|
+
tags=["local", "inventory"],
|
|
2421
|
+
)
|
|
2422
|
+
)
|
|
2423
|
+
steps.append(
|
|
2424
|
+
_s(
|
|
2425
|
+
"record_systemd_timer_run",
|
|
2426
|
+
"Record systemd_timer run metadata in inventory",
|
|
2427
|
+
L,
|
|
2428
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2429
|
+
command="record_run",
|
|
2430
|
+
tags=["local", "inventory"],
|
|
2431
|
+
)
|
|
2432
|
+
)
|
|
2433
|
+
|
|
2434
|
+
return steps
|
|
2435
|
+
|
|
2436
|
+
|
|
2437
|
+
def _plan_http_check(spec: HttpCheckSpec, ctx: NormalizedContext) -> list[Step]:
|
|
2438
|
+
"""Generate steps for a GET-only HTTP readiness probe."""
|
|
2439
|
+
steps: list[Step] = []
|
|
2440
|
+
R = StepScope.REMOTE
|
|
2441
|
+
L = StepScope.LOCAL
|
|
2442
|
+
V = StepScope.VERIFY
|
|
2443
|
+
|
|
2444
|
+
c = spec.check
|
|
2445
|
+
|
|
2446
|
+
# Preflight
|
|
2447
|
+
steps.append(
|
|
2448
|
+
_s(
|
|
2449
|
+
"preflight_connect_admin",
|
|
2450
|
+
f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
|
|
2451
|
+
R,
|
|
2452
|
+
StepKind.SSH_COMMAND,
|
|
2453
|
+
command="echo 'preflight ok'",
|
|
2454
|
+
tags=["preflight"],
|
|
2455
|
+
)
|
|
2456
|
+
)
|
|
2457
|
+
|
|
2458
|
+
# HTTP check gate — executed by agent with retry loop
|
|
2459
|
+
steps.append(
|
|
2460
|
+
_s(
|
|
2461
|
+
"http_check_gate",
|
|
2462
|
+
f"HTTP readiness check: GET {c.url} expecting {c.expected_status}",
|
|
2463
|
+
R,
|
|
2464
|
+
StepKind.GATE,
|
|
2465
|
+
command=f"http_check:{c.url}:{c.expected_status}:{c.retries}:{c.interval}:{c.timeout}",
|
|
2466
|
+
gate=True,
|
|
2467
|
+
tags=["http_check", "always"],
|
|
2468
|
+
)
|
|
2469
|
+
)
|
|
2470
|
+
|
|
2471
|
+
# Postflight checks
|
|
2472
|
+
for check in spec.checks:
|
|
2473
|
+
steps.append(
|
|
2474
|
+
_s(
|
|
2475
|
+
f"postflight_{check.type}",
|
|
2476
|
+
f"Postflight check: {check.type}",
|
|
2477
|
+
V,
|
|
2478
|
+
StepKind.VERIFY,
|
|
2479
|
+
command=_encode_check_command(check, spec),
|
|
2480
|
+
tags=["postflight", check.type],
|
|
2481
|
+
)
|
|
2482
|
+
)
|
|
2483
|
+
|
|
2484
|
+
# Local inventory
|
|
2485
|
+
if spec.local.inventory.enabled:
|
|
2486
|
+
steps.append(
|
|
2487
|
+
_s(
|
|
2488
|
+
"open_or_init_local_inventory",
|
|
2489
|
+
"Open or initialize local inventory database",
|
|
2490
|
+
L,
|
|
2491
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2492
|
+
command="init_inventory",
|
|
2493
|
+
tags=["local", "inventory"],
|
|
2494
|
+
)
|
|
2495
|
+
)
|
|
2496
|
+
steps.append(
|
|
2497
|
+
_s(
|
|
2498
|
+
"update_http_check_metadata",
|
|
2499
|
+
f"Update http_check metadata in inventory for {spec.host.name}",
|
|
2500
|
+
L,
|
|
2501
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2502
|
+
command="upsert_services",
|
|
2503
|
+
tags=["local", "inventory"],
|
|
2504
|
+
)
|
|
2505
|
+
)
|
|
2506
|
+
steps.append(
|
|
2507
|
+
_s(
|
|
2508
|
+
"record_http_check_run",
|
|
2509
|
+
"Record http_check run metadata in inventory",
|
|
2510
|
+
L,
|
|
2511
|
+
StepKind.LOCAL_DB_WRITE,
|
|
2512
|
+
command="record_run",
|
|
2513
|
+
tags=["local", "inventory"],
|
|
2514
|
+
)
|
|
2515
|
+
)
|
|
2516
|
+
|
|
2517
|
+
return steps
|
|
2518
|
+
|
|
2519
|
+
|
|
2520
|
+
def _topo_sort(resources) -> list:
|
|
2521
|
+
"""Topologically sort stack resources by depends_on.
|
|
2522
|
+
|
|
2523
|
+
Returns resources in dependency-first order.
|
|
2524
|
+
"""
|
|
2525
|
+
name_to_res = {r.name: r for r in resources}
|
|
2526
|
+
visited: set[str] = set()
|
|
2527
|
+
result: list = []
|
|
2528
|
+
|
|
2529
|
+
def _visit(name: str) -> None:
|
|
2530
|
+
if name in visited:
|
|
2531
|
+
return
|
|
2532
|
+
visited.add(name)
|
|
2533
|
+
res = name_to_res.get(name)
|
|
2534
|
+
if res is None:
|
|
2535
|
+
return
|
|
2536
|
+
for dep in res.depends_on:
|
|
2537
|
+
_visit(dep)
|
|
2538
|
+
result.append(res)
|
|
2539
|
+
|
|
2540
|
+
for r in resources:
|
|
2541
|
+
_visit(r.name)
|
|
2542
|
+
|
|
2543
|
+
return result
|