loft-cli 0.6.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. loft_cli/__init__.py +3 -0
  2. loft_cli/_builtins.py +269 -0
  3. loft_cli/addons/__init__.py +12 -0
  4. loft_cli/addons/goss/__init__.py +1 -0
  5. loft_cli/addons/goss/generator.py +202 -0
  6. loft_cli/addons/goss/renderer.py +207 -0
  7. loft_cli/addons/goss/shipper.py +174 -0
  8. loft_cli/agent_installer.py +65 -0
  9. loft_cli/checks/__init__.py +0 -0
  10. loft_cli/checks/compose.py +147 -0
  11. loft_cli/checks/container.py +24 -0
  12. loft_cli/checks/http.py +30 -0
  13. loft_cli/checks/nginx.py +21 -0
  14. loft_cli/checks/ports.py +24 -0
  15. loft_cli/checks/postgres.py +21 -0
  16. loft_cli/checks/ssh.py +66 -0
  17. loft_cli/checks/wireguard.py +21 -0
  18. loft_cli/cli.py +1161 -0
  19. loft_cli/compiler/__init__.py +0 -0
  20. loft_cli/compiler/normalizer.py +420 -0
  21. loft_cli/compiler/parser.py +42 -0
  22. loft_cli/compiler/planner.py +2543 -0
  23. loft_cli/local/__init__.py +0 -0
  24. loft_cli/local/ddl/__init__.py +0 -0
  25. loft_cli/local/ddl/bootstrap_tables.py +75 -0
  26. loft_cli/local/ddl/versionize_system.py +374 -0
  27. loft_cli/local/inventory.py +449 -0
  28. loft_cli/local/inventory_db.py +255 -0
  29. loft_cli/local/keys.py +45 -0
  30. loft_cli/local/remove.py +135 -0
  31. loft_cli/local/ssh_config.py +118 -0
  32. loft_cli/local/tunnel.py +228 -0
  33. loft_cli/local/wireguard_store.py +138 -0
  34. loft_cli/logs/__init__.py +0 -0
  35. loft_cli/logs/reader.py +55 -0
  36. loft_cli/logs/writer.py +58 -0
  37. loft_cli/runtime/__init__.py +0 -0
  38. loft_cli/runtime/agent_transport.py +115 -0
  39. loft_cli/runtime/executor.py +798 -0
  40. loft_cli/runtime/fabric_transport.py +68 -0
  41. loft_cli/runtime/secret_rotation.py +99 -0
  42. loft_cli/runtime/ssh.py +147 -0
  43. loft_cli/runtime/steps/__init__.py +0 -0
  44. loft_cli/runtime/steps/backup.py +72 -0
  45. loft_cli/runtime/steps/bootstrap.py +227 -0
  46. loft_cli/runtime/steps/compose.py +43 -0
  47. loft_cli/runtime/steps/container.py +40 -0
  48. loft_cli/runtime/steps/docker.py +15 -0
  49. loft_cli/runtime/steps/file_template.py +25 -0
  50. loft_cli/runtime/steps/nginx.py +126 -0
  51. loft_cli/runtime/steps/postgres.py +101 -0
  52. loft_cli/runtime/steps/postgres_ensure.py +141 -0
  53. loft_cli/runtime/steps/systemd.py +131 -0
  54. loft_cli/runtime/steps/wireguard.py +76 -0
  55. loft_cli/runtime/transport.py +43 -0
  56. loft_cli/updater.py +208 -0
  57. loft_cli-0.6.3.dist-info/METADATA +19 -0
  58. loft_cli-0.6.3.dist-info/RECORD +61 -0
  59. loft_cli-0.6.3.dist-info/WHEEL +5 -0
  60. loft_cli-0.6.3.dist-info/entry_points.txt +5 -0
  61. loft_cli-0.6.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2543 @@
1
+ """Phase 3: Convert normalized spec into an ordered Plan.
2
+
3
+ CRITICAL INVARIANT (SSH lockout prevention):
4
+ Steps 'disable_root_login' and 'disable_password_auth' MUST have
5
+ depends_on referencing the index of 'verify_admin_login_on_new_port'.
6
+ 'verify_admin_login_on_new_port' MUST be gate=True.
7
+ Local steps MUST only be generated after all critical remote steps.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import contextlib
13
+ from datetime import UTC, datetime
14
+
15
+ from loft_cli.compiler.normalizer import NormalizedContext
16
+ from loft_cli_core.plan.models import Plan, Step, StepKind, StepScope
17
+ from loft_cli_core.specs.backup_job_schema import BackupJobSpec
18
+ from loft_cli_core.specs.bootstrap_schema import BootstrapSpec
19
+ from loft_cli_core.specs.compose_project_schema import ComposeProjectSpec
20
+ from loft_cli_core.specs.file_template_schema import FileTemplateSpec
21
+ from loft_cli_core.specs.http_check_schema import HttpCheckSpec
22
+ from loft_cli_core.specs.postgres_ensure_schema import PostgresEnsureSpec
23
+ from loft_cli_core.specs.service_schema import ServiceSpec
24
+ from loft_cli_core.specs.stack_schema import StackSpec
25
+ from loft_cli_core.specs.systemd_timer_schema import SystemdTimerSpec
26
+ from loft_cli_core.specs.systemd_unit_schema import SystemdUnitSpec
27
+ from loft_cli_core.utils.hashing import sha256_string
28
+
29
+ AnySpec = (
30
+ BootstrapSpec
31
+ | ServiceSpec
32
+ | FileTemplateSpec
33
+ | ComposeProjectSpec
34
+ | StackSpec
35
+ | HttpCheckSpec
36
+ | SystemdUnitSpec
37
+ | SystemdTimerSpec
38
+ | BackupJobSpec
39
+ | PostgresEnsureSpec
40
+ )
41
+
42
+
43
+ def _encode_check_command(check, spec) -> str:
44
+ """Encode a CheckBlock into a command string for the executor to dispatch."""
45
+ ctype = check.type
46
+ host = spec.host.address
47
+ if ctype == "ssh_reachable":
48
+ port = check.port or 22
49
+ user = check.user or "root"
50
+ return f"check:ssh_reachable:{host}:{port}:{user}"
51
+ elif ctype == "port_open":
52
+ port = check.port or 22
53
+ return f"check:port_open:{host}:{port}"
54
+ elif ctype == "wireguard_up":
55
+ iface = check.interface or "wg0"
56
+ return f"check:wireguard_up:{iface}"
57
+ elif ctype == "container_running":
58
+ name = check.name or "unknown"
59
+ return f"check:container_running:{name}"
60
+ elif ctype == "http":
61
+ url = check.url or ""
62
+ status = check.expect_status or 200
63
+ return f"check:http:{url}:{status}"
64
+ elif ctype == "postgres_ready":
65
+ return "check:postgres_ready"
66
+ elif ctype == "nginx_ready":
67
+ return "check:nginx_ready"
68
+ return f"check:{ctype}"
69
+
70
+
71
+ def plan(ctx: NormalizedContext) -> Plan:
72
+ """Convert a NormalizedContext into an executable Plan."""
73
+ from loft_cli_core.registry import get_planner, load_addons
74
+
75
+ load_addons()
76
+
77
+ spec = ctx.spec
78
+ planner_fn = get_planner(spec.kind)
79
+ if planner_fn is None:
80
+ raise RuntimeError(f"No planner registered for spec kind '{spec.kind}'")
81
+ steps = planner_fn(spec, ctx)
82
+
83
+ # Re-index all steps
84
+ for i, step in enumerate(steps):
85
+ step.index = i
86
+
87
+ spec_hash = sha256_string(spec.model_dump_json())
88
+ plan_obj = Plan(
89
+ spec_name=spec.meta.name,
90
+ spec_kind=spec.kind,
91
+ target_host=spec.host.address,
92
+ spec_hash=spec_hash,
93
+ plan_hash="",
94
+ steps=steps,
95
+ created_at=datetime.now(UTC).isoformat(),
96
+ )
97
+ plan_obj.plan_hash = sha256_string("".join(s.id + (s.command or "") for s in steps))
98
+ return plan_obj
99
+
100
+
101
+ def _s(
102
+ id: str,
103
+ description: str,
104
+ scope: StepScope,
105
+ kind: StepKind,
106
+ *,
107
+ command: str | None = None,
108
+ file_content: str | None = None,
109
+ target_path: str | None = None,
110
+ sudo: bool = False,
111
+ check_command: str | None = None,
112
+ rollback_hint: str | None = None,
113
+ depends_on: list[int] | None = None,
114
+ gate: bool = False,
115
+ tags: list[str] | None = None,
116
+ ) -> Step:
117
+ """Helper to construct a Step with index placeholder (set later)."""
118
+ return Step(
119
+ id=id,
120
+ index=0, # filled in by plan()
121
+ description=description,
122
+ scope=scope,
123
+ kind=kind,
124
+ command=command,
125
+ file_content=file_content,
126
+ target_path=target_path,
127
+ sudo=sudo,
128
+ check_command=check_command,
129
+ rollback_hint=rollback_hint,
130
+ depends_on=depends_on or [],
131
+ gate=gate,
132
+ tags=tags or [],
133
+ )
134
+
135
+
136
+ def _plan_bootstrap(spec: BootstrapSpec, ctx: NormalizedContext) -> list[Step]:
137
+ from loft_cli.runtime.steps import bootstrap as bs
138
+ from loft_cli.runtime.steps import wireguard as wg
139
+
140
+ steps: list[Step] = []
141
+ R = StepScope.REMOTE
142
+ L = StepScope.LOCAL
143
+ V = StepScope.VERIFY
144
+
145
+ pubkey_content = "\n".join(ctx.pubkey_contents) if ctx.pubkey_contents else ""
146
+
147
+ # ------------------------------------------------------------------ #
148
+ # REMOTE: critical bootstrap path
149
+ # ------------------------------------------------------------------ #
150
+
151
+ # 0: preflight — verify root SSH access
152
+ steps.append(
153
+ _s(
154
+ "preflight_connect_root",
155
+ f"Verify root SSH access to {spec.host.address}:{spec.login.port}",
156
+ R,
157
+ StepKind.SSH_COMMAND,
158
+ command="echo 'preflight ok'",
159
+ rollback_hint="Check SSH credentials and network connectivity.",
160
+ tags=["preflight"],
161
+ )
162
+ )
163
+ idx_preflight = len(steps) - 1
164
+
165
+ # 1: detect OS
166
+ steps.append(
167
+ _s(
168
+ "detect_os",
169
+ "Detect remote OS (assert Debian/Ubuntu)",
170
+ R,
171
+ StepKind.SSH_COMMAND,
172
+ command="cat /etc/os-release",
173
+ depends_on=[idx_preflight],
174
+ tags=["os"],
175
+ )
176
+ )
177
+
178
+ # 2: update apt package index
179
+ steps.append(
180
+ _s(
181
+ "apt_update",
182
+ "Update apt package index",
183
+ R,
184
+ StepKind.SSH_COMMAND,
185
+ command=bs.apt_update(),
186
+ sudo=True,
187
+ rollback_hint="Check apt sources and network connectivity.",
188
+ tags=["packages"],
189
+ )
190
+ )
191
+
192
+ # 3: install base packages
193
+ base_packages = ["ufw"]
194
+ if spec.wireguard.enabled:
195
+ base_packages.append("wireguard")
196
+ pkg_list = " ".join(base_packages)
197
+ steps.append(
198
+ _s(
199
+ "install_base_packages",
200
+ f"Install base packages: {pkg_list}",
201
+ R,
202
+ StepKind.SSH_COMMAND,
203
+ command=bs.install_packages(base_packages),
204
+ sudo=True,
205
+ rollback_hint="Check apt sources and network connectivity.",
206
+ tags=["packages"],
207
+ )
208
+ )
209
+
210
+ # 3: create admin user (idempotent)
211
+ steps.append(
212
+ _s(
213
+ "create_admin_user",
214
+ f"Create admin user '{spec.admin_user.name}' with sudo",
215
+ R,
216
+ StepKind.SSH_COMMAND,
217
+ command=bs.create_admin_user(
218
+ spec.admin_user.name,
219
+ spec.admin_user.groups,
220
+ ),
221
+ sudo=True,
222
+ tags=["user"],
223
+ )
224
+ )
225
+
226
+ # 3b: grant passwordless sudo so the admin can run sudo without a password
227
+ steps.append(
228
+ _s(
229
+ "configure_nopasswd_sudo",
230
+ f"Grant {spec.admin_user.name} passwordless sudo",
231
+ R,
232
+ StepKind.SSH_COMMAND,
233
+ command=bs.nopasswd_sudoers(spec.admin_user.name),
234
+ sudo=True,
235
+ tags=["user"],
236
+ )
237
+ )
238
+ steps.append(
239
+ _s(
240
+ "secure_sudoers_file",
241
+ f"Secure /etc/sudoers.d/{spec.admin_user.name} (chmod 440)",
242
+ R,
243
+ StepKind.SSH_COMMAND,
244
+ command=bs.secure_sudoers(spec.admin_user.name),
245
+ sudo=True,
246
+ tags=["user"],
247
+ )
248
+ )
249
+
250
+ # 4: install authorized keys
251
+ if pubkey_content:
252
+ steps.append(
253
+ _s(
254
+ "install_authorized_keys",
255
+ f"Install SSH authorized keys for {spec.admin_user.name}",
256
+ R,
257
+ StepKind.SSH_COMMAND,
258
+ command=bs.install_authorized_keys(spec.admin_user.name, pubkey_content),
259
+ sudo=True,
260
+ tags=["ssh", "keys"],
261
+ )
262
+ )
263
+ else:
264
+ steps.append(
265
+ _s(
266
+ "install_authorized_keys",
267
+ "No pubkeys configured — skipping authorized_keys install",
268
+ R,
269
+ StepKind.VERIFY,
270
+ command="echo 'no pubkeys configured'",
271
+ tags=["ssh", "keys"],
272
+ )
273
+ )
274
+
275
+ # 4b: ensure PubkeyAuthentication yes + reload sshd
276
+ # Some images (e.g. this VirtualBox Ubuntu) ship with PubkeyAuthentication no.
277
+ # The gate below requires key auth to work, so we must enable it first.
278
+ steps.append(
279
+ _s(
280
+ "enable_pubkey_auth",
281
+ "Enable PubkeyAuthentication in sshd and reload",
282
+ R,
283
+ StepKind.SSH_COMMAND,
284
+ command=bs.enable_pubkey_auth(),
285
+ sudo=True,
286
+ tags=["ssh", "sshd"],
287
+ )
288
+ )
289
+
290
+ # 4c: GATE — verify admin login on current port BEFORE touching sshd
291
+ # This is the critical safety gate: if admin key login doesn't work yet,
292
+ # we must NOT change the SSH port — the server would become unrecoverable.
293
+ if pubkey_content:
294
+ steps.append(
295
+ _s(
296
+ "verify_admin_login_before_port_change",
297
+ f"[GATE] Verify admin SSH login before port change: "
298
+ f"{spec.admin_user.name}@{spec.host.address}:{spec.login.port}",
299
+ V,
300
+ StepKind.GATE,
301
+ command=f"ssh_check:{spec.host.address}:{spec.login.port}:{spec.admin_user.name}",
302
+ rollback_hint=(
303
+ "Admin key login failed — SSH port has NOT been changed. "
304
+ "Safe to re-run. Check that admin user and authorized_keys were created correctly."
305
+ ),
306
+ gate=True,
307
+ tags=["gate", "ssh", "lockout-prevention"],
308
+ )
309
+ )
310
+
311
+ # 5: write sshd config candidate (port change, defer root/password disable)
312
+ steps.append(
313
+ _s(
314
+ "write_sshd_config_candidate",
315
+ f"Configure SSH daemon: port={spec.ssh.port} (root/password hardening deferred)",
316
+ R,
317
+ StepKind.SSH_COMMAND,
318
+ command=bs.write_sshd_config_candidate(spec.ssh.port),
319
+ sudo=True,
320
+ rollback_hint="Restore /etc/ssh/sshd_config from backup: "
321
+ "cp /etc/ssh/sshd_config.bak /etc/ssh/sshd_config",
322
+ tags=["ssh", "sshd"],
323
+ )
324
+ )
325
+
326
+ # 6: open new SSH port in firewall
327
+ steps.append(
328
+ _s(
329
+ "open_new_ssh_port_in_firewall",
330
+ f"Open firewall for new SSH port {spec.ssh.port}/tcp",
331
+ R,
332
+ StepKind.SSH_COMMAND,
333
+ command=bs.open_firewall_port(spec.ssh.port),
334
+ sudo=True,
335
+ tags=["firewall"],
336
+ )
337
+ )
338
+
339
+ # 7: validate sshd config
340
+ steps.append(
341
+ _s(
342
+ "validate_sshd_config",
343
+ "Validate sshd configuration (sshd -t)",
344
+ R,
345
+ StepKind.SSH_COMMAND,
346
+ command="sshd -t",
347
+ sudo=True,
348
+ rollback_hint="Fix /etc/ssh/sshd_config errors before reloading.",
349
+ tags=["ssh", "sshd"],
350
+ )
351
+ )
352
+
353
+ # 8: reload sshd
354
+ steps.append(
355
+ _s(
356
+ "reload_sshd",
357
+ "Reload SSH daemon to apply config",
358
+ R,
359
+ StepKind.SSH_COMMAND,
360
+ command=bs.reload_sshd(),
361
+ sudo=True,
362
+ rollback_hint="If reload fails, check sshd_config syntax with 'sshd -t'.",
363
+ tags=["ssh", "sshd"],
364
+ )
365
+ )
366
+
367
+ # 9: GATE — verify admin login on new port
368
+ # This is the SSH lockout prevention gate.
369
+ # Steps that disable root login and password auth MUST depend on this index.
370
+ steps.append(
371
+ _s(
372
+ "verify_admin_login_on_new_port",
373
+ f"[GATE] Verify admin SSH login: {spec.admin_user.name}@{spec.host.address}:{spec.ssh.port}",
374
+ V,
375
+ StepKind.GATE,
376
+ command=f"ssh_check:{spec.host.address}:{spec.ssh.port}:{spec.admin_user.name}",
377
+ rollback_hint=(
378
+ "Admin login failed. Do NOT disable root login or password auth. "
379
+ "Restore sshd_config: cp /etc/ssh/sshd_config.bak /etc/ssh/sshd_config && "
380
+ "systemctl reload ssh"
381
+ ),
382
+ gate=True,
383
+ tags=["gate", "ssh", "lockout-prevention"],
384
+ )
385
+ )
386
+ idx_gate = len(steps) - 1
387
+
388
+ # 10: disable root login — MUST depend on gate
389
+ steps.append(
390
+ _s(
391
+ "disable_root_login",
392
+ "Disable root SSH login (PermitRootLogin no)",
393
+ R,
394
+ StepKind.SSH_COMMAND,
395
+ command=bs.disable_root_login(),
396
+ sudo=True,
397
+ depends_on=[idx_gate],
398
+ rollback_hint="sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config && systemctl reload ssh",
399
+ tags=["ssh", "hardening", "lockout-prevention"],
400
+ )
401
+ )
402
+
403
+ # 11: disable password auth — MUST depend on gate
404
+ if spec.ssh.disable_password_auth and pubkey_content:
405
+ steps.append(
406
+ _s(
407
+ "disable_password_auth",
408
+ "Disable SSH password authentication",
409
+ R,
410
+ StepKind.SSH_COMMAND,
411
+ command=bs.disable_password_auth(),
412
+ sudo=True,
413
+ depends_on=[idx_gate],
414
+ rollback_hint="sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config && systemctl reload ssh",
415
+ tags=["ssh", "hardening", "lockout-prevention"],
416
+ )
417
+ )
418
+ else:
419
+ steps.append(
420
+ _s(
421
+ "disable_password_auth",
422
+ "Password auth not disabled (no pubkeys or config not set)",
423
+ R,
424
+ StepKind.VERIFY,
425
+ command="echo 'password auth left enabled'",
426
+ depends_on=[idx_gate],
427
+ tags=["ssh", "hardening"],
428
+ )
429
+ )
430
+
431
+ # 12: finalize firewall (three separate steps — no shell chaining)
432
+ steps.append(
433
+ _s(
434
+ "ufw_default_deny_incoming",
435
+ "Firewall: default deny incoming",
436
+ R,
437
+ StepKind.SSH_COMMAND,
438
+ command=bs.ufw_default_deny_incoming(),
439
+ sudo=True,
440
+ tags=["firewall"],
441
+ )
442
+ )
443
+ steps.append(
444
+ _s(
445
+ "ufw_default_allow_outgoing",
446
+ "Firewall: default allow outgoing",
447
+ R,
448
+ StepKind.SSH_COMMAND,
449
+ command=bs.ufw_default_allow_outgoing(),
450
+ sudo=True,
451
+ tags=["firewall"],
452
+ )
453
+ )
454
+ steps.append(
455
+ _s(
456
+ "ufw_force_enable",
457
+ "Enable firewall (ufw --force enable)",
458
+ R,
459
+ StepKind.SSH_COMMAND,
460
+ command=bs.ufw_force_enable(),
461
+ sudo=True,
462
+ tags=["firewall"],
463
+ )
464
+ )
465
+
466
+ # 13: reload sshd again after hardening changes
467
+ steps.append(
468
+ _s(
469
+ "reload_sshd_final",
470
+ "Reload SSH daemon to apply hardening changes",
471
+ R,
472
+ StepKind.SSH_COMMAND,
473
+ command=bs.reload_sshd(),
474
+ sudo=True,
475
+ depends_on=[idx_gate],
476
+ tags=["ssh", "sshd"],
477
+ )
478
+ )
479
+
480
+ # 14-18: WireGuard (remote steps + local state save)
481
+ if spec.wireguard.enabled:
482
+ import ipaddress as _ip
483
+
484
+ wg_port = (
485
+ int(spec.wireguard.endpoint.split(":")[-1]) if ":" in spec.wireguard.endpoint else 51820
486
+ )
487
+ # VPN subnet derived from the server's interface address (e.g. 10.10.0.0/24)
488
+ vpn_subnet = str(_ip.ip_interface(spec.wireguard.address).network)
489
+
490
+ # Server config: [Interface] with ListenPort + [Peer] with client public key
491
+ server_conf = wg.generate_server_config(
492
+ interface=spec.wireguard.interface,
493
+ address=spec.wireguard.address,
494
+ private_key=ctx.wireguard_private_key,
495
+ listen_port=wg_port,
496
+ client_public_key=ctx.wg_client_public_key,
497
+ peer_address=spec.wireguard.peer_address,
498
+ )
499
+ # Client config: saved locally to ~/.wg/loft-cli/{host}/client.conf
500
+ client_conf = wg.generate_client_config(
501
+ client_private_key=ctx.wg_client_private_key,
502
+ peer_address=spec.wireguard.peer_address,
503
+ server_public_key=ctx.wireguard_public_key,
504
+ endpoint=spec.wireguard.endpoint,
505
+ vpn_subnet=vpn_subnet,
506
+ persistent_keepalive=spec.wireguard.persistent_keepalive,
507
+ )
508
+ # Store both on ctx so the executor can persist them after apply
509
+ ctx.wireguard_conf_content = server_conf
510
+ ctx.wg_client_conf_content = client_conf
511
+
512
+ steps.append(
513
+ _s(
514
+ "set_wireguard_dir_permissions",
515
+ "Secure /etc/wireguard directory (chmod 700)",
516
+ R,
517
+ StepKind.SSH_COMMAND,
518
+ command=wg.set_wireguard_dir_permissions(),
519
+ sudo=True,
520
+ tags=["wireguard"],
521
+ )
522
+ )
523
+ steps.append(
524
+ _s(
525
+ "write_wireguard_config",
526
+ f"Write WireGuard server config: /etc/wireguard/{spec.wireguard.interface}.conf",
527
+ R,
528
+ StepKind.SSH_UPLOAD,
529
+ file_content=server_conf,
530
+ target_path=f"/etc/wireguard/{spec.wireguard.interface}.conf",
531
+ sudo=True,
532
+ tags=["wireguard"],
533
+ )
534
+ )
535
+ # Open the WireGuard UDP port in the firewall before starting the interface
536
+ steps.append(
537
+ _s(
538
+ "open_wireguard_port_in_firewall",
539
+ f"Open firewall for WireGuard UDP port {wg_port}",
540
+ R,
541
+ StepKind.SSH_COMMAND,
542
+ command=f"ufw allow {wg_port}/udp",
543
+ sudo=True,
544
+ tags=["wireguard", "firewall"],
545
+ )
546
+ )
547
+ steps.append(
548
+ _s(
549
+ "load_wireguard_module",
550
+ "Load WireGuard kernel module",
551
+ R,
552
+ StepKind.SSH_COMMAND,
553
+ command=wg.load_wireguard_module(),
554
+ sudo=True,
555
+ tags=["wireguard"],
556
+ )
557
+ )
558
+ steps.append(
559
+ _s(
560
+ "enable_wireguard",
561
+ f"Enable and start WireGuard: wg-quick@{spec.wireguard.interface}",
562
+ R,
563
+ StepKind.SSH_COMMAND,
564
+ command=wg.enable_wireguard(spec.wireguard.interface),
565
+ sudo=True,
566
+ tags=["wireguard"],
567
+ )
568
+ )
569
+ steps.append(
570
+ _s(
571
+ "verify_wireguard",
572
+ f"Verify WireGuard interface {spec.wireguard.interface} is up",
573
+ V,
574
+ StepKind.VERIFY,
575
+ command=f"wg show {spec.wireguard.interface}",
576
+ sudo=True,
577
+ tags=["wireguard"],
578
+ )
579
+ )
580
+
581
+ # 17: postflight checks (from spec)
582
+ for check in spec.checks:
583
+ steps.append(
584
+ _s(
585
+ f"postflight_{check.type}",
586
+ f"Postflight check: {check.type}",
587
+ V,
588
+ StepKind.VERIFY,
589
+ command=_encode_check_command(check, spec),
590
+ tags=["postflight", check.type],
591
+ )
592
+ )
593
+
594
+ # ------------------------------------------------------------------ #
595
+ # 18-19: Goss — generate, ship, and run server-spec verification.
596
+ # This is optional but strongly recommended. If the goss generator
597
+ # raises for any reason the steps are emitted as no-ops so the plan
598
+ # never breaks because of test-tooling issues.
599
+ # ------------------------------------------------------------------ #
600
+ goss_content: str | None = None
601
+ try:
602
+ from loft_cli.addons.goss.generator import generate_goss_yaml
603
+
604
+ goss_content = generate_goss_yaml(spec)
605
+ except Exception:
606
+ goss_content = None
607
+
608
+ if goss_content is not None:
609
+ idx_ship = len(steps)
610
+ steps.append(
611
+ _s(
612
+ "ship_goss_file",
613
+ f"Ship goss spec to ~/.goss/{spec.meta.name}.yaml on remote",
614
+ R,
615
+ StepKind.SSH_UPLOAD,
616
+ file_content=goss_content,
617
+ target_path=f"~/.goss/{spec.meta.name}.yaml",
618
+ sudo=False,
619
+ tags=["goss"],
620
+ )
621
+ )
622
+ steps.append(
623
+ _s(
624
+ "run_goss_validate",
625
+ "Run goss validate and display verification results",
626
+ V,
627
+ StepKind.VERIFY,
628
+ command="goss_validate",
629
+ depends_on=[idx_ship],
630
+ tags=["goss", "verify"],
631
+ )
632
+ )
633
+ else:
634
+ # Emit a visible warning step so the operator sees it in the plan output
635
+ steps.append(
636
+ _s(
637
+ "goss_unavailable",
638
+ "[WARNING] No goss spec available — server state will NOT be verified",
639
+ V,
640
+ StepKind.VERIFY,
641
+ command="goss_unavailable",
642
+ tags=["goss", "warning"],
643
+ )
644
+ )
645
+
646
+ # ------------------------------------------------------------------ #
647
+ # WireGuard SSH restriction — MUST be the absolute last remote steps.
648
+ # Adds a WireGuard-restricted SSH rule, verifies SSH through the tunnel
649
+ # from the client side, then removes the open-to-all rule.
650
+ # After delete_open_ssh_rule executes, direct SSH to spec.host.address
651
+ # stops working.
652
+ # ------------------------------------------------------------------ #
653
+ if spec.wireguard.enabled:
654
+ import ipaddress as _ip_wg
655
+
656
+ peer_ip = (
657
+ spec.wireguard.peer_address.split("/")[0]
658
+ if spec.firewall.registered_peers_only
659
+ else None
660
+ )
661
+ label_allow = (
662
+ f"Allow SSH port {spec.ssh.port} on WireGuard peer {peer_ip}"
663
+ if peer_ip
664
+ else f"Allow SSH port {spec.ssh.port} on WireGuard interface {spec.wireguard.interface}"
665
+ )
666
+ steps.append(
667
+ _s(
668
+ "allow_ssh_on_wireguard",
669
+ label_allow,
670
+ R,
671
+ StepKind.SSH_COMMAND,
672
+ command=bs.allow_ssh_on_wireguard(
673
+ spec.ssh.port,
674
+ spec.wireguard.interface,
675
+ peer_ip,
676
+ ),
677
+ sudo=True,
678
+ tags=["ssh", "firewall", "wireguard"],
679
+ )
680
+ )
681
+
682
+ # Tunnel safety gate: bring up WireGuard tunnel locally, verify SSH
683
+ # through the VPN IP, and only then proceed to delete the open SSH rule.
684
+ # If this gate fails, the open SSH rule is NOT deleted — the server
685
+ # remains accessible via public IP.
686
+ wg_server_vpn_ip = str(_ip_wg.ip_interface(spec.wireguard.address).ip)
687
+ steps.append(
688
+ _s(
689
+ "verify_ssh_over_wireguard_tunnel",
690
+ (
691
+ f"Verify SSH through WireGuard tunnel "
692
+ f"({spec.admin_user.name}@{wg_server_vpn_ip}:{spec.ssh.port})"
693
+ ),
694
+ L,
695
+ StepKind.GATE,
696
+ command=(
697
+ f"tunnel_ssh_gate:{spec.host.name}:{wg_server_vpn_ip}"
698
+ f":{spec.ssh.port}:{spec.admin_user.name}"
699
+ ),
700
+ gate=True,
701
+ rollback_hint=(
702
+ "WireGuard tunnel SSH verification failed. The open SSH rule was NOT deleted — "
703
+ "the server is still accessible via its public IP. "
704
+ "Check WireGuard configuration, ensure wg-quick is installed locally, "
705
+ "and verify the tunnel can be established."
706
+ ),
707
+ tags=["wireguard", "ssh", "gate", "tunnel"],
708
+ )
709
+ )
710
+
711
+ steps.append(
712
+ _s(
713
+ "delete_open_ssh_rule",
714
+ f"Remove open-to-all SSH rule for port {spec.ssh.port}/tcp",
715
+ R,
716
+ StepKind.SSH_COMMAND,
717
+ command=bs.delete_open_ssh_rule(spec.ssh.port),
718
+ sudo=True,
719
+ tags=["ssh", "firewall", "wireguard"],
720
+ )
721
+ )
722
+
723
+ # ------------------------------------------------------------------ #
724
+ # LOCAL: only after remote success
725
+ # ------------------------------------------------------------------ #
726
+
727
+ # SSH conf.d entry — only when local.ssh_config.enabled (default: true)
728
+ if spec.local.ssh_config.enabled:
729
+ steps.append(
730
+ _s(
731
+ "backup_local_ssh_config",
732
+ "Backup local ~/.ssh/config",
733
+ L,
734
+ StepKind.LOCAL_COMMAND,
735
+ command="backup_ssh_config",
736
+ tags=["local", "ssh-config"],
737
+ )
738
+ )
739
+ steps.append(
740
+ _s(
741
+ "write_local_ssh_conf_d",
742
+ f"Write local SSH conf.d entry: {ctx.ssh_conf_d_path}",
743
+ L,
744
+ StepKind.LOCAL_FILE_WRITE,
745
+ target_path=str(ctx.ssh_conf_d_path) if ctx.ssh_conf_d_path else "",
746
+ tags=["local", "ssh-config"],
747
+ )
748
+ )
749
+ steps.append(
750
+ _s(
751
+ "ensure_include_directive",
752
+ "Ensure Include directive in ~/.ssh/config",
753
+ L,
754
+ StepKind.LOCAL_COMMAND,
755
+ command="ensure_include",
756
+ tags=["local", "ssh-config"],
757
+ )
758
+ )
759
+
760
+ # WireGuard local state — save key material + metadata after remote success
761
+ if spec.wireguard.enabled:
762
+ steps.append(
763
+ _s(
764
+ "save_local_wireguard_state",
765
+ f"Save WireGuard state to ~/.wg/loft-cli/{spec.host.name}/",
766
+ L,
767
+ StepKind.LOCAL_COMMAND,
768
+ command="save_wireguard_state",
769
+ tags=["local", "wireguard"],
770
+ )
771
+ )
772
+
773
+ # Inventory DB
774
+ if spec.local.inventory.enabled:
775
+ steps.append(
776
+ _s(
777
+ "open_or_init_local_inventory",
778
+ "Open or initialize local SQLCipher inventory database",
779
+ L,
780
+ StepKind.LOCAL_DB_WRITE,
781
+ command="init_inventory",
782
+ tags=["local", "inventory"],
783
+ )
784
+ )
785
+
786
+ # 22: upsert server record
787
+ steps.append(
788
+ _s(
789
+ "upsert_server_inventory",
790
+ f"Upsert server record in inventory: {spec.host.name}",
791
+ L,
792
+ StepKind.LOCAL_DB_WRITE,
793
+ command="upsert_server",
794
+ tags=["local", "inventory"],
795
+ )
796
+ )
797
+
798
+ # 23: record run metadata
799
+ steps.append(
800
+ _s(
801
+ "record_run_metadata",
802
+ "Record bootstrap run metadata in inventory",
803
+ L,
804
+ StepKind.LOCAL_DB_WRITE,
805
+ command="record_run",
806
+ tags=["local", "inventory"],
807
+ )
808
+ )
809
+
810
+ return steps
811
+
812
+
813
+ def _plan_service(spec: ServiceSpec, ctx: NormalizedContext) -> list[Step]:
814
+ from loft_cli.runtime.steps import container as ct
815
+ from loft_cli.runtime.steps import docker as dk
816
+ from loft_cli.runtime.steps import nginx as nx
817
+ from loft_cli.runtime.steps import postgres as pg
818
+
819
+ steps: list[Step] = []
820
+ R = StepScope.REMOTE
821
+ L = StepScope.LOCAL
822
+ V = StepScope.VERIFY
823
+
824
+ # preflight
825
+ steps.append(
826
+ _s(
827
+ "preflight_connect_admin",
828
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
829
+ R,
830
+ StepKind.SSH_COMMAND,
831
+ command="echo 'preflight ok'",
832
+ tags=["preflight"],
833
+ )
834
+ )
835
+
836
+ steps.append(
837
+ _s(
838
+ "detect_os",
839
+ "Detect remote OS",
840
+ R,
841
+ StepKind.SSH_COMMAND,
842
+ command="cat /etc/os-release",
843
+ tags=["os"],
844
+ )
845
+ )
846
+
847
+ # apt update — shared by all service installations that need packages
848
+ needs_apt = (
849
+ (spec.postgres and spec.postgres.enabled)
850
+ or (spec.nginx and spec.nginx.enabled)
851
+ or (spec.docker and spec.docker.enabled)
852
+ or bool(spec.containers)
853
+ )
854
+ if needs_apt:
855
+ steps.append(
856
+ _s(
857
+ "apt_update",
858
+ "Update apt package index",
859
+ R,
860
+ StepKind.SSH_COMMAND,
861
+ command="apt-get update -y",
862
+ sudo=True,
863
+ tags=["packages"],
864
+ )
865
+ )
866
+
867
+ # Postgres
868
+ if spec.postgres and spec.postgres.enabled:
869
+ # PGDG repository — ensures the requested PostgreSQL version is
870
+ # available regardless of the distro's default packages.
871
+ steps.append(
872
+ _s(
873
+ "install_pgdg_prerequisites",
874
+ "Install PGDG repository prerequisites",
875
+ R,
876
+ StepKind.SSH_COMMAND,
877
+ command=pg.install_pgdg_prerequisites(),
878
+ sudo=True,
879
+ tags=["postgres", "pgdg"],
880
+ )
881
+ )
882
+ steps.append(
883
+ _s(
884
+ "add_pgdg_signing_key",
885
+ "Import PostgreSQL PGDG apt signing key",
886
+ R,
887
+ StepKind.SSH_COMMAND,
888
+ command=pg.add_pgdg_signing_key(),
889
+ sudo=True,
890
+ tags=["postgres", "pgdg"],
891
+ )
892
+ )
893
+ steps.append(
894
+ _s(
895
+ "add_pgdg_source_list",
896
+ "Add PGDG apt repository source list",
897
+ R,
898
+ StepKind.SSH_COMMAND,
899
+ command=pg.add_pgdg_source_list(),
900
+ sudo=True,
901
+ tags=["postgres", "pgdg"],
902
+ )
903
+ )
904
+ steps.append(
905
+ _s(
906
+ "apt_update_pgdg",
907
+ "Update apt package index (with PGDG repo)",
908
+ R,
909
+ StepKind.SSH_COMMAND,
910
+ command="apt-get update -y",
911
+ sudo=True,
912
+ tags=["postgres", "pgdg", "packages"],
913
+ )
914
+ )
915
+ steps.append(
916
+ _s(
917
+ "install_postgres",
918
+ f"Install PostgreSQL {spec.postgres.version}",
919
+ R,
920
+ StepKind.SSH_COMMAND,
921
+ command=pg.install_postgres(spec.postgres.version),
922
+ sudo=True,
923
+ tags=["postgres"],
924
+ )
925
+ )
926
+ steps.append(
927
+ _s(
928
+ "configure_postgres_listen",
929
+ "Configure PostgreSQL listen_addresses",
930
+ R,
931
+ StepKind.SSH_COMMAND,
932
+ command=pg.configure_listen(spec.postgres.listen_addresses),
933
+ sudo=True,
934
+ tags=["postgres"],
935
+ )
936
+ )
937
+ steps.append(
938
+ _s(
939
+ "enable_postgres",
940
+ "Enable and start PostgreSQL service",
941
+ R,
942
+ StepKind.SSH_COMMAND,
943
+ command=pg.enable_postgres(),
944
+ sudo=True,
945
+ tags=["postgres"],
946
+ )
947
+ )
948
+ if spec.postgres.create_role:
949
+ steps.append(
950
+ _s(
951
+ "create_db_role",
952
+ f"Create PostgreSQL role: {spec.postgres.create_role.name}",
953
+ R,
954
+ StepKind.SSH_COMMAND,
955
+ command=pg.create_role(
956
+ spec.postgres.create_role.name,
957
+ spec.postgres.create_role.password_env,
958
+ ),
959
+ sudo=True,
960
+ tags=["postgres"],
961
+ )
962
+ )
963
+ if spec.postgres.create_database:
964
+ steps.append(
965
+ _s(
966
+ "create_database",
967
+ f"Create PostgreSQL database: {spec.postgres.create_database.name}",
968
+ R,
969
+ StepKind.SSH_COMMAND,
970
+ command=pg.create_database(
971
+ spec.postgres.create_database.name,
972
+ spec.postgres.create_database.owner,
973
+ ),
974
+ sudo=True,
975
+ tags=["postgres"],
976
+ )
977
+ )
978
+ steps.append(
979
+ _s(
980
+ "postgres_ready_check",
981
+ "Verify PostgreSQL is ready",
982
+ V,
983
+ StepKind.VERIFY,
984
+ command="pg_isready",
985
+ sudo=True,
986
+ tags=["postgres", "verify"],
987
+ )
988
+ )
989
+
990
+ # Nginx
991
+ if spec.nginx and spec.nginx.enabled:
992
+ steps.append(
993
+ _s(
994
+ "install_nginx",
995
+ "Install nginx",
996
+ R,
997
+ StepKind.SSH_COMMAND,
998
+ command=nx.install_nginx(),
999
+ sudo=True,
1000
+ tags=["nginx"],
1001
+ )
1002
+ )
1003
+ steps.append(
1004
+ _s(
1005
+ "enable_nginx",
1006
+ "Enable and start nginx service",
1007
+ R,
1008
+ StepKind.SSH_COMMAND,
1009
+ command=nx.enable_nginx(),
1010
+ sudo=True,
1011
+ tags=["nginx"],
1012
+ )
1013
+ )
1014
+ steps.append(
1015
+ _s(
1016
+ "remove_nginx_default_site",
1017
+ "Remove default nginx site",
1018
+ R,
1019
+ StepKind.SSH_COMMAND,
1020
+ command=nx.remove_default_site(),
1021
+ sudo=True,
1022
+ tags=["nginx"],
1023
+ )
1024
+ )
1025
+ for site in spec.nginx.sites:
1026
+ safe_name = site.domain.replace(".", "_")
1027
+ steps.append(
1028
+ _s(
1029
+ f"write_nginx_site_{safe_name}",
1030
+ f"Write nginx site config for {site.domain}",
1031
+ R,
1032
+ StepKind.SSH_UPLOAD,
1033
+ file_content=nx.site_config_content(site),
1034
+ target_path=nx.site_config_path(site),
1035
+ sudo=True,
1036
+ tags=["nginx", site.domain],
1037
+ )
1038
+ )
1039
+ steps.append(
1040
+ _s(
1041
+ f"enable_nginx_site_{safe_name}",
1042
+ f"Enable nginx site: {site.domain}",
1043
+ R,
1044
+ StepKind.SSH_COMMAND,
1045
+ command=nx.enable_site(site),
1046
+ sudo=True,
1047
+ tags=["nginx", site.domain],
1048
+ )
1049
+ )
1050
+ steps.append(
1051
+ _s(
1052
+ "validate_nginx_config",
1053
+ "Validate nginx configuration",
1054
+ R,
1055
+ StepKind.SSH_COMMAND,
1056
+ command=nx.validate_nginx_config(),
1057
+ sudo=True,
1058
+ rollback_hint="Check nginx config files for syntax errors.",
1059
+ tags=["nginx"],
1060
+ )
1061
+ )
1062
+ steps.append(
1063
+ _s(
1064
+ "reload_nginx",
1065
+ "Reload nginx to apply configuration",
1066
+ R,
1067
+ StepKind.SSH_COMMAND,
1068
+ command=nx.reload_nginx_service(),
1069
+ sudo=True,
1070
+ tags=["nginx"],
1071
+ )
1072
+ )
1073
+ steps.append(
1074
+ _s(
1075
+ "nginx_config_check",
1076
+ "Verify nginx configuration is valid",
1077
+ V,
1078
+ StepKind.VERIFY,
1079
+ command="nginx -t",
1080
+ sudo=True,
1081
+ tags=["nginx", "verify"],
1082
+ )
1083
+ )
1084
+
1085
+ # Docker
1086
+ needs_docker = spec.docker and spec.docker.enabled or bool(spec.containers)
1087
+ if needs_docker:
1088
+ steps.append(
1089
+ _s(
1090
+ "install_docker",
1091
+ "Install Docker",
1092
+ R,
1093
+ StepKind.SSH_COMMAND,
1094
+ command=dk.install_docker(),
1095
+ sudo=True,
1096
+ tags=["docker"],
1097
+ )
1098
+ )
1099
+ steps.append(
1100
+ _s(
1101
+ "enable_docker",
1102
+ "Enable Docker service",
1103
+ R,
1104
+ StepKind.SSH_COMMAND,
1105
+ command=dk.enable_docker(),
1106
+ sudo=True,
1107
+ tags=["docker"],
1108
+ )
1109
+ )
1110
+ steps.append(
1111
+ _s(
1112
+ "docker_version_check",
1113
+ "Verify Docker installation",
1114
+ V,
1115
+ StepKind.VERIFY,
1116
+ command="docker --version",
1117
+ sudo=True,
1118
+ tags=["docker", "verify"],
1119
+ )
1120
+ )
1121
+
1122
+ # Containers
1123
+ for c in spec.containers:
1124
+ steps.append(
1125
+ _s(
1126
+ f"pull_image_{c.name}",
1127
+ f"Pull container image: {c.image}",
1128
+ R,
1129
+ StepKind.SSH_COMMAND,
1130
+ command=ct.pull_image(c.image),
1131
+ sudo=True,
1132
+ tags=["container", c.name],
1133
+ )
1134
+ )
1135
+ steps.append(
1136
+ _s(
1137
+ f"stop_container_{c.name}",
1138
+ f"Stop existing container '{c.name}' (if running)",
1139
+ R,
1140
+ StepKind.SSH_COMMAND,
1141
+ command=ct.stop_container(c.name),
1142
+ sudo=True,
1143
+ tags=["container", c.name],
1144
+ )
1145
+ )
1146
+ steps.append(
1147
+ _s(
1148
+ f"remove_container_{c.name}",
1149
+ f"Remove existing container '{c.name}' (if present)",
1150
+ R,
1151
+ StepKind.SSH_COMMAND,
1152
+ command=ct.remove_container(c.name),
1153
+ sudo=True,
1154
+ tags=["container", c.name],
1155
+ )
1156
+ )
1157
+ steps.append(
1158
+ _s(
1159
+ f"run_container_{c.name}",
1160
+ f"Start container '{c.name}' from {c.image}",
1161
+ R,
1162
+ StepKind.SSH_COMMAND,
1163
+ command=ct.run_container(c),
1164
+ sudo=True,
1165
+ tags=["container", c.name],
1166
+ )
1167
+ )
1168
+ steps.append(
1169
+ _s(
1170
+ f"container_running_check_{c.name}",
1171
+ f"Verify container '{c.name}' is running",
1172
+ V,
1173
+ StepKind.VERIFY,
1174
+ command=f"docker inspect --format='{{{{.State.Running}}}}' {c.name}",
1175
+ sudo=True,
1176
+ tags=["container", c.name, "verify"],
1177
+ )
1178
+ )
1179
+ if c.healthcheck:
1180
+ steps.append(
1181
+ _s(
1182
+ f"http_health_check_{c.name}",
1183
+ f"HTTP health check: {c.healthcheck.url}",
1184
+ V,
1185
+ StepKind.VERIFY,
1186
+ command=f"http_check:{c.healthcheck.url}:{c.healthcheck.expect_status}",
1187
+ tags=["container", c.name, "health"],
1188
+ )
1189
+ )
1190
+
1191
+ # Local inventory
1192
+ if spec.local.inventory.enabled:
1193
+ steps.append(
1194
+ _s(
1195
+ "open_or_init_local_inventory",
1196
+ "Open or initialize local SQLCipher inventory database",
1197
+ L,
1198
+ StepKind.LOCAL_DB_WRITE,
1199
+ command="init_inventory",
1200
+ tags=["local", "inventory"],
1201
+ )
1202
+ )
1203
+ steps.append(
1204
+ _s(
1205
+ "update_server_services_metadata",
1206
+ f"Update service metadata in inventory for {spec.host.name}",
1207
+ L,
1208
+ StepKind.LOCAL_DB_WRITE,
1209
+ command="upsert_services",
1210
+ tags=["local", "inventory"],
1211
+ )
1212
+ )
1213
+ steps.append(
1214
+ _s(
1215
+ "record_service_run_metadata",
1216
+ "Record service run metadata in inventory",
1217
+ L,
1218
+ StepKind.LOCAL_DB_WRITE,
1219
+ command="record_run",
1220
+ tags=["local", "inventory"],
1221
+ )
1222
+ )
1223
+
1224
+ return steps
1225
+
1226
+
1227
+ def _plan_file_template(spec: FileTemplateSpec, ctx: NormalizedContext) -> list[Step]:
1228
+ """Generate steps for rendering and uploading managed configuration files."""
1229
+ from loft_cli.runtime.steps import file_template as ft
1230
+
1231
+ steps: list[Step] = []
1232
+ R = StepScope.REMOTE
1233
+ L = StepScope.LOCAL
1234
+ V = StepScope.VERIFY
1235
+
1236
+ # Preflight
1237
+ steps.append(
1238
+ _s(
1239
+ "preflight_connect_admin",
1240
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
1241
+ R,
1242
+ StepKind.SSH_COMMAND,
1243
+ command="echo 'preflight ok'",
1244
+ tags=["preflight"],
1245
+ )
1246
+ )
1247
+
1248
+ # For each template: mkdir parent -> upload rendered content -> chmod -> chown
1249
+ for t in spec.templates:
1250
+ safe_id = t.dest.replace("/", "_").strip("_")
1251
+ rendered = ctx.rendered_templates.get(t.dest, "")
1252
+
1253
+ # mkdir parent directory
1254
+ steps.append(
1255
+ _s(
1256
+ f"mkdir_{safe_id}",
1257
+ f"Create parent directory for {t.dest}",
1258
+ R,
1259
+ StepKind.SSH_COMMAND,
1260
+ command=ft.mkdir_for_file(t.dest),
1261
+ sudo=True,
1262
+ tags=["file_template", "mkdir"],
1263
+ )
1264
+ )
1265
+
1266
+ # Upload rendered content
1267
+ steps.append(
1268
+ _s(
1269
+ f"upload_{safe_id}",
1270
+ f"Upload rendered template to {t.dest}",
1271
+ R,
1272
+ StepKind.SSH_UPLOAD,
1273
+ file_content=rendered,
1274
+ target_path=t.dest,
1275
+ sudo=True,
1276
+ tags=["file_template", "upload"],
1277
+ )
1278
+ )
1279
+
1280
+ # Set permissions
1281
+ steps.append(
1282
+ _s(
1283
+ f"chmod_{safe_id}",
1284
+ f"Set permissions {t.mode} on {t.dest}",
1285
+ R,
1286
+ StepKind.SSH_COMMAND,
1287
+ command=ft.chmod_file(t.dest, t.mode),
1288
+ sudo=True,
1289
+ tags=["file_template", "chmod"],
1290
+ )
1291
+ )
1292
+
1293
+ # Set ownership
1294
+ steps.append(
1295
+ _s(
1296
+ f"chown_{safe_id}",
1297
+ f"Set ownership {t.owner}:{t.group} on {t.dest}",
1298
+ R,
1299
+ StepKind.SSH_COMMAND,
1300
+ command=ft.chown_file(t.dest, t.owner, t.group),
1301
+ sudo=True,
1302
+ tags=["file_template", "chown"],
1303
+ )
1304
+ )
1305
+
1306
+ # Postflight checks
1307
+ for check in spec.checks:
1308
+ steps.append(
1309
+ _s(
1310
+ f"postflight_{check.type}",
1311
+ f"Postflight check: {check.type}",
1312
+ V,
1313
+ StepKind.VERIFY,
1314
+ command=_encode_check_command(check, spec),
1315
+ tags=["postflight", check.type],
1316
+ )
1317
+ )
1318
+
1319
+ # Local inventory
1320
+ if spec.local.inventory.enabled:
1321
+ steps.append(
1322
+ _s(
1323
+ "open_or_init_local_inventory",
1324
+ "Open or initialize local inventory database",
1325
+ L,
1326
+ StepKind.LOCAL_DB_WRITE,
1327
+ command="init_inventory",
1328
+ tags=["local", "inventory"],
1329
+ )
1330
+ )
1331
+ steps.append(
1332
+ _s(
1333
+ "update_file_template_metadata",
1334
+ f"Update file_template metadata in inventory for {spec.host.name}",
1335
+ L,
1336
+ StepKind.LOCAL_DB_WRITE,
1337
+ command="upsert_services",
1338
+ tags=["local", "inventory"],
1339
+ )
1340
+ )
1341
+ steps.append(
1342
+ _s(
1343
+ "record_file_template_run",
1344
+ "Record file_template run metadata in inventory",
1345
+ L,
1346
+ StepKind.LOCAL_DB_WRITE,
1347
+ command="record_run",
1348
+ tags=["local", "inventory"],
1349
+ )
1350
+ )
1351
+
1352
+ return steps
1353
+
1354
+
1355
+ def _plan_compose_project(spec: ComposeProjectSpec, ctx: NormalizedContext) -> list[Step]:
1356
+ """Generate steps for deploying a Docker Compose project."""
1357
+ from loft_cli.runtime.steps import compose as cp
1358
+ from loft_cli.runtime.steps import docker as dk
1359
+
1360
+ steps: list[Step] = []
1361
+ R = StepScope.REMOTE
1362
+ L = StepScope.LOCAL
1363
+ V = StepScope.VERIFY
1364
+
1365
+ p = spec.project
1366
+
1367
+ # Preflight
1368
+ steps.append(
1369
+ _s(
1370
+ "preflight_connect_admin",
1371
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
1372
+ R,
1373
+ StepKind.SSH_COMMAND,
1374
+ command="echo 'preflight ok'",
1375
+ tags=["preflight"],
1376
+ )
1377
+ )
1378
+
1379
+ # Docker — always required for compose_project
1380
+ steps.append(
1381
+ _s(
1382
+ "apt_update",
1383
+ "Update apt package index",
1384
+ R,
1385
+ StepKind.SSH_COMMAND,
1386
+ command="apt-get update -y",
1387
+ sudo=True,
1388
+ tags=["packages"],
1389
+ )
1390
+ )
1391
+ steps.append(
1392
+ _s(
1393
+ "install_docker",
1394
+ "Install Docker",
1395
+ R,
1396
+ StepKind.SSH_COMMAND,
1397
+ command=dk.install_docker(),
1398
+ sudo=True,
1399
+ tags=["docker"],
1400
+ )
1401
+ )
1402
+ steps.append(
1403
+ _s(
1404
+ "enable_docker",
1405
+ "Enable Docker service",
1406
+ R,
1407
+ StepKind.SSH_COMMAND,
1408
+ command=dk.enable_docker(),
1409
+ sudo=True,
1410
+ tags=["docker"],
1411
+ )
1412
+ )
1413
+ steps.append(
1414
+ _s(
1415
+ "docker_version_check",
1416
+ "Verify Docker installation",
1417
+ V,
1418
+ StepKind.VERIFY,
1419
+ command="docker --version",
1420
+ sudo=True,
1421
+ tags=["docker", "verify"],
1422
+ )
1423
+ )
1424
+
1425
+ # Create project directory
1426
+ steps.append(
1427
+ _s(
1428
+ "mkdir_project_dir",
1429
+ f"Create project directory: {p.directory}",
1430
+ R,
1431
+ StepKind.SSH_COMMAND,
1432
+ command=f"mkdir -p {p.directory}",
1433
+ sudo=True,
1434
+ tags=["compose", "mkdir"],
1435
+ )
1436
+ )
1437
+
1438
+ # Create managed directories
1439
+ for d in p.directories:
1440
+ if d.path.startswith("/"):
1441
+ full_path = d.path
1442
+ else:
1443
+ full_path = f"{p.directory}/{d.path}"
1444
+ safe_id = full_path.replace("/", "_").strip("_")
1445
+ steps.append(
1446
+ _s(
1447
+ f"mkdir_{safe_id}",
1448
+ f"Create managed directory: {full_path}",
1449
+ R,
1450
+ StepKind.SSH_COMMAND,
1451
+ command=cp.mkdir_with_permissions(full_path, d.mode, d.owner, d.group),
1452
+ sudo=True,
1453
+ tags=["compose", "mkdir"],
1454
+ )
1455
+ )
1456
+
1457
+ # Upload rendered templates
1458
+ for t in p.templates:
1459
+ if t.dest.startswith("/"):
1460
+ full_dest = t.dest
1461
+ else:
1462
+ full_dest = f"{p.directory}/{t.dest}"
1463
+ safe_id = full_dest.replace("/", "_").strip("_")
1464
+ rendered = ctx.rendered_templates.get(full_dest, "")
1465
+
1466
+ steps.append(
1467
+ _s(
1468
+ f"upload_template_{safe_id}",
1469
+ f"Upload rendered template: {full_dest}",
1470
+ R,
1471
+ StepKind.SSH_UPLOAD,
1472
+ file_content=rendered,
1473
+ target_path=full_dest,
1474
+ sudo=True,
1475
+ tags=["compose", "template"],
1476
+ )
1477
+ )
1478
+
1479
+ # Upload compose file
1480
+ compose_dest = f"{p.directory}/{p.compose_file}"
1481
+ steps.append(
1482
+ _s(
1483
+ "upload_compose_file",
1484
+ f"Upload compose file: {compose_dest}",
1485
+ R,
1486
+ StepKind.SSH_UPLOAD,
1487
+ file_content=ctx.compose_file_content,
1488
+ target_path=compose_dest,
1489
+ sudo=True,
1490
+ tags=["compose", "compose-file"],
1491
+ )
1492
+ )
1493
+
1494
+ # Validate compose config
1495
+ steps.append(
1496
+ _s(
1497
+ "compose_config_validate",
1498
+ f"Validate compose configuration for project '{p.name}'",
1499
+ R,
1500
+ StepKind.SSH_COMMAND,
1501
+ command=cp.compose_config(p.directory, p.compose_file, p.name),
1502
+ sudo=True,
1503
+ rollback_hint="Check compose file syntax and variable substitution.",
1504
+ tags=["compose", "validate"],
1505
+ )
1506
+ )
1507
+
1508
+ # Pull images (optional)
1509
+ if p.pull_before_up:
1510
+ steps.append(
1511
+ _s(
1512
+ "compose_pull",
1513
+ f"Pull images for project '{p.name}'",
1514
+ R,
1515
+ StepKind.SSH_COMMAND,
1516
+ command=cp.compose_pull(p.directory, p.compose_file, p.name),
1517
+ sudo=True,
1518
+ tags=["compose", "pull"],
1519
+ )
1520
+ )
1521
+
1522
+ # Start the stack
1523
+ steps.append(
1524
+ _s(
1525
+ "compose_up",
1526
+ f"Start compose project '{p.name}' (detached)",
1527
+ R,
1528
+ StepKind.SSH_COMMAND,
1529
+ command=cp.compose_up(p.directory, p.compose_file, p.name),
1530
+ sudo=True,
1531
+ tags=["compose", "up"],
1532
+ )
1533
+ )
1534
+
1535
+ # Health check
1536
+ if p.healthcheck.enabled:
1537
+ # Command encodes the parameters for the handler to parse
1538
+ health_cmd = (
1539
+ f"compose_health:{p.directory}:{p.compose_file}:{p.name}"
1540
+ f":{p.healthcheck.timeout}:{p.healthcheck.interval}"
1541
+ )
1542
+ steps.append(
1543
+ _s(
1544
+ "compose_health_check",
1545
+ f"Wait for containers healthy (timeout={p.healthcheck.timeout}s)",
1546
+ V,
1547
+ StepKind.COMPOSE_HEALTH_CHECK,
1548
+ command=health_cmd,
1549
+ tags=["compose", "health"],
1550
+ )
1551
+ )
1552
+
1553
+ # Postflight checks
1554
+ for check in spec.checks:
1555
+ steps.append(
1556
+ _s(
1557
+ f"postflight_{check.type}",
1558
+ f"Postflight check: {check.type}",
1559
+ V,
1560
+ StepKind.VERIFY,
1561
+ command=_encode_check_command(check, spec),
1562
+ tags=["postflight", check.type],
1563
+ )
1564
+ )
1565
+
1566
+ # Local inventory
1567
+ if spec.local.inventory.enabled:
1568
+ steps.append(
1569
+ _s(
1570
+ "open_or_init_local_inventory",
1571
+ "Open or initialize local inventory database",
1572
+ L,
1573
+ StepKind.LOCAL_DB_WRITE,
1574
+ command="init_inventory",
1575
+ tags=["local", "inventory"],
1576
+ )
1577
+ )
1578
+ steps.append(
1579
+ _s(
1580
+ "update_compose_project_metadata",
1581
+ f"Update compose_project metadata in inventory for {spec.host.name}",
1582
+ L,
1583
+ StepKind.LOCAL_DB_WRITE,
1584
+ command="upsert_services",
1585
+ tags=["local", "inventory"],
1586
+ )
1587
+ )
1588
+ steps.append(
1589
+ _s(
1590
+ "record_compose_project_run",
1591
+ "Record compose_project run metadata in inventory",
1592
+ L,
1593
+ StepKind.LOCAL_DB_WRITE,
1594
+ command="record_run",
1595
+ tags=["local", "inventory"],
1596
+ )
1597
+ )
1598
+
1599
+ return steps
1600
+
1601
+
1602
+ def _plan_stack(spec: StackSpec, ctx: NormalizedContext) -> list[Step]:
1603
+ """Plan a stack by expanding resources in dependency order.
1604
+
1605
+ Stack resources are topologically sorted by ``depends_on``, then each
1606
+ resource is delegated to its kind's registered planner. Steps from
1607
+ each resource are prefixed with the resource name for traceability.
1608
+ """
1609
+ from loft_cli_core.registry import get_normalizer, get_planner, get_spec_model
1610
+
1611
+ R = StepScope.REMOTE
1612
+ L = StepScope.LOCAL
1613
+ V = StepScope.VERIFY
1614
+
1615
+ steps: list[Step] = []
1616
+
1617
+ # Topological sort
1618
+ ordered = _topo_sort(spec.resources)
1619
+
1620
+ for res in ordered:
1621
+ planner_fn = get_planner(res.kind)
1622
+ if planner_fn is None:
1623
+ steps.append(
1624
+ _s(
1625
+ f"stack_{res.name}_error",
1626
+ f"No planner for resource kind '{res.kind}'",
1627
+ R,
1628
+ StepKind.AGENT_COMMAND,
1629
+ command=f"echo 'ERROR: no planner for {res.kind}'",
1630
+ tags=["stack", res.name, "error"],
1631
+ )
1632
+ )
1633
+ continue
1634
+
1635
+ # Build a minimal spec-like object for the resource's planner.
1636
+ model_class = get_spec_model(res.kind)
1637
+ if model_class is None:
1638
+ continue
1639
+
1640
+ # Construct a synthetic spec for the child resource. We merge
1641
+ # the stack-level host/login/local into the resource config.
1642
+ child_data = {
1643
+ "kind": res.kind,
1644
+ "meta": {"name": f"{spec.meta.name}/{res.name}"},
1645
+ "host": spec.host.model_dump(),
1646
+ **res.config,
1647
+ }
1648
+
1649
+ # Inject login and local if the model expects them and they're
1650
+ # not already in the config.
1651
+ if "login" not in child_data:
1652
+ child_data["login"] = spec.login.model_dump()
1653
+ if "local" not in child_data:
1654
+ child_data["local"] = spec.local.model_dump()
1655
+
1656
+ try:
1657
+ child_spec = model_class.model_validate(child_data)
1658
+ except Exception:
1659
+ steps.append(
1660
+ _s(
1661
+ f"stack_{res.name}_validation_error",
1662
+ f"Failed to validate resource '{res.name}' config as {res.kind}",
1663
+ R,
1664
+ StepKind.AGENT_COMMAND,
1665
+ command=f"echo 'ERROR: validation failed for {res.name}'",
1666
+ tags=["stack", res.name, "error"],
1667
+ )
1668
+ )
1669
+ continue
1670
+
1671
+ # Normalize the child spec
1672
+ child_ctx = NormalizedContext(spec=child_spec, spec_dir=ctx.spec_dir)
1673
+ normalizer_fn = get_normalizer(res.kind)
1674
+ if normalizer_fn is not None:
1675
+ with contextlib.suppress(Exception):
1676
+ normalizer_fn(child_spec, child_ctx)
1677
+
1678
+ # Generate steps from the child planner
1679
+ child_steps = planner_fn(child_spec, child_ctx)
1680
+
1681
+ # Prefix step IDs with the resource name for traceability
1682
+ for cs in child_steps:
1683
+ cs.id = f"stack_{res.name}_{cs.id}"
1684
+ cs.tags = ["stack", res.name] + cs.tags
1685
+
1686
+ steps.extend(child_steps)
1687
+
1688
+ # Postflight checks for the stack
1689
+ for check in spec.checks:
1690
+ steps.append(
1691
+ _s(
1692
+ f"stack_postflight_{check.type}",
1693
+ f"Stack postflight check: {check.type}",
1694
+ V,
1695
+ StepKind.VERIFY,
1696
+ command=_encode_check_command(check, spec),
1697
+ tags=["stack", "postflight", check.type],
1698
+ )
1699
+ )
1700
+
1701
+ # Local inventory
1702
+ if spec.local.inventory.enabled:
1703
+ steps.append(
1704
+ _s(
1705
+ "stack_open_or_init_local_inventory",
1706
+ "Open or initialize local inventory database",
1707
+ L,
1708
+ StepKind.LOCAL_DB_WRITE,
1709
+ command="init_inventory",
1710
+ tags=["local", "inventory", "stack"],
1711
+ )
1712
+ )
1713
+
1714
+ return steps
1715
+
1716
+
1717
+ def _plan_postgres_ensure(spec: PostgresEnsureSpec, ctx: NormalizedContext) -> list[Step]:
1718
+ """Generate steps for ensuring PostgreSQL resources exist."""
1719
+ from loft_cli.runtime.steps.postgres_ensure import (
1720
+ ensure_database_cmd,
1721
+ ensure_extension_cmd,
1722
+ ensure_grant_cmd,
1723
+ ensure_user_cmd,
1724
+ pg_isready_cmd,
1725
+ )
1726
+
1727
+ steps: list[Step] = []
1728
+ R = StepScope.REMOTE
1729
+ L = StepScope.LOCAL
1730
+ V = StepScope.VERIFY
1731
+
1732
+ conn = spec.connection
1733
+ conn_kwargs = {
1734
+ "conn_host": conn.host,
1735
+ "conn_port": conn.port,
1736
+ "admin_user": conn.admin_user,
1737
+ "docker_exec": conn.docker_exec,
1738
+ }
1739
+
1740
+ # Preflight
1741
+ steps.append(
1742
+ _s(
1743
+ "preflight_connect_admin",
1744
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
1745
+ R,
1746
+ StepKind.SSH_COMMAND,
1747
+ command="echo 'preflight ok'",
1748
+ tags=["preflight"],
1749
+ )
1750
+ )
1751
+
1752
+ # Gate: verify PostgreSQL is ready
1753
+ steps.append(
1754
+ _s(
1755
+ "pg_isready_gate",
1756
+ "Verify PostgreSQL is accepting connections",
1757
+ R,
1758
+ StepKind.GATE,
1759
+ command=pg_isready_cmd(**conn_kwargs),
1760
+ gate=True,
1761
+ sudo=True,
1762
+ tags=["postgres_ensure", "always"],
1763
+ )
1764
+ )
1765
+
1766
+ # Ensure users
1767
+ for u in spec.users:
1768
+ password = u.password_env if u.password_env else None
1769
+ steps.append(
1770
+ _s(
1771
+ f"ensure_user_{u.name}",
1772
+ f"Ensure PostgreSQL user '{u.name}' exists",
1773
+ R,
1774
+ StepKind.SSH_COMMAND,
1775
+ command=ensure_user_cmd(u.name, password, **conn_kwargs),
1776
+ sudo=True,
1777
+ tags=["postgres_ensure", "user"],
1778
+ )
1779
+ )
1780
+
1781
+ # Ensure databases
1782
+ for d in spec.databases:
1783
+ steps.append(
1784
+ _s(
1785
+ f"ensure_database_{d.name}",
1786
+ f"Ensure PostgreSQL database '{d.name}' exists with owner '{d.owner}'",
1787
+ R,
1788
+ StepKind.SSH_COMMAND,
1789
+ command=ensure_database_cmd(d.name, d.owner, **conn_kwargs),
1790
+ sudo=True,
1791
+ tags=["postgres_ensure", "database"],
1792
+ )
1793
+ )
1794
+
1795
+ # Ensure extensions
1796
+ for e in spec.extensions:
1797
+ steps.append(
1798
+ _s(
1799
+ f"ensure_extension_{e.name}_on_{e.database}",
1800
+ f"Ensure extension '{e.name}' on database '{e.database}'",
1801
+ R,
1802
+ StepKind.SSH_COMMAND,
1803
+ command=ensure_extension_cmd(e.name, e.database, **conn_kwargs),
1804
+ sudo=True,
1805
+ tags=["postgres_ensure", "extension"],
1806
+ )
1807
+ )
1808
+
1809
+ # Ensure grants
1810
+ for g in spec.grants:
1811
+ steps.append(
1812
+ _s(
1813
+ f"grant_{g.privilege.lower()}_{g.on_database}_to_{g.to_user}",
1814
+ f"Grant {g.privilege} on {g.on_database} to {g.to_user}",
1815
+ R,
1816
+ StepKind.SSH_COMMAND,
1817
+ command=ensure_grant_cmd(g.privilege, g.on_database, g.to_user, **conn_kwargs),
1818
+ sudo=True,
1819
+ tags=["postgres_ensure", "grant"],
1820
+ )
1821
+ )
1822
+
1823
+ # Postflight checks
1824
+ for check in spec.checks:
1825
+ steps.append(
1826
+ _s(
1827
+ f"postflight_{check.type}",
1828
+ f"Postflight check: {check.type}",
1829
+ V,
1830
+ StepKind.VERIFY,
1831
+ command=_encode_check_command(check, spec),
1832
+ tags=["postflight", check.type],
1833
+ )
1834
+ )
1835
+
1836
+ # Local inventory
1837
+ if spec.local.inventory.enabled:
1838
+ steps.append(
1839
+ _s(
1840
+ "open_or_init_local_inventory",
1841
+ "Open or initialize local inventory database",
1842
+ L,
1843
+ StepKind.LOCAL_DB_WRITE,
1844
+ command="init_inventory",
1845
+ tags=["local", "inventory"],
1846
+ )
1847
+ )
1848
+ steps.append(
1849
+ _s(
1850
+ "update_postgres_ensure_metadata",
1851
+ f"Update postgres_ensure metadata in inventory for {spec.host.name}",
1852
+ L,
1853
+ StepKind.LOCAL_DB_WRITE,
1854
+ command="upsert_services",
1855
+ tags=["local", "inventory"],
1856
+ )
1857
+ )
1858
+ steps.append(
1859
+ _s(
1860
+ "record_postgres_ensure_run",
1861
+ "Record postgres_ensure run metadata in inventory",
1862
+ L,
1863
+ StepKind.LOCAL_DB_WRITE,
1864
+ command="record_run",
1865
+ tags=["local", "inventory"],
1866
+ )
1867
+ )
1868
+
1869
+ return steps
1870
+
1871
+
1872
+ def _plan_backup_job(spec: BackupJobSpec, ctx: NormalizedContext) -> list[Step]:
1873
+ """Generate steps for deploying a backup job with systemd timer."""
1874
+ from loft_cli.runtime.steps.backup import render_backup_script
1875
+ from loft_cli.runtime.steps.systemd import (
1876
+ daemon_reload,
1877
+ enable_now_unit,
1878
+ is_active,
1879
+ render_service_unit,
1880
+ render_timer_unit,
1881
+ )
1882
+
1883
+ steps: list[Step] = []
1884
+ R = StepScope.REMOTE
1885
+ L = StepScope.LOCAL
1886
+ V = StepScope.VERIFY
1887
+
1888
+ b = spec.backup
1889
+ src = b.source
1890
+ script_name = f"loft-cli-backup-{b.name}"
1891
+ script_path = f"/usr/local/bin/{script_name}.sh"
1892
+
1893
+ # Preflight
1894
+ steps.append(
1895
+ _s(
1896
+ "preflight_connect_admin",
1897
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
1898
+ R,
1899
+ StepKind.SSH_COMMAND,
1900
+ command="echo 'preflight ok'",
1901
+ tags=["preflight"],
1902
+ )
1903
+ )
1904
+
1905
+ # Create backup destination directory
1906
+ steps.append(
1907
+ _s(
1908
+ f"mkdir_backup_dest_{b.name}",
1909
+ f"Create backup destination directory {b.destination.path}",
1910
+ R,
1911
+ StepKind.SSH_COMMAND,
1912
+ command=f"mkdir -p {b.destination.path}",
1913
+ sudo=True,
1914
+ tags=["backup_job", "mkdir"],
1915
+ )
1916
+ )
1917
+
1918
+ # Write backup script
1919
+ script_content = render_backup_script(
1920
+ name=b.name,
1921
+ source_type=src.type,
1922
+ destination_path=b.destination.path,
1923
+ retention_count=b.retention.count,
1924
+ database=src.database,
1925
+ pg_host=src.host,
1926
+ pg_port=src.port,
1927
+ pg_user=src.user,
1928
+ docker_exec=src.docker_exec,
1929
+ source_path=src.path,
1930
+ )
1931
+ steps.append(
1932
+ _s(
1933
+ f"write_backup_script_{b.name}",
1934
+ f"Write backup script {script_path}",
1935
+ R,
1936
+ StepKind.SSH_UPLOAD,
1937
+ file_content=script_content,
1938
+ target_path=script_path,
1939
+ sudo=True,
1940
+ tags=["backup_job", "upload"],
1941
+ )
1942
+ )
1943
+
1944
+ # Make script executable
1945
+ steps.append(
1946
+ _s(
1947
+ f"chmod_backup_script_{b.name}",
1948
+ "Make backup script executable",
1949
+ R,
1950
+ StepKind.SSH_COMMAND,
1951
+ command=f"chmod 0755 {script_path}",
1952
+ sudo=True,
1953
+ tags=["backup_job"],
1954
+ )
1955
+ )
1956
+
1957
+ # Write systemd service (oneshot)
1958
+ service_content = render_service_unit(
1959
+ description=f"Backup job: {b.name}",
1960
+ exec_start=script_path,
1961
+ service_type="oneshot",
1962
+ restart="no",
1963
+ restart_sec=0,
1964
+ wanted_by="",
1965
+ )
1966
+ service_file = f"/etc/systemd/system/{script_name}.service"
1967
+ steps.append(
1968
+ _s(
1969
+ f"write_backup_service_{b.name}",
1970
+ f"Write backup service file {service_file}",
1971
+ R,
1972
+ StepKind.SSH_UPLOAD,
1973
+ file_content=service_content,
1974
+ target_path=service_file,
1975
+ sudo=True,
1976
+ tags=["backup_job", "upload"],
1977
+ )
1978
+ )
1979
+
1980
+ # Write systemd timer
1981
+ timer_content = render_timer_unit(
1982
+ description=f"Backup timer: {b.name}",
1983
+ on_calendar=b.schedule,
1984
+ persistent=True,
1985
+ )
1986
+ timer_file = f"/etc/systemd/system/{script_name}.timer"
1987
+ steps.append(
1988
+ _s(
1989
+ f"write_backup_timer_{b.name}",
1990
+ f"Write backup timer file {timer_file}",
1991
+ R,
1992
+ StepKind.SSH_UPLOAD,
1993
+ file_content=timer_content,
1994
+ target_path=timer_file,
1995
+ sudo=True,
1996
+ tags=["backup_job", "upload"],
1997
+ )
1998
+ )
1999
+
2000
+ # Daemon reload
2001
+ steps.append(
2002
+ _s(
2003
+ "systemd_daemon_reload",
2004
+ "Reload systemd daemon",
2005
+ R,
2006
+ StepKind.SSH_COMMAND,
2007
+ command=daemon_reload(),
2008
+ sudo=True,
2009
+ tags=["backup_job", "always"],
2010
+ )
2011
+ )
2012
+
2013
+ # Enable and start timer
2014
+ steps.append(
2015
+ _s(
2016
+ f"enable_start_backup_timer_{b.name}",
2017
+ f"Enable and start {script_name}.timer",
2018
+ R,
2019
+ StepKind.SSH_COMMAND,
2020
+ command=enable_now_unit(f"{script_name}.timer"),
2021
+ sudo=True,
2022
+ tags=["backup_job", "always"],
2023
+ )
2024
+ )
2025
+
2026
+ # Verify timer active
2027
+ steps.append(
2028
+ _s(
2029
+ f"verify_backup_timer_{b.name}_active",
2030
+ f"Verify {script_name}.timer is active",
2031
+ V,
2032
+ StepKind.VERIFY,
2033
+ command=is_active(f"{script_name}.timer"),
2034
+ tags=["backup_job", "verify"],
2035
+ )
2036
+ )
2037
+
2038
+ # Postflight checks
2039
+ for check in spec.checks:
2040
+ steps.append(
2041
+ _s(
2042
+ f"postflight_{check.type}",
2043
+ f"Postflight check: {check.type}",
2044
+ V,
2045
+ StepKind.VERIFY,
2046
+ command=_encode_check_command(check, spec),
2047
+ tags=["postflight", check.type],
2048
+ )
2049
+ )
2050
+
2051
+ # Local inventory
2052
+ if spec.local.inventory.enabled:
2053
+ steps.append(
2054
+ _s(
2055
+ "open_or_init_local_inventory",
2056
+ "Open or initialize local inventory database",
2057
+ L,
2058
+ StepKind.LOCAL_DB_WRITE,
2059
+ command="init_inventory",
2060
+ tags=["local", "inventory"],
2061
+ )
2062
+ )
2063
+ steps.append(
2064
+ _s(
2065
+ "update_backup_job_metadata",
2066
+ f"Update backup_job metadata in inventory for {spec.host.name}",
2067
+ L,
2068
+ StepKind.LOCAL_DB_WRITE,
2069
+ command="upsert_services",
2070
+ tags=["local", "inventory"],
2071
+ )
2072
+ )
2073
+ steps.append(
2074
+ _s(
2075
+ "record_backup_job_run",
2076
+ "Record backup_job run metadata in inventory",
2077
+ L,
2078
+ StepKind.LOCAL_DB_WRITE,
2079
+ command="record_run",
2080
+ tags=["local", "inventory"],
2081
+ )
2082
+ )
2083
+
2084
+ return steps
2085
+
2086
+
2087
+ def _plan_systemd_unit(spec: SystemdUnitSpec, ctx: NormalizedContext) -> list[Step]:
2088
+ """Generate steps for deploying a systemd service unit."""
2089
+ from loft_cli.runtime.steps.systemd import (
2090
+ daemon_reload,
2091
+ enable_unit,
2092
+ is_active,
2093
+ render_logrotate_config,
2094
+ render_service_unit,
2095
+ restart_unit,
2096
+ )
2097
+
2098
+ steps: list[Step] = []
2099
+ R = StepScope.REMOTE
2100
+ L = StepScope.LOCAL
2101
+ V = StepScope.VERIFY
2102
+
2103
+ u = spec.unit
2104
+ unit_file = f"/etc/systemd/system/{u.unit_name}.service"
2105
+
2106
+ # Preflight
2107
+ steps.append(
2108
+ _s(
2109
+ "preflight_connect_admin",
2110
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
2111
+ R,
2112
+ StepKind.SSH_COMMAND,
2113
+ command="echo 'preflight ok'",
2114
+ tags=["preflight"],
2115
+ )
2116
+ )
2117
+
2118
+ # Write unit file
2119
+ unit_content = render_service_unit(
2120
+ description=u.description or spec.meta.description or u.unit_name,
2121
+ exec_start=u.exec_start,
2122
+ exec_stop=u.exec_stop,
2123
+ working_directory=u.working_directory,
2124
+ user=u.user,
2125
+ group=u.group,
2126
+ restart=u.restart,
2127
+ restart_sec=u.restart_sec,
2128
+ after=u.after,
2129
+ environment=u.environment,
2130
+ environment_file=u.environment_file,
2131
+ service_type=u.type,
2132
+ wanted_by=u.wanted_by,
2133
+ )
2134
+
2135
+ steps.append(
2136
+ _s(
2137
+ f"write_unit_{u.unit_name}",
2138
+ f"Write systemd unit file {unit_file}",
2139
+ R,
2140
+ StepKind.SSH_UPLOAD,
2141
+ file_content=unit_content,
2142
+ target_path=unit_file,
2143
+ sudo=True,
2144
+ tags=["systemd_unit", "upload"],
2145
+ )
2146
+ )
2147
+
2148
+ # Daemon reload
2149
+ steps.append(
2150
+ _s(
2151
+ "systemd_daemon_reload",
2152
+ "Reload systemd daemon",
2153
+ R,
2154
+ StepKind.SSH_COMMAND,
2155
+ command=daemon_reload(),
2156
+ sudo=True,
2157
+ tags=["systemd_unit", "always"],
2158
+ )
2159
+ )
2160
+
2161
+ # Enable
2162
+ steps.append(
2163
+ _s(
2164
+ f"enable_{u.unit_name}",
2165
+ f"Enable {u.unit_name}.service",
2166
+ R,
2167
+ StepKind.SSH_COMMAND,
2168
+ command=enable_unit(u.unit_name),
2169
+ sudo=True,
2170
+ tags=["systemd_unit"],
2171
+ )
2172
+ )
2173
+
2174
+ # Restart
2175
+ steps.append(
2176
+ _s(
2177
+ f"restart_{u.unit_name}",
2178
+ f"Restart {u.unit_name}.service",
2179
+ R,
2180
+ StepKind.SSH_COMMAND,
2181
+ command=restart_unit(u.unit_name),
2182
+ sudo=True,
2183
+ tags=["systemd_unit", "always"],
2184
+ )
2185
+ )
2186
+
2187
+ # Verify active
2188
+ steps.append(
2189
+ _s(
2190
+ f"verify_{u.unit_name}_active",
2191
+ f"Verify {u.unit_name}.service is active",
2192
+ V,
2193
+ StepKind.VERIFY,
2194
+ command=is_active(u.unit_name),
2195
+ tags=["systemd_unit", "verify"],
2196
+ )
2197
+ )
2198
+
2199
+ # Optional logrotate
2200
+ lr = spec.logrotate
2201
+ if lr and lr.enabled:
2202
+ logrotate_content = render_logrotate_config(
2203
+ name=u.unit_name,
2204
+ path=lr.path,
2205
+ rotate=lr.rotate,
2206
+ frequency=lr.frequency,
2207
+ compress=lr.compress,
2208
+ max_size=lr.max_size,
2209
+ )
2210
+ steps.append(
2211
+ _s(
2212
+ f"write_logrotate_{u.unit_name}",
2213
+ f"Write logrotate config for {u.unit_name}",
2214
+ R,
2215
+ StepKind.SSH_UPLOAD,
2216
+ file_content=logrotate_content,
2217
+ target_path=f"/etc/logrotate.d/loft-cli-{u.unit_name}",
2218
+ sudo=True,
2219
+ tags=["systemd_unit", "logrotate"],
2220
+ )
2221
+ )
2222
+
2223
+ # Postflight checks
2224
+ for check in spec.checks:
2225
+ steps.append(
2226
+ _s(
2227
+ f"postflight_{check.type}",
2228
+ f"Postflight check: {check.type}",
2229
+ V,
2230
+ StepKind.VERIFY,
2231
+ command=_encode_check_command(check, spec),
2232
+ tags=["postflight", check.type],
2233
+ )
2234
+ )
2235
+
2236
+ # Local inventory
2237
+ if spec.local.inventory.enabled:
2238
+ steps.append(
2239
+ _s(
2240
+ "open_or_init_local_inventory",
2241
+ "Open or initialize local inventory database",
2242
+ L,
2243
+ StepKind.LOCAL_DB_WRITE,
2244
+ command="init_inventory",
2245
+ tags=["local", "inventory"],
2246
+ )
2247
+ )
2248
+ steps.append(
2249
+ _s(
2250
+ "update_systemd_unit_metadata",
2251
+ f"Update systemd_unit metadata in inventory for {spec.host.name}",
2252
+ L,
2253
+ StepKind.LOCAL_DB_WRITE,
2254
+ command="upsert_services",
2255
+ tags=["local", "inventory"],
2256
+ )
2257
+ )
2258
+ steps.append(
2259
+ _s(
2260
+ "record_systemd_unit_run",
2261
+ "Record systemd_unit run metadata in inventory",
2262
+ L,
2263
+ StepKind.LOCAL_DB_WRITE,
2264
+ command="record_run",
2265
+ tags=["local", "inventory"],
2266
+ )
2267
+ )
2268
+
2269
+ return steps
2270
+
2271
+
2272
+ def _plan_systemd_timer(spec: SystemdTimerSpec, ctx: NormalizedContext) -> list[Step]:
2273
+ """Generate steps for deploying a systemd timer with companion oneshot service."""
2274
+ from loft_cli.runtime.steps.systemd import (
2275
+ daemon_reload,
2276
+ enable_now_unit,
2277
+ is_active,
2278
+ render_service_unit,
2279
+ render_timer_unit,
2280
+ )
2281
+
2282
+ steps: list[Step] = []
2283
+ R = StepScope.REMOTE
2284
+ L = StepScope.LOCAL
2285
+ V = StepScope.VERIFY
2286
+
2287
+ t = spec.timer
2288
+ s = spec.service
2289
+
2290
+ # Preflight
2291
+ steps.append(
2292
+ _s(
2293
+ "preflight_connect_admin",
2294
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
2295
+ R,
2296
+ StepKind.SSH_COMMAND,
2297
+ command="echo 'preflight ok'",
2298
+ tags=["preflight"],
2299
+ )
2300
+ )
2301
+
2302
+ # Write companion .service (Type=oneshot)
2303
+ service_content = render_service_unit(
2304
+ description=t.description or f"Service for timer {t.timer_name}",
2305
+ exec_start=s.exec_start,
2306
+ working_directory=s.working_directory,
2307
+ user=s.user,
2308
+ group=s.group,
2309
+ restart="no",
2310
+ restart_sec=0,
2311
+ environment=s.environment,
2312
+ service_type="oneshot",
2313
+ wanted_by="",
2314
+ )
2315
+ service_file = f"/etc/systemd/system/{t.timer_name}.service"
2316
+ steps.append(
2317
+ _s(
2318
+ f"write_service_{t.timer_name}",
2319
+ f"Write companion service file {service_file}",
2320
+ R,
2321
+ StepKind.SSH_UPLOAD,
2322
+ file_content=service_content,
2323
+ target_path=service_file,
2324
+ sudo=True,
2325
+ tags=["systemd_timer", "upload"],
2326
+ )
2327
+ )
2328
+
2329
+ # Write .timer
2330
+ timer_content = render_timer_unit(
2331
+ description=t.description or f"Timer for {t.timer_name}",
2332
+ on_calendar=t.on_calendar,
2333
+ persistent=t.persistent,
2334
+ accuracy_sec=t.accuracy_sec,
2335
+ )
2336
+ timer_file = f"/etc/systemd/system/{t.timer_name}.timer"
2337
+ steps.append(
2338
+ _s(
2339
+ f"write_timer_{t.timer_name}",
2340
+ f"Write timer unit file {timer_file}",
2341
+ R,
2342
+ StepKind.SSH_UPLOAD,
2343
+ file_content=timer_content,
2344
+ target_path=timer_file,
2345
+ sudo=True,
2346
+ tags=["systemd_timer", "upload"],
2347
+ )
2348
+ )
2349
+
2350
+ # Daemon reload
2351
+ steps.append(
2352
+ _s(
2353
+ "systemd_daemon_reload",
2354
+ "Reload systemd daemon",
2355
+ R,
2356
+ StepKind.SSH_COMMAND,
2357
+ command=daemon_reload(),
2358
+ sudo=True,
2359
+ tags=["systemd_timer", "always"],
2360
+ )
2361
+ )
2362
+
2363
+ # Enable and start timer
2364
+ steps.append(
2365
+ _s(
2366
+ f"enable_start_{t.timer_name}_timer",
2367
+ f"Enable and start {t.timer_name}.timer",
2368
+ R,
2369
+ StepKind.SSH_COMMAND,
2370
+ command=enable_now_unit(f"{t.timer_name}.timer"),
2371
+ sudo=True,
2372
+ tags=["systemd_timer", "always"],
2373
+ )
2374
+ )
2375
+
2376
+ # Verify timer active
2377
+ steps.append(
2378
+ _s(
2379
+ f"verify_{t.timer_name}_timer_active",
2380
+ f"Verify {t.timer_name}.timer is active",
2381
+ V,
2382
+ StepKind.VERIFY,
2383
+ command=is_active(f"{t.timer_name}.timer"),
2384
+ tags=["systemd_timer", "verify"],
2385
+ )
2386
+ )
2387
+
2388
+ # Postflight checks
2389
+ for check in spec.checks:
2390
+ steps.append(
2391
+ _s(
2392
+ f"postflight_{check.type}",
2393
+ f"Postflight check: {check.type}",
2394
+ V,
2395
+ StepKind.VERIFY,
2396
+ command=_encode_check_command(check, spec),
2397
+ tags=["postflight", check.type],
2398
+ )
2399
+ )
2400
+
2401
+ # Local inventory
2402
+ if spec.local.inventory.enabled:
2403
+ steps.append(
2404
+ _s(
2405
+ "open_or_init_local_inventory",
2406
+ "Open or initialize local inventory database",
2407
+ L,
2408
+ StepKind.LOCAL_DB_WRITE,
2409
+ command="init_inventory",
2410
+ tags=["local", "inventory"],
2411
+ )
2412
+ )
2413
+ steps.append(
2414
+ _s(
2415
+ "update_systemd_timer_metadata",
2416
+ f"Update systemd_timer metadata in inventory for {spec.host.name}",
2417
+ L,
2418
+ StepKind.LOCAL_DB_WRITE,
2419
+ command="upsert_services",
2420
+ tags=["local", "inventory"],
2421
+ )
2422
+ )
2423
+ steps.append(
2424
+ _s(
2425
+ "record_systemd_timer_run",
2426
+ "Record systemd_timer run metadata in inventory",
2427
+ L,
2428
+ StepKind.LOCAL_DB_WRITE,
2429
+ command="record_run",
2430
+ tags=["local", "inventory"],
2431
+ )
2432
+ )
2433
+
2434
+ return steps
2435
+
2436
+
2437
+ def _plan_http_check(spec: HttpCheckSpec, ctx: NormalizedContext) -> list[Step]:
2438
+ """Generate steps for a GET-only HTTP readiness probe."""
2439
+ steps: list[Step] = []
2440
+ R = StepScope.REMOTE
2441
+ L = StepScope.LOCAL
2442
+ V = StepScope.VERIFY
2443
+
2444
+ c = spec.check
2445
+
2446
+ # Preflight
2447
+ steps.append(
2448
+ _s(
2449
+ "preflight_connect_admin",
2450
+ f"Verify admin SSH access to {spec.host.address}:{spec.login.port}",
2451
+ R,
2452
+ StepKind.SSH_COMMAND,
2453
+ command="echo 'preflight ok'",
2454
+ tags=["preflight"],
2455
+ )
2456
+ )
2457
+
2458
+ # HTTP check gate — executed by agent with retry loop
2459
+ steps.append(
2460
+ _s(
2461
+ "http_check_gate",
2462
+ f"HTTP readiness check: GET {c.url} expecting {c.expected_status}",
2463
+ R,
2464
+ StepKind.GATE,
2465
+ command=f"http_check:{c.url}:{c.expected_status}:{c.retries}:{c.interval}:{c.timeout}",
2466
+ gate=True,
2467
+ tags=["http_check", "always"],
2468
+ )
2469
+ )
2470
+
2471
+ # Postflight checks
2472
+ for check in spec.checks:
2473
+ steps.append(
2474
+ _s(
2475
+ f"postflight_{check.type}",
2476
+ f"Postflight check: {check.type}",
2477
+ V,
2478
+ StepKind.VERIFY,
2479
+ command=_encode_check_command(check, spec),
2480
+ tags=["postflight", check.type],
2481
+ )
2482
+ )
2483
+
2484
+ # Local inventory
2485
+ if spec.local.inventory.enabled:
2486
+ steps.append(
2487
+ _s(
2488
+ "open_or_init_local_inventory",
2489
+ "Open or initialize local inventory database",
2490
+ L,
2491
+ StepKind.LOCAL_DB_WRITE,
2492
+ command="init_inventory",
2493
+ tags=["local", "inventory"],
2494
+ )
2495
+ )
2496
+ steps.append(
2497
+ _s(
2498
+ "update_http_check_metadata",
2499
+ f"Update http_check metadata in inventory for {spec.host.name}",
2500
+ L,
2501
+ StepKind.LOCAL_DB_WRITE,
2502
+ command="upsert_services",
2503
+ tags=["local", "inventory"],
2504
+ )
2505
+ )
2506
+ steps.append(
2507
+ _s(
2508
+ "record_http_check_run",
2509
+ "Record http_check run metadata in inventory",
2510
+ L,
2511
+ StepKind.LOCAL_DB_WRITE,
2512
+ command="record_run",
2513
+ tags=["local", "inventory"],
2514
+ )
2515
+ )
2516
+
2517
+ return steps
2518
+
2519
+
2520
+ def _topo_sort(resources) -> list:
2521
+ """Topologically sort stack resources by depends_on.
2522
+
2523
+ Returns resources in dependency-first order.
2524
+ """
2525
+ name_to_res = {r.name: r for r in resources}
2526
+ visited: set[str] = set()
2527
+ result: list = []
2528
+
2529
+ def _visit(name: str) -> None:
2530
+ if name in visited:
2531
+ return
2532
+ visited.add(name)
2533
+ res = name_to_res.get(name)
2534
+ if res is None:
2535
+ return
2536
+ for dep in res.depends_on:
2537
+ _visit(dep)
2538
+ result.append(res)
2539
+
2540
+ for r in resources:
2541
+ _visit(r.name)
2542
+
2543
+ return result