loft-cli 0.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loft_cli/__init__.py +3 -0
- loft_cli/_builtins.py +269 -0
- loft_cli/addons/__init__.py +12 -0
- loft_cli/addons/goss/__init__.py +1 -0
- loft_cli/addons/goss/generator.py +202 -0
- loft_cli/addons/goss/renderer.py +207 -0
- loft_cli/addons/goss/shipper.py +174 -0
- loft_cli/agent_installer.py +65 -0
- loft_cli/checks/__init__.py +0 -0
- loft_cli/checks/compose.py +147 -0
- loft_cli/checks/container.py +24 -0
- loft_cli/checks/http.py +30 -0
- loft_cli/checks/nginx.py +21 -0
- loft_cli/checks/ports.py +24 -0
- loft_cli/checks/postgres.py +21 -0
- loft_cli/checks/ssh.py +66 -0
- loft_cli/checks/wireguard.py +21 -0
- loft_cli/cli.py +1161 -0
- loft_cli/compiler/__init__.py +0 -0
- loft_cli/compiler/normalizer.py +420 -0
- loft_cli/compiler/parser.py +42 -0
- loft_cli/compiler/planner.py +2543 -0
- loft_cli/local/__init__.py +0 -0
- loft_cli/local/ddl/__init__.py +0 -0
- loft_cli/local/ddl/bootstrap_tables.py +75 -0
- loft_cli/local/ddl/versionize_system.py +374 -0
- loft_cli/local/inventory.py +449 -0
- loft_cli/local/inventory_db.py +255 -0
- loft_cli/local/keys.py +45 -0
- loft_cli/local/remove.py +135 -0
- loft_cli/local/ssh_config.py +118 -0
- loft_cli/local/tunnel.py +228 -0
- loft_cli/local/wireguard_store.py +138 -0
- loft_cli/logs/__init__.py +0 -0
- loft_cli/logs/reader.py +55 -0
- loft_cli/logs/writer.py +58 -0
- loft_cli/runtime/__init__.py +0 -0
- loft_cli/runtime/agent_transport.py +115 -0
- loft_cli/runtime/executor.py +798 -0
- loft_cli/runtime/fabric_transport.py +68 -0
- loft_cli/runtime/secret_rotation.py +99 -0
- loft_cli/runtime/ssh.py +147 -0
- loft_cli/runtime/steps/__init__.py +0 -0
- loft_cli/runtime/steps/backup.py +72 -0
- loft_cli/runtime/steps/bootstrap.py +227 -0
- loft_cli/runtime/steps/compose.py +43 -0
- loft_cli/runtime/steps/container.py +40 -0
- loft_cli/runtime/steps/docker.py +15 -0
- loft_cli/runtime/steps/file_template.py +25 -0
- loft_cli/runtime/steps/nginx.py +126 -0
- loft_cli/runtime/steps/postgres.py +101 -0
- loft_cli/runtime/steps/postgres_ensure.py +141 -0
- loft_cli/runtime/steps/systemd.py +131 -0
- loft_cli/runtime/steps/wireguard.py +76 -0
- loft_cli/runtime/transport.py +43 -0
- loft_cli/updater.py +208 -0
- loft_cli-0.6.3.dist-info/METADATA +19 -0
- loft_cli-0.6.3.dist-info/RECORD +61 -0
- loft_cli-0.6.3.dist-info/WHEEL +5 -0
- loft_cli-0.6.3.dist-info/entry_points.txt +5 -0
- loft_cli-0.6.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,798 @@
|
|
|
1
|
+
"""Plan execution engine with gate/dependency semantics.
|
|
2
|
+
|
|
3
|
+
Critical invariants enforced here:
|
|
4
|
+
- Steps with gate=True abort the plan on failure
|
|
5
|
+
- Steps in depends_on that failed cause the dependent step to be skipped
|
|
6
|
+
- Local steps run only after all remote critical steps succeed
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import time
|
|
12
|
+
import warnings
|
|
13
|
+
from datetime import UTC, datetime
|
|
14
|
+
from typing import TYPE_CHECKING, Literal
|
|
15
|
+
|
|
16
|
+
from pydantic import BaseModel
|
|
17
|
+
from rich.console import Console
|
|
18
|
+
|
|
19
|
+
from loft_cli_core.plan.models import Plan, Step, StepScope
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from loft_cli.runtime.transport import Transport
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class StepResult(BaseModel):
|
|
26
|
+
step_index: int
|
|
27
|
+
step_id: str
|
|
28
|
+
scope: str
|
|
29
|
+
status: Literal["success", "failed", "skipped"]
|
|
30
|
+
output: str = ""
|
|
31
|
+
error: str = ""
|
|
32
|
+
duration_seconds: float = 0.0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ApplyResult(BaseModel):
|
|
36
|
+
plan: Plan
|
|
37
|
+
step_results: list[StepResult]
|
|
38
|
+
status: Literal["success", "failed", "success_with_local_warnings"]
|
|
39
|
+
aborted_at: int | None = None
|
|
40
|
+
started_at: str
|
|
41
|
+
finished_at: str = ""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Executor:
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
plan: Plan,
|
|
48
|
+
ssh_session=None,
|
|
49
|
+
transport: Transport | None = None,
|
|
50
|
+
inventory_db=None,
|
|
51
|
+
ctx=None,
|
|
52
|
+
spec=None,
|
|
53
|
+
console: Console | None = None,
|
|
54
|
+
effective_port: int | None = None,
|
|
55
|
+
) -> None:
|
|
56
|
+
self._plan = plan
|
|
57
|
+
# Transport-first: prefer transport, fall back to ssh_session (deprecated).
|
|
58
|
+
if transport is not None:
|
|
59
|
+
self._session = transport
|
|
60
|
+
elif ssh_session is not None:
|
|
61
|
+
warnings.warn(
|
|
62
|
+
"Executor(ssh_session=...) is deprecated, use transport= instead.",
|
|
63
|
+
DeprecationWarning,
|
|
64
|
+
stacklevel=2,
|
|
65
|
+
)
|
|
66
|
+
self._session = ssh_session
|
|
67
|
+
else:
|
|
68
|
+
self._session = None
|
|
69
|
+
self._db = inventory_db
|
|
70
|
+
self._ctx = ctx
|
|
71
|
+
self._spec = spec
|
|
72
|
+
self._console = console or Console()
|
|
73
|
+
self._effective_port = effective_port
|
|
74
|
+
|
|
75
|
+
def apply(self, dry_run: bool = False) -> ApplyResult:
|
|
76
|
+
"""Execute all steps in order, respecting gates and dependencies."""
|
|
77
|
+
started_at = datetime.now(UTC).isoformat()
|
|
78
|
+
step_results: list[StepResult] = []
|
|
79
|
+
aborted_at: int | None = None
|
|
80
|
+
remote_failed = False
|
|
81
|
+
local_warnings = False
|
|
82
|
+
|
|
83
|
+
for step in self._plan.steps:
|
|
84
|
+
# Check if any dependency failed — skip if so
|
|
85
|
+
if self._has_failed_dependency(step, step_results):
|
|
86
|
+
result = StepResult(
|
|
87
|
+
step_index=step.index,
|
|
88
|
+
step_id=step.id,
|
|
89
|
+
scope=step.scope.value,
|
|
90
|
+
status="skipped",
|
|
91
|
+
error="Dependency failed",
|
|
92
|
+
)
|
|
93
|
+
step_results.append(result)
|
|
94
|
+
self._print_step(step, result)
|
|
95
|
+
if step.gate:
|
|
96
|
+
aborted_at = step.index
|
|
97
|
+
break
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
if aborted_at is not None:
|
|
101
|
+
# Plan aborted — skip all remaining steps
|
|
102
|
+
step_results.append(
|
|
103
|
+
StepResult(
|
|
104
|
+
step_index=step.index,
|
|
105
|
+
step_id=step.id,
|
|
106
|
+
scope=step.scope.value,
|
|
107
|
+
status="skipped",
|
|
108
|
+
error="Plan aborted",
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
result = self._execute_step(step, dry_run)
|
|
114
|
+
step_results.append(result)
|
|
115
|
+
self._print_step(step, result)
|
|
116
|
+
|
|
117
|
+
if result.status == "failed":
|
|
118
|
+
if step.index == 0:
|
|
119
|
+
# Preflight connection failure — abort immediately with a clear message.
|
|
120
|
+
# Continuing makes no sense: every subsequent step would fail with the
|
|
121
|
+
# same connection error, obscuring the real problem.
|
|
122
|
+
self._console.print(
|
|
123
|
+
f"\n[bold red]✗ Preflight failed: cannot connect to the host.[/bold red]\n"
|
|
124
|
+
f" Check that the host is up, login.port is correct, and credentials are valid.\n"
|
|
125
|
+
f" Error: {result.error or result.output}"
|
|
126
|
+
)
|
|
127
|
+
aborted_at = step.index
|
|
128
|
+
break
|
|
129
|
+
elif step.gate:
|
|
130
|
+
aborted_at = step.index
|
|
131
|
+
self._console.print(
|
|
132
|
+
f"\n[bold red]⛔ GATE FAILED at step {step.index}: {step.id}[/bold red]"
|
|
133
|
+
)
|
|
134
|
+
if step.rollback_hint:
|
|
135
|
+
self._console.print(f"[yellow]Recovery:[/yellow] {step.rollback_hint}")
|
|
136
|
+
break
|
|
137
|
+
elif step.scope == StepScope.LOCAL:
|
|
138
|
+
local_warnings = True
|
|
139
|
+
else:
|
|
140
|
+
remote_failed = True
|
|
141
|
+
|
|
142
|
+
finished_at = datetime.now(UTC).isoformat()
|
|
143
|
+
|
|
144
|
+
if aborted_at is not None or remote_failed:
|
|
145
|
+
final_status = "failed"
|
|
146
|
+
elif local_warnings:
|
|
147
|
+
final_status = "success_with_local_warnings"
|
|
148
|
+
else:
|
|
149
|
+
final_status = "success"
|
|
150
|
+
|
|
151
|
+
return ApplyResult(
|
|
152
|
+
plan=self._plan,
|
|
153
|
+
step_results=step_results,
|
|
154
|
+
status=final_status,
|
|
155
|
+
aborted_at=aborted_at,
|
|
156
|
+
started_at=started_at,
|
|
157
|
+
finished_at=finished_at,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
def _has_failed_dependency(self, step: Step, results: list[StepResult]) -> bool:
|
|
161
|
+
"""Return True if any dependency step failed."""
|
|
162
|
+
for dep_idx in step.depends_on:
|
|
163
|
+
dep_result = next((r for r in results if r.step_index == dep_idx), None)
|
|
164
|
+
if dep_result and dep_result.status == "failed":
|
|
165
|
+
return True
|
|
166
|
+
return False
|
|
167
|
+
|
|
168
|
+
def _execute_step(self, step: Step, dry_run: bool) -> StepResult:
|
|
169
|
+
start = time.monotonic()
|
|
170
|
+
|
|
171
|
+
if dry_run:
|
|
172
|
+
duration = time.monotonic() - start
|
|
173
|
+
return StepResult(
|
|
174
|
+
step_index=step.index,
|
|
175
|
+
step_id=step.id,
|
|
176
|
+
scope=step.scope.value,
|
|
177
|
+
status="success",
|
|
178
|
+
output=f"[dry-run] would execute: {step.command or step.kind}",
|
|
179
|
+
duration_seconds=duration,
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
try:
|
|
183
|
+
from loft_cli_core.registry.executors import get_step_handler
|
|
184
|
+
|
|
185
|
+
handler = get_step_handler(step.kind)
|
|
186
|
+
if handler is not None:
|
|
187
|
+
result = handler(self, step)
|
|
188
|
+
else:
|
|
189
|
+
result = StepResult(
|
|
190
|
+
step_index=step.index,
|
|
191
|
+
step_id=step.id,
|
|
192
|
+
scope=step.scope.value,
|
|
193
|
+
status="failed",
|
|
194
|
+
error=(f"Unknown step kind: '{step.kind}'. Is the required addon installed?"),
|
|
195
|
+
)
|
|
196
|
+
except Exception as e:
|
|
197
|
+
result = StepResult(
|
|
198
|
+
step_index=step.index,
|
|
199
|
+
step_id=step.id,
|
|
200
|
+
scope=step.scope.value,
|
|
201
|
+
status="failed",
|
|
202
|
+
error=str(e),
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
result.duration_seconds = time.monotonic() - start
|
|
206
|
+
return result
|
|
207
|
+
|
|
208
|
+
def _execute_ssh_command(self, step: Step) -> StepResult:
|
|
209
|
+
if self._session is None:
|
|
210
|
+
raise RuntimeError("No SSH session available")
|
|
211
|
+
cmd_result = self._session.run(step.command or "", sudo=step.sudo, warn=True)
|
|
212
|
+
return StepResult(
|
|
213
|
+
step_index=step.index,
|
|
214
|
+
step_id=step.id,
|
|
215
|
+
scope=step.scope.value,
|
|
216
|
+
status="success" if cmd_result.ok else "failed",
|
|
217
|
+
output=cmd_result.stdout,
|
|
218
|
+
error=cmd_result.stderr if not cmd_result.ok else "",
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
def _execute_ssh_upload(self, step: Step) -> StepResult:
|
|
222
|
+
if self._session is None:
|
|
223
|
+
raise RuntimeError("No SSH session available")
|
|
224
|
+
if step.file_content and step.target_path:
|
|
225
|
+
# Expand ~ on the remote side so paths like ~/.goss/... resolve
|
|
226
|
+
# correctly for the connecting user (not necessarily root).
|
|
227
|
+
target = step.target_path
|
|
228
|
+
if target.startswith("~/") or target == "~":
|
|
229
|
+
expand_result = self._session.run("echo $HOME", sudo=False, warn=True)
|
|
230
|
+
if expand_result.ok:
|
|
231
|
+
home = expand_result.stdout.strip()
|
|
232
|
+
target = home + target[1:] # replace leading ~ with $HOME
|
|
233
|
+
|
|
234
|
+
# For goss files: ensure the parent directory exists first
|
|
235
|
+
if "/.goss/" in target:
|
|
236
|
+
parent = target.rsplit("/", 1)[0]
|
|
237
|
+
self._session.run(f"mkdir -p {parent}", sudo=False, warn=True)
|
|
238
|
+
|
|
239
|
+
upload_result = self._session.upload_content(step.file_content, target, sudo=step.sudo)
|
|
240
|
+
if not upload_result.ok:
|
|
241
|
+
return StepResult(
|
|
242
|
+
step_index=step.index,
|
|
243
|
+
step_id=step.id,
|
|
244
|
+
scope=step.scope.value,
|
|
245
|
+
status="failed",
|
|
246
|
+
error=f"Upload failed: {upload_result.stderr}",
|
|
247
|
+
)
|
|
248
|
+
return StepResult(
|
|
249
|
+
step_index=step.index,
|
|
250
|
+
step_id=step.id,
|
|
251
|
+
scope=step.scope.value,
|
|
252
|
+
status="success",
|
|
253
|
+
output=f"Uploaded to {step.target_path}",
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
def _execute_gate(self, step: Step) -> StepResult:
|
|
257
|
+
"""Execute a gate step — typically an SSH login verification.
|
|
258
|
+
|
|
259
|
+
When ``effective_port`` is set (credential fallback activated because
|
|
260
|
+
the server was already bootstrapped), all ``ssh_check:`` gate commands
|
|
261
|
+
have their port replaced with the effective port. This prevents the
|
|
262
|
+
gate from trying to connect to the original login port (e.g. 22) that
|
|
263
|
+
may already be firewalled off.
|
|
264
|
+
"""
|
|
265
|
+
from loft_cli.checks.ssh import check_ssh_reachable
|
|
266
|
+
|
|
267
|
+
# Tunnel SSH gate: bring up the WireGuard tunnel locally, verify SSH
|
|
268
|
+
# through the VPN IP, and tear down the tunnel on failure.
|
|
269
|
+
if step.command and step.command.startswith("tunnel_ssh_gate:"):
|
|
270
|
+
from loft_cli.local.tunnel import tunnel_down, tunnel_up
|
|
271
|
+
from loft_cli.local.wireguard_store import save_wireguard_state
|
|
272
|
+
|
|
273
|
+
_, host_name, vpn_ip, port_str, user = step.command.split(":", 4)
|
|
274
|
+
|
|
275
|
+
# Save WireGuard state first so tunnel_up can find client.conf
|
|
276
|
+
if self._spec and self._ctx:
|
|
277
|
+
spec = self._spec
|
|
278
|
+
ctx = self._ctx
|
|
279
|
+
save_wireguard_state(
|
|
280
|
+
host_name=spec.host.name,
|
|
281
|
+
spec_name=spec.meta.name,
|
|
282
|
+
private_key=ctx.wireguard_private_key,
|
|
283
|
+
public_key=ctx.wireguard_public_key,
|
|
284
|
+
wg_conf_content=ctx.wireguard_conf_content,
|
|
285
|
+
client_private_key=ctx.wg_client_private_key,
|
|
286
|
+
client_public_key=ctx.wg_client_public_key,
|
|
287
|
+
client_conf_content=ctx.wg_client_conf_content,
|
|
288
|
+
interface=spec.wireguard.interface,
|
|
289
|
+
address=spec.wireguard.address,
|
|
290
|
+
endpoint=spec.wireguard.endpoint,
|
|
291
|
+
peer_address=spec.wireguard.peer_address,
|
|
292
|
+
persistent_keepalive=spec.wireguard.persistent_keepalive,
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
# Bring up the tunnel
|
|
296
|
+
up_ok, up_msg = tunnel_up(host_name)
|
|
297
|
+
if not up_ok:
|
|
298
|
+
return StepResult(
|
|
299
|
+
step_index=step.index,
|
|
300
|
+
step_id=step.id,
|
|
301
|
+
scope=step.scope.value,
|
|
302
|
+
status="failed",
|
|
303
|
+
error=f"Failed to bring up WireGuard tunnel: {up_msg}",
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
# Verify SSH through the tunnel
|
|
307
|
+
key_path = None
|
|
308
|
+
if self._ctx and self._ctx.admin_key_path:
|
|
309
|
+
key_path = str(self._ctx.admin_key_path)
|
|
310
|
+
check = check_ssh_reachable(
|
|
311
|
+
vpn_ip, int(port_str), user, key_path=key_path, retries=5, retry_delay=2.0
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
if check.passed:
|
|
315
|
+
# Tear down the tunnel after verification — the user can bring
|
|
316
|
+
# it up again with `loft-cli tunnel up <host>`
|
|
317
|
+
tunnel_down(host_name)
|
|
318
|
+
return StepResult(
|
|
319
|
+
step_index=step.index,
|
|
320
|
+
step_id=step.id,
|
|
321
|
+
scope=step.scope.value,
|
|
322
|
+
status="success",
|
|
323
|
+
output=(
|
|
324
|
+
f"SSH through WireGuard tunnel verified: " f"{user}@{vpn_ip}:{port_str}"
|
|
325
|
+
),
|
|
326
|
+
)
|
|
327
|
+
else:
|
|
328
|
+
# Gate failed — tear down the broken tunnel
|
|
329
|
+
tunnel_down(host_name)
|
|
330
|
+
return StepResult(
|
|
331
|
+
step_index=step.index,
|
|
332
|
+
step_id=step.id,
|
|
333
|
+
scope=step.scope.value,
|
|
334
|
+
status="failed",
|
|
335
|
+
error=(
|
|
336
|
+
f"SSH through WireGuard tunnel failed: {check.message}. "
|
|
337
|
+
f"The open SSH rule will NOT be deleted — server remains "
|
|
338
|
+
f"accessible via public IP."
|
|
339
|
+
),
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
if step.command and step.command.startswith("ssh_check:"):
|
|
343
|
+
_, host, port_str, user = step.command.split(":", 3)
|
|
344
|
+
# When credential fallback is active, override the compiled port
|
|
345
|
+
# so gates don't try to connect to the now-firewalled original port.
|
|
346
|
+
if self._effective_port is not None:
|
|
347
|
+
port_str = str(self._effective_port)
|
|
348
|
+
key_path = None
|
|
349
|
+
password = None
|
|
350
|
+
if self._ctx:
|
|
351
|
+
# Use admin key (no password) when the gate target is the admin user.
|
|
352
|
+
# Use root login credentials (key or password) for all other gates.
|
|
353
|
+
admin_name = (
|
|
354
|
+
self._spec.admin_user.name
|
|
355
|
+
if self._spec and hasattr(self._spec, "admin_user")
|
|
356
|
+
else None
|
|
357
|
+
)
|
|
358
|
+
if user == admin_name and self._ctx.admin_key_path:
|
|
359
|
+
key_path = str(self._ctx.admin_key_path)
|
|
360
|
+
else:
|
|
361
|
+
key_path = str(self._ctx.login_key_path) if self._ctx.login_key_path else None
|
|
362
|
+
password = self._ctx.login_password
|
|
363
|
+
check = check_ssh_reachable(
|
|
364
|
+
host, int(port_str), user, key_path=key_path, password=password
|
|
365
|
+
)
|
|
366
|
+
return StepResult(
|
|
367
|
+
step_index=step.index,
|
|
368
|
+
step_id=step.id,
|
|
369
|
+
scope=step.scope.value,
|
|
370
|
+
status="success" if check.passed else "failed",
|
|
371
|
+
output=check.message,
|
|
372
|
+
error="" if check.passed else check.message,
|
|
373
|
+
)
|
|
374
|
+
if step.command and step.command.startswith("http_check:"):
|
|
375
|
+
# The URL may contain colons (e.g. http://host:port/), so parse
|
|
376
|
+
# from the right: the trailing 4 fields are always integers.
|
|
377
|
+
rest = step.command[len("http_check:") :]
|
|
378
|
+
parts = rest.rsplit(":", 4)
|
|
379
|
+
if len(parts) == 5:
|
|
380
|
+
url, status_str, retries_str, interval_str, timeout_str = parts
|
|
381
|
+
# Translate to a bash retry loop executed via SSH on the target.
|
|
382
|
+
curl_cmd = (
|
|
383
|
+
f"for i in $(seq 1 {retries_str}); do "
|
|
384
|
+
f"code=$(curl -s -o /dev/null -w '%{{http_code}}' -m {timeout_str} '{url}' 2>/dev/null); "
|
|
385
|
+
f'[ "$code" = "{status_str}" ] && '
|
|
386
|
+
f'echo "HTTP {url}: $code (attempt $i/{retries_str})" && exit 0; '
|
|
387
|
+
f"[ $i -lt {retries_str} ] && sleep {interval_str}; "
|
|
388
|
+
f"done; "
|
|
389
|
+
f'echo "HTTP check failed after {retries_str} attempts: last code=$code"; exit 1'
|
|
390
|
+
)
|
|
391
|
+
step = step.model_copy(update={"command": curl_cmd})
|
|
392
|
+
return self._execute_ssh_command(step)
|
|
393
|
+
|
|
394
|
+
# Default verify: run the command
|
|
395
|
+
return self._execute_ssh_command(step)
|
|
396
|
+
|
|
397
|
+
def _execute_compose_health_check(self, step: Step) -> StepResult:
|
|
398
|
+
"""Execute a compose health check step — polls container health."""
|
|
399
|
+
from loft_cli.checks.compose import check_compose_health
|
|
400
|
+
|
|
401
|
+
if self._session is None:
|
|
402
|
+
return StepResult(
|
|
403
|
+
step_index=step.index,
|
|
404
|
+
step_id=step.id,
|
|
405
|
+
scope=step.scope.value,
|
|
406
|
+
status="success",
|
|
407
|
+
output="[dry-run or no session — compose health check skipped]",
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
# Parse command: compose_health:<dir>:<compose_file>:<project>:<timeout>:<interval>
|
|
411
|
+
if step.command and step.command.startswith("compose_health:"):
|
|
412
|
+
parts = step.command.split(":", 5)
|
|
413
|
+
if len(parts) == 6:
|
|
414
|
+
_, directory, compose_file, project_name, timeout_s, interval_s = parts
|
|
415
|
+
check = check_compose_health(
|
|
416
|
+
session=self._session,
|
|
417
|
+
directory=directory,
|
|
418
|
+
compose_file=compose_file,
|
|
419
|
+
project_name=project_name,
|
|
420
|
+
timeout=int(timeout_s),
|
|
421
|
+
interval=int(interval_s),
|
|
422
|
+
)
|
|
423
|
+
return StepResult(
|
|
424
|
+
step_index=step.index,
|
|
425
|
+
step_id=step.id,
|
|
426
|
+
scope=step.scope.value,
|
|
427
|
+
status="success" if check.passed else "failed",
|
|
428
|
+
output=check.message,
|
|
429
|
+
error="" if check.passed else check.message,
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
return StepResult(
|
|
433
|
+
step_index=step.index,
|
|
434
|
+
step_id=step.id,
|
|
435
|
+
scope=step.scope.value,
|
|
436
|
+
status="failed",
|
|
437
|
+
error=f"Invalid compose_health_check command: {step.command}",
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
def _execute_verify(self, step: Step) -> StepResult:
|
|
441
|
+
"""Execute a verify step (non-gate)."""
|
|
442
|
+
# ── goss: no spec was generated (generator failed) ──────────────
|
|
443
|
+
if step.command == "goss_unavailable":
|
|
444
|
+
self._console.print(
|
|
445
|
+
"\n[bold yellow]⚠ No goss spec is available for this run.[/bold yellow]\n"
|
|
446
|
+
" Server state will NOT be automatically verified.\n"
|
|
447
|
+
" Ensure loft_cli.goss.generator can import and run cleanly.\n"
|
|
448
|
+
)
|
|
449
|
+
return StepResult(
|
|
450
|
+
step_index=step.index,
|
|
451
|
+
step_id=step.id,
|
|
452
|
+
scope=step.scope.value,
|
|
453
|
+
status="success", # warning, not a hard failure
|
|
454
|
+
output="[WARNING] goss spec unavailable — server state not verified",
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
# ── goss: ship succeeded, now run validate ───────────────────────
|
|
458
|
+
if step.command == "goss_validate":
|
|
459
|
+
return self._execute_goss_validate(step)
|
|
460
|
+
|
|
461
|
+
if step.command and step.command.startswith("check:"):
|
|
462
|
+
return self._execute_postflight_check(step)
|
|
463
|
+
if step.command and not step.command.startswith(("echo ", "ssh_check:")):
|
|
464
|
+
return self._execute_ssh_command(step)
|
|
465
|
+
return StepResult(
|
|
466
|
+
step_index=step.index,
|
|
467
|
+
step_id=step.id,
|
|
468
|
+
scope=step.scope.value,
|
|
469
|
+
status="success",
|
|
470
|
+
output="ok",
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
def _execute_postflight_check(self, step: Step) -> StepResult:
|
|
474
|
+
"""Dispatch a postflight check encoded as check:{type}:{params...}."""
|
|
475
|
+
parts = step.command.split(":", maxsplit=2) if step.command else []
|
|
476
|
+
check_type = parts[1] if len(parts) > 1 else ""
|
|
477
|
+
params = parts[2] if len(parts) > 2 else ""
|
|
478
|
+
|
|
479
|
+
try:
|
|
480
|
+
if check_type == "ssh_reachable":
|
|
481
|
+
from loft_cli.checks.ssh import check_ssh_reachable
|
|
482
|
+
|
|
483
|
+
host, port_str, user = params.split(":", 2)
|
|
484
|
+
key_path = None
|
|
485
|
+
if self._ctx and self._ctx.admin_key_path:
|
|
486
|
+
key_path = str(self._ctx.admin_key_path)
|
|
487
|
+
check = check_ssh_reachable(host, int(port_str), user, key_path=key_path)
|
|
488
|
+
|
|
489
|
+
elif check_type == "port_open":
|
|
490
|
+
from loft_cli.checks.ports import check_port_open
|
|
491
|
+
|
|
492
|
+
host, port_str = params.split(":", 1)
|
|
493
|
+
check = check_port_open(host, int(port_str))
|
|
494
|
+
|
|
495
|
+
elif check_type == "wireguard_up":
|
|
496
|
+
from loft_cli.checks.wireguard import check_wireguard_up
|
|
497
|
+
|
|
498
|
+
if self._session is None:
|
|
499
|
+
raise RuntimeError("No SSH session for wireguard check")
|
|
500
|
+
check = check_wireguard_up(self._session, params)
|
|
501
|
+
|
|
502
|
+
elif check_type == "container_running":
|
|
503
|
+
from loft_cli.checks.container import check_container_running
|
|
504
|
+
|
|
505
|
+
if self._session is None:
|
|
506
|
+
raise RuntimeError("No SSH session for container check")
|
|
507
|
+
check = check_container_running(self._session, params)
|
|
508
|
+
|
|
509
|
+
elif check_type == "http":
|
|
510
|
+
from loft_cli.checks.http import check_http
|
|
511
|
+
|
|
512
|
+
url, status_str = params.rsplit(":", 1)
|
|
513
|
+
check = check_http(url, expect_status=int(status_str))
|
|
514
|
+
|
|
515
|
+
elif check_type == "postgres_ready":
|
|
516
|
+
from loft_cli.checks.postgres import check_postgres_ready
|
|
517
|
+
|
|
518
|
+
if self._session is None:
|
|
519
|
+
raise RuntimeError("No SSH session for postgres check")
|
|
520
|
+
check = check_postgres_ready(self._session)
|
|
521
|
+
|
|
522
|
+
elif check_type == "nginx_ready":
|
|
523
|
+
from loft_cli.checks.nginx import check_nginx_ready
|
|
524
|
+
|
|
525
|
+
if self._session is None:
|
|
526
|
+
raise RuntimeError("No SSH session for nginx check")
|
|
527
|
+
check = check_nginx_ready(self._session)
|
|
528
|
+
|
|
529
|
+
else:
|
|
530
|
+
return StepResult(
|
|
531
|
+
step_index=step.index,
|
|
532
|
+
step_id=step.id,
|
|
533
|
+
scope=step.scope.value,
|
|
534
|
+
status="failed",
|
|
535
|
+
error=f"Unknown check type: {check_type}",
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
return StepResult(
|
|
539
|
+
step_index=step.index,
|
|
540
|
+
step_id=step.id,
|
|
541
|
+
scope=step.scope.value,
|
|
542
|
+
status="success" if check.passed else "failed",
|
|
543
|
+
output=check.message,
|
|
544
|
+
error="" if check.passed else check.message,
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
except Exception as exc:
|
|
548
|
+
return StepResult(
|
|
549
|
+
step_index=step.index,
|
|
550
|
+
step_id=step.id,
|
|
551
|
+
scope=step.scope.value,
|
|
552
|
+
status="failed",
|
|
553
|
+
error=f"Postflight check error: {exc}",
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
def _execute_goss_validate(self, step: Step) -> StepResult:
|
|
557
|
+
"""Install goss on the remote, update the master gossfile, run validate."""
|
|
558
|
+
from loft_cli.addons.goss.renderer import render_goss_results
|
|
559
|
+
from loft_cli.addons.goss.shipper import ship_and_run
|
|
560
|
+
|
|
561
|
+
if self._session is None:
|
|
562
|
+
return StepResult(
|
|
563
|
+
step_index=step.index,
|
|
564
|
+
step_id=step.id,
|
|
565
|
+
scope=step.scope.value,
|
|
566
|
+
status="success",
|
|
567
|
+
output="[dry-run or no session — goss validate skipped]",
|
|
568
|
+
)
|
|
569
|
+
|
|
570
|
+
spec_name = self._plan.spec_name
|
|
571
|
+
admin_user = self._spec.admin_user.name if self._spec else "admin"
|
|
572
|
+
|
|
573
|
+
# Retrieve the goss file content that was embedded in the ship step
|
|
574
|
+
goss_content: str | None = None
|
|
575
|
+
for s in self._plan.steps:
|
|
576
|
+
if s.id == "ship_goss_file":
|
|
577
|
+
goss_content = s.file_content
|
|
578
|
+
break
|
|
579
|
+
|
|
580
|
+
if not goss_content:
|
|
581
|
+
self._console.print(
|
|
582
|
+
"[bold yellow]⚠ goss_validate: could not find ship_goss_file content "
|
|
583
|
+
"in plan — skipping.[/bold yellow]"
|
|
584
|
+
)
|
|
585
|
+
return StepResult(
|
|
586
|
+
step_index=step.index,
|
|
587
|
+
step_id=step.id,
|
|
588
|
+
scope=step.scope.value,
|
|
589
|
+
status="success",
|
|
590
|
+
output="[WARNING] goss content missing in plan — validate skipped",
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
goss_result = ship_and_run(
|
|
594
|
+
session=self._session,
|
|
595
|
+
spec_name=spec_name,
|
|
596
|
+
goss_yaml_content=goss_content,
|
|
597
|
+
admin_user=admin_user,
|
|
598
|
+
)
|
|
599
|
+
|
|
600
|
+
# Always render — even on error the renderer shows a clear message
|
|
601
|
+
self._console.print()
|
|
602
|
+
render_goss_results(goss_result, console=self._console)
|
|
603
|
+
|
|
604
|
+
# A goss failure is "success_with_warnings" territory — the server WAS
|
|
605
|
+
# configured; goss surfaces discrepancies for the operator to review.
|
|
606
|
+
# We mark the step failed so the final apply status reflects reality,
|
|
607
|
+
# but we do NOT abort the plan (step.gate is False).
|
|
608
|
+
if goss_result.get("error"):
|
|
609
|
+
return StepResult(
|
|
610
|
+
step_index=step.index,
|
|
611
|
+
step_id=step.id,
|
|
612
|
+
scope=step.scope.value,
|
|
613
|
+
status="failed",
|
|
614
|
+
error=goss_result["error"],
|
|
615
|
+
output=goss_result.get("raw_output", ""),
|
|
616
|
+
)
|
|
617
|
+
|
|
618
|
+
status = "success" if goss_result["exit_ok"] else "failed"
|
|
619
|
+
summary = goss_result.get("summary", {})
|
|
620
|
+
output = (
|
|
621
|
+
f"goss: {summary.get('test-count', '?')} checks, "
|
|
622
|
+
f"{summary.get('success-count', '?')} passed, "
|
|
623
|
+
f"{summary.get('failed-count', '?')} failed"
|
|
624
|
+
)
|
|
625
|
+
return StepResult(
|
|
626
|
+
step_index=step.index,
|
|
627
|
+
step_id=step.id,
|
|
628
|
+
scope=step.scope.value,
|
|
629
|
+
status=status,
|
|
630
|
+
output=output,
|
|
631
|
+
error=(
|
|
632
|
+
""
|
|
633
|
+
if goss_result["exit_ok"]
|
|
634
|
+
else f"{summary.get('failed-count', '?')} check(s) failed"
|
|
635
|
+
),
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
def _execute_local_file_write(self, step: Step) -> StepResult:
|
|
639
|
+
from loft_cli.local.ssh_config import write_ssh_conf_d
|
|
640
|
+
|
|
641
|
+
if step.id == "write_local_ssh_conf_d" and self._spec and self._ctx:
|
|
642
|
+
spec = self._spec
|
|
643
|
+
|
|
644
|
+
# When WireGuard is enabled, SSH is locked to the VPN interface.
|
|
645
|
+
# Use the server's WireGuard VPN IP instead of the public address
|
|
646
|
+
# so that `ssh {host}` connects through the tunnel.
|
|
647
|
+
address = spec.host.address
|
|
648
|
+
tunnel_comment = None
|
|
649
|
+
if hasattr(spec, "wireguard") and spec.wireguard.enabled and spec.wireguard.address:
|
|
650
|
+
import ipaddress as _ip_ssh
|
|
651
|
+
|
|
652
|
+
address = str(_ip_ssh.ip_interface(spec.wireguard.address).ip)
|
|
653
|
+
tunnel_comment = f"# Requires: loft-cli tunnel up {spec.host.name}"
|
|
654
|
+
|
|
655
|
+
conf_file = write_ssh_conf_d(
|
|
656
|
+
host_name=spec.host.name,
|
|
657
|
+
address=address,
|
|
658
|
+
user=spec.admin_user.name,
|
|
659
|
+
port=spec.ssh.port,
|
|
660
|
+
identity_file=spec.login.private_key,
|
|
661
|
+
tunnel_comment=tunnel_comment,
|
|
662
|
+
)
|
|
663
|
+
return StepResult(
|
|
664
|
+
step_index=step.index,
|
|
665
|
+
step_id=step.id,
|
|
666
|
+
scope=step.scope.value,
|
|
667
|
+
status="success",
|
|
668
|
+
output=f"Written: {conf_file}",
|
|
669
|
+
)
|
|
670
|
+
return StepResult(
|
|
671
|
+
step_index=step.index,
|
|
672
|
+
step_id=step.id,
|
|
673
|
+
scope=step.scope.value,
|
|
674
|
+
status="success",
|
|
675
|
+
output="[local file write ok]",
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
def _execute_local_command(self, step: Step) -> StepResult:
|
|
679
|
+
from pathlib import Path
|
|
680
|
+
|
|
681
|
+
from loft_cli.local.ssh_config import backup_ssh_config, ensure_include
|
|
682
|
+
|
|
683
|
+
if step.command == "backup_ssh_config" and self._spec:
|
|
684
|
+
config_path = Path(self._spec.local.ssh_config.config_path).expanduser()
|
|
685
|
+
backup = backup_ssh_config(config_path)
|
|
686
|
+
return StepResult(
|
|
687
|
+
step_index=step.index,
|
|
688
|
+
step_id=step.id,
|
|
689
|
+
scope=step.scope.value,
|
|
690
|
+
status="success",
|
|
691
|
+
output=f"Backup: {backup}" if backup else "No existing config to backup",
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
elif step.command == "ensure_include" and self._spec:
|
|
695
|
+
config_path = Path(self._spec.local.ssh_config.config_path).expanduser()
|
|
696
|
+
ensure_include(config_path)
|
|
697
|
+
return StepResult(
|
|
698
|
+
step_index=step.index,
|
|
699
|
+
step_id=step.id,
|
|
700
|
+
scope=step.scope.value,
|
|
701
|
+
status="success",
|
|
702
|
+
output="Include directive ensured",
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
elif step.command == "save_wireguard_state" and self._spec and self._ctx:
|
|
706
|
+
from loft_cli.local.wireguard_store import save_wireguard_state
|
|
707
|
+
|
|
708
|
+
spec = self._spec
|
|
709
|
+
ctx = self._ctx
|
|
710
|
+
host_dir = save_wireguard_state(
|
|
711
|
+
host_name=spec.host.name,
|
|
712
|
+
spec_name=spec.meta.name,
|
|
713
|
+
private_key=ctx.wireguard_private_key,
|
|
714
|
+
public_key=ctx.wireguard_public_key,
|
|
715
|
+
wg_conf_content=ctx.wireguard_conf_content,
|
|
716
|
+
client_private_key=ctx.wg_client_private_key,
|
|
717
|
+
client_public_key=ctx.wg_client_public_key,
|
|
718
|
+
client_conf_content=ctx.wg_client_conf_content,
|
|
719
|
+
interface=spec.wireguard.interface,
|
|
720
|
+
address=spec.wireguard.address,
|
|
721
|
+
endpoint=spec.wireguard.endpoint,
|
|
722
|
+
peer_address=spec.wireguard.peer_address,
|
|
723
|
+
persistent_keepalive=spec.wireguard.persistent_keepalive,
|
|
724
|
+
)
|
|
725
|
+
return StepResult(
|
|
726
|
+
step_index=step.index,
|
|
727
|
+
step_id=step.id,
|
|
728
|
+
scope=step.scope.value,
|
|
729
|
+
status="success",
|
|
730
|
+
output=f"WireGuard state saved to: {host_dir}",
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
return StepResult(
|
|
734
|
+
step_index=step.index,
|
|
735
|
+
step_id=step.id,
|
|
736
|
+
scope=step.scope.value,
|
|
737
|
+
status="success",
|
|
738
|
+
output=f"[local command: {step.command}]",
|
|
739
|
+
)
|
|
740
|
+
|
|
741
|
+
def _execute_local_db_write(self, step: Step) -> StepResult:
|
|
742
|
+
if self._db is None:
|
|
743
|
+
return StepResult(
|
|
744
|
+
step_index=step.index,
|
|
745
|
+
step_id=step.id,
|
|
746
|
+
scope=step.scope.value,
|
|
747
|
+
status="success",
|
|
748
|
+
output="[inventory disabled — skipped]",
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
if step.command == "init_inventory":
|
|
752
|
+
self._db.open()
|
|
753
|
+
self._db.initialize()
|
|
754
|
+
return StepResult(
|
|
755
|
+
step_index=step.index,
|
|
756
|
+
step_id=step.id,
|
|
757
|
+
scope=step.scope.value,
|
|
758
|
+
status="success",
|
|
759
|
+
output="Inventory database initialized",
|
|
760
|
+
)
|
|
761
|
+
elif step.command == "upsert_server" and self._spec and self._ctx:
|
|
762
|
+
# Will be called after apply completes with full result
|
|
763
|
+
return StepResult(
|
|
764
|
+
step_index=step.index,
|
|
765
|
+
step_id=step.id,
|
|
766
|
+
scope=step.scope.value,
|
|
767
|
+
status="success",
|
|
768
|
+
output="Server record queued for upsert",
|
|
769
|
+
)
|
|
770
|
+
elif step.command == "record_run":
|
|
771
|
+
return StepResult(
|
|
772
|
+
step_index=step.index,
|
|
773
|
+
step_id=step.id,
|
|
774
|
+
scope=step.scope.value,
|
|
775
|
+
status="success",
|
|
776
|
+
output="Run metadata queued for recording",
|
|
777
|
+
)
|
|
778
|
+
|
|
779
|
+
return StepResult(
|
|
780
|
+
step_index=step.index,
|
|
781
|
+
step_id=step.id,
|
|
782
|
+
scope=step.scope.value,
|
|
783
|
+
status="success",
|
|
784
|
+
output=f"[db write: {step.command}]",
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
def _print_step(self, step: Step, result: StepResult) -> None:
|
|
788
|
+
icons = {"success": "✓", "failed": "✗", "skipped": "○"}
|
|
789
|
+
colors = {"success": "green", "failed": "red", "skipped": "dim"}
|
|
790
|
+
icon = icons.get(result.status, "?")
|
|
791
|
+
color = colors.get(result.status, "white")
|
|
792
|
+
duration = f"{result.duration_seconds:.1f}s" if result.duration_seconds else ""
|
|
793
|
+
self._console.print(
|
|
794
|
+
f" [{color}]{icon}[/{color}] [{step.index:>2}] {step.description[:60]}"
|
|
795
|
+
+ (f" [{duration}]" if duration else ""),
|
|
796
|
+
)
|
|
797
|
+
if result.status == "failed" and result.error:
|
|
798
|
+
self._console.print(f" [red]{result.error[:120]}[/red]")
|