mayhem-cli 0.5.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. mayhem/agent/__init__.py +1 -0
  2. mayhem/agent/cli.py +36 -0
  3. mayhem/agents/__init__.py +1 -0
  4. mayhem/agents/capabilities.py +106 -0
  5. mayhem/agents/executors.py +430 -0
  6. mayhem/agents/impact.py +729 -0
  7. mayhem/agents/lease_client.py +141 -0
  8. mayhem/agents/probes.py +284 -0
  9. mayhem/agents/protocol.py +134 -0
  10. mayhem/agents/server.py +281 -0
  11. mayhem/agents/sinks.py +60 -0
  12. mayhem/agents/transports.py +134 -0
  13. mayhem/agents/watchdog.py +140 -0
  14. mayhem/cli/__init__.py +11 -0
  15. mayhem/cli/app.py +154 -0
  16. mayhem/cli/campaign.py +496 -0
  17. mayhem/cli/config_cmd.py +47 -0
  18. mayhem/cli/context.py +23 -0
  19. mayhem/cli/dependency.py +429 -0
  20. mayhem/cli/exit_codes.py +24 -0
  21. mayhem/cli/experiment.py +24 -0
  22. mayhem/cli/lifecycle.py +805 -0
  23. mayhem/cli/resolver.py +72 -0
  24. mayhem/cli/services.py +459 -0
  25. mayhem/cli/style.py +101 -0
  26. mayhem/cli/toolkit.py +41 -0
  27. mayhem/cli/topology.py +127 -0
  28. mayhem/config.py +208 -0
  29. mayhem/controller/__init__.py +1 -0
  30. mayhem/controller/compensation.py +2156 -0
  31. mayhem/controller/executor.py +1719 -0
  32. mayhem/controller/janitor.py +196 -0
  33. mayhem/controller/observability_collector.py +382 -0
  34. mayhem/controller/observations.py +102 -0
  35. mayhem/controller/planner.py +715 -0
  36. mayhem/controller/recovery.py +245 -0
  37. mayhem/controller/resilience_report.py +585 -0
  38. mayhem/controller/resource_manager.py +457 -0
  39. mayhem/controller/safety.py +392 -0
  40. mayhem/domain/__init__.py +6 -0
  41. mayhem/domain/campaigns.py +118 -0
  42. mayhem/domain/cancellation.py +110 -0
  43. mayhem/domain/candidates.py +101 -0
  44. mayhem/domain/capabilities.py +86 -0
  45. mayhem/domain/catalog.py +727 -0
  46. mayhem/domain/checks.py +173 -0
  47. mayhem/domain/common.py +104 -0
  48. mayhem/domain/coverage.py +106 -0
  49. mayhem/domain/decisions.py +57 -0
  50. mayhem/domain/errors.py +87 -0
  51. mayhem/domain/events.py +61 -0
  52. mayhem/domain/execution_context.py +120 -0
  53. mayhem/domain/execution_loci.py +94 -0
  54. mayhem/domain/experiments.py +370 -0
  55. mayhem/domain/faults.py +239 -0
  56. mayhem/domain/identity.py +200 -0
  57. mayhem/domain/k8s_adapter.py +132 -0
  58. mayhem/domain/leases.py +186 -0
  59. mayhem/domain/load_strategy.py +98 -0
  60. mayhem/domain/m5_campaign.py +120 -0
  61. mayhem/domain/maniac.py +93 -0
  62. mayhem/domain/observability.py +146 -0
  63. mayhem/domain/outcomes.py +92 -0
  64. mayhem/domain/remote_agent_interface.py +70 -0
  65. mayhem/domain/resources.py +245 -0
  66. mayhem/domain/risks.py +61 -0
  67. mayhem/domain/run_outcome.py +146 -0
  68. mayhem/domain/runtime_adapter.py +256 -0
  69. mayhem/domain/success.py +329 -0
  70. mayhem/domain/topology.py +452 -0
  71. mayhem/infra/__init__.py +1 -0
  72. mayhem/infra/campaign_engine.py +205 -0
  73. mayhem/infra/candidate_gates.py +124 -0
  74. mayhem/infra/candidate_generator.py +110 -0
  75. mayhem/infra/coverage_repository.py +101 -0
  76. mayhem/infra/lease_repository.py +129 -0
  77. mayhem/infra/maniac.py +103 -0
  78. mayhem/infra/migrations.py +596 -0
  79. mayhem/infra/migrator.py +149 -0
  80. mayhem/infra/report.py +227 -0
  81. mayhem/infra/store.py +200 -0
  82. mayhem/py.typed +0 -0
  83. mayhem/spec.py +52 -0
  84. mayhem/toolkit/__init__.py +1 -0
  85. mayhem/toolkit/fingerprint.py +69 -0
  86. mayhem/toolkit/hashing.py +32 -0
  87. mayhem/toolkit/manifests/docker.yaml +11 -0
  88. mayhem/toolkit/manifests/podman.yaml +11 -0
  89. mayhem/toolkit/manifests/stress-ng.yaml +11 -0
  90. mayhem/toolkit/manifests/tc-netem.yaml +11 -0
  91. mayhem/toolkit/manifests/toxiproxy.yaml +10 -0
  92. mayhem/toolkit/registry.py +185 -0
  93. mayhem/toolkit/tool_runner.py +129 -0
  94. mayhem/topology/__init__.py +10 -0
  95. mayhem/topology/providers/__init__.py +0 -0
  96. mayhem/topology/providers/adapter_registry.py +60 -0
  97. mayhem/topology/providers/base.py +31 -0
  98. mayhem/topology/providers/compose.py +207 -0
  99. mayhem/topology/providers/docker_adapter.py +277 -0
  100. mayhem/topology/providers/docker_runtime.py +461 -0
  101. mayhem/topology/providers/podman_adapter.py +328 -0
  102. mayhem/topology/resolve.py +196 -0
  103. mayhem/topology/service.py +158 -0
  104. mayhem_cli-0.5.1.dist-info/METADATA +555 -0
  105. mayhem_cli-0.5.1.dist-info/RECORD +107 -0
  106. mayhem_cli-0.5.1.dist-info/WHEEL +4 -0
  107. mayhem_cli-0.5.1.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,715 @@
1
+ """Planner — compiles drill specs into frozen ExecutionPlans.
2
+
3
+ Drill plans resolve every container name against the topology snapshot and
4
+ attach a compensation contract to every fault (write-ahead undo). Wait and
5
+ check blocks compile to non-fault steps; parallel blocks emit one step per
6
+ container so the executor can run them concurrently (Phase 5, ADR-0019/0021).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import uuid
12
+ from pathlib import Path
13
+ from typing import TYPE_CHECKING, cast
14
+
15
+ from mayhem.controller.compensation import compensated
16
+ from mayhem.domain.capabilities import Capability
17
+ from mayhem.domain.catalog import all_definitions, definition_for
18
+ from mayhem.domain.common import parse_duration
19
+ from mayhem.domain.decisions import (
20
+ DECISION_M4_1_ADDITIVE_DSL,
21
+ DECISION_M4_3_SUCCESS_CRITERIA,
22
+ DECISION_M4_4_OBSERVABILITY,
23
+ DECISION_M4_5_SCHEMA_FREEZE,
24
+ DECISION_M5_1_MANIAC,
25
+ DecisionRef,
26
+ )
27
+ from mayhem.domain.errors import (
28
+ InvariantViolationError,
29
+ SchemaValidationError,
30
+ )
31
+ from mayhem.domain.experiments import (
32
+ CheckHttp,
33
+ CheckSpecStep,
34
+ DrillContainer,
35
+ DrillFault,
36
+ DrillSpec,
37
+ ExecutionPlan,
38
+ ExecutionStep,
39
+ ExperimentKind,
40
+ GroupMode,
41
+ InjectFault,
42
+ ManiacCfg,
43
+ OnFailure,
44
+ PlannedFault,
45
+ PlannedStep,
46
+ ResolvedTarget,
47
+ Wait,
48
+ )
49
+ from mayhem.domain.faults import FaultDefinition, ParamType
50
+ from mayhem.domain.identity import RuntimeIdentity
51
+ from mayhem.domain.maniac import draw_maniac_rounds
52
+ from mayhem.domain.topology import (
53
+ PortBinding,
54
+ TargetSelector,
55
+ TopologyNode,
56
+ )
57
+
58
+ if TYPE_CHECKING:
59
+ from mayhem.domain.topology import TopologyGraph
60
+
61
+
62
+ class PlanningError(Exception):
63
+ """Raised when a spec cannot be compiled into an honest plan."""
64
+
65
+
66
+ #: Capabilities only a Kubernetes runtime can provide. ``mayhem maniac``
67
+ #: synthesizes its config from a compose topology, so catalog faults gated on
68
+ #: these are never pooled (a compose graph cannot satisfy them at execution
69
+ #: time). Everything else — NET_ADMIN, PROCESS_CONTROL, engine flags, … — stays
70
+ #: in the pool and is resolved by the impact gate, which probes each container
71
+ #: and bypasses only the inert draws.
72
+ _MANIAC_EXCLUDED_CAPS = frozenset({Capability.KUBERNETES_ENGINE})
73
+
74
+
75
+ def synthesize_maniac_spec(graph: TopologyGraph, *, name: str = "maniac") -> DrillSpec:
76
+ """Derive a drill spec purely from a compose-derived topology graph.
77
+
78
+ ``mayhem maniac`` runs without an authored config file: this builder makes
79
+ the draw pool self-describing. Every container in the topology authors the
80
+ full container-addressable fault catalog, so
81
+ :func:`draw_maniac_rounds` keeps a pure random walk over (container,
82
+ fault) exactly like a hand-authored spec. The impact gate probes actual
83
+ tooling at execution time and bypasses only the draws proven inert, so
84
+ the compile never fails because an image lacks a binary.
85
+
86
+ Safety is inherited, not authored: the synthesized spec carries no policy
87
+ of its own (``config.maniac`` is left unset so the layered ``mayhem.yaml``
88
+ ``maniac:`` block — or the CLI defaults — tune the draw), and every
89
+ planned fault still passes the unchanged planner safety stack (risk
90
+ ceiling, blast radius, ``max_faults``, timeout, compensation contract).
91
+
92
+ Each pooled fault is authored with compile-ready parameters: catalog
93
+ defaults are reused verbatim, and parameters the catalog declares
94
+ mandatory are derived from the topology where an honest value exists
95
+ (``port`` from the container's own published bindings, ``host`` from the
96
+ container name, ``connections``/``delay_ms``/``rate``/``offset_ms`` at
97
+ their least-disruptive minimums). Faults with a mandatory parameter that
98
+ carries no derivable value (``net.bandwidth``'s host-side ``rate`` string,
99
+ or a portless container) are left out of that container's pool — a draw
100
+ must never name a fault it cannot compile. Faults that require a
101
+ Kubernetes-only capability are pooled for no container.
102
+ """
103
+ containers: dict[str, DrillContainer] = {}
104
+ for container_name in sorted(_container_names(graph)):
105
+ matched = _find_container_nodes(graph, container_name)
106
+ kinds = frozenset(node.kind for node in matched)
107
+ ports = tuple(
108
+ binding
109
+ for node in matched
110
+ for binding in (getattr(node, "exposed_ports", ()) or getattr(node, "ports", ()))
111
+ )
112
+ faults: list[DrillFault] = []
113
+ for definition in all_definitions():
114
+ if not definition.applicable_node_kinds & kinds:
115
+ continue
116
+ if not definition.required_caps.isdisjoint(_MANIAC_EXCLUDED_CAPS):
117
+ continue
118
+ params = _synthesized_params(definition, container_name, ports)
119
+ if params is None:
120
+ continue
121
+ # ``params`` is an undeclared pydantic extra field on DrillFault
122
+ # (extra="allow"), so construct through model_validate.
123
+ faults.append(DrillFault.model_validate({"fault": definition.id, "params": params}))
124
+ containers[container_name] = DrillContainer(faults=tuple(faults))
125
+ return DrillSpec(
126
+ kind="drill",
127
+ name=name,
128
+ hypothesis=(
129
+ f"maniac: synthesized config for {len(containers)} container(s) "
130
+ "from the compose topology"
131
+ ),
132
+ containers=containers,
133
+ # ``plan_drill`` needs one execution step; the maniac planner replaces
134
+ # the authored execution wholesale, so this placeholder never runs.
135
+ execution=(ExecutionStep(wait="1s"),),
136
+ )
137
+
138
+
139
+ def _synthesized_params(
140
+ definition: FaultDefinition,
141
+ container_name: str,
142
+ ports: tuple[PortBinding, ...],
143
+ ) -> dict[str, object] | None:
144
+ """Auto-author a compile-ready param set for a synthesized fault.
145
+
146
+ Catalog defaults win; the handful of parameters the catalog marks
147
+ mandatory are filled from topology where an honest value exists, else
148
+ ``None`` (the caller drops the fault from the pool).
149
+ """
150
+ params: dict[str, object] = {}
151
+ mandatory: list[tuple[str, ParamType]] = []
152
+ for spec in definition.params_schema:
153
+ if spec.default is not None:
154
+ params[spec.name] = spec.default
155
+ elif spec.required:
156
+ mandatory.append((spec.name, spec.type))
157
+ for name, param_type in mandatory:
158
+ if param_type is ParamType.STRING and name == "host":
159
+ params[name] = container_name
160
+ continue
161
+ if param_type is ParamType.STRING:
162
+ return None
163
+ if name == "port":
164
+ port = next(
165
+ (binding.container_port for binding in ports if binding.protocol == "tcp"),
166
+ next((binding.container_port for binding in ports), None),
167
+ )
168
+ if port is None:
169
+ return None
170
+ params[name] = port
171
+ elif name in {"connections", "delay_ms", "rate", "offset_ms"}:
172
+ params[name] = {"connections": 1, "delay_ms": 1000, "rate": 1, "offset_ms": 60000}[name]
173
+ else:
174
+ return None
175
+ return params
176
+
177
+
178
+ def _embed_load_script(
179
+ params: dict[str, object], fault_id: str, spec_dir: str | None
180
+ ) -> dict[str, object]:
181
+ """Embed a ``net.load`` ``script`` param into the frozen plan as content."""
182
+ script_ref = params.get("script")
183
+ if fault_id != "net.load" or not isinstance(script_ref, str) or not script_ref.strip():
184
+ return params
185
+ base = Path(spec_dir) if spec_dir else Path.cwd()
186
+ script_path = Path(script_ref) if Path(script_ref).is_absolute() else base / script_ref
187
+ try:
188
+ content = script_path.read_text(encoding="utf-8")
189
+ except OSError as exc:
190
+ raise PlanningError(f"fault net.load script {script_path!s} unreadable: {exc}") from None
191
+ except UnicodeDecodeError as exc:
192
+ raise PlanningError(
193
+ f"fault net.load script {script_path!s} must be UTF-8 text: {exc}"
194
+ ) from None
195
+ return {**params, "script_content": content}
196
+
197
+
198
+ def plan_drill(
199
+ run_id: str,
200
+ spec: DrillSpec,
201
+ graph: TopologyGraph,
202
+ *,
203
+ config_snapshot_id: str,
204
+ topology_snapshot_id: str,
205
+ environment_fingerprint: str,
206
+ engine: str = "podman",
207
+ spec_dir: str | None = None,
208
+ ) -> ExecutionPlan:
209
+ """Compile a :class:`DrillSpec` into an :class:`ExecutionPlan`.
210
+
211
+ Drill plans are authored around container names rather than target
212
+ selectors. Each fault on a container is resolved to its topology node and
213
+ the relevant compensation contract is attached (write-ahead undo). Wait and
214
+ check blocks compile to non-fault steps; parallel blocks emit one step per
215
+ container so the executor can run them concurrently (Phase 5). ``spec_dir``
216
+ anchors assets referenced by the spec (e.g. the ``net.load`` ``script``
217
+ parameter) so relative paths resolve against the drill file.
218
+ """
219
+ _validate_container_names(spec, graph)
220
+
221
+ steps: list[PlannedStep] = []
222
+ seq = 0
223
+ for block in spec.execution:
224
+ if block.parallel:
225
+ # All containers in a parallel block share one `seq` so the
226
+ # executor groups and runs them concurrently (Phase 5); the block
227
+ # advances by the longest container's fault chain.
228
+ emitted = 0
229
+ for container_name in block.parallel:
230
+ container = spec.containers[container_name]
231
+ planned = _plan_container_faults(
232
+ container_name,
233
+ container,
234
+ graph,
235
+ steps,
236
+ seq,
237
+ recovery_default=spec.config.recovery,
238
+ on_failure_default=spec.config.on_failure,
239
+ spec_dir=spec_dir,
240
+ )
241
+ emitted = max(emitted, planned)
242
+ seq += emitted
243
+ elif block.sequential:
244
+ for container_name in block.sequential:
245
+ container = spec.containers[container_name]
246
+ seq += _plan_container_faults(
247
+ container_name,
248
+ container,
249
+ graph,
250
+ steps,
251
+ seq,
252
+ recovery_default=spec.config.recovery,
253
+ on_failure_default=spec.config.on_failure,
254
+ spec_dir=spec_dir,
255
+ )
256
+ elif block.wait is not None:
257
+ steps.append(
258
+ PlannedStep(
259
+ id=f"wait-{seq:04d}",
260
+ seq=seq,
261
+ raw_action=Wait(type="wait", duration=block.wait),
262
+ )
263
+ )
264
+ seq += 1
265
+ elif block.check:
266
+ for i, probe in enumerate(block.check):
267
+ expected = probe.expect.status if probe.expect else None
268
+ steps.append(
269
+ PlannedStep(
270
+ id=f"check-{seq:04d}-{i}",
271
+ seq=seq,
272
+ raw_action=CheckHttp(
273
+ type="check_http",
274
+ url=probe.http or "",
275
+ expected_status=expected,
276
+ ),
277
+ )
278
+ )
279
+ seq += 1
280
+ elif block.check_spec:
281
+ for i, cspec in enumerate(block.check_spec):
282
+ steps.append(
283
+ PlannedStep(
284
+ id=f"check-{seq:04d}-{i}",
285
+ seq=seq,
286
+ raw_action=CheckSpecStep(
287
+ type="check_spec",
288
+ check_id=cspec.id,
289
+ probe=cspec.probe,
290
+ execution=cspec.execution,
291
+ target=cspec.target,
292
+ ),
293
+ )
294
+ )
295
+ seq += 1
296
+
297
+ if not any(s.fault for s in steps):
298
+ raise PlanningError(f"drill {spec.name!r} contains no fault injection")
299
+
300
+ return ExecutionPlan(
301
+ run_id=run_id,
302
+ kind=ExperimentKind.DRILL,
303
+ steps=tuple(steps),
304
+ config_snapshot_id=config_snapshot_id,
305
+ topology_snapshot_id=topology_snapshot_id,
306
+ environment_fingerprint=environment_fingerprint,
307
+ success=spec.success,
308
+ observability=spec.observability,
309
+ decision_refs=_governing_decisions(spec),
310
+ )
311
+
312
+
313
+ def plan_maniac(
314
+ run_id: str,
315
+ spec: DrillSpec,
316
+ graph: TopologyGraph,
317
+ *,
318
+ config_snapshot_id: str,
319
+ topology_snapshot_id: str,
320
+ environment_fingerprint: str,
321
+ engine: str = "podman",
322
+ spec_dir: str | None = None,
323
+ maniac: ManiacCfg,
324
+ ) -> ExecutionPlan:
325
+ """Compile a maniac (random) drill into an :class:`ExecutionPlan` (ADR-M5-1).
326
+
327
+ ``mayhem maniac`` compiles the spec exactly like ``plan_drill`` — container
328
+ names resolved against the topology, compensation contracts attached — but
329
+ replaces the authored ``execution`` with ``maniac.run_level`` random
330
+ (container, fault) rounds drawn from the spec's own container map. Each
331
+ round injects one fault, then replays the spec's authored check/check_spec
332
+ steps so the M4 success criteria and observability sources produce the same
333
+ machine verdict and evidence as a deterministic run. The spec's safety
334
+ gates (risk ceiling, blast radius, ``max_faults``, timeout) apply unchanged
335
+ to every round (ADR-M5-1).
336
+ """
337
+ _validate_container_names(spec, graph)
338
+ draws = draw_maniac_rounds(
339
+ spec, level=maniac.level, run_level=maniac.run_level, seed=maniac.seed
340
+ )
341
+
342
+ steps: list[PlannedStep] = []
343
+ seq = 0
344
+ for draw in draws:
345
+ matched = tuple(_find_container_nodes(graph, draw.container))
346
+ steps.append(
347
+ _plan_fault_step(
348
+ draw.container,
349
+ matched,
350
+ draw.fault,
351
+ graph,
352
+ seq,
353
+ execution_group_id=f"grp-{uuid.uuid4().hex[:12]}",
354
+ group_mode=GroupMode.SEQUENTIAL,
355
+ group_path=f"/{draw.container}",
356
+ recovery=(
357
+ draw.fault.recovery if draw.fault.recovery is not None else spec.config.recovery
358
+ ),
359
+ on_failure=(
360
+ draw.fault.on_failure
361
+ if draw.fault.on_failure is not None
362
+ else spec.config.on_failure
363
+ ),
364
+ spec_dir=spec_dir,
365
+ )
366
+ )
367
+ seq += 1
368
+ seq = _plan_maniac_checks(steps, spec, seq)
369
+
370
+ if not any(s.fault for s in steps):
371
+ raise PlanningError(f"maniac drill {spec.name!r} drew no fault injection")
372
+
373
+ return ExecutionPlan(
374
+ run_id=run_id,
375
+ kind=ExperimentKind.DRILL,
376
+ steps=tuple(steps),
377
+ config_snapshot_id=config_snapshot_id,
378
+ topology_snapshot_id=topology_snapshot_id,
379
+ environment_fingerprint=environment_fingerprint,
380
+ success=spec.success,
381
+ observability=spec.observability,
382
+ decision_refs=(*_governing_decisions(spec), DECISION_M5_1_MANIAC),
383
+ )
384
+
385
+
386
+ def _plan_maniac_checks(steps: list[PlannedStep], spec: DrillSpec, seq: int) -> int:
387
+ """Replay the spec's authored check steps after a maniac round (ADR-M5-1).
388
+
389
+ Waits, parallel/sequential grouping and the faults themselves are maniac's
390
+ own business; only the check/check_spec blocks carry over so the success
391
+ criteria can be evaluated per round. Ids keep the ``check-{seq:04d}-{i}``
392
+ shape used by deterministic plans.
393
+ """
394
+ for block in spec.execution:
395
+ if block.check:
396
+ for i, probe in enumerate(block.check):
397
+ expected = probe.expect.status if probe.expect else None
398
+ steps.append(
399
+ PlannedStep(
400
+ id=f"check-{seq:04d}-{i}",
401
+ seq=seq,
402
+ raw_action=CheckHttp(
403
+ type="check_http",
404
+ url=probe.http or "",
405
+ expected_status=expected,
406
+ ),
407
+ )
408
+ )
409
+ seq += 1
410
+ elif block.check_spec:
411
+ for i, cspec in enumerate(block.check_spec):
412
+ steps.append(
413
+ PlannedStep(
414
+ id=f"check-{seq:04d}-{i}",
415
+ seq=seq,
416
+ raw_action=CheckSpecStep(
417
+ type="check_spec",
418
+ check_id=cspec.id,
419
+ probe=cspec.probe,
420
+ execution=cspec.execution,
421
+ target=cspec.target,
422
+ ),
423
+ )
424
+ )
425
+ seq += 1
426
+ return seq
427
+
428
+
429
+ def _governing_decisions(spec: DrillSpec) -> tuple[DecisionRef, ...]:
430
+ """Decision ids + approved timestamps captured onto the plan (ADR-M4-1).
431
+
432
+ The M4 decisions are the additivity/schema-freeze contracts every plan is
433
+ compiled under; success and observability sections are included only when
434
+ the spec actually exercises them, so each outcome records exactly the
435
+ decisions that produced it.
436
+ """
437
+ refs = [
438
+ DECISION_M4_1_ADDITIVE_DSL,
439
+ DECISION_M4_5_SCHEMA_FREEZE,
440
+ ]
441
+ if spec.success is not None and not spec.success.empty:
442
+ refs.append(DECISION_M4_3_SUCCESS_CRITERIA)
443
+ if spec.observability is not None and not spec.observability.empty:
444
+ refs.append(DECISION_M4_4_OBSERVABILITY)
445
+ return tuple(refs)
446
+
447
+
448
+ def _container_names(graph: TopologyGraph) -> set[str]:
449
+ """Every ``container_name`` declared across the topology (services + containers)."""
450
+ names: set[str] = set()
451
+ for node in graph.nodes:
452
+ value = getattr(node, "container_name", None)
453
+ if isinstance(value, str):
454
+ names.add(value)
455
+ return names
456
+
457
+
458
+ def _validate_container_names(spec: DrillSpec, graph: TopologyGraph) -> None:
459
+ """Every container name in the spec must exist in the topology."""
460
+ graph_names = _container_names(graph)
461
+ missing = [name for name in spec.containers if name not in graph_names]
462
+ if missing:
463
+ raise PlanningError(
464
+ f"container(s) not found in topology: {sorted(missing)} — "
465
+ f"available: {sorted(graph_names)}"
466
+ )
467
+ # Also validate execution-block references name defined containers.
468
+ for block in spec.execution:
469
+ referenced = list(block.parallel or ()) + list(block.sequential or ())
470
+ for name in referenced:
471
+ if name not in spec.containers:
472
+ raise PlanningError(
473
+ f"execution references container {name!r} which is not defined "
474
+ f"under 'containers'"
475
+ )
476
+
477
+
478
+ def _find_container_nodes(graph: TopologyGraph, container_name: str) -> tuple[TopologyNode, ...]:
479
+ """All topology nodes whose ``container_name`` matches the drill name."""
480
+ matched = tuple(
481
+ node for node in graph.nodes if getattr(node, "container_name", None) == container_name
482
+ )
483
+ if not matched:
484
+ raise PlanningError(f"container {container_name!r} not found in topology graph")
485
+ return matched
486
+
487
+
488
+ def _plan_container_faults(
489
+ container_name: str,
490
+ container: DrillContainer,
491
+ graph: TopologyGraph,
492
+ out: list[PlannedStep],
493
+ seq: int,
494
+ *,
495
+ recovery_default: bool = True,
496
+ on_failure_default: OnFailure = OnFailure.ABORT_AND_RECOVER,
497
+ spec_dir: str | None = None,
498
+ ) -> int:
499
+ """Plan every fault on the container as its own compensatable step.
500
+
501
+ Each fault becomes an independently compensatable step (write-ahead undo
502
+ per fault); steps advance ``seq`` by one so a multi-fault container never
503
+ double-injects concurrently. Returns the number of steps emitted so the
504
+ caller can advance its sequence counter.
505
+ """
506
+ if not container.faults:
507
+ # Container referenced in execution but defines no faults: emit a
508
+ # no-op placeholder step so the plan stays traceable.
509
+ out.append(
510
+ PlannedStep(
511
+ id=f"{container_name}-{seq:04d}",
512
+ seq=seq,
513
+ raw_action=Wait(type="wait", duration=0.0),
514
+ )
515
+ )
516
+ return 1
517
+
518
+ # A container name matches the whole subtree (service, container, process).
519
+ matched = tuple(_find_container_nodes(graph, container_name))
520
+ # Every fault on this container belongs to one persistent group (ADR-M2-1):
521
+ # members share an execution_group_id and run sequentially one-at-a-time so
522
+ # a multi-fault container never double-injects concurrently.
523
+ group_id = f"grp-{uuid.uuid4().hex[:12]}"
524
+ mode = GroupMode.SEQUENTIAL
525
+ path = f"/{container_name}"
526
+ for i, drill_fault in enumerate(container.faults):
527
+ out.append(
528
+ _plan_fault_step(
529
+ container_name,
530
+ matched,
531
+ drill_fault,
532
+ graph,
533
+ seq + i,
534
+ execution_group_id=group_id,
535
+ group_mode=mode,
536
+ spec_dir=spec_dir,
537
+ group_path=path,
538
+ recovery=(
539
+ drill_fault.recovery if drill_fault.recovery is not None else recovery_default
540
+ ),
541
+ on_failure=(
542
+ drill_fault.on_failure
543
+ if drill_fault.on_failure is not None
544
+ else on_failure_default
545
+ ),
546
+ )
547
+ )
548
+ return len(container.faults)
549
+
550
+
551
+ def _plan_fault_step(
552
+ container_name: str,
553
+ matched: tuple[TopologyNode, ...],
554
+ drill_fault: DrillFault,
555
+ graph: TopologyGraph,
556
+ seq: int,
557
+ *,
558
+ execution_group_id: str | None = None,
559
+ group_mode: GroupMode | None = None,
560
+ group_path: str | None = None,
561
+ recovery: bool = True,
562
+ on_failure: OnFailure | None = None,
563
+ spec_dir: str | None = None,
564
+ ) -> PlannedStep:
565
+ """Compile one drill fault into a compensatable :class:`PlannedStep`."""
566
+ try:
567
+ definition = definition_for(drill_fault.fault)
568
+ except (SchemaValidationError, LookupError) as exc:
569
+ raise PlanningError(str(exc)) from None
570
+
571
+ # The fault applies to a specific kind subset, so target only the nodes it
572
+ # can actually act on.
573
+ nodes = tuple(n for n in matched if n.kind in definition.applicable_node_kinds)
574
+ if not nodes:
575
+ kinds = ", ".join(sorted({n.kind.value for n in matched}))
576
+ raise PlanningError(
577
+ f"fault {definition.id!r} does not apply to any node matched by "
578
+ f"container {container_name!r} (matched kinds: {kinds})"
579
+ )
580
+
581
+ # Compensation runs against the drill container's whole subtree: the
582
+ # fault's own matching nodes plus the container/process address the undo op
583
+ # executes in. Drill faults name containers; e.g. cpu.saturate targets
584
+ # SERVICE only, but its payload undo must still reach the container that
585
+ # backs the service (ADR-0020).
586
+ compensation_nodes = list(matched)
587
+ seen = {id(node) for node in compensation_nodes}
588
+ for node in matched:
589
+ for proc in graph.connected_processes(node.id):
590
+ if id(proc) not in seen:
591
+ seen.add(id(proc))
592
+ compensation_nodes.append(proc)
593
+
594
+ # ``duration`` is a Duration (float); the ``"10s"`` class default reaches
595
+ # the runtime as an unvalidated str unless explicitly passed through
596
+ # validation, so resolve both forms before comparing against the cap.
597
+ raw_duration: int | float | str = cast("int | float | str", drill_fault.duration)
598
+ duration_s = (
599
+ parse_duration(raw_duration) if isinstance(raw_duration, str) else float(raw_duration)
600
+ )
601
+ if duration_s > definition.max_duration_s:
602
+ raise PlanningError(
603
+ f"fault {definition.id!r} duration {duration_s}s exceeds cap "
604
+ f"{definition.max_duration_s}s"
605
+ )
606
+ # DrillFault accepts fault parameters as extra YAML keys on the fault
607
+ # entry (e.g. ``percent: 80``), collected via ``model_extra``; an explicit
608
+ # ``params:`` mapping is honored as well. ``duration``, ``on_failure`` and
609
+ # ``targets`` are declared fields and never treated as parameters.
610
+ raw_params: dict[str, object] = {}
611
+ explicit = getattr(drill_fault, "params", None)
612
+ if isinstance(explicit, dict):
613
+ raw_params.update(explicit)
614
+ for key, value in (getattr(drill_fault, "model_extra", None) or {}).items():
615
+ if key == "params":
616
+ continue # already merged from the explicit ``params:`` group
617
+ if value is not None:
618
+ raw_params.setdefault(key, value)
619
+ params = definition.validate_params(raw_params)
620
+
621
+ # ``net.load`` accepts a host-side k6 ``script.js`` (param ``script``). The
622
+ # script runs on the drill host at the target container's ip:port, so its
623
+ # content is embedded into the frozen plan here — resolved relative to
624
+ # the drill spec directory — rather than read at execution time.
625
+ params = _embed_load_script(params, definition.id, spec_dir)
626
+
627
+ selectors = tuple(TargetSelector(kind=node.kind, expr=node.name) for node in nodes)
628
+ planned = PlannedFault(
629
+ fault_id=definition.id,
630
+ targets=tuple(
631
+ ResolvedTarget(selector=selector, node_ids=frozenset({node.id}))
632
+ for selector, node in zip(selectors, nodes, strict=True)
633
+ ),
634
+ params=params,
635
+ duration=drill_fault.duration,
636
+ backend=None,
637
+ runtime_identity=_resolve_planned_identity(matched),
638
+ recovery=recovery,
639
+ on_failure=on_failure if on_failure is not None else OnFailure.ABORT_AND_RECOVER,
640
+ )
641
+ planned = compensated(planned, tuple(compensation_nodes))
642
+ if not planned.undo_ops:
643
+ msg = f"fault {definition.id!r} compiled without undo contract"
644
+ raise InvariantViolationError("plan_write_ahead_undo", msg)
645
+ return PlannedStep(
646
+ id=f"{container_name}-{seq:04d}",
647
+ seq=seq,
648
+ fault=planned,
649
+ runtime_identity=_resolve_planned_identity(matched),
650
+ execution_group_id=execution_group_id,
651
+ group_mode=group_mode,
652
+ group_path=group_path,
653
+ raw_action=InjectFault(
654
+ fault=definition.id,
655
+ selectors=selectors,
656
+ params=params,
657
+ duration=drill_fault.duration,
658
+ ),
659
+ )
660
+
661
+
662
+ def _resolve_planned_identity(nodes: tuple[TopologyNode, ...]) -> RuntimeIdentity | None:
663
+ """The canonical identity backing the authored name (ADR-M1-1).
664
+
665
+ The plan targets the whole matched subtree (service/container/process), but
666
+ the canonical ``RuntimeIdentity`` is contributed by the container node —
667
+ the only node kind that carries it (topology.py). Resolver-key nodes
668
+ (service/process) contribute no identity; their plan identity stays ``None``
669
+ until a runtime adapter supplies one. This satisfies the milestone bar:
670
+ identity is populated for a container-targeted fault whenever a provider is
671
+ available.
672
+ """
673
+ for node in nodes:
674
+ identity: object = getattr(node, "runtime_identity", None)
675
+ if isinstance(identity, RuntimeIdentity):
676
+ return identity
677
+ return None
678
+
679
+
680
+ def restrict_plan_to_container(
681
+ plan: ExecutionPlan, container_name: str, graph: TopologyGraph
682
+ ) -> ExecutionPlan:
683
+ """Scope a compiled plan to one container subtree (``mayhem --ctr``).
684
+
685
+ Fault steps survive when any resolved target belongs to the container's
686
+ service/container/process subtree; the subtree's own orchestration steps
687
+ (``group_path == "/<container_name>"``) survive too. Faults, waits and
688
+ checks for every other container are removed, so the run executes only
689
+ against the requested container. All remaining steps keep their original
690
+ ids and sequence — skipped ids are intentional (ADR-0021 traceability).
691
+
692
+ Raises :class:`PlanningError` when the graph has no such container or the
693
+ plan carries no fault step against it.
694
+ """
695
+ subtree = graph.node_ids_for_container(container_name)
696
+ if not subtree:
697
+ available = ", ".join(graph.container_names()) or "<none>"
698
+ raise PlanningError(
699
+ f"container {container_name!r} not found in topology graph (available: {available})"
700
+ )
701
+
702
+ kept: list[PlannedStep] = []
703
+ for step in plan.steps:
704
+ if step.fault is None:
705
+ if step.group_path == f"/{container_name}":
706
+ kept.append(step)
707
+ continue
708
+ if any(target.node_ids & subtree for target in step.fault.targets):
709
+ kept.append(step)
710
+
711
+ if not any(step.fault is not None for step in kept):
712
+ raise PlanningError(
713
+ f"plan {plan.run_id!r} has no fault steps targeting container {container_name!r}"
714
+ )
715
+ return plan.model_copy(update={"steps": tuple(kept)})