mayhem-cli 0.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mayhem/agent/__init__.py +1 -0
- mayhem/agent/cli.py +36 -0
- mayhem/agents/__init__.py +1 -0
- mayhem/agents/capabilities.py +106 -0
- mayhem/agents/executors.py +430 -0
- mayhem/agents/impact.py +729 -0
- mayhem/agents/lease_client.py +141 -0
- mayhem/agents/probes.py +284 -0
- mayhem/agents/protocol.py +134 -0
- mayhem/agents/server.py +281 -0
- mayhem/agents/sinks.py +60 -0
- mayhem/agents/transports.py +134 -0
- mayhem/agents/watchdog.py +140 -0
- mayhem/cli/__init__.py +11 -0
- mayhem/cli/app.py +154 -0
- mayhem/cli/campaign.py +496 -0
- mayhem/cli/config_cmd.py +47 -0
- mayhem/cli/context.py +23 -0
- mayhem/cli/dependency.py +429 -0
- mayhem/cli/exit_codes.py +24 -0
- mayhem/cli/experiment.py +24 -0
- mayhem/cli/lifecycle.py +805 -0
- mayhem/cli/resolver.py +72 -0
- mayhem/cli/services.py +459 -0
- mayhem/cli/style.py +101 -0
- mayhem/cli/toolkit.py +41 -0
- mayhem/cli/topology.py +127 -0
- mayhem/config.py +208 -0
- mayhem/controller/__init__.py +1 -0
- mayhem/controller/compensation.py +2156 -0
- mayhem/controller/executor.py +1719 -0
- mayhem/controller/janitor.py +196 -0
- mayhem/controller/observability_collector.py +382 -0
- mayhem/controller/observations.py +102 -0
- mayhem/controller/planner.py +715 -0
- mayhem/controller/recovery.py +245 -0
- mayhem/controller/resilience_report.py +585 -0
- mayhem/controller/resource_manager.py +457 -0
- mayhem/controller/safety.py +392 -0
- mayhem/domain/__init__.py +6 -0
- mayhem/domain/campaigns.py +118 -0
- mayhem/domain/cancellation.py +110 -0
- mayhem/domain/candidates.py +101 -0
- mayhem/domain/capabilities.py +86 -0
- mayhem/domain/catalog.py +727 -0
- mayhem/domain/checks.py +173 -0
- mayhem/domain/common.py +104 -0
- mayhem/domain/coverage.py +106 -0
- mayhem/domain/decisions.py +57 -0
- mayhem/domain/errors.py +87 -0
- mayhem/domain/events.py +61 -0
- mayhem/domain/execution_context.py +120 -0
- mayhem/domain/execution_loci.py +94 -0
- mayhem/domain/experiments.py +370 -0
- mayhem/domain/faults.py +239 -0
- mayhem/domain/identity.py +200 -0
- mayhem/domain/k8s_adapter.py +132 -0
- mayhem/domain/leases.py +186 -0
- mayhem/domain/load_strategy.py +98 -0
- mayhem/domain/m5_campaign.py +120 -0
- mayhem/domain/maniac.py +93 -0
- mayhem/domain/observability.py +146 -0
- mayhem/domain/outcomes.py +92 -0
- mayhem/domain/remote_agent_interface.py +70 -0
- mayhem/domain/resources.py +245 -0
- mayhem/domain/risks.py +61 -0
- mayhem/domain/run_outcome.py +146 -0
- mayhem/domain/runtime_adapter.py +256 -0
- mayhem/domain/success.py +329 -0
- mayhem/domain/topology.py +452 -0
- mayhem/infra/__init__.py +1 -0
- mayhem/infra/campaign_engine.py +205 -0
- mayhem/infra/candidate_gates.py +124 -0
- mayhem/infra/candidate_generator.py +110 -0
- mayhem/infra/coverage_repository.py +101 -0
- mayhem/infra/lease_repository.py +129 -0
- mayhem/infra/maniac.py +103 -0
- mayhem/infra/migrations.py +596 -0
- mayhem/infra/migrator.py +149 -0
- mayhem/infra/report.py +227 -0
- mayhem/infra/store.py +200 -0
- mayhem/py.typed +0 -0
- mayhem/spec.py +52 -0
- mayhem/toolkit/__init__.py +1 -0
- mayhem/toolkit/fingerprint.py +69 -0
- mayhem/toolkit/hashing.py +32 -0
- mayhem/toolkit/manifests/docker.yaml +11 -0
- mayhem/toolkit/manifests/podman.yaml +11 -0
- mayhem/toolkit/manifests/stress-ng.yaml +11 -0
- mayhem/toolkit/manifests/tc-netem.yaml +11 -0
- mayhem/toolkit/manifests/toxiproxy.yaml +10 -0
- mayhem/toolkit/registry.py +185 -0
- mayhem/toolkit/tool_runner.py +129 -0
- mayhem/topology/__init__.py +10 -0
- mayhem/topology/providers/__init__.py +0 -0
- mayhem/topology/providers/adapter_registry.py +60 -0
- mayhem/topology/providers/base.py +31 -0
- mayhem/topology/providers/compose.py +207 -0
- mayhem/topology/providers/docker_adapter.py +277 -0
- mayhem/topology/providers/docker_runtime.py +461 -0
- mayhem/topology/providers/podman_adapter.py +328 -0
- mayhem/topology/resolve.py +196 -0
- mayhem/topology/service.py +158 -0
- mayhem_cli-0.5.1.dist-info/METADATA +555 -0
- mayhem_cli-0.5.1.dist-info/RECORD +107 -0
- mayhem_cli-0.5.1.dist-info/WHEEL +4 -0
- mayhem_cli-0.5.1.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,715 @@
|
|
|
1
|
+
"""Planner — compiles drill specs into frozen ExecutionPlans.
|
|
2
|
+
|
|
3
|
+
Drill plans resolve every container name against the topology snapshot and
|
|
4
|
+
attach a compensation contract to every fault (write-ahead undo). Wait and
|
|
5
|
+
check blocks compile to non-fault steps; parallel blocks emit one step per
|
|
6
|
+
container so the executor can run them concurrently (Phase 5, ADR-0019/0021).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import uuid
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import TYPE_CHECKING, cast
|
|
14
|
+
|
|
15
|
+
from mayhem.controller.compensation import compensated
|
|
16
|
+
from mayhem.domain.capabilities import Capability
|
|
17
|
+
from mayhem.domain.catalog import all_definitions, definition_for
|
|
18
|
+
from mayhem.domain.common import parse_duration
|
|
19
|
+
from mayhem.domain.decisions import (
|
|
20
|
+
DECISION_M4_1_ADDITIVE_DSL,
|
|
21
|
+
DECISION_M4_3_SUCCESS_CRITERIA,
|
|
22
|
+
DECISION_M4_4_OBSERVABILITY,
|
|
23
|
+
DECISION_M4_5_SCHEMA_FREEZE,
|
|
24
|
+
DECISION_M5_1_MANIAC,
|
|
25
|
+
DecisionRef,
|
|
26
|
+
)
|
|
27
|
+
from mayhem.domain.errors import (
|
|
28
|
+
InvariantViolationError,
|
|
29
|
+
SchemaValidationError,
|
|
30
|
+
)
|
|
31
|
+
from mayhem.domain.experiments import (
|
|
32
|
+
CheckHttp,
|
|
33
|
+
CheckSpecStep,
|
|
34
|
+
DrillContainer,
|
|
35
|
+
DrillFault,
|
|
36
|
+
DrillSpec,
|
|
37
|
+
ExecutionPlan,
|
|
38
|
+
ExecutionStep,
|
|
39
|
+
ExperimentKind,
|
|
40
|
+
GroupMode,
|
|
41
|
+
InjectFault,
|
|
42
|
+
ManiacCfg,
|
|
43
|
+
OnFailure,
|
|
44
|
+
PlannedFault,
|
|
45
|
+
PlannedStep,
|
|
46
|
+
ResolvedTarget,
|
|
47
|
+
Wait,
|
|
48
|
+
)
|
|
49
|
+
from mayhem.domain.faults import FaultDefinition, ParamType
|
|
50
|
+
from mayhem.domain.identity import RuntimeIdentity
|
|
51
|
+
from mayhem.domain.maniac import draw_maniac_rounds
|
|
52
|
+
from mayhem.domain.topology import (
|
|
53
|
+
PortBinding,
|
|
54
|
+
TargetSelector,
|
|
55
|
+
TopologyNode,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if TYPE_CHECKING:
|
|
59
|
+
from mayhem.domain.topology import TopologyGraph
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class PlanningError(Exception):
|
|
63
|
+
"""Raised when a spec cannot be compiled into an honest plan."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
#: Capabilities only a Kubernetes runtime can provide. ``mayhem maniac``
|
|
67
|
+
#: synthesizes its config from a compose topology, so catalog faults gated on
|
|
68
|
+
#: these are never pooled (a compose graph cannot satisfy them at execution
|
|
69
|
+
#: time). Everything else — NET_ADMIN, PROCESS_CONTROL, engine flags, … — stays
|
|
70
|
+
#: in the pool and is resolved by the impact gate, which probes each container
|
|
71
|
+
#: and bypasses only the inert draws.
|
|
72
|
+
_MANIAC_EXCLUDED_CAPS = frozenset({Capability.KUBERNETES_ENGINE})
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def synthesize_maniac_spec(graph: TopologyGraph, *, name: str = "maniac") -> DrillSpec:
|
|
76
|
+
"""Derive a drill spec purely from a compose-derived topology graph.
|
|
77
|
+
|
|
78
|
+
``mayhem maniac`` runs without an authored config file: this builder makes
|
|
79
|
+
the draw pool self-describing. Every container in the topology authors the
|
|
80
|
+
full container-addressable fault catalog, so
|
|
81
|
+
:func:`draw_maniac_rounds` keeps a pure random walk over (container,
|
|
82
|
+
fault) exactly like a hand-authored spec. The impact gate probes actual
|
|
83
|
+
tooling at execution time and bypasses only the draws proven inert, so
|
|
84
|
+
the compile never fails because an image lacks a binary.
|
|
85
|
+
|
|
86
|
+
Safety is inherited, not authored: the synthesized spec carries no policy
|
|
87
|
+
of its own (``config.maniac`` is left unset so the layered ``mayhem.yaml``
|
|
88
|
+
``maniac:`` block — or the CLI defaults — tune the draw), and every
|
|
89
|
+
planned fault still passes the unchanged planner safety stack (risk
|
|
90
|
+
ceiling, blast radius, ``max_faults``, timeout, compensation contract).
|
|
91
|
+
|
|
92
|
+
Each pooled fault is authored with compile-ready parameters: catalog
|
|
93
|
+
defaults are reused verbatim, and parameters the catalog declares
|
|
94
|
+
mandatory are derived from the topology where an honest value exists
|
|
95
|
+
(``port`` from the container's own published bindings, ``host`` from the
|
|
96
|
+
container name, ``connections``/``delay_ms``/``rate``/``offset_ms`` at
|
|
97
|
+
their least-disruptive minimums). Faults with a mandatory parameter that
|
|
98
|
+
carries no derivable value (``net.bandwidth``'s host-side ``rate`` string,
|
|
99
|
+
or a portless container) are left out of that container's pool — a draw
|
|
100
|
+
must never name a fault it cannot compile. Faults that require a
|
|
101
|
+
Kubernetes-only capability are pooled for no container.
|
|
102
|
+
"""
|
|
103
|
+
containers: dict[str, DrillContainer] = {}
|
|
104
|
+
for container_name in sorted(_container_names(graph)):
|
|
105
|
+
matched = _find_container_nodes(graph, container_name)
|
|
106
|
+
kinds = frozenset(node.kind for node in matched)
|
|
107
|
+
ports = tuple(
|
|
108
|
+
binding
|
|
109
|
+
for node in matched
|
|
110
|
+
for binding in (getattr(node, "exposed_ports", ()) or getattr(node, "ports", ()))
|
|
111
|
+
)
|
|
112
|
+
faults: list[DrillFault] = []
|
|
113
|
+
for definition in all_definitions():
|
|
114
|
+
if not definition.applicable_node_kinds & kinds:
|
|
115
|
+
continue
|
|
116
|
+
if not definition.required_caps.isdisjoint(_MANIAC_EXCLUDED_CAPS):
|
|
117
|
+
continue
|
|
118
|
+
params = _synthesized_params(definition, container_name, ports)
|
|
119
|
+
if params is None:
|
|
120
|
+
continue
|
|
121
|
+
# ``params`` is an undeclared pydantic extra field on DrillFault
|
|
122
|
+
# (extra="allow"), so construct through model_validate.
|
|
123
|
+
faults.append(DrillFault.model_validate({"fault": definition.id, "params": params}))
|
|
124
|
+
containers[container_name] = DrillContainer(faults=tuple(faults))
|
|
125
|
+
return DrillSpec(
|
|
126
|
+
kind="drill",
|
|
127
|
+
name=name,
|
|
128
|
+
hypothesis=(
|
|
129
|
+
f"maniac: synthesized config for {len(containers)} container(s) "
|
|
130
|
+
"from the compose topology"
|
|
131
|
+
),
|
|
132
|
+
containers=containers,
|
|
133
|
+
# ``plan_drill`` needs one execution step; the maniac planner replaces
|
|
134
|
+
# the authored execution wholesale, so this placeholder never runs.
|
|
135
|
+
execution=(ExecutionStep(wait="1s"),),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _synthesized_params(
|
|
140
|
+
definition: FaultDefinition,
|
|
141
|
+
container_name: str,
|
|
142
|
+
ports: tuple[PortBinding, ...],
|
|
143
|
+
) -> dict[str, object] | None:
|
|
144
|
+
"""Auto-author a compile-ready param set for a synthesized fault.
|
|
145
|
+
|
|
146
|
+
Catalog defaults win; the handful of parameters the catalog marks
|
|
147
|
+
mandatory are filled from topology where an honest value exists, else
|
|
148
|
+
``None`` (the caller drops the fault from the pool).
|
|
149
|
+
"""
|
|
150
|
+
params: dict[str, object] = {}
|
|
151
|
+
mandatory: list[tuple[str, ParamType]] = []
|
|
152
|
+
for spec in definition.params_schema:
|
|
153
|
+
if spec.default is not None:
|
|
154
|
+
params[spec.name] = spec.default
|
|
155
|
+
elif spec.required:
|
|
156
|
+
mandatory.append((spec.name, spec.type))
|
|
157
|
+
for name, param_type in mandatory:
|
|
158
|
+
if param_type is ParamType.STRING and name == "host":
|
|
159
|
+
params[name] = container_name
|
|
160
|
+
continue
|
|
161
|
+
if param_type is ParamType.STRING:
|
|
162
|
+
return None
|
|
163
|
+
if name == "port":
|
|
164
|
+
port = next(
|
|
165
|
+
(binding.container_port for binding in ports if binding.protocol == "tcp"),
|
|
166
|
+
next((binding.container_port for binding in ports), None),
|
|
167
|
+
)
|
|
168
|
+
if port is None:
|
|
169
|
+
return None
|
|
170
|
+
params[name] = port
|
|
171
|
+
elif name in {"connections", "delay_ms", "rate", "offset_ms"}:
|
|
172
|
+
params[name] = {"connections": 1, "delay_ms": 1000, "rate": 1, "offset_ms": 60000}[name]
|
|
173
|
+
else:
|
|
174
|
+
return None
|
|
175
|
+
return params
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _embed_load_script(
|
|
179
|
+
params: dict[str, object], fault_id: str, spec_dir: str | None
|
|
180
|
+
) -> dict[str, object]:
|
|
181
|
+
"""Embed a ``net.load`` ``script`` param into the frozen plan as content."""
|
|
182
|
+
script_ref = params.get("script")
|
|
183
|
+
if fault_id != "net.load" or not isinstance(script_ref, str) or not script_ref.strip():
|
|
184
|
+
return params
|
|
185
|
+
base = Path(spec_dir) if spec_dir else Path.cwd()
|
|
186
|
+
script_path = Path(script_ref) if Path(script_ref).is_absolute() else base / script_ref
|
|
187
|
+
try:
|
|
188
|
+
content = script_path.read_text(encoding="utf-8")
|
|
189
|
+
except OSError as exc:
|
|
190
|
+
raise PlanningError(f"fault net.load script {script_path!s} unreadable: {exc}") from None
|
|
191
|
+
except UnicodeDecodeError as exc:
|
|
192
|
+
raise PlanningError(
|
|
193
|
+
f"fault net.load script {script_path!s} must be UTF-8 text: {exc}"
|
|
194
|
+
) from None
|
|
195
|
+
return {**params, "script_content": content}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def plan_drill(
|
|
199
|
+
run_id: str,
|
|
200
|
+
spec: DrillSpec,
|
|
201
|
+
graph: TopologyGraph,
|
|
202
|
+
*,
|
|
203
|
+
config_snapshot_id: str,
|
|
204
|
+
topology_snapshot_id: str,
|
|
205
|
+
environment_fingerprint: str,
|
|
206
|
+
engine: str = "podman",
|
|
207
|
+
spec_dir: str | None = None,
|
|
208
|
+
) -> ExecutionPlan:
|
|
209
|
+
"""Compile a :class:`DrillSpec` into an :class:`ExecutionPlan`.
|
|
210
|
+
|
|
211
|
+
Drill plans are authored around container names rather than target
|
|
212
|
+
selectors. Each fault on a container is resolved to its topology node and
|
|
213
|
+
the relevant compensation contract is attached (write-ahead undo). Wait and
|
|
214
|
+
check blocks compile to non-fault steps; parallel blocks emit one step per
|
|
215
|
+
container so the executor can run them concurrently (Phase 5). ``spec_dir``
|
|
216
|
+
anchors assets referenced by the spec (e.g. the ``net.load`` ``script``
|
|
217
|
+
parameter) so relative paths resolve against the drill file.
|
|
218
|
+
"""
|
|
219
|
+
_validate_container_names(spec, graph)
|
|
220
|
+
|
|
221
|
+
steps: list[PlannedStep] = []
|
|
222
|
+
seq = 0
|
|
223
|
+
for block in spec.execution:
|
|
224
|
+
if block.parallel:
|
|
225
|
+
# All containers in a parallel block share one `seq` so the
|
|
226
|
+
# executor groups and runs them concurrently (Phase 5); the block
|
|
227
|
+
# advances by the longest container's fault chain.
|
|
228
|
+
emitted = 0
|
|
229
|
+
for container_name in block.parallel:
|
|
230
|
+
container = spec.containers[container_name]
|
|
231
|
+
planned = _plan_container_faults(
|
|
232
|
+
container_name,
|
|
233
|
+
container,
|
|
234
|
+
graph,
|
|
235
|
+
steps,
|
|
236
|
+
seq,
|
|
237
|
+
recovery_default=spec.config.recovery,
|
|
238
|
+
on_failure_default=spec.config.on_failure,
|
|
239
|
+
spec_dir=spec_dir,
|
|
240
|
+
)
|
|
241
|
+
emitted = max(emitted, planned)
|
|
242
|
+
seq += emitted
|
|
243
|
+
elif block.sequential:
|
|
244
|
+
for container_name in block.sequential:
|
|
245
|
+
container = spec.containers[container_name]
|
|
246
|
+
seq += _plan_container_faults(
|
|
247
|
+
container_name,
|
|
248
|
+
container,
|
|
249
|
+
graph,
|
|
250
|
+
steps,
|
|
251
|
+
seq,
|
|
252
|
+
recovery_default=spec.config.recovery,
|
|
253
|
+
on_failure_default=spec.config.on_failure,
|
|
254
|
+
spec_dir=spec_dir,
|
|
255
|
+
)
|
|
256
|
+
elif block.wait is not None:
|
|
257
|
+
steps.append(
|
|
258
|
+
PlannedStep(
|
|
259
|
+
id=f"wait-{seq:04d}",
|
|
260
|
+
seq=seq,
|
|
261
|
+
raw_action=Wait(type="wait", duration=block.wait),
|
|
262
|
+
)
|
|
263
|
+
)
|
|
264
|
+
seq += 1
|
|
265
|
+
elif block.check:
|
|
266
|
+
for i, probe in enumerate(block.check):
|
|
267
|
+
expected = probe.expect.status if probe.expect else None
|
|
268
|
+
steps.append(
|
|
269
|
+
PlannedStep(
|
|
270
|
+
id=f"check-{seq:04d}-{i}",
|
|
271
|
+
seq=seq,
|
|
272
|
+
raw_action=CheckHttp(
|
|
273
|
+
type="check_http",
|
|
274
|
+
url=probe.http or "",
|
|
275
|
+
expected_status=expected,
|
|
276
|
+
),
|
|
277
|
+
)
|
|
278
|
+
)
|
|
279
|
+
seq += 1
|
|
280
|
+
elif block.check_spec:
|
|
281
|
+
for i, cspec in enumerate(block.check_spec):
|
|
282
|
+
steps.append(
|
|
283
|
+
PlannedStep(
|
|
284
|
+
id=f"check-{seq:04d}-{i}",
|
|
285
|
+
seq=seq,
|
|
286
|
+
raw_action=CheckSpecStep(
|
|
287
|
+
type="check_spec",
|
|
288
|
+
check_id=cspec.id,
|
|
289
|
+
probe=cspec.probe,
|
|
290
|
+
execution=cspec.execution,
|
|
291
|
+
target=cspec.target,
|
|
292
|
+
),
|
|
293
|
+
)
|
|
294
|
+
)
|
|
295
|
+
seq += 1
|
|
296
|
+
|
|
297
|
+
if not any(s.fault for s in steps):
|
|
298
|
+
raise PlanningError(f"drill {spec.name!r} contains no fault injection")
|
|
299
|
+
|
|
300
|
+
return ExecutionPlan(
|
|
301
|
+
run_id=run_id,
|
|
302
|
+
kind=ExperimentKind.DRILL,
|
|
303
|
+
steps=tuple(steps),
|
|
304
|
+
config_snapshot_id=config_snapshot_id,
|
|
305
|
+
topology_snapshot_id=topology_snapshot_id,
|
|
306
|
+
environment_fingerprint=environment_fingerprint,
|
|
307
|
+
success=spec.success,
|
|
308
|
+
observability=spec.observability,
|
|
309
|
+
decision_refs=_governing_decisions(spec),
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def plan_maniac(
|
|
314
|
+
run_id: str,
|
|
315
|
+
spec: DrillSpec,
|
|
316
|
+
graph: TopologyGraph,
|
|
317
|
+
*,
|
|
318
|
+
config_snapshot_id: str,
|
|
319
|
+
topology_snapshot_id: str,
|
|
320
|
+
environment_fingerprint: str,
|
|
321
|
+
engine: str = "podman",
|
|
322
|
+
spec_dir: str | None = None,
|
|
323
|
+
maniac: ManiacCfg,
|
|
324
|
+
) -> ExecutionPlan:
|
|
325
|
+
"""Compile a maniac (random) drill into an :class:`ExecutionPlan` (ADR-M5-1).
|
|
326
|
+
|
|
327
|
+
``mayhem maniac`` compiles the spec exactly like ``plan_drill`` — container
|
|
328
|
+
names resolved against the topology, compensation contracts attached — but
|
|
329
|
+
replaces the authored ``execution`` with ``maniac.run_level`` random
|
|
330
|
+
(container, fault) rounds drawn from the spec's own container map. Each
|
|
331
|
+
round injects one fault, then replays the spec's authored check/check_spec
|
|
332
|
+
steps so the M4 success criteria and observability sources produce the same
|
|
333
|
+
machine verdict and evidence as a deterministic run. The spec's safety
|
|
334
|
+
gates (risk ceiling, blast radius, ``max_faults``, timeout) apply unchanged
|
|
335
|
+
to every round (ADR-M5-1).
|
|
336
|
+
"""
|
|
337
|
+
_validate_container_names(spec, graph)
|
|
338
|
+
draws = draw_maniac_rounds(
|
|
339
|
+
spec, level=maniac.level, run_level=maniac.run_level, seed=maniac.seed
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
steps: list[PlannedStep] = []
|
|
343
|
+
seq = 0
|
|
344
|
+
for draw in draws:
|
|
345
|
+
matched = tuple(_find_container_nodes(graph, draw.container))
|
|
346
|
+
steps.append(
|
|
347
|
+
_plan_fault_step(
|
|
348
|
+
draw.container,
|
|
349
|
+
matched,
|
|
350
|
+
draw.fault,
|
|
351
|
+
graph,
|
|
352
|
+
seq,
|
|
353
|
+
execution_group_id=f"grp-{uuid.uuid4().hex[:12]}",
|
|
354
|
+
group_mode=GroupMode.SEQUENTIAL,
|
|
355
|
+
group_path=f"/{draw.container}",
|
|
356
|
+
recovery=(
|
|
357
|
+
draw.fault.recovery if draw.fault.recovery is not None else spec.config.recovery
|
|
358
|
+
),
|
|
359
|
+
on_failure=(
|
|
360
|
+
draw.fault.on_failure
|
|
361
|
+
if draw.fault.on_failure is not None
|
|
362
|
+
else spec.config.on_failure
|
|
363
|
+
),
|
|
364
|
+
spec_dir=spec_dir,
|
|
365
|
+
)
|
|
366
|
+
)
|
|
367
|
+
seq += 1
|
|
368
|
+
seq = _plan_maniac_checks(steps, spec, seq)
|
|
369
|
+
|
|
370
|
+
if not any(s.fault for s in steps):
|
|
371
|
+
raise PlanningError(f"maniac drill {spec.name!r} drew no fault injection")
|
|
372
|
+
|
|
373
|
+
return ExecutionPlan(
|
|
374
|
+
run_id=run_id,
|
|
375
|
+
kind=ExperimentKind.DRILL,
|
|
376
|
+
steps=tuple(steps),
|
|
377
|
+
config_snapshot_id=config_snapshot_id,
|
|
378
|
+
topology_snapshot_id=topology_snapshot_id,
|
|
379
|
+
environment_fingerprint=environment_fingerprint,
|
|
380
|
+
success=spec.success,
|
|
381
|
+
observability=spec.observability,
|
|
382
|
+
decision_refs=(*_governing_decisions(spec), DECISION_M5_1_MANIAC),
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _plan_maniac_checks(steps: list[PlannedStep], spec: DrillSpec, seq: int) -> int:
|
|
387
|
+
"""Replay the spec's authored check steps after a maniac round (ADR-M5-1).
|
|
388
|
+
|
|
389
|
+
Waits, parallel/sequential grouping and the faults themselves are maniac's
|
|
390
|
+
own business; only the check/check_spec blocks carry over so the success
|
|
391
|
+
criteria can be evaluated per round. Ids keep the ``check-{seq:04d}-{i}``
|
|
392
|
+
shape used by deterministic plans.
|
|
393
|
+
"""
|
|
394
|
+
for block in spec.execution:
|
|
395
|
+
if block.check:
|
|
396
|
+
for i, probe in enumerate(block.check):
|
|
397
|
+
expected = probe.expect.status if probe.expect else None
|
|
398
|
+
steps.append(
|
|
399
|
+
PlannedStep(
|
|
400
|
+
id=f"check-{seq:04d}-{i}",
|
|
401
|
+
seq=seq,
|
|
402
|
+
raw_action=CheckHttp(
|
|
403
|
+
type="check_http",
|
|
404
|
+
url=probe.http or "",
|
|
405
|
+
expected_status=expected,
|
|
406
|
+
),
|
|
407
|
+
)
|
|
408
|
+
)
|
|
409
|
+
seq += 1
|
|
410
|
+
elif block.check_spec:
|
|
411
|
+
for i, cspec in enumerate(block.check_spec):
|
|
412
|
+
steps.append(
|
|
413
|
+
PlannedStep(
|
|
414
|
+
id=f"check-{seq:04d}-{i}",
|
|
415
|
+
seq=seq,
|
|
416
|
+
raw_action=CheckSpecStep(
|
|
417
|
+
type="check_spec",
|
|
418
|
+
check_id=cspec.id,
|
|
419
|
+
probe=cspec.probe,
|
|
420
|
+
execution=cspec.execution,
|
|
421
|
+
target=cspec.target,
|
|
422
|
+
),
|
|
423
|
+
)
|
|
424
|
+
)
|
|
425
|
+
seq += 1
|
|
426
|
+
return seq
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _governing_decisions(spec: DrillSpec) -> tuple[DecisionRef, ...]:
|
|
430
|
+
"""Decision ids + approved timestamps captured onto the plan (ADR-M4-1).
|
|
431
|
+
|
|
432
|
+
The M4 decisions are the additivity/schema-freeze contracts every plan is
|
|
433
|
+
compiled under; success and observability sections are included only when
|
|
434
|
+
the spec actually exercises them, so each outcome records exactly the
|
|
435
|
+
decisions that produced it.
|
|
436
|
+
"""
|
|
437
|
+
refs = [
|
|
438
|
+
DECISION_M4_1_ADDITIVE_DSL,
|
|
439
|
+
DECISION_M4_5_SCHEMA_FREEZE,
|
|
440
|
+
]
|
|
441
|
+
if spec.success is not None and not spec.success.empty:
|
|
442
|
+
refs.append(DECISION_M4_3_SUCCESS_CRITERIA)
|
|
443
|
+
if spec.observability is not None and not spec.observability.empty:
|
|
444
|
+
refs.append(DECISION_M4_4_OBSERVABILITY)
|
|
445
|
+
return tuple(refs)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _container_names(graph: TopologyGraph) -> set[str]:
|
|
449
|
+
"""Every ``container_name`` declared across the topology (services + containers)."""
|
|
450
|
+
names: set[str] = set()
|
|
451
|
+
for node in graph.nodes:
|
|
452
|
+
value = getattr(node, "container_name", None)
|
|
453
|
+
if isinstance(value, str):
|
|
454
|
+
names.add(value)
|
|
455
|
+
return names
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def _validate_container_names(spec: DrillSpec, graph: TopologyGraph) -> None:
|
|
459
|
+
"""Every container name in the spec must exist in the topology."""
|
|
460
|
+
graph_names = _container_names(graph)
|
|
461
|
+
missing = [name for name in spec.containers if name not in graph_names]
|
|
462
|
+
if missing:
|
|
463
|
+
raise PlanningError(
|
|
464
|
+
f"container(s) not found in topology: {sorted(missing)} — "
|
|
465
|
+
f"available: {sorted(graph_names)}"
|
|
466
|
+
)
|
|
467
|
+
# Also validate execution-block references name defined containers.
|
|
468
|
+
for block in spec.execution:
|
|
469
|
+
referenced = list(block.parallel or ()) + list(block.sequential or ())
|
|
470
|
+
for name in referenced:
|
|
471
|
+
if name not in spec.containers:
|
|
472
|
+
raise PlanningError(
|
|
473
|
+
f"execution references container {name!r} which is not defined "
|
|
474
|
+
f"under 'containers'"
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _find_container_nodes(graph: TopologyGraph, container_name: str) -> tuple[TopologyNode, ...]:
|
|
479
|
+
"""All topology nodes whose ``container_name`` matches the drill name."""
|
|
480
|
+
matched = tuple(
|
|
481
|
+
node for node in graph.nodes if getattr(node, "container_name", None) == container_name
|
|
482
|
+
)
|
|
483
|
+
if not matched:
|
|
484
|
+
raise PlanningError(f"container {container_name!r} not found in topology graph")
|
|
485
|
+
return matched
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _plan_container_faults(
|
|
489
|
+
container_name: str,
|
|
490
|
+
container: DrillContainer,
|
|
491
|
+
graph: TopologyGraph,
|
|
492
|
+
out: list[PlannedStep],
|
|
493
|
+
seq: int,
|
|
494
|
+
*,
|
|
495
|
+
recovery_default: bool = True,
|
|
496
|
+
on_failure_default: OnFailure = OnFailure.ABORT_AND_RECOVER,
|
|
497
|
+
spec_dir: str | None = None,
|
|
498
|
+
) -> int:
|
|
499
|
+
"""Plan every fault on the container as its own compensatable step.
|
|
500
|
+
|
|
501
|
+
Each fault becomes an independently compensatable step (write-ahead undo
|
|
502
|
+
per fault); steps advance ``seq`` by one so a multi-fault container never
|
|
503
|
+
double-injects concurrently. Returns the number of steps emitted so the
|
|
504
|
+
caller can advance its sequence counter.
|
|
505
|
+
"""
|
|
506
|
+
if not container.faults:
|
|
507
|
+
# Container referenced in execution but defines no faults: emit a
|
|
508
|
+
# no-op placeholder step so the plan stays traceable.
|
|
509
|
+
out.append(
|
|
510
|
+
PlannedStep(
|
|
511
|
+
id=f"{container_name}-{seq:04d}",
|
|
512
|
+
seq=seq,
|
|
513
|
+
raw_action=Wait(type="wait", duration=0.0),
|
|
514
|
+
)
|
|
515
|
+
)
|
|
516
|
+
return 1
|
|
517
|
+
|
|
518
|
+
# A container name matches the whole subtree (service, container, process).
|
|
519
|
+
matched = tuple(_find_container_nodes(graph, container_name))
|
|
520
|
+
# Every fault on this container belongs to one persistent group (ADR-M2-1):
|
|
521
|
+
# members share an execution_group_id and run sequentially one-at-a-time so
|
|
522
|
+
# a multi-fault container never double-injects concurrently.
|
|
523
|
+
group_id = f"grp-{uuid.uuid4().hex[:12]}"
|
|
524
|
+
mode = GroupMode.SEQUENTIAL
|
|
525
|
+
path = f"/{container_name}"
|
|
526
|
+
for i, drill_fault in enumerate(container.faults):
|
|
527
|
+
out.append(
|
|
528
|
+
_plan_fault_step(
|
|
529
|
+
container_name,
|
|
530
|
+
matched,
|
|
531
|
+
drill_fault,
|
|
532
|
+
graph,
|
|
533
|
+
seq + i,
|
|
534
|
+
execution_group_id=group_id,
|
|
535
|
+
group_mode=mode,
|
|
536
|
+
spec_dir=spec_dir,
|
|
537
|
+
group_path=path,
|
|
538
|
+
recovery=(
|
|
539
|
+
drill_fault.recovery if drill_fault.recovery is not None else recovery_default
|
|
540
|
+
),
|
|
541
|
+
on_failure=(
|
|
542
|
+
drill_fault.on_failure
|
|
543
|
+
if drill_fault.on_failure is not None
|
|
544
|
+
else on_failure_default
|
|
545
|
+
),
|
|
546
|
+
)
|
|
547
|
+
)
|
|
548
|
+
return len(container.faults)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _plan_fault_step(
|
|
552
|
+
container_name: str,
|
|
553
|
+
matched: tuple[TopologyNode, ...],
|
|
554
|
+
drill_fault: DrillFault,
|
|
555
|
+
graph: TopologyGraph,
|
|
556
|
+
seq: int,
|
|
557
|
+
*,
|
|
558
|
+
execution_group_id: str | None = None,
|
|
559
|
+
group_mode: GroupMode | None = None,
|
|
560
|
+
group_path: str | None = None,
|
|
561
|
+
recovery: bool = True,
|
|
562
|
+
on_failure: OnFailure | None = None,
|
|
563
|
+
spec_dir: str | None = None,
|
|
564
|
+
) -> PlannedStep:
|
|
565
|
+
"""Compile one drill fault into a compensatable :class:`PlannedStep`."""
|
|
566
|
+
try:
|
|
567
|
+
definition = definition_for(drill_fault.fault)
|
|
568
|
+
except (SchemaValidationError, LookupError) as exc:
|
|
569
|
+
raise PlanningError(str(exc)) from None
|
|
570
|
+
|
|
571
|
+
# The fault applies to a specific kind subset, so target only the nodes it
|
|
572
|
+
# can actually act on.
|
|
573
|
+
nodes = tuple(n for n in matched if n.kind in definition.applicable_node_kinds)
|
|
574
|
+
if not nodes:
|
|
575
|
+
kinds = ", ".join(sorted({n.kind.value for n in matched}))
|
|
576
|
+
raise PlanningError(
|
|
577
|
+
f"fault {definition.id!r} does not apply to any node matched by "
|
|
578
|
+
f"container {container_name!r} (matched kinds: {kinds})"
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
# Compensation runs against the drill container's whole subtree: the
|
|
582
|
+
# fault's own matching nodes plus the container/process address the undo op
|
|
583
|
+
# executes in. Drill faults name containers; e.g. cpu.saturate targets
|
|
584
|
+
# SERVICE only, but its payload undo must still reach the container that
|
|
585
|
+
# backs the service (ADR-0020).
|
|
586
|
+
compensation_nodes = list(matched)
|
|
587
|
+
seen = {id(node) for node in compensation_nodes}
|
|
588
|
+
for node in matched:
|
|
589
|
+
for proc in graph.connected_processes(node.id):
|
|
590
|
+
if id(proc) not in seen:
|
|
591
|
+
seen.add(id(proc))
|
|
592
|
+
compensation_nodes.append(proc)
|
|
593
|
+
|
|
594
|
+
# ``duration`` is a Duration (float); the ``"10s"`` class default reaches
|
|
595
|
+
# the runtime as an unvalidated str unless explicitly passed through
|
|
596
|
+
# validation, so resolve both forms before comparing against the cap.
|
|
597
|
+
raw_duration: int | float | str = cast("int | float | str", drill_fault.duration)
|
|
598
|
+
duration_s = (
|
|
599
|
+
parse_duration(raw_duration) if isinstance(raw_duration, str) else float(raw_duration)
|
|
600
|
+
)
|
|
601
|
+
if duration_s > definition.max_duration_s:
|
|
602
|
+
raise PlanningError(
|
|
603
|
+
f"fault {definition.id!r} duration {duration_s}s exceeds cap "
|
|
604
|
+
f"{definition.max_duration_s}s"
|
|
605
|
+
)
|
|
606
|
+
# DrillFault accepts fault parameters as extra YAML keys on the fault
|
|
607
|
+
# entry (e.g. ``percent: 80``), collected via ``model_extra``; an explicit
|
|
608
|
+
# ``params:`` mapping is honored as well. ``duration``, ``on_failure`` and
|
|
609
|
+
# ``targets`` are declared fields and never treated as parameters.
|
|
610
|
+
raw_params: dict[str, object] = {}
|
|
611
|
+
explicit = getattr(drill_fault, "params", None)
|
|
612
|
+
if isinstance(explicit, dict):
|
|
613
|
+
raw_params.update(explicit)
|
|
614
|
+
for key, value in (getattr(drill_fault, "model_extra", None) or {}).items():
|
|
615
|
+
if key == "params":
|
|
616
|
+
continue # already merged from the explicit ``params:`` group
|
|
617
|
+
if value is not None:
|
|
618
|
+
raw_params.setdefault(key, value)
|
|
619
|
+
params = definition.validate_params(raw_params)
|
|
620
|
+
|
|
621
|
+
# ``net.load`` accepts a host-side k6 ``script.js`` (param ``script``). The
|
|
622
|
+
# script runs on the drill host at the target container's ip:port, so its
|
|
623
|
+
# content is embedded into the frozen plan here — resolved relative to
|
|
624
|
+
# the drill spec directory — rather than read at execution time.
|
|
625
|
+
params = _embed_load_script(params, definition.id, spec_dir)
|
|
626
|
+
|
|
627
|
+
selectors = tuple(TargetSelector(kind=node.kind, expr=node.name) for node in nodes)
|
|
628
|
+
planned = PlannedFault(
|
|
629
|
+
fault_id=definition.id,
|
|
630
|
+
targets=tuple(
|
|
631
|
+
ResolvedTarget(selector=selector, node_ids=frozenset({node.id}))
|
|
632
|
+
for selector, node in zip(selectors, nodes, strict=True)
|
|
633
|
+
),
|
|
634
|
+
params=params,
|
|
635
|
+
duration=drill_fault.duration,
|
|
636
|
+
backend=None,
|
|
637
|
+
runtime_identity=_resolve_planned_identity(matched),
|
|
638
|
+
recovery=recovery,
|
|
639
|
+
on_failure=on_failure if on_failure is not None else OnFailure.ABORT_AND_RECOVER,
|
|
640
|
+
)
|
|
641
|
+
planned = compensated(planned, tuple(compensation_nodes))
|
|
642
|
+
if not planned.undo_ops:
|
|
643
|
+
msg = f"fault {definition.id!r} compiled without undo contract"
|
|
644
|
+
raise InvariantViolationError("plan_write_ahead_undo", msg)
|
|
645
|
+
return PlannedStep(
|
|
646
|
+
id=f"{container_name}-{seq:04d}",
|
|
647
|
+
seq=seq,
|
|
648
|
+
fault=planned,
|
|
649
|
+
runtime_identity=_resolve_planned_identity(matched),
|
|
650
|
+
execution_group_id=execution_group_id,
|
|
651
|
+
group_mode=group_mode,
|
|
652
|
+
group_path=group_path,
|
|
653
|
+
raw_action=InjectFault(
|
|
654
|
+
fault=definition.id,
|
|
655
|
+
selectors=selectors,
|
|
656
|
+
params=params,
|
|
657
|
+
duration=drill_fault.duration,
|
|
658
|
+
),
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _resolve_planned_identity(nodes: tuple[TopologyNode, ...]) -> RuntimeIdentity | None:
|
|
663
|
+
"""The canonical identity backing the authored name (ADR-M1-1).
|
|
664
|
+
|
|
665
|
+
The plan targets the whole matched subtree (service/container/process), but
|
|
666
|
+
the canonical ``RuntimeIdentity`` is contributed by the container node —
|
|
667
|
+
the only node kind that carries it (topology.py). Resolver-key nodes
|
|
668
|
+
(service/process) contribute no identity; their plan identity stays ``None``
|
|
669
|
+
until a runtime adapter supplies one. This satisfies the milestone bar:
|
|
670
|
+
identity is populated for a container-targeted fault whenever a provider is
|
|
671
|
+
available.
|
|
672
|
+
"""
|
|
673
|
+
for node in nodes:
|
|
674
|
+
identity: object = getattr(node, "runtime_identity", None)
|
|
675
|
+
if isinstance(identity, RuntimeIdentity):
|
|
676
|
+
return identity
|
|
677
|
+
return None
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def restrict_plan_to_container(
|
|
681
|
+
plan: ExecutionPlan, container_name: str, graph: TopologyGraph
|
|
682
|
+
) -> ExecutionPlan:
|
|
683
|
+
"""Scope a compiled plan to one container subtree (``mayhem --ctr``).
|
|
684
|
+
|
|
685
|
+
Fault steps survive when any resolved target belongs to the container's
|
|
686
|
+
service/container/process subtree; the subtree's own orchestration steps
|
|
687
|
+
(``group_path == "/<container_name>"``) survive too. Faults, waits and
|
|
688
|
+
checks for every other container are removed, so the run executes only
|
|
689
|
+
against the requested container. All remaining steps keep their original
|
|
690
|
+
ids and sequence — skipped ids are intentional (ADR-0021 traceability).
|
|
691
|
+
|
|
692
|
+
Raises :class:`PlanningError` when the graph has no such container or the
|
|
693
|
+
plan carries no fault step against it.
|
|
694
|
+
"""
|
|
695
|
+
subtree = graph.node_ids_for_container(container_name)
|
|
696
|
+
if not subtree:
|
|
697
|
+
available = ", ".join(graph.container_names()) or "<none>"
|
|
698
|
+
raise PlanningError(
|
|
699
|
+
f"container {container_name!r} not found in topology graph (available: {available})"
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
kept: list[PlannedStep] = []
|
|
703
|
+
for step in plan.steps:
|
|
704
|
+
if step.fault is None:
|
|
705
|
+
if step.group_path == f"/{container_name}":
|
|
706
|
+
kept.append(step)
|
|
707
|
+
continue
|
|
708
|
+
if any(target.node_ids & subtree for target in step.fault.targets):
|
|
709
|
+
kept.append(step)
|
|
710
|
+
|
|
711
|
+
if not any(step.fault is not None for step in kept):
|
|
712
|
+
raise PlanningError(
|
|
713
|
+
f"plan {plan.run_id!r} has no fault steps targeting container {container_name!r}"
|
|
714
|
+
)
|
|
715
|
+
return plan.model_copy(update={"steps": tuple(kept)})
|