mayhem-cli 0.5.1__py3-none-any.whl

mayhem/domain/execution_context.py

@@ -0,0 +1,120 @@
+"""Execution context model (ADR-0014).
+
+A fault must declare *where* it runs — not just *what* it targets.  The execution
+context distinguishes host-level, container-level, process-level, network-
+namespace, and remote execution so the planner can validate feasibility before
+any mutation occurs.
+
+Backward compatibility: the ``execution`` field on ``InjectFault`` is optional.
+When absent the planner infers the context from the target node kind, which
+matches the pre-0.2.0 behaviour.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, ConfigDict
+
+from mayhem.domain.errors import InvariantViolationError
+
+if TYPE_CHECKING:
+    from mayhem.domain.execution_loci import ThreeLocusContext
+    from mayhem.domain.topology import NodeKind
+
+
+class ExecutionContext(StrEnum):
+    """Where a fault is injected relative to the target."""
+
+    HOST = "host"
+    CONTAINER = "container"
+    PROCESS = "process"
+    NETWORK_NAMESPACE = "network_namespace"
+    REMOTE_HOST = "remote_host"
+    REMOTE_CONTAINER = "remote_container"
+
+
+# Which topology node-kinds are compatible with each context.
+# The planner uses this to reject impossible combinations *before* execution.
+_CONTEXT_COMPATIBILITY: dict[ExecutionContext, frozenset[NodeKind]] = {}
+
+
+def _build_compatibility() -> dict[ExecutionContext, frozenset[NodeKind]]:
+    from mayhem.domain.topology import NodeKind  # local import to break cycles
+
+    return {
+        ExecutionContext.HOST: frozenset({NodeKind.HOST}),
+        ExecutionContext.CONTAINER: frozenset({NodeKind.CONTAINER, NodeKind.SERVICE}),
+        ExecutionContext.PROCESS: frozenset({NodeKind.PROCESS, NodeKind.CONTAINER, NodeKind.HOST}),
+        ExecutionContext.NETWORK_NAMESPACE: frozenset({NodeKind.CONTAINER, NodeKind.HOST}),
+        ExecutionContext.REMOTE_HOST: frozenset({NodeKind.HOST, NodeKind.EXTERNAL_DEPENDENCY}),
+        ExecutionContext.REMOTE_CONTAINER: frozenset({NodeKind.CONTAINER, NodeKind.SERVICE}),
+    }
+
+
+def _compatibility() -> dict[ExecutionContext, frozenset[NodeKind]]:
+    """Lazy-init the compatibility map on first access."""
+    if not _CONTEXT_COMPATIBILITY:
+        _CONTEXT_COMPATIBILITY.update(_build_compatibility())
+    return _CONTEXT_COMPATIBILITY
+
+
+class ExecutionContextSpec(BaseModel):
+    """Declared execution context for a fault step.
+
+    Optional on ``InjectFault`` — when absent the planner infers from the
+    resolved target node kinds.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    context: ExecutionContext
+
+    def assert_compatible(self, node_kinds: frozenset[NodeKind]) -> None:
+        """Raise if *none* of the resolved target node-kinds are compatible.
+
+        Raises:
+            InvariantViolationError: When no target kind matches this context.
+        """
+        compat = _compatibility()
+        allowed = compat.get(self.context, frozenset())
+        if not allowed.intersection(node_kinds):
+            kinds_str = ", ".join(sorted(k.value for k in node_kinds))
+            raise InvariantViolationError(
+                "execution_context_incompatible",
+                f"context '{self.context.value}' cannot execute against "
+                f"node kinds {{{kinds_str}}}; "
+                f"allowed: {{{', '.join(sorted(k.value for k in allowed))}}}",
+            )
+
+    def to_three_locus(self) -> ThreeLocusContext:
+        """Bridge to the three-locus model (ADR-M3-3).
+
+        Returns a ``ThreeLocusContext`` derived from this legacy single-locus
+        spec.  Imported lazily to keep this module dependency-light.
+        """
+        from mayhem.domain.execution_loci import ThreeLocusContext
+
+        return ThreeLocusContext.from_single(self.context)
+
+
+def infer_context_for_node(node_kind: NodeKind) -> ExecutionContext:
+    """Best-default execution context for a given node kind.
+
+    This is the heuristic the planner uses when the author omits an explicit
+    ``execution`` block — it must never be more permissive than the explicit
+    path.
+    """
+    from mayhem.domain.topology import NodeKind as NK  # local import
+
+    mapping: dict[NodeKind, ExecutionContext] = {
+        NK.HOST: ExecutionContext.HOST,
+        NK.CONTAINER: ExecutionContext.CONTAINER,
+        NK.SERVICE: ExecutionContext.CONTAINER,
+        NK.PROCESS: ExecutionContext.PROCESS,
+        NK.EXTERNAL_DEPENDENCY: ExecutionContext.REMOTE_HOST,
+        NK.POD: ExecutionContext.REMOTE_HOST,
+        NK.K8S_NODE: ExecutionContext.REMOTE_HOST,
+    }
+    return mapping.get(node_kind, ExecutionContext.HOST)

mayhem/domain/execution_loci.py

@@ -0,0 +1,94 @@
+"""Three-locus execution context (ADR-M3-3).
+
+Replaces the single-locus ``ExecutionContext`` with three loci that let the
+engine reason about *where the mutation lands* (target locus) vs *where the
+agent/tool actually runs* (agent/tool locus) — local vs remote vs container.
+
+Backward compatibility: ``from_single()`` derives all three loci from the
+legacy single-locus value so pre-0.3.0 specs keep working unchanged.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+from mayhem.domain.execution_context import ExecutionContext
+
+
+class Locus(StrEnum):
+    """Where a given locus physically runs."""
+
+    LOCAL = "local"
+    REMOTE = "remote"
+    CONTAINER = "container"
+    NETWORK_NAMESPACE = "network_namespace"
+    KUBERNETES = "kubernetes"
+
+
+@dataclass(frozen=True)
+class LocusSpec:
+    """A single locus: the kind (where) plus the resolved identifier (what).
+
+    For example a container locus carries the container id; a network-namespace
+    locus carries the ns path.  The identifier is optional because the planner
+    may not yet have resolved it.
+    """
+
+    kind: Locus
+    identifier: str | None = None
+
+
+@dataclass(frozen=True)
+class ThreeLocusContext:
+    """Target / agent / tool loci for one planned fault.
+
+    ``plan_context`` is the required legacy ``ExecutionContext`` retained for
+    M2 drift detection and planner compatibility.  The three loci trade it for
+    higher precision where available.
+    """
+
+    target_locus: LocusSpec
+    agent_locus: LocusSpec
+    tool_locus: LocusSpec
+    plan_context: ExecutionContext
+
+    @classmethod
+    def from_single(cls, ctx: ExecutionContext) -> ThreeLocusContext:
+        """Derive a three-locus context from a legacy single locus.
+
+        For host/remote contexts the agent and tool run on the same node as the
+        target.  For container/process contexts the agent and tool run via the
+        engine (inside the container), while the target is the container or its
+        network namespace.
+        """
+        if ctx in (ExecutionContext.HOST, ExecutionContext.REMOTE_HOST):
+            kind = Locus.LOCAL if ctx == ExecutionContext.HOST else Locus.REMOTE
+            return cls(
+                target_locus=LocusSpec(kind),
+                agent_locus=LocusSpec(kind),
+                tool_locus=LocusSpec(kind),
+                plan_context=ctx,
+            )
+        if ctx in (ExecutionContext.CONTAINER, ExecutionContext.PROCESS):
+            return cls(
+                target_locus=LocusSpec(Locus.CONTAINER),
+                agent_locus=LocusSpec(Locus.CONTAINER),
+                tool_locus=LocusSpec(Locus.CONTAINER),
+                plan_context=ctx,
+            )
+        if ctx == ExecutionContext.NETWORK_NAMESPACE:
+            return cls(
+                target_locus=LocusSpec(Locus.NETWORK_NAMESPACE),
+                agent_locus=LocusSpec(Locus.LOCAL),
+                tool_locus=LocusSpec(Locus.LOCAL),
+                plan_context=ctx,
+            )
+        if ctx == ExecutionContext.REMOTE_CONTAINER:
+            return cls(
+                target_locus=LocusSpec(Locus.CONTAINER),
+                agent_locus=LocusSpec(Locus.REMOTE),
+                tool_locus=LocusSpec(Locus.REMOTE),
+                plan_context=ctx,
+            )
+        raise ValueError(f"unsupported execution context: {ctx!r}")

mayhem/domain/experiments.py

@@ -0,0 +1,370 @@
+"""Experiment specs and execution plans.
+
+``DrillSpec`` is the authored input format (ADR-0019); ``ExecutionPlan`` is
+the frozen, validated output of compilation. The plan pins config/topology
+snapshot ids so any later analysis knows what the controller knew when it
+committed. Since the clean break (ADR-0021) the only supported kind is
+``drill`` — the deterministic/random authoring surfaces were removed.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from mayhem.domain.capabilities import Identifier
+from mayhem.domain.checks import CheckLocus, CheckSpec, Probe
+from mayhem.domain.common import Duration
+from mayhem.domain.decisions import DecisionRef
+from mayhem.domain.errors import InvariantViolationError
+from mayhem.domain.execution_context import ExecutionContextSpec
+from mayhem.domain.faults import FaultCategory
+from mayhem.domain.identity import RuntimeIdentity
+from mayhem.domain.leases import UndoOp, VerifyProbe
+from mayhem.domain.observability import ObservabilityConfig
+from mayhem.domain.risks import RiskLevel
+from mayhem.domain.success import SuccessCriteria
+from mayhem.domain.topology import TargetSelector
+
+
+class ExperimentKind(StrEnum):
+    DETERMINISTIC = "deterministic"
+    RANDOM = "random"
+    DRILL = "drill"
+
+
+class OnFailure(StrEnum):
+    ABORT_AND_RECOVER = "abort_and_recover"
+    CONTINUE = "continue"
+
+
+class BlastRadiusBudget(BaseModel):
+    """Topology-derived limits enforced by the scheduler (ADR-0012 §5)."""
+
+    model_config = ConfigDict(frozen=True)
+
+    max_services_pct: float = Field(default=50.0, gt=0, le=100)
+    max_hosts: int = Field(default=2, ge=1)
+    max_concurrent_faults: int = Field(default=3, ge=1)
+    max_duration_per_fault_s: float = 300.0
+    forbidden_fault_pairs: frozenset[frozenset[str]] = Field(default_factory=frozenset)
+
+
+class Constraints(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    duration_cap: Duration | None = None
+    abort_on_violation: bool = True
+    require_dry_run_first: bool = True
+    risk_ceiling: RiskLevel | None = None  # may only tighten policy ceiling
+    blast_radius: BlastRadiusBudget | None = None
+
+
+# -- step actions ------------------------------------------------------------------
+
+
+class InjectFault(BaseModel):
+    type: Literal["inject_fault"] = "inject_fault"
+    fault: str
+    selectors: tuple[TargetSelector, ...]
+    params: dict[str, object] = Field(default_factory=dict)
+    duration: Duration
+    backend: Identifier | None = None
+    execution: ExecutionContextSpec | None = None  # ADR-0014; None → infer from target
+
+    @field_validator("fault")
+    @classmethod
+    def _known_category(cls, value: str) -> str:
+        FaultCategory.from_fault_id(value)
+        return value
+
+    @field_validator("selectors")
+    @classmethod
+    def _non_empty_selectors(cls, value: tuple[TargetSelector, ...]) -> tuple[TargetSelector, ...]:
+        if not value:
+            raise InvariantViolationError(
+                "step_requires_targets", "inject_fault needs >= 1 selector"
+            )
+        return value
+
+
+class Wait(BaseModel):
+    type: Literal["wait"] = "wait"
+    duration: Duration | None = None
+    until_check_passes: str | None = None  # check id
+    timeout: Duration = 120.0
+
+
+class CheckHttp(BaseModel):
+    """Inline HTTP health check for drill plans (ADR-0019)."""
+
+    type: Literal["check_http"] = "check_http"
+    url: str
+    expected_status: int | None = None
+
+
+class CheckSpecStep(BaseModel):
+    """A drill check step compiled from a :class:`CheckSpec` (ADR-M4-2).
+
+    Carries the fully-resolved probe and its execution locus so the executor
+    can evaluate the check where it is declared to run.
+    """
+
+    type: Literal["check_spec"] = "check_spec"
+    check_id: str
+    probe: Probe
+    execution: CheckLocus | None = None  # None → infer from fault target
+    target: str | None = None
+
+
+# The raw action of a planned step. Drill plans only ever emit inject_fault,
+# wait and check_http — the start_load/stop_load/check/notify/parallel action
+# types were authoring-only and removed with the deterministic/random surfaces.
+StepAction = Annotated[InjectFault | Wait | CheckHttp | CheckSpecStep, Field(discriminator="type")]
+
+
+# -- drill spec (ADR-0019) ------------------------------------------------------------------
+
+
+class ManiacCfg(BaseModel):
+    """Maniac-mode tuning — ``config.maniac`` in a drill spec or ``maniac:``
+    in the layered ``mayhem.yaml`` config (spec-level wins, ADR-M5-1).
+
+    ``level`` — the "level of randomness" dial:
+
+    =====  ==================================================================
+    level  behaviour
+    =====  ==================================================================
+    1      random container, first authored fault on it; no duration jitter
+    2      random container, random one of its authored faults; no jitter
+    3      random container, any fault from the whole spec (cross-locus); no jitter
+    4      cross-locus pool + duration jitter of ±10 %
+    5      cross-locus pool + duration jitter of ±20 % (full chaos)
+    =====  ==================================================================
+
+    Jitter is clamped to the fault's catalog maximum duration and never drops
+    below 1 second; the spec's own safety gates (risk ceiling, blast radius,
+    ``max_faults``, timeout) still apply to every round.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    level: int = Field(default=2, ge=1, le=5)
+    run_level: int = Field(default=10, ge=1, le=500)  # injection rounds
+    seed: int | None = Field(default=None, ge=0)  # reproducible draws
+
+
+class DrillConfig(BaseModel):
+    """Configuration for a drill spec — replaces the separate mayhem.yml."""
+
+    model_config = ConfigDict(frozen=True)
+
+    risk_ceiling: RiskLevel = RiskLevel.HIGH
+    max_faults: int = Field(default=1, ge=0)
+    timeout: Duration = "30m"
+    log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO"
+    # True: auto-recover after each fault (undo contract runs, container restored).
+    # False: keep the perturbation in place after injection — the container stays
+    # faulted so downstream checks observe whether the stack self-heals.
+    recovery: bool = True
+    # Drill-wide failure policy: when a fault round fails, `abort_and_recover`
+    # cancels the remaining steps and recovers, `continue` records the failure
+    # and keeps testing the remaining faults (the run still ends `failed`).
+    # A fault can override this per-fault via its own `on_failure`.
+    on_failure: OnFailure = OnFailure.ABORT_AND_RECOVER
+    # ADR-M5-1: when set, `mayhem maniac` replaces the authored execution with
+    # `maniac.run_level` random (container, fault) rounds dialed by `maniac.level`.
+    # Leave unset to keep `mayhem run` fully deterministic.
+    maniac: ManiacCfg | None = None
+
+
+class DrillFault(BaseModel):
+    """A single fault to inject on a container."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    fault: str
+    duration: Duration = "10s"
+    # Per-fault override of ``config.on_failure``; None ⇒ inherit config.
+    on_failure: OnFailure | None = None
+    targets: tuple[str, ...] = ()  # for network faults: container names to partition
+    network_path: str | None = None  # ADR-M3-7 optional path target for network faults
+    # Optional per-fault override of ``config.recovery``; None ⇒ inherit config.
+    recovery: bool | None = None
+
+
+class DrillContainer(BaseModel):
+    """Faults to run on a specific container (identified by container_name)."""
+
+    model_config = ConfigDict(frozen=True)
+
+    faults: tuple[DrillFault, ...] = ()
+
+
+class CheckExpectation(BaseModel):
+    """What to verify in a check probe."""
+
+    model_config = ConfigDict(frozen=True)
+
+    status: int | None = None
+
+
+class CheckProbe(BaseModel):
+    """A health check to run between execution rounds."""
+
+    model_config = ConfigDict(frozen=True)
+
+    http: str | None = None
+    expect: CheckExpectation = CheckExpectation()
+
+
+class ExecutionStep(BaseModel):
+    """A single step in the execution plan — parallel, sequential, wait, or check."""
+
+    model_config = ConfigDict(frozen=True)
+
+    parallel: tuple[str, ...] | None = None  # container names to run concurrently
+    sequential: tuple[str, ...] | None = None  # container names to run in order
+    wait: Duration | None = None  # seconds to wait after this step
+    check: tuple[CheckProbe, ...] | None = None  # health checks to run
+    check_spec: tuple[CheckSpec, ...] | None = None  # locus-aware checks (ADR-M4-2)
+
+
+class DrillSpec(BaseModel):
+    """Unified drill spec — single YAML file replacing config + fault spec (ADR-0019)."""
+
+    model_config = ConfigDict(frozen=True)
+
+    kind: Literal["drill"]
+    name: str
+    hypothesis: str = ""
+    config: DrillConfig = Field(default_factory=DrillConfig)
+    containers: dict[str, DrillContainer]  # key = container_name from docker-compose
+    execution: tuple[ExecutionStep, ...]
+    success: SuccessCriteria | None = None  # optional machine verdict (ADR-M4-3)
+    observability: ObservabilityConfig | None = None  # optional evidence sources (ADR-M4-4)
+
+    @field_validator("containers")
+    @classmethod
+    def _at_least_one_container(cls, value: dict[str, DrillContainer]) -> dict[str, DrillContainer]:
+        if not value:
+            raise InvariantViolationError("drill_requires_containers", "no containers defined")
+        return value
+
+    @field_validator("execution")
+    @classmethod
+    def _at_least_one_step(cls, value: tuple[ExecutionStep, ...]) -> tuple[ExecutionStep, ...]:
+        if not value:
+            raise InvariantViolationError("drill_requires_execution", "no execution steps defined")
+        return value
+
+
+# -- compiled plan ------------------------------------------------------------------------
+
+
+class ResolvedTarget(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    selector: TargetSelector
+    node_ids: frozenset[str]
+
+    @field_validator("node_ids")
+    @classmethod
+    def _resolved(cls, value: frozenset[str]) -> frozenset[str]:
+        if not value:
+            raise InvariantViolationError("plan_targets_resolved", "selector matched no nodes")
+        return value
+
+
+class PlannedFault(BaseModel):
+    """A fault invocation with compile-time resolution done.
+
+    Carries its compensation contract (write-ahead undo ops + verify probes)
+    decided at planning time — never discovered mid-execution.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    fault_id: str
+    targets: tuple[ResolvedTarget, ...]
+    undo_ops: tuple[UndoOp, ...] = ()
+    verify_probes: tuple[VerifyProbe, ...] = ()
+    params: dict[str, object] = Field(default_factory=dict)
+    duration: Duration
+    backend: Identifier | None = None
+    execution_context: ExecutionContextSpec | None = None  # ADR-0014
+    runtime_identity: RuntimeIdentity | None = None  # planned identity (ADR-M1-1/1-3)
+    execution_loci: dict[str, object] | None = None  # ADR-M3-3 target/agent/tool loci
+    # False ⇒ executor keeps the perturbation in place instead of undoing it
+    # after injection (self-healing observation mode).
+    recovery: bool = True
+    # Resolved failure policy (config default overridden per-fault at planning
+    # time): abort_and_recover cancels the remaining steps on the first failing
+    # round; continue records the failure and keeps testing the rest.
+    on_failure: OnFailure = OnFailure.ABORT_AND_RECOVER
+
+
+class ExecutionPlan(BaseModel):
+    """Frozen contract handed from planning to execution."""
+
+    model_config = ConfigDict(frozen=True)
+
+    run_id: str
+    kind: ExperimentKind
+    steps: tuple[PlannedStep, ...]
+    config_snapshot_id: str
+    topology_snapshot_id: str
+    environment_fingerprint: str
+    seed: int | None = None
+    success: SuccessCriteria | None = None  # copied from the spec (ADR-M4-3)
+    observability: ObservabilityConfig | None = None  # copied from the spec (ADR-M4-4)
+    decision_refs: tuple[DecisionRef, ...] = ()  # governing ADR ids + timestamps
+
+    @model_validator(mode="after")
+    def _check_plan(self) -> ExecutionPlan:
+        return self
+
+
+class GroupMode(StrEnum):
+    """Execution semantics of a fault group (ADR-M2-1)."""
+
+    PARALLEL = "parallel"  # members run concurrently
+    SEQUENTIAL = "sequential"  # members run one-at-a-time in order
+    BEST_EFFORT = "best_effort"  # continue past member failures
+
+
+class FaultGroup(BaseModel):
+    """A set of fault members executed under one persistent identity.
+
+    v1 semantics (ADR-M2-1): ``parallel`` runs members concurrently;
+    ``sequential`` runs them in order; ``best_effort`` continues past member
+    failures. ``atomic`` (strong all-or-nothing) is demoted to
+    compensate-on-failure in v1. Every executed group carries a persistent
+    ``execution_group_id`` and a ``group_path``; member faults share the id.
+    Partial failure is a first-class result (which members succeeded/failed).
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    execution_group_id: str
+    parent_group_id: str | None = None
+    mode: GroupMode = GroupMode.SEQUENTIAL
+    path: str = "/"
+    fault_ids: tuple[str, ...] = ()
+
+
+class PlannedStep(BaseModel):
+    """A step whose fault actions have been fully resolved against topology."""
+
+    model_config = ConfigDict(frozen=True)
+
+    id: str
+    seq: int
+    fault: PlannedFault | None = None
+    raw_action: StepAction  # for non-fault steps (wait/check)
+    runtime_identity: RuntimeIdentity | None = None  # planned identity (ADR-M1-1/1-3)
+    execution_group_id: str | None = None  # group attribution (ADR-M2-1/2-2)
+    group_mode: GroupMode | None = None  # parallel|sequential|best_effort
+    group_path: str | None = None  # hierarchical group location