mayhem-cli 0.5.1__py3-none-any.whl

mayhem/controller/safety.py

@@ -0,0 +1,392 @@
+"""Mechanical safety gates between planner and executor (ADR-0012, ADR-0014,
+architecture/safety.md).
+
+Gate stack enforced here:
+  G1 config policy      — allowlists/denylists, risk ladder, critical opt-in
+  G2 plan validation    — budgets fit the topology graph, fingerprint match
+  G3 pre-exec assertion — resolved targets re-checked against *live* topology
+  G4 execution context  — declared context must be feasible for target node kinds
+
+Precedence: denylist beats allowlist beats selector beats default.
+Every refusal is typed and carries a machine-readable reason.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from mayhem.domain.errors import InvariantViolationError, TargetResolutionError
+from mayhem.domain.execution_context import ExecutionContext
+from mayhem.domain.risks import RiskLevel
+from mayhem.domain.runtime_adapter import (
+    CapabilityRequirements,
+    CapabilityVerdict,
+    RuntimeAdapter,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from mayhem.config import PolicyCfg
+    from mayhem.domain.experiments import BlastRadiusBudget, ExecutionPlan, PlannedFault
+    from mayhem.domain.topology import NodeKind, TargetSelector, TopologyGraph
+
+
+class SafetyRefusedError(InvariantViolationError):
+    """A mechanical safety gate refused a fault or plan (``safety.refused``)."""
+
+    def __init__(self, reason_code: str, message: str) -> None:
+        super().__init__(reason_code, message)
+        self.reason_code = reason_code
+
+
+def environment_fingerprint(
+    *,
+    host_names: Iterable[str],
+    compose_digest: str,
+    profile: str | None = None,
+) -> str:
+    """SHA256(sorted host set + compose digest + profile name)."""
+    payload = "|".join(
+        (
+            ",".join(sorted(host_names)),
+            compose_digest,
+            profile or "default",
+        )
+    )
+    return hashlib.sha256(payload.encode()).hexdigest()
+
+
+@dataclass(frozen=True)
+class SafetyContext:
+    """Everything G1/G2 need; built once per CLI invocation."""
+
+    policy: PolicyCfg
+    budget: BlastRadiusBudget
+    fingerprint: str
+    allow_critical_cli: bool = False
+    warnings: list[str] = field(default_factory=list)
+
+
+DEFAULT_DENY = frozenset({"node.reboot"})
+"""Host-reboot class faults are forbidden unless explicitly allowlisted
+([ADR-0012]: an explicit allowlist overrides default deny; the bare default
+policy never grants them)."""
+
+
+def check_fault_admission(fault_id: str, definition_risk: RiskLevel, ctx: SafetyContext) -> None:
+    """G1: denylist → allowlist → risk ceiling → critical double-opt-in."""
+    if fault_id in ctx.policy.deny_faults:
+        raise SafetyRefusedError("safety.refused", f"{fault_id}: denied by policy denylist")
+    if ctx.policy.allow_faults is not None:
+        if fault_id not in ctx.policy.allow_faults:
+            raise SafetyRefusedError(
+                "safety.refused", f"{fault_id}: not present in policy allowlist"
+            )
+    elif fault_id in DEFAULT_DENY:
+        raise SafetyRefusedError(
+            "safety.refused",
+            f"{fault_id}: denied by default; add it to policy.allow_faults to opt in",
+        )
+    ceiling = ctx.policy.risk_ceiling
+    if ceiling is not None and definition_risk.at_least(ceiling.next_higher()):
+        raise SafetyRefusedError(
+            "safety.refused",
+            f"{fault_id}: risk {definition_risk.value} exceeds policy ceiling {ceiling.value}",
+        )
+    if definition_risk is RiskLevel.CRITICAL and not (
+        ctx.policy.allow_critical and ctx.allow_critical_cli
+    ):
+        raise SafetyRefusedError(
+            "safety.refused",
+            f"{fault_id}: critical risk requires config policy.allow_critical AND --allow-critical",
+        )
+
+
+def _affected_node_ids(graph: TopologyGraph, targets: Iterable[str]) -> frozenset[str]:
+    affected: set[str] = set()
+    for node_id in targets:
+        affected.add(node_id)
+        affected |= graph.dependents_closure(node_id)
+    return frozenset(affected)
+
+
+def check_blast_radius(
+    graph: TopologyGraph,
+    target_node_ids: Iterable[str],
+    duration_s: float,
+    fault_ids_so_far: tuple[str, ...],
+    new_fault_id: str,
+    *,
+    ctx: SafetyContext,
+) -> dict[str, float]:
+    """G2 (budget half): topology-derived blast radius must fit; returns measured stats."""
+    budget = ctx.budget
+    affected = _affected_node_ids(graph, target_node_ids)
+    services_total = len(graph.of_kind(_service_kind()))
+    services_hit = sum(1 for n in graph.of_kind(_service_kind()) if n.id in affected)
+    hosts_hit = sum(1 for n in graph.of_kind(_host_kind()) if n.id in affected)
+    pct = (services_hit / services_total * 100.0) if services_total else 0.0
+    stats = {
+        "services_pct": round(pct, 1),
+        "hosts": float(hosts_hit),
+        "concurrent_faults": float(len(fault_ids_so_far) + 1),
+        "duration_per_fault": duration_s,
+    }
+    if pct > budget.max_services_pct:
+        raise SafetyRefusedError(
+            "safety.refused",
+            f"blast radius: {new_fault_id} would affect {pct:.0f}% of services"
+            f" > budget {budget.max_services_pct}%",
+        )
+    if hosts_hit > budget.max_hosts:
+        raise SafetyRefusedError(
+            "safety.refused",
+            f"blast radius: {new_fault_id} touches {hosts_hit} hosts > budget {budget.max_hosts}",
+        )
+    if len(fault_ids_so_far) + 1 > budget.max_concurrent_faults:
+        raise SafetyRefusedError("safety.refused", "blast radius: max_concurrent_faults exceeded")
+    if duration_s > budget.max_duration_per_fault_s:
+        raise SafetyRefusedError(
+            "safety.refused",
+            f"{new_fault_id} duration {duration_s:.0f}s exceeds per-fault cap"
+            f" {budget.max_duration_per_fault_s:.0f}s",
+        )
+    pair = frozenset((*fault_ids_so_far, new_fault_id)) if fault_ids_so_far else None
+    if pair and pair in budget.forbidden_fault_pairs:
+        raise SafetyRefusedError(
+            "safety.refused",
+            f"blast radius: forbidden fault pair {sorted(pair)}",
+        )
+    return stats
+
+
+def _check_execution_context(fault: PlannedFault, graph: TopologyGraph) -> None:
+    """G4: validate that the declared execution context is feasible for all targets.
+
+    Prevents accidental host-level execution when the experiment intended
+    container-level execution, or vice versa.
+    """
+    if fault.execution_context is None:
+        return
+    node_kinds: set[NodeKind] = set()
+    for target in fault.targets:
+        for node_id in target.node_ids:
+            node = graph.by_id(node_id)
+            if node is not None:
+                node_kinds.add(node.kind)
+    if not node_kinds:
+        return  # no nodes resolved — G3 will catch this
+    try:
+        fault.execution_context.assert_compatible(frozenset(node_kinds))
+    except InvariantViolationError as exc:
+        raise SafetyRefusedError(
+            "execution_context.refused",
+            f"{fault.fault_id}: {exc}",
+        ) from exc
+
+
+_K8S_NODE_KIND_VALUES: frozenset[str] = frozenset({"pod", "k8s_node"})
+_K8S_REFUSE_MSG = (
+    "kubernetes execution not yet supported; see the RuntimeAdapter contract at ADR-M7-1"
+)
+
+_REMOTE_NODE_KIND_VALUES: frozenset[str] = frozenset({"external_dependency"})
+_REMOTE_REFUSE_MSG = (
+    "remote execution not yet supported; ADR-M3-5 ships only the "
+    "RemoteAgentInterface contract — no transport is wired in this milestone"
+)
+
+
+def _check_k8s_targets(plan: ExecutionPlan, graph: TopologyGraph) -> None:
+    """Refuse any plan that targets K8s node kinds without a live driver (ADR-M7).
+
+    K8s execution is out-of-scope for this milestone — the adapter contract
+    exists so future drivers can implement against a stable seam, but no
+    live-cluster fault injection is wired yet.  A plan targeting K8s nodes
+    must fail loud and early with an actionable message.
+    """
+    for step in plan.steps:
+        fault = step.fault
+        if fault is None:
+            continue
+        for target in fault.targets:
+            for node_id in target.node_ids:
+                node = graph.by_id(node_id)
+                if node is not None and node.kind in _K8S_NODE_KIND_VALUES:
+                    raise SafetyRefusedError(
+                        "k8s.unsupported",
+                        f"{fault.fault_id}: {_K8S_REFUSE_MSG}",
+                    )
+
+
+def _check_remote_targets(plan: ExecutionPlan, graph: TopologyGraph) -> None:
+    """Hard planning gate for remote targets (ADR-M3-5) — defect register #4.
+
+    Remote execution is interface-only (``RemoteAgentInterface``): the adapter
+    rejects requirements at ``evaluate`` time, but that refusal was never
+    auto-wired into ``validate_plan``, so a remote spec would plan and only
+    fail mid-execution.  This gate mirrors the K8s one: a plan targeting an
+    ``external_dependency`` node fails loud and early at plan time.
+    """
+    for step in plan.steps:
+        fault = step.fault
+        if fault is None:
+            continue
+        for target in fault.targets:
+            for node_id in target.node_ids:
+                node = graph.by_id(node_id)
+                if node is not None and node.kind in _REMOTE_NODE_KIND_VALUES:
+                    raise SafetyRefusedError(
+                        "remote.unsupported",
+                        f"{fault.fault_id}: {_REMOTE_REFUSE_MSG}",
+                    )
+
+
+def validate_plan(
+    plan: ExecutionPlan,
+    graph: TopologyGraph,
+    ctx: SafetyContext,
+    adapter: RuntimeAdapter | None = None,
+) -> None:
+    """G1+G2+G4 over every fault step of an already-compiled plan.
+
+    When *adapter* is provided, capability requirements are derived from the
+    plan's execution contexts and evaluated against the adapter.  UNSUPPORTED
+    verdicts block the plan; ALTERNATIVE verdicts are tolerated but emit a
+    warning (ADR-M3-2).
+    """
+    if plan.environment_fingerprint != ctx.fingerprint:
+        raise SafetyRefusedError(
+            "environment.mismatch",
+            "plan fingerprint does not match current environment identity;"
+            " re-plan against live topology",
+        )
+    _check_k8s_targets(plan, graph)
+    _check_remote_targets(plan, graph)
+    if adapter is not None:
+        _validate_capability_requirements(plan, adapter, ctx)
+    seen_faults: list[str] = []
+    for step in plan.steps:
+        fault = step.fault
+        if fault is None:
+            continue
+        definition_risk = _risk_of(fault.fault_id)
+        check_fault_admission(fault.fault_id, definition_risk, ctx)
+        target_ids = frozenset().union(*(t.node_ids for t in fault.targets))
+        check_blast_radius(
+            graph,
+            target_ids,
+            float(fault.duration),
+            tuple(seen_faults),
+            fault.fault_id,
+            ctx=ctx,
+        )
+        seen_faults.append(fault.fault_id)
+        # G4: validate execution context compatibility
+        if fault.execution_context is not None:
+            _check_execution_context(fault, graph)
+
+
+def _validate_capability_requirements(
+    plan: ExecutionPlan,
+    adapter: RuntimeAdapter,
+    ctx: SafetyContext,
+) -> None:
+    """ADR-M3-2: evaluate plan capability requirements against an adapter.
+
+    Unsupported verdicts block the plan; alternatives warn.
+    """
+    namespaces: set[str] = set()
+    tools: set[str] = set()
+    permissions: set[str] = set()
+    for step in plan.steps:
+        fault = step.fault
+        if fault is None:
+            continue
+        if fault.execution_loci is not None:
+            target = fault.execution_loci.get("target")
+            if isinstance(target, str) and target.startswith("network_namespace"):
+                namespaces.add(target)
+        if fault.execution_context is not None and (
+            fault.execution_context.context
+            in (ExecutionContext.NETWORK_NAMESPACE, ExecutionContext.PROCESS)
+        ):
+            namespaces.add(fault.execution_context.context.value)
+        for target in fault.targets:
+            for node_id in target.node_ids:
+                if node_id.startswith("net"):
+                    namespaces.add(node_id)
+                if node_id.startswith(("p-", "proc")):
+                    permissions.add("limit")
+
+    reqs = CapabilityRequirements(
+        namespaces=frozenset(namespaces),
+        tools=frozenset(tools),
+        permissions=frozenset(permissions),
+    )
+    fallback = CapabilityRequirements(
+        namespaces=frozenset({"network"}),
+        tools=frozenset({"tool"}),
+        permissions=frozenset({"limit"}),
+    )
+    if not (namespaces or tools or permissions):
+        result = adapter.evaluate(fallback)
+    else:
+        result = adapter.evaluate(reqs)
+    if result.blocking:
+        message = result.refuse_with_message() or "unsupported capability requirements"
+        raise SafetyRefusedError("capability.unsupported", f"{adapter.id}: {message}")
+    for key, verdict in result.verdicts.items():
+        if verdict == CapabilityVerdict.ALTERNATIVE:
+            ctx.warnings.append(f"{adapter.id}: capability '{key}' satisfied via ALTERNATIVE path")
+
+
+def pre_exec_assertion(
+    target_selector_pairs: Iterable[tuple[TargetSelector, frozenset[str] | tuple[str, ...]]],
+    live_graph: TopologyGraph,
+) -> None:
+    """G3: seconds before injection, re-check selectors against live topology."""
+    for selector, expected_ids in target_selector_pairs:
+        try:
+            live = live_graph.resolve(selector)
+        except TargetResolutionError as exc:
+            raise SafetyRefusedError(
+                "target.drift", f"G3 drift refusal for {selector}: {exc}"
+            ) from None
+        live_ids = frozenset(n.id for n in live)
+        missing = set(expected_ids) - live_ids
+        if missing:
+            raise SafetyRefusedError(
+                "target.drift",
+                f"G3 drift refusal: nodes vanished since planning: {sorted(missing)}",
+            )
+
+
+# -- helpers ---------------------------------------------------------------------------
+
+
+def _risk_of(fault_id: str) -> RiskLevel:
+    from mayhem.domain.catalog import (
+        definition_for,
+    )  # local: keep module import graph flat
+    from mayhem.domain.errors import SchemaValidationError
+
+    try:
+        return definition_for(fault_id).risk
+    except SchemaValidationError:
+        return RiskLevel.LOW
+
+
+def _service_kind() -> NodeKind:
+    from mayhem.domain.topology import NodeKind
+
+    return NodeKind.SERVICE
+
+
+def _host_kind() -> NodeKind:
+    from mayhem.domain.topology import NodeKind
+
+    return NodeKind.HOST

mayhem/domain/__init__.py

@@ -0,0 +1,6 @@
+"""Tgondi domain models.
+
+Pure, importable-by-everything data layer (ADR-0002): no IO, no upward imports.
+All models serialize deterministically via ``model_dump(mode="json")``; enums
+serialize as their string values and timestamps are ISO-8601 UTC.
+"""

mayhem/domain/campaigns.py

@@ -0,0 +1,118 @@
+"""Campaign model — orchestrating multiple experiments (ADR-0022).
+
+A ``Campaign`` groups multiple experiments under a single scheduling and
+execution umbrella, with start/stop times, concurrency limits, and a
+policy for what happens when an experiment fails.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from mayhem.domain.common import Duration
+from mayhem.domain.errors import InvariantViolationError
+from mayhem.domain.risks import RiskLevel
+
+
+class CampaignStatus(StrEnum):
+    DRAFT = "draft"
+    SCHEDULED = "scheduled"
+    RUNNING = "running"
+    PAUSED = "paused"
+    COMPLETED = "completed"
+    ABORTED = "aborted"
+
+
+class ExperimentOnFailure(StrEnum):
+    """What happens when an experiment in a campaign fails."""
+
+    ABORT_CAMPAIGN = "abort_campaign"
+    SKIP_AND_CONTINUE = "skip_and_continue"
+    RETRY_THEN_ABORT = "retry_then_abort"
+
+
+class CampaignExperiment(BaseModel):
+    """A single experiment entry within a campaign.
+
+    References an experiment spec by name or ID. The actual spec is resolved
+    at scheduling time.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    experiment_ref: str  # name or ID of the experiment
+    priority: int = Field(default=0, ge=0)  # higher = runs first
+    delay_seconds: Duration = 0.0  # delay after previous experiment completes
+    weight: float = 1.0  # for weighted random selection
+
+
+class CampaignWindow(BaseModel):
+    """Time window for campaign execution."""
+
+    model_config = ConfigDict(frozen=True)
+
+    start_epoch_s: float | None = None
+    end_epoch_s: float | None = None
+    max_duration_s: Duration = 3600.0  # hard stop
+    cooldown_between_experiments_s: Duration = 5.0
+
+
+class CampaignPolicy(BaseModel):
+    """Execution policies for a campaign."""
+
+    model_config = ConfigDict(frozen=True)
+
+    on_experiment_failure: ExperimentOnFailure = ExperimentOnFailure.ABORT_CAMPAIGN
+    max_concurrent_experiments: int = Field(default=1, ge=1)
+    max_risk_level: RiskLevel | None = None  # risk ceiling for all experiments
+    total_budget_usd: float | None = None  # cost ceiling (future use)
+    require_approval_above: RiskLevel | None = RiskLevel.HIGH
+
+
+class Campaign(BaseModel):
+    """A campaign is a named, schedulable collection of experiments."""
+
+    model_config = ConfigDict(frozen=True)
+
+    id: str
+    name: str
+    description: str = ""
+    status: CampaignStatus = CampaignStatus.DRAFT
+    experiments: tuple[CampaignExperiment, ...] = ()
+    window: CampaignWindow = Field(default_factory=CampaignWindow)
+    policy: CampaignPolicy = Field(default_factory=CampaignPolicy)
+    labels: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("experiments")
+    @classmethod
+    def _non_empty_experiments(
+        cls, value: tuple[CampaignExperiment, ...]
+    ) -> tuple[CampaignExperiment, ...]:
+        if not value:
+            raise InvariantViolationError(
+                "campaign_requires_experiments",
+                "a campaign must have at least one experiment",
+            )
+        return value
+
+    def sorted_experiments(self) -> tuple[CampaignExperiment, ...]:
+        """Experiments ordered by priority (descending)."""
+        return tuple(sorted(self.experiments, key=lambda e: e.priority, reverse=True))
+
+    def total_weight(self) -> float:
+        """Sum of all experiment weights."""
+        return sum(e.weight for e in self.experiments)
+
+
+class CampaignSchedule(BaseModel):
+    """A scheduled campaign with execution metadata."""
+
+    model_config = ConfigDict(frozen=True)
+
+    campaign: Campaign
+    scheduled_epoch_s: float | None = None
+    last_run_epoch_s: float | None = None
+    run_count: int = 0
+    next_experiment_idx: int = 0

mayhem/domain/cancellation.py

@@ -0,0 +1,110 @@
+"""Cancellation escalation ladder (ADR-M2 Phase 2.5).
+
+Users can request abort/cancel of a running run and escalate if the current
+mechanism fails. The ladder is monotonic:
+
+    grace  ->  term  ->  kill
+
+* ``grace`` — cooperative, safe-abort at the next checkpoint (between steps /
+  fault boundaries). In-flight faults are allowed to finish their undo so the
+  run never leaves a mutated system behind.
+* ``term`` — SIGTERM to live payload toolkit processes; the engine re-asserts
+  after a short grace wait.
+* ``kill`` — SIGKILL to payload toolkit processes; immediate abort.
+
+A :class:`CancellationToken` is the shared, thread-safe object the signal
+handler escalates and the agent loop reads at every safe point. Levels never
+decrease: requesting ``term`` after ``kill`` is a no-op.
+"""
+
+from __future__ import annotations
+
+from enum import IntEnum
+from threading import Lock
+
+_LADDER: tuple[CancellationLevel, ...] = (
+    "grace",
+    "term",
+    "kill",
+)
+
+
+class CancellationLevel(IntEnum):
+    """Monotonic cancellation intensity; IntEnum so ``>=`` comparisons order it."""
+
+    NONE = 0
+    GRACE = 1
+    TERM = 2
+    KILL = 3
+
+    @property
+    def next(self) -> CancellationLevel | None:
+        idx = int(self)
+        nxt = _LADDER[idx] if idx < len(_LADDER) else None
+        return CancellationLevel[nxt.upper()] if nxt is not None else None
+
+    def __str__(self) -> str:  # paint nicely in logs/events
+        return self.name.lower()
+
+
+class CancellationToken:
+    """Thread-safe cancellation signal readable by the agent loop.
+
+    The level only ever rises. Readers observe ``level`` and the convenience
+    flags; the signal layer escalates via :meth:`request` / :meth:`escalate`.
+    """
+
+    __slots__ = ("_level", "_lock", "_mutations")
+
+    def __init__(self, level: CancellationLevel = CancellationLevel.NONE) -> None:
+        self._level: CancellationLevel = CancellationLevel(level)
+        self._lock = Lock()
+        self._mutations = 0
+
+    # -- escalation ----------------------------------------------------------
+
+    def request(self, level: CancellationLevel) -> bool:
+        """Raise the level to *level* (never lower it). Idempotent.
+
+        Returns ``True`` if the level actually changed.
+        """
+        level = CancellationLevel(level)
+        with self._lock:
+            if level > self._level:
+                self._level = level
+                self._mutations += 1
+                return True
+            return False
+
+    def escalate(self) -> CancellationLevel:
+        """Advance one rung up the ladder; ``kill`` is the ceiling.
+
+        Returns the new (effective) level.
+        """
+        with self._lock:
+            nxt = self._level.next
+            if nxt is not None:
+                self._level = nxt
+                self._mutations += 1
+            return self._level
+
+    # -- observation ---------------------------------------------------------
+
+    @property
+    def level(self) -> CancellationLevel:
+        with self._lock:
+            return self._level
+
+    @property
+    def cancelled(self) -> bool:
+        """True at any non-NONE level — agents must stop cooperative work."""
+        return self.level != CancellationLevel.NONE
+
+    @property
+    def is_kill(self) -> bool:
+        return self.level == CancellationLevel.KILL
+
+    def snapshot(self) -> tuple[CancellationLevel, int]:
+        """Atomically read ``(level, mutation_count)`` for change detection."""
+        with self._lock:
+            return self._level, self._mutations