mayhem-cli 0.5.1__py3-none-any.whl

mayhem/domain/outcomes.py

@@ -0,0 +1,92 @@
+"""Shared outcome/state taxonomy (ADR-M1-3 Phase 1.4).
+
+The outcome vocabulary is a single source of truth that the executor,
+planner, CLI exit mapping, recovery state machine, and a future drift
+*publisher event* all name the same way.
+
+The three constants are deliberately siblings with sharply distinct
+semantics:
+
+* :data:`TARGET_DRIFT` — the plan targeted an identity that no longer exists.
+  The planned ``RuntimeIdentity`` does not match the live one. A drifted
+  target is *mismatched*, never failed.
+* :data:`FAILED_TO_APPLY` — a capability/permission/mutation failure on a
+  *present* target.
+* :data:`RESOURCE_CONFLICT` — ownership/lease contention on a *present*
+  target.
+
+``TARGET_DRIFT`` is declared here (M1) but *detected* in M2. This module
+ships no detection logic; it only fixes the shared vocabulary.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from mayhem.domain.identity import RuntimeIdentity
+
+
+class StepOutcome(StrEnum):
+    """Lead outcome for a single step at execution time (ADR-M1-3).
+
+    Values are exactly the ``step_runs.status`` vocabulary admitted by the
+    persisted schema (migration M0006): ``completed``, ``failed``,
+    ``skipped``, ``cancelled``, ``bypassed``, and ``target_drift``.
+    :attr:`TARGET_DRIFT` is a first-class persisted step state — it is *not* a
+    failure of injection, not a dirty state, and not ownership contention; it
+    means the object the plan intended to touch is no longer the object
+    present.
+    """
+
+    COMPLETED = "completed"
+    FAILED = "failed"
+    TARGET_DRIFT = "target_drift"
+
+
+class TargetOutcome(StrEnum):
+    """Why a particular fault never reached its target (failure taxonomy).
+
+    These classify the *reason* a step did not apply to its target. They
+    refine the raw ``failed`` step outcome, and are the vocabulary used to
+    distinguish drift from capability failures from contention:
+
+    * :attr:`FAILED_TO_APPLY` — capability/permission/mutation failure on a
+      *present* target.
+    * :attr:`RESOURCE_CONFLICT` — ownership/lease contention on a *present*
+      target.
+    * :attr:`TARGET_DRIFT` — the planned identity no longer matches the live
+      identity; the target was *mismatched, not failed*.
+    """
+
+    FAILED_TO_APPLY = "failed_to_apply"
+    RESOURCE_CONFLICT = "resource_conflict"
+    TARGET_DRIFT = "target_drift"
+
+
+@dataclass(frozen=True)
+class DriftEvent:
+    """Payload of a future drift publisher event (declared M1, fired M2).
+
+    No detection logic ships here; this dataclass is the contract consumed by
+    a publisher callback so recovery, reporting, and the CLI all agree on the
+    shape of a drift notification.
+    """
+
+    run_id: str
+    step_id: str
+    planned: RuntimeIdentity
+    live: RuntimeIdentity
+
+
+class DriftPublisher(Protocol):
+    """Callback signature for the drift event publisher (ADR-M1-3).
+
+    A conforming publisher accepts a :class:`DriftEvent` and returns
+    ``None``. The controller may use this to emit an observation, persist a
+    recovery hint, or notify an external surface — enforcement is wired in M2.
+    """
+
+    def __call__(self, event: DriftEvent) -> None: ...

mayhem/domain/remote_agent_interface.py

@@ -0,0 +1,70 @@
+"""Remote-agent interface contract (ADR-M3-5).
+
+Defines the seam a remote execution transport must satisfy.  **No transport is
+implemented in this milestone** — a remote target in a spec fails planning with
+``UNSUPPORTED`` until a future milestone implements a transport (Q11).
+
+The methods below describe the *capability handshake* and *tool run* lifecycle.
+Because no SSH/agent transport exists yet, ``capabilities()`` returns all
+verdicts as ``UNSUPPORTED`` so the planner can refuse remote plans with a clear
+message rather than fail unpredictably mid-run.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from mayhem.domain.runtime_adapter import (
+    CapabilityRequirements,
+    CapabilityVerdict,
+    VerdictResult,
+)
+
+
+@runtime_checkable
+class RemoteAgentInterface(Protocol):
+    """Contract for a remote execution agent (ADR-M3-5).
+
+    Implementations provide a transport (SSH, WireGuard, managed agent, ...).
+    This milestone ships the interface only.
+    """
+
+    def connect(self) -> None: ...
+
+    def capability_handshake(self, reqs: CapabilityRequirements) -> VerdictResult: ...
+
+    def target_resolution(self, target_id: str) -> str: ...
+
+    def tool_run(self, command: list[str], *, timeout_s: float) -> str: ...
+
+    def cancel(self, run_id: str) -> None: ...
+
+    def teardown(self) -> None: ...
+
+
+class RemoteAgentAdapter:
+    """Placeholder adapter so the planner can statically refuse remote plans.
+
+    ``evaluate`` returns UNSUPPORTED for every requirement, which is what the
+    planner uses to reject remote execution with a clear message.
+    """
+
+    def __init__(self, remote_id: str = "remote") -> None:
+        self._remote_id = remote_id
+
+    @property
+    def id(self) -> str:
+        return self._remote_id
+
+    def evaluate(self, reqs: CapabilityRequirements) -> VerdictResult:
+        blocking = True
+        verdicts = {
+            "remote_execution": CapabilityVerdict.UNSUPPORTED.value,
+            "transport": CapabilityVerdict.UNSUPPORTED.value,
+        }
+        return VerdictResult(
+            engine=self._remote_id,
+            requirements=reqs,
+            verdicts=verdicts,
+            blocking=blocking,
+        )

mayhem/domain/resources.py

@@ -0,0 +1,245 @@
+"""Resource ownership and conflict management (ADR-0015).
+
+Every fault that mutates system state must declare exactly which resources it
+creates or modifies.  ``TrackedResource`` is the canonical record; the
+``ResourceOwnershipGraph`` provides in-memory conflict detection; and
+``ResourceManager`` persists + recovers through the SQLite store.
+
+Design principles:
+  * ownership is scoped to (run_id, step_id, fault_id)
+  * recovery is ownership-aware — experiment A never touches experiment B's resources
+  * every resource must have a verify probe — recovery is not complete until verified
+  * orphan detection runs through the Janitor which inspects the ownership graph
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from mayhem.domain.common import utc_now
+from mayhem.domain.leases import UndoOp, VerifyProbe
+
+
+class ResourceType(StrEnum):
+    """Kinds of system resources a fault may create or modifies."""
+
+    TC_RULE = "tc_rule"
+    IPTABLES_RULE = "iptables_rule"
+    NFTABLES_RULE = "nftables_rule"
+    PROCESS_SIGNAL = "process_signal"
+    CGROUP_LIMIT = "cgroup_limit"
+    TOXIPROXY_TOXIC = "toxiproxy_toxic"
+    CONTAINER_STATE = "container_state"
+    TEMPORARY_FILE = "temporary_file"
+    LOAD_GENERATOR = "load_generator"
+    NETWORK_NAMESPACE = "network_namespace"
+    NETWORK_FAULT = "network_fault"
+    FILESYSTEM_MOUNT = "filesystem_mount"
+    PROCESS_SPAWN = "process_spawn"
+    RESOURCE_LIMIT = "resource_limit"
+    GENERIC = "generic"
+
+
+class ResourceState(StrEnum):
+    """Lifecycle of a tracked resource."""
+
+    PENDING = "pending"  # declared, not yet created
+    ACTIVE = "active"  # created and tracked
+    RECOVERING = "recovering"  # cleanup in progress
+    RECOVERED = "recovered"  # verified cleaned
+    ORPHANED = "orphaned"  # owner experiment lost
+    DIRTY = "dirty"  # cleanup failed — needs operator
+
+
+class ConflictKind(StrEnum):
+    """Classification of resource conflicts between experiments."""
+
+    COEXIST = "coexist"  # different resource types on same target — OK
+    SERIALIZE = "serialize"  # same resource type on same target — must not overlap
+    REJECT = "reject"  # irreconcilable — one experiment must not run
+
+
+class TrackedResource(BaseModel):
+    """A single resource created or modified by a fault injection.
+
+    Immutable after creation except for state transitions managed by
+    ``ResourceManager``.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    id: str  # uuid4
+    resource_type: ResourceType
+    owner_run_id: str
+    owner_step_id: str
+    owner_fault_id: str
+    state: ResourceState = ResourceState.PENDING
+    target_identity: str  # immutable reference to the target node
+    cleanup_op: UndoOp
+    verify_probe: VerifyProbe
+    created_at: datetime = Field(default_factory=utc_now)
+    recovered_at: datetime | None = None
+    metadata: dict[str, object] = Field(default_factory=dict)
+    fingerprint: str = ""  # ADR-M3-7 fault-path fingerprint
+
+
+class MutationJournalEntry(BaseModel):
+    """ADR-M2 Phase 2.6 — mutation-boundary journal entry.
+
+    Recorded at the exact boundary when the last undo-fallible op is applied
+    (injection succeeds), not as a post-hoc probe result. Carries the resource
+    owner, the mutation's defining op (e.g. ``kill -9``), and the lease
+    reference so downstream can tell *which* lease holds the mutation and
+    *what* actually mutated the system.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    id: str  # uuid4
+    lease_id: str
+    resource_id: str
+    run_id: str
+    step_id: str
+    fault_id: str
+    defining_op: UndoOp  # the op that actually mutated the system
+    target_identity: str
+    journaled_at: datetime = Field(default_factory=utc_now)
+
+
+class ResourceConflict(BaseModel):
+    """A detected conflict between a new resource and an existing one."""
+
+    model_config = ConfigDict(frozen=True)
+
+    kind: ConflictKind
+    new_resource: TrackedResource
+    existing_resource: TrackedResource
+    reason: str
+
+
+class RecoveryResult(BaseModel):
+    """Outcome of a single resource recovery attempt."""
+
+    model_config = ConfigDict(frozen=True)
+
+    resource_id: str
+    success: bool
+    verified: bool = False
+    error: str | None = None
+
+
+class ResourceOwnershipGraph:
+    """In-memory graph of active resource ownership.
+
+    Not a persistence layer — the ``ResourceManager`` stores rows in SQLite
+    and loads them into this graph at startup.  The graph provides fast
+    conflict detection and ownership queries without touching the database
+    on every check.
+    """
+
+    def __init__(self) -> None:
+        self._resources: dict[str, TrackedResource] = {}
+        # target_identity -> set of resource ids for fast conflict lookup
+        self._by_target: dict[str, set[str]] = {}
+
+    def add(self, resource: TrackedResource) -> None:
+        """Register a resource in the graph."""
+        self._resources[resource.id] = resource
+        by_target = self._by_target.setdefault(resource.target_identity, set())
+        by_target.add(resource.id)
+
+    def remove(self, resource_id: str) -> TrackedResource | None:
+        """Remove a resource from the graph. Returns the resource if present."""
+        resource = self._resources.pop(resource_id, None)
+        if resource is None:
+            return None
+        by_target = self._by_target.get(resource.target_identity)
+        if by_target:
+            by_target.discard(resource_id)
+            if not by_target:
+                del self._by_target[resource.target_identity]
+        return resource
+
+    def get(self, resource_id: str) -> TrackedResource | None:
+        return self._resources.get(resource_id)
+
+    def active_for_target(self, target_identity: str) -> list[TrackedResource]:
+        """All active/pending resources on a given target."""
+        ids = self._by_target.get(target_identity, set())
+        return [
+            r
+            for r in (self._resources.get(rid) for rid in ids)
+            if r is not None and r.state in (ResourceState.ACTIVE, ResourceState.PENDING)
+        ]
+
+    def active_for_run(self, run_id: str) -> list[TrackedResource]:
+        """All active/pending resources owned by a specific run."""
+        return [
+            r
+            for r in self._resources.values()
+            if r.owner_run_id == run_id and r.state in (ResourceState.ACTIVE, ResourceState.PENDING)
+        ]
+
+    def orphans(self) -> list[TrackedResource]:
+        """Resources that have no active owner run (stale after controller crash)."""
+        return [
+            r
+            for r in self._resources.values()
+            if r.state in (ResourceState.ACTIVE, ResourceState.PENDING, ResourceState.RECOVERING)
+        ]
+
+    @property
+    def size(self) -> int:
+        return len(self._resources)
+
+    def detect_conflicts(self, new_resource: TrackedResource) -> list[ResourceConflict]:
+        """Check whether a proposed resource conflicts with existing ones.
+
+        Conflict rules:
+          * Same resource_type + same target → SERIALIZE
+          * Different resource_type + same target → COEXIST
+          * Irreconcilable (e.g. tc qdisc + tc class on same device) → REJECT
+        """
+        conflicts: list[ResourceConflict] = []
+        existing = self.active_for_target(new_resource.target_identity)
+
+        for ex in existing:
+            if ex.owner_run_id == new_resource.owner_run_id:
+                continue  # same run — no conflict
+            if ex.resource_type == new_resource.resource_type:
+                conflicts.append(
+                    ResourceConflict(
+                        kind=ConflictKind.SERIALIZE,
+                        new_resource=new_resource,
+                        existing_resource=ex,
+                        reason=(
+                            f"same resource type '{new_resource.resource_type.value}' "
+                            f"on target '{new_resource.target_identity}' "
+                            f"owned by run '{ex.owner_run_id}'"
+                        ),
+                    )
+                )
+            else:
+                conflicts.append(
+                    ResourceConflict(
+                        kind=ConflictKind.COEXIST,
+                        new_resource=new_resource,
+                        existing_resource=ex,
+                        reason=(
+                            f"different resource type '{new_resource.resource_type.value}' "
+                            f"coexists with '{ex.resource_type.value}' "
+                            f"on target '{new_resource.target_identity}'"
+                        ),
+                    )
+                )
+        return conflicts
+
+    def has_serializable_conflicts(self, new_resource: TrackedResource) -> bool:
+        """True if a SERIALIZE or REJECT conflict exists."""
+        return any(
+            c.kind in (ConflictKind.SERIALIZE, ConflictKind.REJECT)
+            for c in self.detect_conflicts(new_resource)
+        )

mayhem/domain/risks.py

@@ -0,0 +1,61 @@
+"""Risk ladder and environment classification (ADR-0012).
+
+The ladder is *ordered* so gates can compare; enforcement lives in the safety
+engine, but the ordering itself is a domain fact.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class RiskLevel(StrEnum):
+    """Ordered severity ladder: low < medium < high < critical."""
+
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+    @property
+    def rank(self) -> int:
+        return _RANKS[self]
+
+    def at_least(self, other: RiskLevel) -> bool:
+        """True when this level is greater than or equal to ``other``."""
+        return self.rank >= other.rank
+
+    def exceeds(self, ceiling: RiskLevel) -> bool:
+        """True when this level is strictly above the policy ceiling."""
+        return self.rank > ceiling.rank
+
+    def next_higher(self) -> RiskLevel:
+        """One step up the ladder; critical returns itself."""
+        return _NEXT_HIGHER[self]
+
+
+_RANKS: dict[RiskLevel, int] = {
+    RiskLevel.LOW: 0,
+    RiskLevel.MEDIUM: 1,
+    RiskLevel.HIGH: 2,
+    RiskLevel.CRITICAL: 3,
+}
+
+_NEXT_HIGHER: dict[RiskLevel, RiskLevel] = {
+    RiskLevel.LOW: RiskLevel.MEDIUM,
+    RiskLevel.MEDIUM: RiskLevel.HIGH,
+    RiskLevel.HIGH: RiskLevel.CRITICAL,
+    RiskLevel.CRITICAL: RiskLevel.CRITICAL,
+}
+
+
+class EnvironmentClass(StrEnum):
+    """Deployment class driving safety defaults (ADR-0012 §3)."""
+
+    DEV = "dev"
+    STAGING = "staging"
+    PRODUCTION = "production"
+
+    @property
+    def is_production(self) -> bool:
+        return self is EnvironmentClass.PRODUCTION

mayhem/domain/run_outcome.py

@@ -0,0 +1,146 @@
+"""Run and Outcome domain models (ADR-M5-1).
+
+A ``RunRecord`` is *what was executed* — the experiment spec, faults, groups,
+cancellation, journal refs, verdict, duration, and tool evidence.
+
+An ``Outcome`` is *what happened* — observed post-run system state, checks
+passed/failed, metric deltas, residual effect, and stability/recovery signal.
+
+They persist separately and link by a ``run_id → outcome`` reference.
+A ``RunRecord`` is never conflated with its ``Outcome``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import StrEnum
+from typing import Any
+
+
+class RunVerdict(StrEnum):
+    """Top-level verdict for a completed run."""
+
+    PASS = "pass"
+    FAIL = "fail"
+    ERROR = "error"
+    ABORTED = "aborted"
+    BYPASSED = "bypassed"
+
+
+class RunStatus(StrEnum):
+    """Lifecycle status of a run."""
+
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    ABORTED = "aborted"
+
+
+@dataclass(frozen=True)
+class RunRecord:
+    """What was executed in a single drill run.
+
+    Persisted separately from its ``Outcome`` and linked by ``run_id``.
+    """
+
+    run_id: str
+    experiment_name: str
+    spec_json: str
+    plan_json: str
+    seed: int | None = None
+    status: RunStatus = RunStatus.PENDING
+    environment_fingerprint: str = ""
+    config_snapshot_id: str = ""
+    started_at: str = ""
+    ended_at: str = ""
+    description: str = ""
+    verdict: RunVerdict = RunVerdict.PASS
+    tags: tuple[str, ...] = ()
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def wall_seconds(self) -> float:
+        """Duration in seconds (requires both timestamps set)."""
+        if not self.started_at or not self.ended_at:
+            return 0.0
+        try:
+            start = datetime.fromisoformat(self.started_at)
+            end = datetime.fromisoformat(self.ended_at)
+            return max(0.0, (end - start).total_seconds())
+        except ValueError:
+            return 0.0
+
+    def summary_md(self) -> str:
+        """Markdown summary of the run."""
+        lines = [
+            f"# Run {self.run_id}",
+            "",
+            f"**experiment**: {self.experiment_name}",
+            f"**status**: {self.status.value}",
+            f"**verdict**: {self.verdict.value}",
+        ]
+        if self.started_at:
+            lines.append(f"**started**: {self.started_at}")
+        if self.ended_at:
+            lines.append(f"**ended**: {self.ended_at}")
+        if self.seed is not None:
+            lines.append(f"**seed**: {self.seed}")
+        if self.tags:
+            lines.append(f"**tags**: {', '.join(self.tags)}")
+        if self.description:
+            lines.append("")
+            lines.append(self.description)
+        return "\n".join(lines)
+
+
+@dataclass(frozen=True)
+class Outcome:
+    """What happened after a run — observed post-run system state.
+
+    Persisted separately from the ``RunRecord`` and linked by ``run_id``.
+    """
+
+    run_id: str
+    body_json: str = "{}"
+    body_hash: str = ""
+    checks_passed: int = 0
+    checks_failed: int = 0
+    metric_deltas: dict[str, float] = field(default_factory=dict)
+    residual_effect: str = ""
+    stability_signal: str = ""
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def all_checks_passed(self) -> bool:
+        """True when there were checks and every one passed."""
+        return self.checks_passed > 0 and self.checks_failed == 0
+
+    @property
+    def total_checks(self) -> int:
+        return self.checks_passed + self.checks_failed
+
+    def summary_md(self) -> str:
+        """Markdown summary of the outcome."""
+        total = self.total_checks
+        if total == 0:
+            status_str = "no checks"
+        elif self.all_checks_passed:
+            status_str = f"all {total} checks passed"
+        else:
+            status_str = f"{self.checks_failed}/{total} checks failed"
+        lines = [
+            f"# Outcome for Run {self.run_id}",
+            "",
+            f"**checks**: {status_str}",
+        ]
+        if self.metric_deltas:
+            lines.append("**metric deltas**:")
+            for name, delta in self.metric_deltas.items():
+                lines.append(f"  - {name}: {delta:+.4f}")
+        if self.residual_effect:
+            lines.append(f"**residual**: {self.residual_effect}")
+        if self.stability_signal:
+            lines.append(f"**stability**: {self.stability_signal}")
+        return "\n".join(lines)