mayhem-cli 0.5.1__py3-none-any.whl

mayhem/infra/candidate_gates.py

@@ -0,0 +1,124 @@
+"""Candidate gate pipeline (ADR-M5-2, M5 Phase 5.3).
+
+Routes an ``ExperimentCandidate`` through the three gates:
+
+    Safety → Feasibility → Resource-conflict
+
+A candidate that fails a gate is rejected with that gate and a reason; it is
+never executed. Each gate is an injectable checker so the pipeline is fully
+deterministic and unit-testable without a live runtime, while the concrete
+checkers mirror the M2 (resource-conflict), M3 (capability verdicts), and
+safety machinery already present in the codebase.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from mayhem.domain.candidates import (
+    CandidateDecision,
+    CandidateGate,
+    ExperimentCandidate,
+    accept,
+    reject,
+)
+
+
+class GateChecker(Protocol):
+    """A single gate: return ``None`` to pass, or a reason string to reject."""
+
+    def check(self, candidate: ExperimentCandidate) -> str | None:
+        """Return ``None`` if the candidate passes, else the rejection reason."""
+        ...
+
+
+class SafetyGate:
+    """Gate 1 — safety (ADR-M5 / controller safety checks)."""
+
+    def __init__(self, *, forbidden_faults: tuple[str, ...] = ()) -> None:
+        # forbidden faults are e.g. host reboot / node kill not allowlisted.
+        self._forbidden_faults = set(forbidden_faults)
+
+    def check(self, candidate: ExperimentCandidate) -> str | None:
+        for fault in candidate.fault_kinds:
+            if fault in self._forbidden_faults:
+                return f"fault {fault!r} is forbidden by safety policy"
+        if not candidate.target:
+            return "candidate has no target"
+        return None
+
+
+class FeasibilityGate:
+    """Gate 2 — feasibility (M3 runtime-capability verdicts).
+
+    ``S``/``A``/``U`` mark each fault as Supported / Alternative / Unsupported.
+    Any ``U`` (unsupported) fault is not feasible.
+    """
+
+    def __init__(self, supported: tuple[str, ...], unsupported: tuple[str, ...] = ()) -> None:
+        self._supported = set(supported)
+        self._unsupported = set(unsupported)
+
+    def check(self, candidate: ExperimentCandidate) -> str | None:
+        for fault in candidate.fault_kinds:
+            if fault in self._unsupported:
+                return f"fault {fault!r} is UNSUPPORTED by the runtime (not feasible)"
+            if fault not in self._supported:
+                return f"fault {fault!r} is not in the supported capability set"
+        return None
+
+
+class ResourceConflictGate:
+    """Gate 3 — resource conflict (M2).
+
+    ``busy_targets`` are targets that already hold an in-flight mutation, so a
+    new experiment against them would collide.
+    """
+
+    def __init__(self, busy_targets: tuple[str, ...] = ()) -> None:
+        self._busy = set(busy_targets)
+
+    def check(self, candidate: ExperimentCandidate) -> str | None:
+        if candidate.target in self._busy:
+            return f"target {candidate.target!r} has an in-flight resource conflict"
+        return None
+
+
+class CandidateGatePipeline:
+    """Composes the three gates; returns the first rejection, else acceptance."""
+
+    def __init__(
+        self,
+        *,
+        safety: GateChecker | None = None,
+        feasibility: GateChecker | None = None,
+        resource_conflict: GateChecker | None = None,
+    ) -> None:
+        self._safety = safety if safety is not None else SafetyGate()
+        self._feasibility = (
+            feasibility if feasibility is not None else FeasibilityGate(supported=())
+        )
+        self._resource_conflict = (
+            resource_conflict if resource_conflict is not None else ResourceConflictGate()
+        )
+
+    def gate(self, candidate: ExperimentCandidate) -> CandidateDecision:
+        """Route a candidate through the three gates in fixed order."""
+        # 1. Safety
+        reason = self._safety.check(candidate)
+        if reason is not None:
+            return reject(candidate, CandidateGate.SAFETY, reason)
+        # 2. Feasibility
+        reason = self._feasibility.check(candidate)
+        if reason is not None:
+            return reject(candidate, CandidateGate.FEASIBILITY, reason)
+        # 3. Resource-conflict
+        reason = self._resource_conflict.check(candidate)
+        if reason is not None:
+            return reject(candidate, CandidateGate.RESOURCE_CONFLICT, reason)
+        return accept(candidate)
+
+    def gate_many(
+        self, candidates: tuple[ExperimentCandidate, ...]
+    ) -> tuple[CandidateDecision, ...]:
+        return tuple(self.gate(c) for c in candidates)

mayhem/infra/candidate_generator.py

@@ -0,0 +1,110 @@
+"""Seeded, bounded candidate generator (ADR-M5-2, M5 Phase 5.3).
+
+Produces ``ExperimentCandidate`` proposals over the
+(target, fault_kind, execution_context, parameter_band) landscape. Generation
+is deterministic for a fixed seed, and honors a per-band risk ceiling so no
+candidate above the campaign's risk tolerance is ever proposed.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+from typing import Any
+
+from mayhem.domain.candidates import ExperimentCandidate
+from mayhem.domain.risks import RiskLevel
+
+# Risk of a fault_kind, keyed by fault id prefix (mirrors safety projection).
+_DEFAULT_FAULT_RISK: dict[str, RiskLevel] = {
+    "net": RiskLevel.MEDIUM,
+    "cpu": RiskLevel.MEDIUM,
+    "mem": RiskLevel.MEDIUM,
+    "fs": RiskLevel.HIGH,
+    "disk": RiskLevel.HIGH,
+    "storage": RiskLevel.HIGH,
+    "process": RiskLevel.LOW,
+    "container": RiskLevel.LOW,
+    "node": RiskLevel.CRITICAL,
+    "http_api": RiskLevel.LOW,
+    "database": RiskLevel.HIGH,
+    "load": RiskLevel.MEDIUM,
+    "fuzz": RiskLevel.HIGH,
+    "dns": RiskLevel.LOW,
+    "tls": RiskLevel.LOW,
+    "clock": RiskLevel.MEDIUM,
+    "fd": RiskLevel.MEDIUM,
+}
+
+
+def _risk_of_fault(fault_kind: str) -> RiskLevel:
+    prefix = fault_kind.split(".", 1)[0]
+    return _DEFAULT_FAULT_RISK.get(prefix, RiskLevel.MEDIUM)
+
+
+@dataclass
+class CandidateLandscape:
+    """The space the generator draws from, plus the allowed risk ceiling."""
+
+    targets: tuple[str, ...] = ()
+    fault_kinds: tuple[str, ...] = ()
+    execution_contexts: tuple[str, ...] = ("container",)
+    parameter_bands: tuple[str, ...] = ("default",)
+    risk_ceiling: RiskLevel = RiskLevel.HIGH
+
+
+@dataclass
+class SeededCandidateGenerator:
+    """Deterministic generator; the same seed yields the same candidates."""
+
+    landscape: CandidateLandscape
+    seed: int = 0
+    rng: random.Random | None = None
+
+    def __post_init__(self) -> None:
+        if self.rng is None:
+            self.rng = random.Random(self.seed)
+
+    def _in_ceiling(self, risk: RiskLevel) -> bool:
+        return _risk_rank(risk) <= _risk_rank(self.landscape.risk_ceiling)
+
+    def generate(self, *, limit: int | None = None) -> tuple[ExperimentCandidate, ...]:
+        """Yield candidates until the landscape is exhausted or ``limit`` reached.
+
+        Candidates above the risk ceiling are skipped, never emitted.
+        """
+        out: list[ExperimentCandidate] = []
+        for target in self.landscape.targets:
+            for fault_kind in self.landscape.fault_kinds:
+                risk = _risk_of_fault(fault_kind)
+                if not self._in_ceiling(risk):
+                    continue
+                for ctx in self.landscape.execution_contexts:
+                    for band in self.landscape.parameter_bands:
+                        params: dict[str, Any] = {"band": band}
+                        candidate = ExperimentCandidate(
+                            target=target,
+                            fault_kinds=(fault_kind,),
+                            params=params,
+                            execution_context=ctx,
+                            expected_effect=f"{fault_kind} on {target} in {band}",
+                            risk_band=risk,
+                            seed_hint=self.seed,
+                        )
+                        out.append(candidate)
+                        if limit is not None and len(out) >= limit:
+                            return tuple(out)
+        # Deterministic shuffle so selection order isn't trivially the
+        # landscape enumeration order — still fully reproducible from seed.
+        rng = self.rng if self.rng is not None else random.Random(self.seed)
+        rng.shuffle(out)
+        return tuple(out)
+
+
+def _risk_rank(level: RiskLevel) -> int:
+    return {
+        RiskLevel.LOW: 0,
+        RiskLevel.MEDIUM: 1,
+        RiskLevel.HIGH: 2,
+        RiskLevel.CRITICAL: 3,
+    }[level]

mayhem/infra/coverage_repository.py

@@ -0,0 +1,101 @@
+"""SQLite-backed coverage accounting (ADR-M5-3, M5 Phase 5.2).
+
+A recorded ``Outcome`` marks the run's coverage cell as *seen*. Marking the
+same cell again (a re-run) is idempotent — ``INSERT OR IGNORE`` on the primary
+key means re-running a cell never double-counts. Maniac can then enumerate the
+``UNKNOWN`` cells of a landscape (cells not yet covered) to prioritize coverage
+gain.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from mayhem.domain.coverage import (
+    CoverageCell,
+    CoverageRecord,
+    CoverageSummary,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from mayhem.infra.store import Store
+
+
+class SQLiteCoverageRepository:
+    """Persists seen coverage cells over the ``m5_coverage`` table."""
+
+    def __init__(self, store: Store) -> None:
+        self._store = store
+
+    def mark_seen(
+        self,
+        cell: CoverageCell,
+        run_id: str,
+        *,
+        extra: dict[str, object] | None = None,
+    ) -> None:
+        """Record that ``cell`` was observed by ``run_id``.
+
+        Idempotent per cell: re-running the same cell does not double-count;
+        ``CREATE TABLE ... PRIMARY KEY`` + ``INSERT OR IGNORE`` guarantees it.
+        """
+        with self._store.write() as conn:
+            conn.execute(
+                """
+                INSERT OR IGNORE INTO m5_coverage (
+                    cell_key, target, fault_kind, execution_context,
+                    parameter_band, run_id, covered, extra_json
+                ) VALUES (?, ?, ?, ?, ?, ?, 1, ?)
+                """,
+                (
+                    cell.key,
+                    cell.target,
+                    cell.fault_kind,
+                    cell.execution_context,
+                    cell.parameter_band,
+                    run_id,
+                    json.dumps(extra or {}),
+                ),
+            )
+
+    def is_covered(self, cell: CoverageCell) -> bool:
+        rows = self._store.query(
+            "SELECT 1 FROM m5_coverage WHERE cell_key = ? AND covered = 1",
+            (cell.key,),
+        )
+        return bool(rows)
+
+    def covered_keys(self) -> frozenset[str]:
+        rows = self._store.query("SELECT cell_key FROM m5_coverage WHERE covered = 1")
+        return frozenset(r["cell_key"] for r in rows)
+
+    def covered_records(self) -> tuple[CoverageRecord, ...]:
+        rows = self._store.query(
+            "SELECT target, fault_kind, execution_context, parameter_band, run_id, extra_json "
+            "FROM m5_coverage WHERE covered = 1"
+        )
+        records: list[CoverageRecord] = []
+        for r in rows:
+            cell = CoverageCell(
+                target=r["target"],
+                fault_kind=r["fault_kind"],
+                execution_context=r["execution_context"],
+                parameter_band=r["parameter_band"],
+            )
+            records.append(
+                CoverageRecord(cell=cell, run_id=r["run_id"], extra=json.loads(r["extra_json"]))
+            )
+        return tuple(records)
+
+    def summary(self, landscape: Iterable[CoverageCell]) -> CoverageSummary:
+        """Aggregate covered vs UNKNOWN cells over ``landscape``."""
+        landscape_tuple = tuple(landscape)
+        covered = self.covered_keys()
+        return CoverageSummary(covered_keys=covered, total_cells=len(landscape_tuple))
+
+    def unknown_cells(self, landscape: Iterable[CoverageCell]) -> tuple[CoverageCell, ...]:
+        """Cells in ``landscape`` not yet covered, in landscape order."""
+        return self.summary(landscape).unknown_cells(tuple(landscape))

mayhem/infra/lease_repository.py

@@ -0,0 +1,129 @@
+"""SQLite-backed LeaseSink — the durable half of the lease protocol (ADR-0007).
+
+The controller is the single writer; agents receive this sink through the
+SDK boundary and never open the database themselves. Rows are self-contained:
+run/fault/targets context is denormalized onto the lease row (migration 002)
+so janitor and watchdog sweeps never need joins.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING
+
+from mayhem.domain.leases import FaultLease, LeaseState
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from mayhem.infra.store import Store
+
+_TERMINAL_STATES = ("released", "expired")
+
+
+class SQLiteLeaseSink:
+    """Implements ``agents.sinks.LeaseSink`` over the fault_leases table."""
+
+    def __init__(self, store: Store) -> None:
+        self._store = store
+
+    def save(self, lease: FaultLease) -> None:
+        created = lease.created_at
+        expires_at = created + timedelta(seconds=float(lease.ttl_seconds))
+        with self._store.write() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO fault_leases (
+                    id, state, owner_agent, undo_json, verify_json,
+                    ttl_seconds, expires_at, injected_at, released_at,
+                    release_mechanism, escalation_notes,
+                    run_id, fault_id, targets_json, created_epoch_s,
+                    runtime_identity
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    lease.id,
+                    lease.state.value,
+                    lease.owner_agent,
+                    json.dumps([op.model_dump(mode="json") for op in lease.undo_ops]),
+                    json.dumps([probe.model_dump(mode="json") for probe in lease.verify_probes]),
+                    float(lease.ttl_seconds),
+                    expires_at.isoformat(),
+                    _iso_or_none(lease.injected_at),
+                    _iso_or_none(lease.released_at),
+                    lease.release_mechanism,
+                    lease.escalation_notes,
+                    lease.run_id,
+                    lease.fault_id,
+                    json.dumps(sorted(lease.targets)),
+                    created.timestamp(),
+                    lease.runtime_identity,
+                ),
+            )
+
+    def load(self, lease_id: str) -> FaultLease | None:
+        rows = self._store.query("SELECT * FROM fault_leases WHERE id = ?", (lease_id,))
+        return _row_to_lease(dict(rows[0])) if rows else None
+
+    def active_leases(self) -> tuple[FaultLease, ...]:
+        placeholders = ", ".join("?" for _ in _TERMINAL_STATES)
+        rows = self._store.query(
+            f"SELECT * FROM fault_leases WHERE state NOT IN ({placeholders})"
+            " ORDER BY created_epoch_s",
+            _TERMINAL_STATES,
+        )
+        return tuple(_row_to_lease(dict(row)) for row in rows)
+
+    def expired_pending(self, now_epoch_s: float) -> tuple[FaultLease, ...]:
+        rows = self._store.query(
+            "SELECT * FROM fault_leases WHERE created_epoch_s + ttl_seconds < ? "
+            "AND state IN ('pending', 'active')",
+            (now_epoch_s,),
+        )
+        return tuple(_row_to_lease(dict(row)) for row in rows)
+
+    def next_sequence(self) -> int:
+        # Max trailing integer of every `l-<n>` id ever persisted. Seed the next
+        # run's counter here so ids never collide in a shared DB.
+        rows = self._store.query(
+            "SELECT MAX(CAST(substr(id, 3) AS INTEGER)) AS seq FROM fault_leases"
+            " WHERE id LIKE 'l-%'"
+        )
+        if not rows:
+            return 0
+        seq = rows[0]["seq"]
+        return int(seq) if seq is not None else 0
+
+
+def _row_to_lease(row: Mapping[str, object]) -> FaultLease:
+    return FaultLease.model_validate(
+        {
+            "id": str(row["id"]),
+            "state": LeaseState(str(row["state"])),
+            "owner_agent": str(row["owner_agent"]),
+            "undo_ops": json.loads(str(row["undo_json"])),
+            "verify_probes": json.loads(str(row["verify_json"])),
+            "ttl_seconds": float(str(row["ttl_seconds"])),
+            "injected_at": _dt_or_none(row.get("injected_at")),
+            "released_at": _dt_or_none(row.get("released_at")),
+            "release_mechanism": row["release_mechanism"],
+            "escalation_notes": row["escalation_notes"],
+            "run_id": str(row["run_id"] or ""),
+            "fault_id": str(row["fault_id"] or ""),
+            "targets": frozenset(json.loads(str(row["targets_json"]))),
+            "created_at": datetime.fromtimestamp(float(str(row["created_epoch_s"])), tz=UTC),
+            "runtime_identity": row.get("runtime_identity"),
+        }
+    )
+
+
+def _iso_or_none(moment: datetime | None) -> str | None:
+    return moment.isoformat() if moment is not None else None
+
+
+def _dt_or_none(raw: object) -> datetime | None:
+    if raw is None:
+        return None
+    parsed = datetime.fromisoformat(str(raw))
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)

mayhem/infra/maniac.py

@@ -0,0 +1,103 @@
+"""Maniac selection/optimization engine (ADR-M5-3, M5 Phase 5.5).
+
+Maniac chooses *which experiment to run next* from the candidate set, grounded
+in recorded outcome coverage and the campaign's bounds. It is:
+
+- **stateless** across campaign runs (no in-memory carry-over; every selection
+  is re-derived from its explicit inputs),
+- **deterministic and reproducible**: identical inputs (candidates, covered
+  cells, gates, bounds, seed) always yield the identical selection,
+- **knowledge-transferable**: recorded coverage from prior campaigns is passed
+  in explicitly, so learnings carry across campaign boundaries.
+
+Selection is a greedy coverage-maximizing walk: gate every candidate, prefer
+cells not yet covered, and stop once the budget or coverage target is met.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from mayhem.domain.coverage import CoverageCell
+from mayhem.infra.candidate_gates import CandidateGatePipeline
+
+if TYPE_CHECKING:
+    from mayhem.domain.candidates import CandidateDecision, ExperimentCandidate
+
+
+def coverage_cell_for_candidate(candidate: ExperimentCandidate) -> CoverageCell:
+    """The coverage cell a candidate explores, derived deterministically."""
+    band = str(candidate.params.get("band") or "default") if candidate.params else "default"
+    return CoverageCell(
+        candidate.target,
+        candidate.primary_fault,
+        candidate.execution_context,
+        band,
+    )
+
+
+@dataclass(frozen=True)
+class SelectionInputs:
+    """Every input Maniac is allowed to look at (no hidden state)."""
+
+    candidates: tuple[ExperimentCandidate, ...]
+    covered_keys: frozenset[str] = frozenset()
+    gates: CandidateGatePipeline | None = None
+    max_runs: int = 100
+    coverage_target: int = 1
+    seed: int = 0
+
+
+@dataclass(frozen=True)
+class SelectionResult:
+    selected: tuple[ExperimentCandidate, ...]
+    rejected: tuple[CandidateDecision, ...] = ()
+    predicted_new_coverage: int = 0
+
+    @property
+    def selected_count(self) -> int:
+        return len(self.selected)
+
+
+def select_next(inputs: SelectionInputs) -> SelectionResult:
+    """Pure, deterministic greedy selection (stateless, reproducible)."""
+    gates = inputs.gates if inputs.gates is not None else CandidateGatePipeline()
+    covered = set(inputs.covered_keys)
+
+    accepted: list[ExperimentCandidate] = []
+    rejected: list[CandidateDecision] = []
+    for cand in inputs.candidates:
+        decision = gates.gate(cand)
+        if decision.accepted:
+            accepted.append(cand)
+        else:
+            rejected.append(decision)
+
+    # Deterministic ordering: seeded random tie-break over the candidates,
+    # then rank so that already-covered cells come last (coverage first).
+    rng = random.Random(inputs.seed)
+    ordered = sorted(accepted, key=lambda c: rng.random())
+    uncovered_first = sorted(
+        ordered,
+        key=lambda c: coverage_cell_for_candidate(c).key in covered,
+    )
+
+    selected: list[ExperimentCandidate] = []
+    covered_now = set(covered)
+    for cand in uncovered_first:
+        if len(selected) >= inputs.max_runs:
+            break
+        if len(covered_now) >= inputs.coverage_target:
+            break
+        cell = coverage_cell_for_candidate(cand)
+        covered_now.add(cell.key)
+        selected.append(cand)
+
+    projected = len(covered_now) - len(covered)
+    return SelectionResult(
+        selected=tuple(selected),
+        rejected=tuple(rejected),
+        predicted_new_coverage=projected,
+    )