mayhem-cli 0.5.1__py3-none-any.whl

mayhem/infra/migrator.py

@@ -0,0 +1,149 @@
+"""Forward migration runner with down-migration support (ADR-M4-5).
+
+Every migration is an ordered, immutable Python module exposing ``version``,
+``name``, ``statements``, and optionally ``down_statements``. Applied versions
+are recorded in ``_schema_migrations``; ``run_migrations`` applies pending ones
+in a single transaction each and refuses out-of-order application
+(testing-strategy §5). ``run_down_migrations`` reverses applied migrations
+back to an older version in descending order — the mechanism ADR-M4-5's
+"down-migration restores the baseline" acceptance relies on.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from mayhem.domain.errors import DomainError
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+class MigrationError(DomainError):
+    """Migration machinery failure."""
+
+
+@dataclass(frozen=True)
+class Migration:
+    version: int
+    name: str
+    statements: tuple[str, ...]
+    down_statements: tuple[str, ...] = ()
+
+    @property
+    def migration_id(self) -> str:
+        return f"{self.version:04d}_{self.name}"
+
+
+def run_migrations(conn: sqlite3.Connection, migrations: Sequence[Migration]) -> list[str]:
+    """Apply all pending migrations; returns ids applied this call."""
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS _schema_migrations (
+            version INTEGER PRIMARY KEY,
+            name TEXT NOT NULL,
+            applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+        )
+        """)
+    applied = {int(row[0]) for row in conn.execute("SELECT version FROM _schema_migrations")}
+    applied_now: list[str] = []
+    previous = -1
+    for migration in migrations:
+        if migration.version <= previous:
+            raise MigrationError(
+                f"migrations must be strictly increasing; got {migration.version} after {previous}"
+            )
+        previous = migration.version
+        if migration.version in applied:
+            continue
+        try:
+            # Schema changes may rebuild parent tables (e.g. extending a CHECK
+            # constraint); FK enforcement blocks those DROP/RENAME steps, so it
+            # is disabled per-migration and re-enabled afterwards. A migration
+            # runs as the single writer with no concurrent readers, so this is
+            # safe and matches the SQLite table-rebuild procedure.
+            conn.execute("PRAGMA foreign_keys=OFF")
+            with conn:
+                for statement in migration.statements:
+                    conn.execute(statement)
+                conn.execute(
+                    "INSERT INTO _schema_migrations (version, name) VALUES (?, ?)",
+                    (migration.version, migration.name),
+                )
+            conn.execute("PRAGMA foreign_keys=ON")
+        except sqlite3.Error as exc:
+            conn.execute("PRAGMA foreign_keys=ON")
+            raise MigrationError(f"migration {migration.migration_id} failed: {exc}") from exc
+        applied_now.append(migration.migration_id)
+    return applied_now
+
+
+def current_version(conn: sqlite3.Connection) -> int | None:
+    """Latest applied schema version; None when database is fresh."""
+    exists = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='_schema_migrations'"
+    ).fetchone()
+    if exists is None:
+        return None
+    row = conn.execute("SELECT MAX(version) FROM _schema_migrations").fetchone()
+    return int(row[0]) if row and row[0] is not None else None
+
+
+def run_down_migrations(
+    conn: sqlite3.Connection,
+    migrations: Sequence[Migration],
+    target_version: int,
+) -> list[str]:
+    """Roll an upgraded schema back to ``target_version`` (ADR-M4-5).
+
+    Applied migrations newer than ``target_version`` are reversed in descending
+    order using their ``down_statements``; each reversal runs in its own
+    transaction and the ``_schema_migrations`` row is deleted with it. A
+    migration without a down path refuses to roll back rather than truncate
+    history. Returns the migration ids reversed.
+    """
+    by_version = {m.version: m for m in migrations}
+    applied = {
+        int(row[0]): str(row[1])
+        for row in conn.execute("SELECT version, name FROM _schema_migrations")
+    }
+    current = max(applied, default=-1)
+    if current == -1:
+        return []  # fresh database has nothing to roll back
+    if target_version >= current:
+        raise MigrationError(
+            f"target version {target_version} is not below current schema {current}; "
+            "nothing to roll back"
+        )
+    missing = set(applied) - set(by_version)
+    if missing:
+        raise MigrationError(
+            f"database has applied migrations unknown to this store: {sorted(missing)}"
+        )
+    reversed_now: list[str] = []
+    ordered = sorted(applied, reverse=True)
+    for version in ordered:
+        if version <= target_version:
+            continue
+        migration = by_version[version]
+        if not migration.down_statements:
+            raise MigrationError(
+                f"migration {migration.migration_id} has no down path; "
+                "refusing to truncate schema history"
+            )
+        try:
+            conn.execute("PRAGMA foreign_keys=OFF")
+            with conn:
+                for statement in migration.down_statements:
+                    conn.execute(statement)
+                conn.execute(
+                    "DELETE FROM _schema_migrations WHERE version = ?",
+                    (version,),
+                )
+            conn.execute("PRAGMA foreign_keys=ON")
+        except sqlite3.Error as exc:
+            conn.execute("PRAGMA foreign_keys=ON")
+            raise MigrationError(f"down-migration {migration.migration_id} failed: {exc}") from exc
+        reversed_now.append(migration.migration_id)
+    return reversed_now

mayhem/infra/report.py

@@ -0,0 +1,227 @@
+"""M5 report + experiment guidance builder (M5 Phase 5.7).
+
+Consumes Run/Outcome history, coverage, and the Maniac candidate backlog to
+produce a rich, model-only report:
+  - a coverage heatmap (rendered ASCII grid),
+  - per-cell verdicts with supported-by evidence (run links),
+  - the candidate backlog ranked by Maniac score,
+  - the guided "what to run next" list (untouched/``UNKNOWN`` cells).
+
+The builder is deterministic and pure: given the same history it always
+renders the same report. Assertions in tests are model/string level.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from mayhem.infra.candidate_gates import CandidateGatePipeline
+from mayhem.infra.maniac import SelectionInputs, select_next
+
+if TYPE_CHECKING:
+    from mayhem.domain.candidates import ExperimentCandidate
+    from mayhem.domain.coverage import CoverageCell, CoverageRecord
+    from mayhem.domain.run_outcome import Outcome, RunRecord
+
+# Verdict / heatmap symbols.
+_CELL_COVERED = "█"
+_CELL_UNKNOWN = "·"
+
+
+class _PermissiveGate:
+    """A gate that passes every candidate (used when no gates are given)."""
+
+    def check(self, candidate: object) -> str | None:
+        return None
+
+
+def _permissive_gates() -> CandidateGatePipeline:
+    return CandidateGatePipeline(safety=_PermissiveGate(), feasibility=_PermissiveGate())
+
+
+@dataclass(frozen=True)
+class CellVerdictReport:
+    """One cell's verdict plus the runs that support/evidence it."""
+
+    cell: CoverageCell
+    covered: bool
+    run_ids: tuple[str, ...] = ()
+    verdict: str = "UNKNOWN"
+
+
+@dataclass(frozen=True)
+class M5Report:
+    """Rendered report over a landscape + recorded history."""
+
+    landscape: tuple[CoverageCell, ...] = ()
+    heatmap: str = ""
+    cell_verdicts: tuple[CellVerdictReport, ...] = ()
+    coverage_fraction: float = 0.0
+    next_to_run: tuple[CoverageCell, ...] = ()
+    ranked_backlog: tuple[ExperimentCandidate, ...] = ()
+
+    def render_markdown(self) -> str:
+        """Render the whole report as a markdown string."""
+        lines = [
+            "# M5 Campaign Report",
+            "",
+            "## Coverage",
+            "",
+            self.heatmap,
+            "",
+            f"**coverage**: {self.coverage_fraction:.1%} of the landscape covered",
+            "",
+            "## Per-cell verdicts",
+            "",
+        ]
+        if not self.cell_verdicts:
+            lines.append("_no cells recorded_")
+        else:
+            lines.append("| cell | verdict | evidence runs |")
+            lines.append("| --- | --- | --- |")
+            for v in self.cell_verdicts:
+                ev = ", ".join(v.run_ids) if v.run_ids else "—"
+                lines.append(f"| `{v.cell.key}` | {v.verdict} | {ev} |")
+        lines.append("")
+        lines.append("## What to run next")
+        lines.append("")
+        if self.next_to_run:
+            for cell in self.next_to_run:
+                lines.append(f"- `{cell.key}` (UNKNOWN)")
+        else:
+            lines.append("_no gap remains_")
+        lines.append("")
+        lines.append("## Candidate backlog (Maniac-ranked)")
+        lines.append("")
+        if self.ranked_backlog:
+            for i, cand in enumerate(self.ranked_backlog, 1):
+                lines.append(f"{i}. `{cand.id}` -> {cand.target} [{cand.primary_fault}]")
+        else:
+            lines.append("_backlog empty_")
+        return "\n".join(lines) + "\n"
+
+
+def _cell_verdict(
+    covered: bool,
+    run_ids: tuple[str, ...],
+    runs: dict[str, RunRecord],
+    outcomes: dict[str, Outcome],
+) -> str:
+    """Derive a verdict for a cell from its evidence (Runs/Outcomes)."""
+    if not covered or not run_ids:
+        return "UNKNOWN"
+    # Use the most recent run's outcome as the cell verdict.
+    for run_id in reversed(run_ids):
+        outcome = outcomes.get(run_id)
+        if outcome is not None and outcome.total_checks > 0:
+            return "PASS" if outcome.all_checks_passed else "FAIL"
+    # No outcome with checks; fall back to the run verdict if it FAILED.
+    for run_id in reversed(run_ids):
+        run = runs.get(run_id)
+        if run is not None and run.verdict.value == "fail":
+            return "FAIL"
+    return "PASS"
+
+
+def render_heatmap(
+    landscape: tuple[CoverageCell, ...],
+    covered_keys: frozenset[str],
+) -> str:
+    """Render a compact ASCII heatmap: targets (rows) x faults (cols).
+
+    A cell is marked covered when any landscape cell in that (target, fault)
+    group is covered. Unknown cells use ``·``, covered cells use ``█``.
+    """
+    targets: list[str] = []
+    faults: list[str] = []
+    seen_cells: dict[tuple[str, str], bool] = {}
+    for cell in landscape:
+        group = (cell.target, cell.fault_kind)
+        covered = cell.key in covered_keys
+        if group not in seen_cells or (covered and not seen_cells[group]):
+            seen_cells[group] = covered
+        if cell.target not in targets:
+            targets.append(cell.target)
+        if cell.fault_kind not in faults:
+            faults.append(cell.fault_kind)
+
+    header = "        " + " ".join(f"{f:<6}" for f in faults)
+    lines = [header]
+    for t in targets:
+        row_parts = []
+        for f in faults:
+            covered = seen_cells.get((t, f), False)
+            row_parts.append(f"{_CELL_COVERED if covered else _CELL_UNKNOWN:<6}")
+        lines.append(f"{t:<8} " + " ".join(row_parts))
+    lines.append("")
+    lines.append(f"legend: {_CELL_COVERED}=covered  {_CELL_UNKNOWN}=unknown")
+    return "\n".join(lines)
+
+
+def build_m5_report(
+    *,
+    landscape: tuple[CoverageCell, ...],
+    covered_records: tuple[CoverageRecord, ...] = (),
+    runs: dict[str, RunRecord] | None = None,
+    outcomes: dict[str, Outcome] | None = None,
+    candidates: tuple[ExperimentCandidate, ...] = (),
+    gates: CandidateGatePipeline | None = None,
+    seed: int = 0,
+    max_runs: int = 50,
+) -> M5Report:
+    """Assemble the report from recorded history (pure/deterministic)."""
+    runs = runs or {}
+    outcomes = outcomes or {}
+
+    covered_keys: set[str] = set()
+    run_ids_by_cell: dict[str, list[str]] = {}
+    for rec in covered_records:
+        covered_keys.add(rec.cell.key)
+        run_ids_by_cell.setdefault(rec.cell.key, []).append(rec.run_id)
+
+    covered_count = len(covered_keys)
+    fraction = covered_count / len(landscape) if landscape else 0.0
+
+    verdicts = tuple(
+        CellVerdictReport(
+            cell=cell,
+            covered=cell.key in covered_keys,
+            run_ids=tuple(run_ids_by_cell.get(cell.key, ())),
+            verdict=_cell_verdict(
+                cell.key in covered_keys,
+                tuple(run_ids_by_cell.get(cell.key, ())),
+                runs,
+                outcomes,
+            ),
+        )
+        for cell in landscape
+    )
+
+    unknown = [cell for cell in landscape if cell.key not in covered_keys]
+
+    # Candidate backlog ranked by Maniac (deterministic greedy coverage).
+    ranked: tuple[ExperimentCandidate, ...] = ()
+    if candidates:
+        result = select_next(
+            SelectionInputs(
+                candidates=candidates,
+                covered_keys=frozenset(covered_keys),
+                gates=gates if gates is not None else _permissive_gates(),
+                max_runs=max_runs,
+                coverage_target=max(len(landscape), 1),
+                seed=seed,
+            )
+        )
+        ranked = result.selected
+
+    heatmap = render_heatmap(landscape, frozenset(covered_keys))
+
+    return M5Report(
+        landscape=landscape,
+        heatmap=heatmap,
+        cell_verdicts=verdicts,
+        coverage_fraction=fraction,
+        next_to_run=tuple(unknown),
+        ranked_backlog=ranked,
+    )

mayhem/infra/store.py

@@ -0,0 +1,200 @@
+"""SQLite store: one connection, WAL, disciplined pragmas (ADR-0007).
+
+The controller is the single writer. Agents never open this file.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+from contextlib import contextmanager
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from mayhem.domain.common import utc_now
+from mayhem.domain.run_outcome import Outcome, RunRecord, RunStatus, RunVerdict
+from mayhem.infra.migrations import ALL_MIGRATIONS
+from mayhem.infra.migrator import Migration, current_version, run_down_migrations, run_migrations
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+_BUSY_TIMEOUT_MS = 5_000
+
+
+class Store:
+    """Owns the SQLite connection and migration lifecycle."""
+
+    def __init__(self, path: Path | str) -> None:
+        self._path = Path(path)
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        # One connection shared across threads (parallel fault execution, ADR-0022).
+        # Access is serialized by ``_lock``; WAL + busy_timeout handle cross-process writers.
+        self._conn = sqlite3.connect(
+            self._path, timeout=_BUSY_TIMEOUT_MS / 1000, check_same_thread=False
+        )
+        self._lock = threading.RLock()
+        self._conn.row_factory = sqlite3.Row
+        for pragma in (
+            "PRAGMA journal_mode=WAL",
+            f"PRAGMA busy_timeout={_BUSY_TIMEOUT_MS}",
+            "PRAGMA foreign_keys=ON",
+            "PRAGMA synchronous=NORMAL",
+        ):
+            self._conn.execute(pragma)
+
+    @classmethod
+    def open_migrated(
+        cls, path: Path | str, migrations: tuple[Migration, ...] = ALL_MIGRATIONS
+    ) -> Store:
+        store = cls(path)
+        store.migrate(migrations)
+        return store
+
+    def migrate(self, migrations: tuple[Migration, ...] = ALL_MIGRATIONS) -> list[str]:
+        with self._lock:
+            return run_migrations(self._conn, migrations)
+
+    def migrate_down(
+        self, target_version: int, migrations: tuple[Migration, ...] = ALL_MIGRATIONS
+    ) -> list[str]:
+        """Roll the schema back to ``target_version`` (ADR-M4-5)."""
+        with self._lock:
+            return run_down_migrations(self._conn, migrations, target_version)
+
+    @property
+    def schema_version(self) -> int | None:
+        with self._lock:
+            return current_version(self._conn)
+
+    @contextmanager
+    def write(self) -> Iterator[sqlite3.Connection]:
+        """Single-writer transaction boundary; commits or rolls back atomically."""
+        try:
+            with self._lock, self._conn:
+                yield self._conn
+        except sqlite3.Error:
+            raise
+
+    def query(self, sql: str, params: tuple[object, ...] = ()) -> list[sqlite3.Row]:
+        with self._lock:
+            return list(self._conn.execute(sql, params))
+
+    def close(self) -> None:
+        self._conn.close()
+
+    # ── Run / Outcome persistence (ADR-M5-1) ──────────────────────────
+
+    def save_run_record(self, run: RunRecord) -> None:
+        """Persist a RunRecord to the m5_runs table."""
+        with self.write() as conn:
+            conn.execute(
+                """INSERT OR REPLACE INTO m5_runs
+                   (id, experiment_name, spec_json, plan_json, seed,
+                    status, environment_fingerprint, config_snapshot_id,
+                    started_at, ended_at, description, verdict,
+                    tags_json, extra_json)
+                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                (
+                    run.run_id,
+                    run.experiment_name,
+                    run.spec_json,
+                    run.plan_json,
+                    run.seed,
+                    run.status.value,
+                    run.environment_fingerprint,
+                    run.config_snapshot_id,
+                    run.started_at,
+                    run.ended_at,
+                    run.description,
+                    run.verdict.value,
+                    json.dumps(list(run.tags)),
+                    json.dumps(run.extra),
+                ),
+            )
+
+    def load_run_record(self, run_id: str) -> RunRecord | None:
+        """Load a RunRecord by id, or None if absent."""
+        rows = self.query("SELECT * FROM m5_runs WHERE id = ?", (run_id,))
+        if not rows:
+            return None
+        row = rows[0]
+        return RunRecord(
+            run_id=row["id"],
+            experiment_name=row["experiment_name"],
+            spec_json=row["spec_json"],
+            plan_json=row["plan_json"],
+            seed=row["seed"],
+            status=RunStatus(row["status"]),
+            environment_fingerprint=row["environment_fingerprint"],
+            config_snapshot_id=row["config_snapshot_id"],
+            started_at=row["started_at"],
+            ended_at=row["ended_at"],
+            description=row["description"],
+            verdict=RunVerdict(row["verdict"]),
+            tags=tuple(json.loads(row["tags_json"])),
+            extra=json.loads(row["extra_json"]),
+        )
+
+    def save_outcome(self, outcome: Outcome) -> None:
+        """Persist an Outcome to the m5_outcomes table."""
+        with self.write() as conn:
+            conn.execute(
+                """INSERT OR REPLACE INTO m5_outcomes
+                   (run_id, body_json, body_hash, checks_passed, checks_failed,
+                    metric_deltas_json, residual_effect, stability_signal,
+                    extra_json)
+                   VALUES (?,?,?,?,?,?,?,?,?)""",
+                (
+                    outcome.run_id,
+                    outcome.body_json,
+                    outcome.body_hash,
+                    outcome.checks_passed,
+                    outcome.checks_failed,
+                    json.dumps(outcome.metric_deltas),
+                    outcome.residual_effect,
+                    outcome.stability_signal,
+                    json.dumps(outcome.extra),
+                ),
+            )
+
+    def save_observation(
+        self,
+        kind: str,
+        *,
+        run_id: str = "",
+        source: str = "",
+        data: dict[str, object] | None = None,
+    ) -> None:
+        """Persist one row to the observations table (ADR-M5 phase-gated)."""
+        with self.write() as conn:
+            conn.execute(
+                """INSERT INTO observations (kind, run_id, source, data_json, timestamp)
+                   VALUES (?,?,?,?,?)""",
+                (
+                    kind,
+                    run_id,
+                    source,
+                    json.dumps(data or {}),
+                    utc_now().isoformat(),
+                ),
+            )
+
+    def load_outcome(self, run_id: str) -> Outcome | None:
+        """Load an Outcome by run_id, or None if absent."""
+        rows = self.query("SELECT * FROM m5_outcomes WHERE run_id = ?", (run_id,))
+        if not rows:
+            return None
+        row = rows[0]
+        return Outcome(
+            run_id=row["run_id"],
+            body_json=row["body_json"],
+            body_hash=row["body_hash"],
+            checks_passed=row["checks_passed"],
+            checks_failed=row["checks_failed"],
+            metric_deltas=json.loads(row["metric_deltas_json"]),
+            residual_effect=row["residual_effect"],
+            stability_signal=row["stability_signal"],
+            extra=json.loads(row["extra_json"]),
+        )

mayhem/spec.py

@@ -0,0 +1,52 @@
+"""YAML drill-spec loading — the authored entry point into the domain.
+
+Spec files stay close to the domain models: keys map 1:1 onto pydantic
+fields (durations as ``10s`` strings, enums as lowercase values), so the
+loader is a thin validate-and-discriminate layer, not a second language.
+
+``load_drill`` / ``parse_drill`` handle the ``kind: drill`` format (ADR-0019)
+— the only supported spec kind since the clean break (ADR-0021).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import ValidationError
+
+from mayhem.domain.errors import SchemaValidationError
+from mayhem.domain.experiments import DrillSpec
+
+
+def _flatten(exc: ValidationError) -> str:
+    parts = []
+    for err in exc.errors()[:8]:
+        loc = ".".join(str(part) for part in err["loc"]) or "<root>"
+        parts.append(f"{loc}: {err['msg']}")
+    return "; ".join(parts)
+
+
+def load_drill(path: str | Path) -> DrillSpec:
+    """Load a drill spec from YAML; refuse anything the domain refuses."""
+    raw_path = Path(path)
+    if not raw_path.is_file():
+        raise FileNotFoundError(f"spec file not found: {raw_path}")
+    try:
+        data = yaml.safe_load(raw_path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise SchemaValidationError("drill", f"invalid YAML: {exc}") from None
+    return parse_drill(data)
+
+
+def parse_drill(data: Any) -> DrillSpec:
+    """Parse raw YAML data into a DrillSpec."""
+    if not isinstance(data, dict):
+        raise SchemaValidationError("drill", "top level must be a mapping")
+    if data.get("kind") != "drill":
+        raise SchemaValidationError("drill", f"expected kind: drill, got: {data.get('kind')}")
+    try:
+        return DrillSpec.model_validate(data)
+    except ValidationError as exc:
+        raise SchemaValidationError("drill", _flatten(exc)) from None

mayhem/toolkit/__init__.py

@@ -0,0 +1 @@
+"""Shared toolkit primitives (Phase 2): tool_runner, snapshots, fingerprinting."""