mayhem-cli 0.5.1__py3-none-any.whl

mayhem/controller/janitor.py

@@ -0,0 +1,196 @@
+"""Janitor — background sweep that enforces lease TTLs.
+
+A fault left past its TTL is by definition unattended. PENDING leases are
+expired (never injected, nothing to undo); ACTIVE leases are orphaned and
+then released through their write-ahead undo contract; ORPHANED/RELEASING
+leases stuck mid-compensation are finalized; DIRTY leases (compensation
+already failed) are surrendered to EXPIRED — the janitor never leaves a
+fault running because its owner vanished.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from mayhem.domain.common import utc_now
+from mayhem.domain.errors import DomainError
+from mayhem.domain.leases import LeaseState
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from datetime import datetime
+
+    from mayhem.agents.sinks import LeaseSink
+    from mayhem.domain.leases import FaultLease
+
+_LOST_OWNER = "owner run no longer live; reclaimed before TTL"
+
+
+def _owner_gone(run_liveness: Callable[[str], bool | None], run_id: str) -> bool:
+    """True only when the resolver can *prove* the owning controller is gone.
+
+    Unknown (``None``) or an absent owner row never triggers the early reclaim —
+    TTL policy stays in charge of those, and a LiveLonger cycling pid can only
+    read "alive", which delays cleanup rather than wrongly reclaiming a lease
+    a live owner still needs.
+    """
+    try:
+        verdict = run_liveness(run_id)
+    except LookupError:
+        return False
+    return verdict is False
+
+
+@dataclass(frozen=True)
+class SweepResult:
+    expired: tuple[str, ...]
+    recovered: tuple[str, ...]  # orphaned -> released via undo path marker
+    dirty: tuple[str, ...]
+
+    @property
+    def quiet(self) -> bool:
+        return not (self.expired or self.recovered or self.dirty)
+
+
+class Janitor:
+    """TTL enforcement over any LeaseSink; state transitions only.
+
+    ``run_liveness`` is an optional resolver (run_id -> bool | None) the
+    caller supplies when it can see the runs table. It returns ``True`` while
+    the owning controller is alive, ``False`` when the owner is provably gone,
+    and ``None`` when unknown. A lease whose owner is provably gone is
+    reclaimed *before* its TTL — without this, a crashed ``run`` wedges its
+    targets for the whole TTL and the next ``run`` conflicts with a lease the
+    janitor "did nothing about".
+    """
+
+    def __init__(self, sink: LeaseSink) -> None:
+        self._sink = sink
+
+    def sweep(
+        self,
+        *,
+        now_epoch_s: float | None = None,
+        run_liveness: Callable[[str], bool | None] | None = None,
+    ) -> SweepResult:
+        now = utc_now()
+        current = now.timestamp() if now_epoch_s is None else now_epoch_s
+        expired: list[str] = []
+        recovered: list[str] = []
+        dirty: list[str] = []
+        for lease in self._sink.active_leases():
+            deadline = lease.created_at.timestamp() + float(lease.ttl_seconds)
+            owner_gone = run_liveness is not None and _owner_gone(run_liveness, lease.run_id)
+            if deadline >= current and not owner_gone:
+                continue
+            notes = _LOST_OWNER if owner_gone else None
+            if lease.state is LeaseState.PENDING:
+                self._expire(lease, now, expired, notes=notes)
+            elif lease.state is LeaseState.DIRTY:
+                self._surrender(lease, now, expired, dirty)
+            elif lease.state is LeaseState.RELEASING:
+                self._finalize(lease, now, recovered, dirty)
+            else:  # ACTIVE or ORPHANED
+                self._recover_orphan(lease, now, recovered, dirty, notes=notes)
+        return SweepResult(tuple(expired), tuple(recovered), tuple(dirty))
+
+    def _expire(
+        self, lease: FaultLease, now: datetime, expired: list[str], *, notes: str | None = None
+    ) -> None:
+        # Never injected, nothing to undo: straight to EXPIRED.
+        expired.append(lease.id)
+        self._save_quietly(
+            lease.transition(
+                LeaseState.EXPIRED,
+                mechanism="janitor",
+                now=now,
+                escalation_notes=notes,
+            )
+        )
+
+    def _surrender(
+        self, lease: FaultLease, now: datetime, expired: list[str], dirty: list[str]
+    ) -> None:
+        # Compensation already failed and nobody is coming back for this lease
+        # past its TTL — record the surrender, unblock the targets next run.
+        try:
+            surrendered = lease.transition(LeaseState.EXPIRED, mechanism="janitor", now=now)
+            self._sink.save(surrendered)
+        except (DomainError, OSError):
+            dirty.append(lease.id)  # stays DIRTY, still wedged
+        else:
+            expired.append(lease.id)
+
+    def _finalize(
+        self, lease: FaultLease, now: datetime, recovered: list[str], dirty: list[str]
+    ) -> None:
+        # A lease stuck mid-compensation finalizes straight to RELEASED — the
+        # owner is gone, no second RELEASING hop.
+        try:
+            final = lease.transition(LeaseState.RELEASED, mechanism="janitor", now=now)
+            self._sink.save(final)
+        except (DomainError, OSError) as exc:
+            self._dirty_from(lease, exc)
+            dirty.append(lease.id)
+        else:
+            recovered.append(lease.id)
+
+    def _recover_orphan(
+        self,
+        lease: FaultLease,
+        now: datetime,
+        recovered: list[str],
+        dirty: list[str],
+        *,
+        notes: str | None = None,
+    ) -> None:
+        # Orphan ACTIVE leases first (honest record), then finalize the
+        # compensation the owner started (stuck ORPHANED leases keep their
+        # targets wedged without this path).
+        if lease.state is LeaseState.ACTIVE:
+            orphaned = lease.transition(
+                LeaseState.ORPHANED,
+                mechanism="janitor",
+                now=now,
+                escalation_notes=notes or f"lease {lease.id} exceeded TTL without release",
+            )
+        else:
+            orphaned = lease
+        try:
+            released = orphaned.transition(LeaseState.RELEASING, mechanism="janitor", now=now)
+            self._sink.save(released)
+            final = released.transition(LeaseState.RELEASED, mechanism="janitor", now=now)
+            self._sink.save(final)
+            recovered.append(orphaned.id)
+        except (DomainError, OSError) as exc:
+            # Persistence failed: never claim recovery we could not record.
+            self._dirty_from(orphaned, exc)
+            dirty.append(orphaned.id)
+
+    def _save_quietly(self, lease: FaultLease) -> None:
+        with contextlib.suppress(Exception):  # sweep must survive sink flakiness
+            self._sink.save(lease)
+
+    def _dirty_from(self, lease: FaultLease, exc: Exception) -> None:
+        try:
+            if lease.state is LeaseState.RELEASING:
+                stuck = lease.transition(
+                    LeaseState.DIRTY,
+                    mechanism="janitor",
+                    now=utc_now(),
+                    escalation_notes=f"orphan recovery failed: {exc}",
+                )
+            else:
+                stuck = lease.transition(
+                    LeaseState.RELEASING, mechanism="janitor", now=utc_now()
+                ).transition(
+                    LeaseState.DIRTY,
+                    mechanism="janitor",
+                    now=utc_now(),
+                    escalation_notes=f"orphan recovery failed: {exc}",
+                )
+            self._save_quietly(stuck)
+        except DomainError:
+            pass

mayhem/controller/observability_collector.py

@@ -0,0 +1,382 @@
+"""Declarative observability collectors (ADR-M4-4).
+
+Each declared source is collected within its own bounded timeout and the whole
+pass respects the config's ``total_timeout``. Collection is **best-effort**:
+a failing source is recorded as a failed collection with a note, never raised
+to the run — evidence gathering must never break the drill.
+
+Collected evidence is returned as :class:`SourceCollection` objects the
+executor persists onto the run row and spools into the journal, so an observer
+can replay exactly what was seen.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+import urllib.request
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from mayhem.agents.probes import run_probe
+from mayhem.domain.checks import Probe, ProbeType
+from mayhem.domain.common import utc_now
+from mayhem.domain.leases import VerifyProbe
+from mayhem.domain.observability import (
+    InspectionSource,
+    LogsSource,
+    MetricsSource,
+    ObservabilityConfig,
+    ObservabilitySourceKind,
+    ProbeSource,
+    duration_seconds,
+)
+from mayhem.toolkit.tool_runner import run_tool
+
+if TYPE_CHECKING:
+    from mayhem.domain.observability import ObservabilitySource
+
+
+_LEADING_NUMBER = re.compile(r"[-+]?[0-9]*\.?[0-9]+")
+
+
+@dataclass(frozen=True)
+class ObservabilitySample:
+    """One measured value with its wall-clock timestamp."""
+
+    at: str
+    value: object
+
+    @classmethod
+    def now(cls, value: object) -> ObservabilitySample:
+        return cls(at=utc_now().isoformat(), value=value)
+
+    def to_jsonable(self) -> dict[str, object]:
+        return {"at": self.at, "value": self.value}
+
+
+@dataclass(frozen=True)
+class SourceCollection:
+    """Outcome of collecting one declared source."""
+
+    source_id: str
+    kind: ObservabilitySourceKind
+    ok: bool
+    note: str
+    latency_ms: float
+    samples: tuple[ObservabilitySample, ...] = ()
+    skipped: bool = False
+
+    def to_jsonable(self) -> dict[str, object]:
+        return {
+            "source_id": self.source_id,
+            "kind": self.kind.value,
+            "ok": self.ok,
+            "note": self.note,
+            "latency_ms": self.latency_ms,
+            "skipped": self.skipped,
+            "samples": [s.to_jsonable() for s in self.samples],
+        }
+
+
+def probe_to_verify(probe: Probe) -> VerifyProbe:
+    """Canonical mapping of a domain ``Probe`` onto the agents ``VerifyProbe``.
+
+    Lifted out of the executor so observability probe sources and check steps
+    share the exact same runtime semantics (ADR-M4-2/4-4).
+    """
+    if probe.type is ProbeType.EXEC:
+        return VerifyProbe(
+            probe="exec",
+            args={"cmd": list(probe.cmd), "timeout_s": float(probe.timeout)},
+            expect_present=True,
+        )
+    if probe.type is ProbeType.TCP:
+        return VerifyProbe(
+            probe="tcp",
+            args={
+                "host": probe.host,
+                "port": int(probe.port),
+                "timeout_s": float(probe.timeout),
+            },
+            expect_present=True,
+        )
+    if probe.type is ProbeType.HTTP:
+        return VerifyProbe(
+            probe="http",
+            args={
+                "url": probe.url,
+                "expect_status": int(probe.expected_status),
+                "timeout_s": float(probe.timeout),
+            },
+            expect_present=True,
+        )
+    if probe.type is ProbeType.PROCESS:
+        return VerifyProbe(
+            probe="process",
+            args={
+                "name": probe.name,
+                "pid": probe.pid,
+                "timeout_s": float(probe.timeout),
+            },
+            expect_present=True,
+        )
+    if probe.type is ProbeType.METRIC:
+        return VerifyProbe(
+            probe="metric",
+            args={
+                "endpoint": probe.endpoint,
+                "query": probe.query,
+                "threshold": probe.threshold,
+                "timeout_s": float(probe.timeout),
+            },
+            expect_present=True,
+        )
+    if probe.type is ProbeType.FILE:
+        return VerifyProbe(
+            probe="file",
+            args={
+                "path": probe.path,
+                "contains": probe.contains,
+                "timeout_s": float(probe.timeout),
+            },
+            expect_present=True,
+        )
+    raise AssertionError(f"unhandled probe type {probe.type!r}")
+
+
+def _default_logs_runner(engine: str, timeout_s: float) -> Any:
+    def run_container_logs(container: str, tail: int, since: str) -> tuple[str, float]:
+        argv = [engine, "logs", "--tail", str(tail)]
+        if since:
+            argv += ["--since", since]
+        argv.append(container)
+        started = time.monotonic()
+        result = run_tool(argv, timeout_s=timeout_s)
+        latency_ms = round((time.monotonic() - started) * 1000, 3)
+        if result.exit_code != 0:
+            raise RuntimeError(f"{engine} logs exited {result.exit_code}: {result.stderr.strip()}")
+        text = f"{result.stdout}\n{result.stderr}".strip()
+        return text, latency_ms
+
+    return run_container_logs
+
+
+def _default_inspect_runner(engine: str, timeout_s: float) -> Any:
+    def run_container_inspect(container: str) -> tuple[str, float]:
+        started = time.monotonic()
+        result = run_tool([engine, "inspect", container], timeout_s=timeout_s)
+        latency_ms = round((time.monotonic() - started) * 1000, 3)
+        if result.exit_code != 0:
+            raise RuntimeError(
+                f"{engine} inspect exited {result.exit_code}: {result.stderr.strip()}"
+            )
+        return result.stdout.strip(), latency_ms
+
+    return run_container_inspect
+
+
+def _default_probe_runner(timeout_s: float) -> Any:
+    def run_domain_probe(probe: Probe) -> tuple[bool, str, float]:
+        started = time.monotonic()
+        result = run_probe(probe_to_verify(probe))
+        latency_ms = round((time.monotonic() - started) * 1000, 3)
+        return result.satisfied, result.detail, latency_ms
+
+    return run_domain_probe
+
+
+def _parse_prometheus_text(body: str, metric: str) -> float | None:
+    """Resolve one metric's last sample value from Prometheus text format."""
+    found: list[float] = []
+    prefix = metric + "{"
+    for raw in body.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split(" ", 1)
+        if len(parts) != 2:
+            continue
+        name = parts[0]
+        if name == metric or name.startswith(prefix):
+            match = _LEADING_NUMBER.match(parts[1])
+            if match is not None:
+                found.append(float(match.group(0)))
+    return found[-1] if found else None
+
+
+def _default_metrics_runner(timeout_s: float) -> Any:
+    def scrape_metric(endpoint: str, metric: str) -> tuple[float | None, str, float]:
+        started = time.monotonic()
+        with urllib.request.urlopen(endpoint, timeout=timeout_s) as response:
+            body = response.read(1_048_576).decode("utf-8", errors="replace")
+        latency_ms = round((time.monotonic() - started) * 1000, 3)
+        value = _parse_prometheus_text(body, metric)
+        note = "" if value is not None else f"metric {metric!r} not present in scrape"
+        return value, note, latency_ms
+
+    return scrape_metric
+
+
+def collect_observability(
+    cfg: ObservabilityConfig,
+    *,
+    engine: str = "podman",
+) -> tuple[SourceCollection, ...]:
+    """Collect every declared source within ``total_timeout`` (ADR-M4-4).
+
+    Runners are module-level defaults that call ``toolkit.run_tool`` and the
+    agents probe runner so real runs capture full tool evidence; tests inject
+    fakes via the ``_collect_*`` helpers or monkeypatch the primitives.
+    """
+    if cfg.empty:
+        return ()
+    deadline = time.monotonic() + duration_seconds(cfg.total_timeout)
+    default_cadence = duration_seconds(cfg.cadence)
+    collections: list[SourceCollection] = []
+    for source in cfg.sources:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            collections.append(
+                SourceCollection(
+                    source_id=source.source_id,
+                    kind=ObservabilitySourceKind(source.kind),
+                    ok=False,
+                    note="pass total_timeout exhausted before this source",
+                    latency_ms=0.0,
+                    skipped=True,
+                )
+            )
+            continue
+        timeout_s = min(duration_seconds(getattr(source, "timeout", "10s")), remaining)
+        cadence = duration_seconds(getattr(source, "cadence", "0s"))
+        if cadence <= 0:
+            cadence = default_cadence  # config-level cadence applies unless overridden
+        collections.append(
+            _collect_source(source, timeout_s=timeout_s, cadence=cadence, deadline=deadline)
+        )
+    return tuple(collections)
+
+
+def _collect_source(
+    source: ObservabilitySource,
+    *,
+    timeout_s: float,
+    cadence: float,
+    deadline: float,
+) -> SourceCollection:
+    if isinstance(source, LogsSource):
+        return _collect_logs(source, timeout_s)
+    if isinstance(source, InspectionSource):
+        return _collect_inspection(source, timeout_s)
+    if isinstance(source, ProbeSource):
+        return _collect_probe(source, timeout_s, cadence, deadline)
+    if isinstance(source, MetricsSource):
+        return _collect_metrics(source, timeout_s, cadence, deadline)
+    raise AssertionError(f"unhandled observability source {source!r}")
+
+
+def _fail(source: ObservabilitySource, note: str) -> SourceCollection:
+    return SourceCollection(
+        source_id=source.source_id,
+        kind=ObservabilitySourceKind(source.kind),
+        ok=False,
+        note=note,
+        latency_ms=0.0,
+    )
+
+
+def _collect_logs(source: LogsSource, timeout_s: float) -> SourceCollection:
+    try:
+        text, latency = _default_logs_runner("podman", timeout_s)(
+            source.container, source.tail, source.since
+        )
+        return SourceCollection(
+            source_id=source.source_id,
+            kind=ObservabilitySourceKind.LOGS,
+            ok=True,
+            note=f"collected {len(text)} chars",
+            latency_ms=latency,
+            samples=(ObservabilitySample.now(text),),
+        )
+    except Exception as exc:
+        return _fail(source, f"collection failed: {type(exc).__name__}: {exc}")
+
+
+def _collect_inspection(source: InspectionSource, timeout_s: float) -> SourceCollection:
+    try:
+        text, latency = _default_inspect_runner("podman", timeout_s)(source.container)
+        return SourceCollection(
+            source_id=source.source_id,
+            kind=ObservabilitySourceKind.INSPECTION,
+            ok=True,
+            note=f"inspect {len(text)} chars",
+            latency_ms=latency,
+            samples=(ObservabilitySample.now(text),),
+        )
+    except Exception as exc:
+        return _fail(source, f"collection failed: {type(exc).__name__}: {exc}")
+
+
+def _collect_probe(
+    source: ProbeSource, timeout_s: float, cadence: float, deadline: float
+) -> SourceCollection:
+    latency_ms_total = 0.0
+    samples: list[ObservabilitySample] = []
+    try:
+        runner = _default_probe_runner(timeout_s)
+        while True:
+            satisfied, detail, latency = runner(source.probe)
+            latency_ms_total += latency
+            samples.append(ObservabilitySample.now({"satisfied": satisfied, "detail": detail}))
+            if time.monotonic() >= deadline:
+                break
+            sleep_for = min(cadence, max(0.0, deadline - time.monotonic()))
+            time.sleep(sleep_for)
+            if time.monotonic() >= deadline:
+                break
+        return SourceCollection(
+            source_id=source.source_id,
+            kind=ObservabilitySourceKind.PROBE,
+            ok=True,
+            note=f"{len(samples)} probe sample(s)",
+            latency_ms=round(latency_ms_total, 3),
+            samples=tuple(samples),
+        )
+    except Exception as exc:
+        return _fail(
+            source,
+            f"collection failed: {type(exc).__name__}: {exc}",
+        )
+
+
+def _collect_metrics(
+    source: MetricsSource, timeout_s: float, cadence: float, deadline: float
+) -> SourceCollection:
+    latency_ms_total = 0.0
+    samples: list[ObservabilitySample] = []
+    note = ""
+    try:
+        runner = _default_metrics_runner(timeout_s)
+        while True:
+            value, scrape_note, latency = runner(source.endpoint, source.metric)
+            latency_ms_total += latency
+            samples.append(ObservabilitySample.now(value))
+            if value is None and scrape_note:
+                note = scrape_note
+            if time.monotonic() >= deadline:
+                break
+            sleep_for = min(cadence, max(0.0, deadline - time.monotonic()))
+            time.sleep(sleep_for)
+            if time.monotonic() >= deadline:
+                break
+        return SourceCollection(
+            source_id=source.source_id,
+            kind=ObservabilitySourceKind.METRICS,
+            ok=note == "" or bool(samples),
+            note=note or f"{len(samples)} metric sample(s)",
+            latency_ms=round(latency_ms_total, 3),
+            samples=tuple(samples),
+        )
+    except Exception as exc:
+        return _fail(source, f"collection failed: {type(exc).__name__}: {exc}")

mayhem/controller/observations.py

@@ -0,0 +1,102 @@
+"""Observation engine — structured event log for experiment runs (ADR-0020).
+
+An ``Observation`` is a timestamped, typed event that occurred during an
+experiment — fault injected, probe measured, recovery attempted, threshold
+breached, etc. The ``ObservationLog`` is an append-only, in-memory store
+that can be serialized to JSON for persistence or display.
+
+Observations are the raw material that the evaluation system consumes.
+They are never mutated after creation.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from mayhem.domain.common import utc_now
+
+
+class ObservationKind(StrEnum):
+    """Categories of observations that can occur during an experiment."""
+
+    FAULT_INJECTED = "fault.injected"
+    FAULT_UNDONE = "fault.undone"
+    PROBE_MEASURED = "probe.measured"
+    THRESHOLD_BREACHED = "threshold.breached"
+    RECOVERY_ATTEMPTED = "recovery.attempted"
+    RECOVERY_SUCCEEDED = "recovery.succeeded"
+    RECOVERY_FAILED = "recovery.failed"
+    STEP_STARTED = "step.started"
+    STEP_COMPLETED = "step.completed"
+    STEP_FAILED = "step.failed"
+    ANOMALY_DETECTED = "anomaly.detected"
+
+
+class Observation(BaseModel):
+    """A single immutable observation event."""
+
+    model_config = ConfigDict(frozen=True)
+
+    kind: ObservationKind
+    run_id: str
+    timestamp: str = Field(default_factory=lambda: utc_now().isoformat())
+    source: str = ""  # e.g. "executor", "janitor", "probe_runner"
+    data: dict[str, Any] = Field(default_factory=dict)
+    runtime_identity: str | None = None  # canonical identity key (ADR-M1-1/1-3)
+
+
+class ObservationLog:
+    """Append-only observation log with query helpers.
+
+    The log lives in memory for the duration of a run. Periodic snapshots
+    can be flushed to SQLite or JSON for durability.
+    """
+
+    def __init__(self) -> None:
+        self._observations: list[Observation] = []
+
+    def record(self, observation: Observation) -> None:
+        """Append an observation to the log."""
+        self._observations.append(observation)
+
+    def emit(
+        self,
+        kind: ObservationKind,
+        run_id: str,
+        *,
+        source: str = "",
+        **data: Any,
+    ) -> None:
+        """Shorthand: create and record in one call."""
+        self.record(Observation(kind=kind, run_id=run_id, source=source, data=data))
+
+    @property
+    def observations(self) -> tuple[Observation, ...]:
+        return tuple(self._observations)
+
+    def for_run(self, run_id: str) -> tuple[Observation, ...]:
+        """All observations for a specific run."""
+        return tuple(o for o in self._observations if o.run_id == run_id)
+
+    def for_kind(
+        self, kind: ObservationKind, *, run_id: str | None = None
+    ) -> tuple[Observation, ...]:
+        """All observations of a given kind, optionally filtered by run."""
+        result = (o for o in self._observations if o.kind == kind)
+        if run_id is not None:
+            result = (o for o in result if o.run_id == run_id)
+        return tuple(result)
+
+    def anomalies_for_run(self, run_id: str) -> tuple[Observation, ...]:
+        """All anomaly observations for a run."""
+        return self.for_kind(ObservationKind.ANOMALY_DETECTED, run_id=run_id)
+
+    def snapshot(self) -> tuple[dict[str, Any], ...]:
+        """Serialize all observations to JSON-compatible dicts."""
+        return tuple(o.model_dump(mode="json") for o in self._observations)
+
+    def __len__(self) -> int:
+        return len(self._observations)