mayhem-cli 0.5.1__py3-none-any.whl

mayhem/agents/lease_client.py

@@ -0,0 +1,141 @@
+"""LeaseClient — the only way an agent touches lease state.
+
+Every mutation is a legal state-machine transition (the domain module enforces
+it) followed by an immediate sink write, so a crash between inject and release
+still leaves a durable trail for the janitor.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from typing import TYPE_CHECKING, Any
+
+from mayhem.domain.common import utc_now
+from mayhem.domain.leases import FaultLease, LeaseState
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from mayhem.agents.sinks import LeaseSink
+from mayhem.toolkit.fingerprint import interpreter_marker
+
+
+def _lease_id(sequence: int) -> str:
+    return f"l-{sequence:08d}"
+
+
+class LeaseConflictError(Exception):
+    """Two agents raced for the same target; the second one loses."""
+
+
+class LeaseClient:
+    def __init__(self, sink: LeaseSink, agent_id: str = f"ag-{interpreter_marker()}") -> None:
+        self._sink = sink
+        self._agent_id = agent_id
+        # Continue the counter from the store's high-water mark so lease ids stay
+        # unique across runs against a shared DB (the sink is single-writer per
+        # run; parallel steps within a run share this client).
+        self._sequence = sink.next_sequence()
+
+    @property
+    def agent_id(self) -> str:
+        return self._agent_id
+
+    def acquire(
+        self,
+        *,
+        run_id: str,
+        fault_id: str,
+        targets: frozenset[str] | set[str],
+        undo_ops: tuple[dict[str, Any], ...],
+        verify_probes: tuple[dict[str, Any], ...] = (),
+        ttl_seconds: float = 120.0,
+        runtime_identity: str | None = None,
+    ) -> FaultLease:
+        """Create a PENDING lease; the caller must activate() before injecting."""
+        self._sequence += 1
+        lease = FaultLease.model_validate(
+            {
+                "id": _lease_id(self._sequence),
+                "run_id": run_id,
+                "fault_id": fault_id,
+                "owner_agent": self._agent_id,
+                "targets": sorted(targets),
+                "undo_ops": list(undo_ops),
+                "verify_probes": list(verify_probes),
+                "ttl_seconds": ttl_seconds,
+                "runtime_identity": runtime_identity,
+            }
+        )
+        now = utc_now()
+        live: list[FaultLease] = []
+        overlap = [x for x in self._sink.active_leases() if x.targets & set(targets)]
+        for existing in overlap:
+            deadline = existing.created_at.timestamp() + float(existing.ttl_seconds)
+            if deadline < now.timestamp():
+                # Past TTL is abandoned by definition (ADR-0007 / janitor
+                # contract) — expire it durably so a crashed run cannot wedge
+                # every later run on the same targets.
+                self._expire_quietly(existing, now)
+            else:
+                live.append(existing)
+        if live:
+            holders = ", ".join(f"{x.id} ({x.owner_agent})" for x in live)
+            raise LeaseConflictError(
+                f"targets {sorted(targets)} already leased by {holders} "
+                "-- the owner will release them, or run `mayhem janitor` "
+                "to reclaim leases whose controller is gone"
+            )
+        self._sink.save(lease)
+        return lease
+
+    def _expire_quietly(self, lease: FaultLease, now: datetime) -> None:
+        """Best-effort TTL reap; a failed reap just gets retried next acquire."""
+        with contextlib.suppress(Exception):
+            expired = lease.transition(LeaseState.EXPIRED, mechanism="past-ttl", now=now)
+            self._sink.save(expired)
+
+    def activate(self, lease_id: str) -> FaultLease:
+        lease = self._require(lease_id)
+        activated = lease.transition(LeaseState.ACTIVE)
+        self._sink.save(activated)
+        return activated
+
+    def mark_releasing(self, lease_id: str) -> FaultLease:
+        lease = self._require(lease_id)
+        releasing = lease.transition(LeaseState.RELEASING)
+        self._sink.save(releasing)
+        return releasing
+
+    def confirm_release(self, lease_id: str, *, mechanism: str) -> FaultLease:
+        lease = self._require(lease_id)
+        released = lease.transition(LeaseState.RELEASED, mechanism=mechanism)
+        self._sink.save(released)
+        return released
+
+    def mark_dirty(self, lease_id: str, *, notes: str) -> FaultLease:
+        lease = self._require(lease_id)
+        dirty = lease.transition(LeaseState.DIRTY, escalation_notes=notes)
+        self._sink.save(dirty)
+        return dirty
+
+    def mark_orphaned(self, lease_id: str, *, notes: str | None = None) -> FaultLease:
+        lease = self._require(lease_id)
+        orphaned = lease.transition(
+            LeaseState.ORPHANED,
+            escalation_notes=notes or f"owner {lease.owner_agent} missed its heartbeat",
+        )
+        self._sink.save(orphaned)
+        return orphaned
+
+    def get(self, lease_id: str) -> FaultLease | None:
+        return self._sink.load(lease_id)
+
+    def active_leases(self) -> tuple[FaultLease, ...]:
+        return self._sink.active_leases()
+
+    def _require(self, lease_id: str) -> FaultLease:
+        lease = self._sink.load(lease_id)
+        if lease is None:
+            raise KeyError(f"unknown lease {lease_id!r}")
+        return lease

mayhem/agents/probes.py

@@ -0,0 +1,284 @@
+"""Verify probes — proof the world returned to normal (or that it didn't).
+
+A probe result is evidence, not a judgement; ``satisfied`` is computed from
+``expect_present`` so negative probes (chain absent) work identically to
+positive ones (service healthy).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import socket
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from mayhem.toolkit.tool_runner import run_tool
+
+if TYPE_CHECKING:
+    from mayhem.domain.leases import VerifyProbe
+
+
+@dataclass(frozen=True)
+class ProbeResult:
+    probe: str
+    satisfied: bool
+    detail: str
+
+    def to_row(self, lease_id: str) -> dict[str, object]:
+        return {
+            "lease_id": lease_id,
+            "probe": self.probe,
+            "satisfied": int(self.satisfied),
+            "detail": self.detail,
+        }
+
+
+def _arg_float(args: dict[str, object], key: str, default: float) -> float:
+    raw = args.get(key)
+    if not isinstance(raw, (int, float, str)) or isinstance(raw, bool):
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def _arg_str_list(args: dict[str, object], key: str) -> list[str] | None:
+    value = args.get(key)
+    if isinstance(value, list) and all(isinstance(item, str) for item in value):
+        return value
+    return None
+
+
+@dataclass(frozen=True)
+class VerifyReport:
+    results: tuple[ProbeResult, ...]
+
+    @property
+    def all_satisfied(self) -> bool:
+        return bool(self.results) and all(r.satisfied for r in self.results)
+
+
+def _run_exec(args: dict[str, object]) -> ProbeResult:
+    cmd = _arg_str_list(args, "cmd")
+    if cmd is None or not cmd:
+        return ProbeResult("exec", False, "exec probe requires cmd: list[str]")
+    engine = args.get("engine")
+    cont = args.get("cont")
+    if engine and cont:
+        # Container-addressed presence check. The resolved pid is a pid inside
+        # the runtime VM (podman-machine on macOS), which a host ``ps`` cannot
+        # see. ``<engine> inspect`` addresses the container main process across
+        # the VM boundary, so a nonzero/inspectable container means "present".
+        if args.get("incontainer"):
+            argv = [str(engine), "exec", str(cont), *cmd]
+            result = run_tool(argv, timeout_s=_arg_float(args, "timeout_s", 10.0))
+            detail = f"exec {cont} exit={result.exit_code} stderr={result.stderr[:120]!r}"
+            return ProbeResult("exec", result.succeeded, detail)
+        argv = [str(engine), "inspect", "--format", "{{.State.Pid}}", str(cont)]
+        result = run_tool(argv, timeout_s=_arg_float(args, "timeout_s", 10.0))
+        pid = result.stdout.strip()
+        present = result.succeeded and pid.isdigit() and int(pid) > 0
+        return ProbeResult("exec", present, f"inspect {cont} pid={pid!r}")
+    result = run_tool(cmd, timeout_s=_arg_float(args, "timeout_s", 10.0))
+    detail = f"exit={result.exit_code} stderr={result.stderr[:120]!r}"
+    out = (result.stdout or "").strip()
+    if out:
+        detail = f"{detail} stdout={out[:120]!r}"
+    return ProbeResult("exec", result.succeeded, detail)
+
+
+def _run_tcp(args: dict[str, object]) -> ProbeResult:
+    host = str(args.get("host", "127.0.0.1"))
+    raw_port = args.get("port")
+    if not isinstance(raw_port, int):
+        return ProbeResult("tcp.open", False, "tcp probe requires port:int")
+    timeout = _arg_float(args, "timeout_s", 3.0)
+    try:
+        with socket.create_connection((host, raw_port), timeout=timeout):
+            return ProbeResult("tcp.open", True, f"{host}:{raw_port} accepted")
+    except OSError as exc:
+        return ProbeResult("tcp.open", False, f"{host}:{raw_port} refused ({exc})")
+
+
+def _run_http(args: dict[str, object]) -> ProbeResult:
+    url = str(args.get("url", ""))
+    expected_raw = args.get("expect_status", 200)
+    if not isinstance(expected_raw, int):
+        return ProbeResult("http.status", False, "expect_status must be an int")
+    timeout = _arg_float(args, "timeout_s", 5.0)
+    if not url.startswith(("http://", "https://")):
+        return ProbeResult("http.status", False, f"bad url {url!r}")
+    try:
+        with urllib.request.urlopen(url, timeout=timeout) as response:
+            status = response.status
+    except urllib.error.HTTPError as exc:
+        status = exc.code
+    except (urllib.error.URLError, OSError) as exc:
+        return ProbeResult("http.status", False, f"{url} unreachable ({exc})")
+    return ProbeResult("http.status", status == expected_raw, f"{url} -> {status}")
+
+
+def _process_state(pid: int) -> str | None:
+    """Kernel state of ``pid`` ('R'/'S'/'T'/'Z'/…), or None when it is gone.
+
+    Reads ``/proc/<pid>/stat`` on Linux (field 3, after the parenthesised
+    comm); falls back to ``ps -o state=`` which is portable (macOS included).
+    A process that is gone — reaped, or never there — yields None.
+    """
+    try:
+        content = Path(f"/proc/{pid}/stat").read_text(errors="replace")
+    except OSError:
+        content = None
+    if content:
+        fields = content.rsplit(")", 1)
+        if len(fields) == 2:
+            parts = fields[1].split()
+            if parts:
+                return parts[0]
+    result = run_tool(["ps", "-o", "stat=", "-p", str(pid)], timeout_s=5.0)
+    if not result.succeeded:
+        return None
+    state = (result.stdout or "").strip()
+    return state[0] if state else None
+
+
+def _pid_alive(pid: int) -> bool:
+    """True when ``pid`` is a live, non-zombie process.
+
+    ``os.kill(pid, 0)`` still succeeds for a zombie — a terminated child that
+    is merely awaiting reap — which would defeat negative process probes.
+    Treat zombies (Z/x) as not alive: the target *was* terminated.
+    """
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+    return _process_state(pid) not in (None, "Z", "X")
+
+
+def _run_process(args: dict[str, object]) -> ProbeResult:
+    name = str(args.get("name", ""))
+    raw_pid = args.get("pid")
+    engine = args.get("engine")
+    cont = args.get("cont")
+    if raw_pid is not None and not isinstance(raw_pid, bool):
+        if isinstance(raw_pid, int):
+            pid: int = raw_pid
+        elif isinstance(raw_pid, str) and raw_pid.strip().lstrip("-").isdigit():
+            pid = int(raw_pid)
+        else:
+            return ProbeResult("process", False, f"bad pid {raw_pid!r}")
+        if engine and cont:
+            # Container-addressed: the resolved pid lives inside the runtime
+            # VM (podman-machine) which a host ``kill -0`` cannot reach; the
+            # container's main process is inspected host-side instead.
+            argv = [str(engine), "inspect", "--format", "{{.State.Pid}}", str(cont)]
+            result = run_tool(argv, timeout_s=_arg_float(args, "timeout_s", 5.0))
+            pid_out = result.stdout.strip()
+            present = result.succeeded and pid_out.isdigit() and int(pid_out) > 0
+            return ProbeResult("process", present, f"inspect {cont} pid={pid_out!r}")
+        alive = _pid_alive(pid)
+        return ProbeResult(
+            "process", alive, f"pid {pid} {'alive' if alive else 'gone (or zombie)'}"
+        )
+    if name:
+        result = run_tool(["pgrep", "-f", name], timeout_s=_arg_float(args, "timeout_s", 5.0))
+        found = result.succeeded and bool((result.stdout or "").strip())
+        return ProbeResult("process", found, f"pgrep {name!r} -> {result.stdout!r}")
+    return ProbeResult("process", False, "process probe requires name or pid")
+
+
+def _run_metric(args: dict[str, object]) -> ProbeResult:  # noqa: PLR0911 (one branch per failure mode)
+    endpoint = str(args.get("endpoint", ""))
+    query = str(args.get("query", ""))
+    if not endpoint:
+        return ProbeResult("metric", False, "metric probe requires endpoint")
+    try:
+        with urllib.request.urlopen(
+            endpoint, timeout=_arg_float(args, "timeout_s", 5.0)
+        ) as response:
+            body = response.read(10_000).decode("utf-8", errors="replace")
+    except Exception as exc:
+        return ProbeResult("metric", False, f"{endpoint} unreachable ({exc})")
+    found = (not query) or (query in body)
+    if not found:
+        return ProbeResult("metric", False, f"metric {query!r} not present")
+    threshold_raw = args.get("threshold")
+    threshold_num: float | None = None
+    if (
+        threshold_raw is not None
+        and not isinstance(threshold_raw, bool)
+        and isinstance(threshold_raw, (int, float, str))
+    ):
+        try:
+            threshold_num = float(threshold_raw)
+        except ValueError:
+            return ProbeResult("metric", False, f"bad threshold {threshold_raw!r}")
+    elif threshold_raw is not None and not isinstance(threshold_raw, bool):
+        return ProbeResult("metric", False, f"bad threshold {threshold_raw!r}")
+    if threshold_num is not None:
+        try:
+            match = re.search(rf"{re.escape(query)}\s+([-+0-9.eE]+)", body)
+            if match is None:
+                return ProbeResult("metric", False, f"metric {query!r} lacks a value")
+            value = float(match.group(1))
+            ok = value >= threshold_num
+            return ProbeResult("metric", ok, f"{query}={value} (threshold {threshold_num})")
+        except (ValueError, IndexError):
+            return ProbeResult("metric", False, f"cannot parse {query!r} value")
+    return ProbeResult("metric", True, f"metric {query!r} present")
+
+
+def _run_file(args: dict[str, object]) -> ProbeResult:
+    path = str(args.get("path", ""))
+    if not path:
+        return ProbeResult("file", False, "file probe requires path")
+    p = Path(path)
+    if not p.exists():
+        return ProbeResult("file", False, f"{path} missing")
+    contains = args.get("contains")
+    if contains:
+        try:
+            content = p.read_text(errors="replace")
+        except OSError as exc:
+            return ProbeResult("file", False, f"{path} unreadable ({exc})")
+        if str(contains) not in content:
+            return ProbeResult("file", False, f"{path} lacks {contains!r}")
+    return ProbeResult("file", True, f"{path} present")
+
+
+_HANDLERS = {
+    "exec": _run_exec,
+    "tcp": _run_tcp,
+    "http": _run_http,
+    "process": _run_process,
+    "metric": _run_metric,
+    "file": _run_file,
+}
+
+
+def run_probe(probe: VerifyProbe) -> ProbeResult:
+    handler = _HANDLERS.get(probe.probe.split(".", 1)[0])
+    if handler is None:
+        return ProbeResult(probe.probe, False, f"unknown probe kind {probe.probe!r}")
+    observed = handler(dict(probe.args))
+    # expect_present=True  → want the thing to exist → satisfied iff observed.
+    # expect_present=False → want absence (e.g. chain gone) → satisfied iff NOT.
+    satisfied = observed.satisfied if probe.expect_present else not observed.satisfied
+    return ProbeResult(probe.probe, satisfied, observed.detail)
+
+
+def verify_all(probes: tuple[VerifyProbe, ...], lease_id: str) -> VerifyReport:
+    """Run every probe; never raises — a crashing probe is a failed probe."""
+    results = []
+    for probe in probes:
+        try:
+            results.append(run_probe(probe))
+        except Exception as exc:  # probe crashes are evidence too
+            results.append(ProbeResult(probe.probe, False, f"probe crashed: {exc}"))
+    return VerifyReport(tuple(results))

mayhem/agents/protocol.py

@@ -0,0 +1,134 @@
+"""ndjson JSON-RPC 2.0 framing for controller↔agent communication (ADR-0003).
+
+One JSON object per line. Requests carry an id; notifications do not.
+Every params payload carries ``run_id``/``agent_id`` context for correlation —
+enforced here at the frame level so no handler can forget it.
+"""
+
+from __future__ import annotations
+
+import json
+from enum import IntEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+PROTOCOL_VERSION = "mayhem/1"
+
+
+class RpcErrorCode(IntEnum):
+    PARSE_ERROR = -32700
+    INVALID_REQUEST = -32600
+    METHOD_NOT_FOUND = -32601
+    INVALID_PARAMS = -32602
+    INTERNAL_ERROR = -32603
+    TASK_FAILED = -32000  # implementation-defined server error range
+    SAFETY_REFUSED = -32001
+
+
+class ProtocolError(Exception):
+    def __init__(self, code: int, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+
+
+class _Frame(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    jsonrpc: str = "2.0"
+
+    @model_validator(mode="after")
+    def _version(self) -> Any:
+        if self.jsonrpc != "2.0":
+            raise ValueError("jsonrpc must be '2.0'")
+        return self
+
+
+def _require_context(params: dict[str, Any]) -> None:
+    missing = [key for key in ("run_id", "agent_id") if key not in params]
+    if missing:
+        raise ProtocolError(
+            RpcErrorCode.INVALID_PARAMS,
+            f"params missing correlation context: {', '.join(missing)}",
+        )
+
+
+class RpcRequest(_Frame):
+    id: int | str
+    method: str
+    params: dict[str, Any] = Field(default_factory=dict)
+
+
+class RpcNotification(_Frame):
+    method: str
+    params: dict[str, Any] = Field(default_factory=dict)
+
+
+class RpcResponse(_Frame):
+    id: int | str | None
+    result: dict[str, Any] | None = None
+    error: dict[str, Any] | None = None  # {"code": int, "message": str}
+
+    @model_validator(mode="after")
+    def _exactly_one(self) -> Any:
+        if (self.result is None) == (self.error is None):
+            raise ValueError("response must carry exactly one of result|error")
+        return self
+
+
+def encode(frame: RpcRequest | RpcNotification | RpcResponse) -> bytes:
+    """Serialize one frame as a single ndjson line (newline included)."""
+    return (frame.model_dump_json(exclude_none=True) + "\n").encode("utf-8")
+
+
+def decode(line: str | bytes) -> RpcRequest | RpcNotification | RpcResponse:
+    """Parse one ndjson line into a typed frame.
+
+    Raises ProtocolError(PARSE_ERROR) on malformed JSON and
+    ProtocolError(INVALID_REQUEST) on structurally invalid frames.
+    """
+    try:
+        raw = json.loads(line)
+    except json.JSONDecodeError as exc:
+        raise ProtocolError(RpcErrorCode.PARSE_ERROR, f"bad json: {exc.msg}") from exc
+    if not isinstance(raw, dict):
+        raise ProtocolError(RpcErrorCode.INVALID_REQUEST, "frame must be a JSON object")
+    try:
+        if "method" in raw:
+            if "id" in raw:
+                return RpcRequest.model_validate(raw)
+            return RpcNotification.model_validate(raw)
+        return RpcResponse.model_validate(raw)
+    except ValueError as exc:
+        raise ProtocolError(RpcErrorCode.INVALID_REQUEST, str(exc)) from exc
+
+
+def error_response(request_id: int | str | None, code: int, message: str) -> RpcResponse:
+    return RpcResponse(id=request_id, error={"code": int(code), "message": message})
+
+
+def result_response(request_id: int | str | None, result: dict[str, Any]) -> RpcResponse:
+    return RpcResponse(id=request_id, result=result)
+
+
+METHOD_HANDSHAKE = "handshake"
+METHOD_CAPABILITIES_QUERY = "capabilities.query"
+METHOD_TASK_EXECUTE = "task.execute"
+METHOD_TASK_CANCEL = "task.cancel"
+METHOD_TASK_STATUS = "task.status"
+METHOD_LEASE_EXTEND = "lease.extend"
+METHOD_HEALTH_PING = "health.ping"
+NOTIFICATION_EVENT_EMIT = "event.emit"
+NOTIFICATION_LOG_EMIT = "log.emit"
+KNOWN_METHODS = frozenset(
+    {
+        METHOD_HANDSHAKE,
+        METHOD_CAPABILITIES_QUERY,
+        METHOD_TASK_EXECUTE,
+        METHOD_TASK_CANCEL,
+        METHOD_TASK_STATUS,
+        METHOD_LEASE_EXTEND,
+        METHOD_HEALTH_PING,
+    }
+)
+KNOWN_NOTIFICATIONS = frozenset({NOTIFICATION_EVENT_EMIT, NOTIFICATION_LOG_EMIT})