mayhem-cli 0.5.1__py3-none-any.whl

mayhem/domain/leases.py

@@ -0,0 +1,186 @@
+"""Fault leases: the recovery guarantee's core object (ADR-0005).
+
+A lease is created *before* any mutation and persisted with write-ahead undo
+data. The state machine below is the single source of truth for legal
+transitions; the janitor, watchdog, and release paths all go through it.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from mayhem.domain.common import Duration, utc_now
+from mayhem.domain.errors import InvalidTransitionError, InvariantViolationError
+
+
+class LeaseState(StrEnum):
+    PENDING = "pending"  # undo written, injection not yet started
+    ACTIVE = "active"  # fault applied
+    RELEASING = "releasing"  # compensation in progress
+    RELEASED = "released"  # compensated AND verified (safe terminal)
+    EXPIRED = "expired"  # watchdog TTL fired, agent self-compensated (safe terminal)
+    ORPHANED = "orphaned"  # owner unreachable; janitor will reclaim
+    DIRTY = "dirty"  # compensation failed; LOUD escalation (ack required)
+
+
+_TRANSITIONS: dict[LeaseState, frozenset[LeaseState]] = {
+    LeaseState.PENDING: frozenset({LeaseState.ACTIVE, LeaseState.EXPIRED}),
+    LeaseState.ACTIVE: frozenset({LeaseState.RELEASING, LeaseState.EXPIRED, LeaseState.ORPHANED}),
+    LeaseState.ORPHANED: frozenset({LeaseState.RELEASING}),
+    LeaseState.RELEASING: frozenset({LeaseState.RELEASED, LeaseState.DIRTY}),
+    # Safe terminals: released, expired. Dirty may only be *expired* past its
+    # TTL by the janitor — the fault is then abandoned (ADR-0007 surrender).
+    LeaseState.RELEASED: frozenset(),
+    LeaseState.EXPIRED: frozenset(),
+    LeaseState.DIRTY: frozenset({LeaseState.EXPIRED}),
+}
+
+_SAFE_TERMINALS: frozenset[LeaseState] = frozenset({LeaseState.RELEASED, LeaseState.EXPIRED})
+
+
+class UndoOp(BaseModel):
+    """One compensation primitive; MUST be idempotent-first by convention."""
+
+    model_config = ConfigDict(frozen=True)
+
+    op: str  # executor operation name, e.g. "tc.del_qdisc"
+    args: dict[str, str] = Field(default_factory=dict)
+    idempotent: bool = True
+
+
+class VerifyProbe(BaseModel):
+    """Post-recovery assertion proving the fault is really gone."""
+
+    model_config = ConfigDict(frozen=True)
+
+    probe: str  # e.g. "exec", "tc.qdisc_absent", "iptables.chain_absent"
+    args: dict[str, object] = Field(default_factory=dict)
+    expect_present: bool = False
+
+
+class FaultLease(BaseModel):
+    """Immutable lease value object; transitions produce new instances."""
+
+    model_config = ConfigDict(frozen=True)
+
+    id: str  # l-<hex>
+    run_id: str
+    fault_id: str
+    owner_agent: str
+    targets: frozenset[str]
+    undo_ops: tuple[UndoOp, ...] = ()
+    verify_probes: tuple[VerifyProbe, ...] = ()
+    ttl_seconds: Duration = 120.0
+    state: LeaseState = LeaseState.PENDING
+    created_at: datetime = Field(default_factory=utc_now)
+    injected_at: datetime | None = None
+    released_at: datetime | None = None
+    release_mechanism: str | None = None  # normal|watchdog|janitor|manual
+    escalation_notes: str | None = None
+    runtime_identity: str | None = None  # canonical identity key (ADR-M1-1/1-3)
+
+    @field_validator("id")
+    @classmethod
+    def _id_prefix(cls, value: str) -> str:
+        if not value.startswith("l-"):
+            msg = f"lease ids start with 'l-', got {value!r}"
+            raise InvariantViolationError("lease_id_prefix", msg)
+        return value
+
+    @model_validator(mode="after")
+    def _check_invariants(self) -> FaultLease:
+        if self.state is LeaseState.ACTIVE and not self.undo_ops:
+            raise InvariantViolationError(
+                "undo_required_before_active",
+                f"lease {self.id} cannot be ACTIVE without write-ahead undo ops",
+            )
+        if self.state in (LeaseState.ACTIVE, LeaseState.RELEASING) and not self.verify_probes:
+            raise InvariantViolationError(
+                "verify_required_before_release",
+                f"lease {self.id} in {self.state.value} needs verify probes",
+            )
+        if self.state is LeaseState.DIRTY and not self.escalation_notes:
+            raise InvariantViolationError(
+                "dirty_requires_escalation_notes",
+                f"lease {self.id} marked dirty without escalation notes",
+            )
+        if self.injected_at is not None and self.created_at > self.injected_at:
+            raise InvariantViolationError(
+                "lease_time_ordering", f"lease {self.id} injected before creation"
+            )
+        if self.released_at is not None and self.state not in _SAFE_TERMINALS:
+            raise InvariantViolationError(
+                "lease_time_ordering",
+                f"lease {self.id} has released_at while in state {self.state.value}",
+            )
+        return self
+
+    # -- state machine ------------------------------------------------------------
+    def can_transition(self, target: LeaseState) -> bool:
+        return target in _TRANSITIONS[self.state]
+
+    def transition(
+        self,
+        target: LeaseState,
+        *,
+        mechanism: str | None = None,
+        now: datetime | None = None,
+        escalation_notes: str | None = None,
+    ) -> FaultLease:
+        """Return a new lease advanced to ``target``.
+
+        Raises:
+            InvalidTransitionError: If the transition is illegal.
+            InvariantViolationError: If target-specific preconditions are unmet.
+        """
+        if not self.can_transition(target):
+            raise InvalidTransitionError("fault_lease", self.id, self.state.value, target.value)
+        updates: dict[str, object] = {"state": target}
+        moment = now if now is not None else utc_now()
+        if target is LeaseState.ACTIVE:
+            updates["injected_at"] = moment
+        if target in (LeaseState.RELEASED, LeaseState.EXPIRED):
+            updates["released_at"] = moment
+        if mechanism is not None:
+            updates["release_mechanism"] = mechanism
+        if escalation_notes is not None:
+            updates["escalation_notes"] = escalation_notes
+        # model_copy skips validators; transitions MUST re-check every invariant.
+        return self.__class__.model_validate({**self.model_dump(), **updates})
+
+    # -- classification -------------------------------------------------------------
+    @property
+    def is_terminal(self) -> bool:
+        return not _TRANSITIONS[self.state]
+
+    @property
+    def is_safe_terminal(self) -> bool:
+        return self.state in _SAFE_TERMINALS
+
+    @property
+    def needs_janitor_attention(self) -> bool:
+        return self.state in (
+            LeaseState.PENDING,
+            LeaseState.ACTIVE,
+            LeaseState.ORPHANED,
+            LeaseState.RELEASING,
+            LeaseState.DIRTY,
+        )
+
+
+def assert_all_recovered(leases: list[FaultLease]) -> None:
+    """Run-completion invariant: no non-safe-terminal leases may remain.
+
+    Raises:
+        InvariantViolationError: Naming every offending lease.
+    """
+    offenders = [lease for lease in leases if not lease.is_safe_terminal]
+    if offenders:
+        detail = ", ".join(f"{o.id}:{o.state.value}" for o in offenders)
+        raise InvariantViolationError(
+            "all_leases_recovered_before_run_completion",
+            f"non-recovered leases remain: {detail}",
+        )

mayhem/domain/load_strategy.py

@@ -0,0 +1,98 @@
+"""Load generation strategy models (ADR-0021).
+
+Defines load patterns that the controller can orchestrate during chaos
+experiments — ramping, steady, burst, and spike profiles with configurable
+concurrency, duration, and ramp phases.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from mayhem.domain.common import Duration
+
+
+class LoadPattern(StrEnum):
+    """Named load patterns for experiment orchestration."""
+
+    CONSTANT = "constant"  # fixed rate for the entire duration
+    RAMP_UP = "ramp_up"  # linearly increase from min to max
+    RAMP_DOWN = "ramp_down"  # linearly decrease from max to min
+    BURST = "burst"  # short high-intensity burst
+    SPIKE = "spike"  # sudden jump then return
+    STEP = "step"  # step function: low → high → low
+
+
+class LoadPhase(BaseModel):
+    """One phase in a multi-phase load profile."""
+
+    model_config = ConfigDict(frozen=True)
+
+    pattern: LoadPattern
+    vus: int = Field(ge=1, default=1)  # virtual users / concurrency
+    rps: int | None = Field(default=None, ge=1)  # requests per second (alternative to VUs)
+    duration: Duration = 10.0
+    ramp_seconds: Duration = 0.0  # ramp time for RAMP_UP/RAMP_DOWN
+    target_vus: int | None = Field(default=None, ge=1)  # endpoint for ramp patterns
+    target_rps: int | None = Field(default=None, ge=1)
+
+
+class LoadStrategy(BaseModel):
+    """A complete load generation strategy: one or more phases.
+
+    Phases execute sequentially. The total experiment duration is the sum
+    of all phase durations. Between phases, there is no pause (zero-load gap
+    is explicit via a phase with vus=1, duration=gap).
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str = ""
+    phases: tuple[LoadPhase, ...] = (LoadPhase(pattern=LoadPattern.CONSTANT),)
+    endpoint: str = ""  # target URL or service
+    method: str = "GET"
+    headers: dict[str, str] = Field(default_factory=dict)
+    timeout: Duration = 10.0
+
+    def total_duration(self) -> float:
+        """Sum of all phase durations in seconds."""
+        return sum(float(p.duration) for p in self.phases)
+
+    def max_concurrency(self) -> int:
+        """Peak VUs across all phases."""
+        return max(p.vus for p in self.phases)
+
+    @classmethod
+    def constant_profile(cls, vus: int, duration: Duration) -> LoadStrategy:
+        """Shorthand: single constant-rate phase."""
+        return cls(phases=(LoadPhase(pattern=LoadPattern.CONSTANT, vus=vus, duration=duration),))
+
+    @classmethod
+    def ramp_profile(cls, start_vus: int, end_vus: int, duration: Duration) -> LoadStrategy:
+        """Shorthand: single ramp-up phase."""
+        return cls(
+            phases=(
+                LoadPhase(
+                    pattern=LoadPattern.RAMP_UP,
+                    vus=start_vus,
+                    target_vus=end_vus,
+                    duration=duration,
+                    ramp_seconds=duration,
+                ),
+            )
+        )
+
+
+class FuzzStrategy(BaseModel):
+    """Configuration for protocol fuzzing during an experiment."""
+
+    model_config = ConfigDict(frozen=True)
+
+    target_field: str = ""  # which field to mutate (empty = all)
+    mutations_per_request: int = Field(default=1, ge=1)
+    max_requests: int = Field(default=100, ge=1)
+    timeout: Duration = 30.0
+    seed: int | None = None  # deterministic fuzzing
+    dictionary: tuple[str, ...] = ()  # custom mutation dictionary

mayhem/domain/m5_campaign.py

@@ -0,0 +1,120 @@
+"""M5 campaign semantics (ADR-M5-4, M5 Phase 5.4).
+
+A campaign is a *goal + bounds*: a target set, a coverage target or stop
+condition, an execution budget (blast-radius bound), a ``mode``
+(``supervised`` | ``autonomous``), an optional deadline, and a per-campaign
+risk ceiling. A campaign produces many Run/Outcome pairs and completes when the
+stop condition is reached, the budget is exhausted, or the deadline passes.
+
+This is a distinct model from the ADR-0022 ``Campaign`` used by the scheduler;
+ADR-M5-4 campaigns are the intelligence-loop container.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+from mayhem.domain.risks import RiskLevel
+
+
+class CampaignMode(StrEnum):
+    SUPERVISED = "supervised"  # human approves each drill (default)
+    AUTONOMOUS = "autonomous"  # auto-run within bounds, explicit opt-in
+
+
+class CampaignState(StrEnum):
+    IDLE = "idle"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    STOPPED = "stopped"
+    ABORTED = "aborted"
+
+
+class StopReason(StrEnum):
+    COVERAGE_REACHED = "coverage_reached"
+    BUDGET_EXHAUSTED = "budget_exhausted"
+    DEADLINE_PASSED = "deadline_passed"
+    STOP_CONDITION = "stop_condition"
+    RESOURCE_CONFLICT = "resource_conflict"  # hard bound-abort; never executed
+    NOT_STARTED = "not_started"
+
+
+class ApproveDecision(StrEnum):
+    PENDING = "pending"
+    APPROVED = "approved"
+    DENIED = "denied"
+
+
+@dataclass(frozen=True)
+class CampaignLogEntry:
+    """One recorded decision in a campaign's approve/deny trail."""
+
+    candidate_id: str
+    decision: ApproveDecision
+    rationale: str = ""
+    run_id: str | None = None
+    outcome_id: str | None = None
+
+
+@dataclass(frozen=True)
+class M5Campaign:
+    """Goal + bounds for one intelligence-loop campaign (ADR-M5-4)."""
+
+    id: str
+    name: str
+    targets: tuple[str, ...] = ()
+    coverage_target: int = 1  # distinct cells to cover
+    max_runs: int = 100  # blast-radius budget = max executed runs
+    mode: CampaignMode = CampaignMode.SUPERVISED
+    risk_ceiling: RiskLevel = RiskLevel.HIGH
+    deadline_epoch_s: float | None = None
+    stop_condition: str = ""  # free-form durable note; caller enforces it
+
+    def with_defaults(self, **kwargs: Any) -> M5Campaign:
+        """Return a copy with overridden bounds (immutable-equivalent helper)."""
+        return M5Campaign(**{**self.__dict__, **kwargs})
+
+
+@dataclass
+class CampaignProgress:
+    """Live counters of one campaign iteration."""
+
+    runs_executed: int = 0
+    cells_covered: int = 0
+    candidates_rejected: int = 0
+    candidates_pending_approval: int = 0
+    candidates_denied: int = 0
+    candidates_aborted: int = 0
+    log: list[CampaignLogEntry] = field(default_factory=list)
+
+    def snapshot(self) -> dict[str, int]:
+        return {
+            "runs_executed": self.runs_executed,
+            "cells_covered": self.cells_covered,
+            "candidates_rejected": self.candidates_rejected,
+            "candidates_pending_approval": self.candidates_pending_approval,
+            "candidates_denied": self.candidates_denied,
+            "candidates_aborted": self.candidates_aborted,
+        }
+
+    def record(
+        self,
+        candidate_id: str,
+        decision: ApproveDecision,
+        *,
+        rationale: str = "",
+        run_id: str | None = None,
+        outcome_id: str | None = None,
+    ) -> None:
+        """Append one decision to the approve/deny trail."""
+        self.log.append(
+            CampaignLogEntry(
+                candidate_id=candidate_id,
+                decision=decision,
+                rationale=rationale,
+                run_id=run_id,
+                outcome_id=outcome_id,
+            )
+        )

mayhem/domain/maniac.py

@@ -0,0 +1,93 @@
+"""Maniac mode — random fault injection (ADR-M5-1).
+
+``mayhem maniac`` compiles a drill spec exactly like ``mayhem run``, then
+replaces the authored execution with a random draw: ``run_level`` rounds, each
+picking a random container (from the spec's own ``containers`` map) and a
+random fault. The spec keeps providing the container pool, the fault catalog
+entries, the success criteria and the observability sources, so the machine
+verdict and evidence are produced exactly as in a deterministic run.
+
+The ``level`` dial (1-5) scales how far each round strays from the spec's
+authored intent (see the table below). Drawing is reproducible when ``seed``
+is set.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from mayhem.domain.catalog import definition_for
+from mayhem.domain.common import parse_duration
+from mayhem.domain.errors import SchemaValidationError
+
+if TYPE_CHECKING:
+    from mayhem.domain.experiments import DrillFault, DrillSpec
+
+
+class ManiacError(Exception):
+    """Raised when maniac mode cannot be compiled."""
+
+
+@dataclass(frozen=True)
+class ManiacDraw:
+    """One maniac round: a fault to inject on a container."""
+
+    container: str  # target container name (matches the topology)
+    fault: DrillFault  # duration may be jittered (levels 4-5)
+    round: int  # 1-based injection round
+
+
+def _jitter_duration(fault: DrillFault, pct: float, rng: random.Random) -> DrillFault:
+    """Jitter ``fault.duration`` by ``±pct``, clamped to the catalog cap / 1 s."""
+    raw = (
+        parse_duration(fault.duration) if isinstance(fault.duration, str) else float(fault.duration)
+    )
+    jittered = raw * (1.0 + rng.uniform(-pct, pct))
+    try:
+        cap = definition_for(fault.fault).max_duration_s
+        jittered = min(jittered, cap)
+    except (SchemaValidationError, LookupError):
+        pass  # the planner surfaces unknown-fault errors with authority
+    return fault.model_copy(update={"duration": round(max(jittered, 1.0), 3)})
+
+
+def draw_maniac_rounds(
+    spec: DrillSpec,
+    *,
+    level: int,
+    run_level: int,
+    seed: int | None = None,
+) -> tuple[ManiacDraw, ...]:
+    """Draw ``run_level`` random (container, fault) rounds from ``spec``.
+
+    Only containers that actually author faults are in the pool; cross-locus
+    draws (``level >= 3``) may apply any spec-wide fault to any pool container.
+    Durations are jittered at ``level >= 4`` (ADR-M5-1 §semantics).
+    """
+    pool = {name: container for name, container in spec.containers.items() if container.faults}
+    if not pool:
+        raise ManiacError(
+            "maniac mode needs at least one container with faults — "
+            "none of the spec's containers define any"
+        )
+    rng = random.Random(seed)
+    names = sorted(pool)
+    all_faults = tuple(f for c in pool.values() for f in c.faults)
+    jitter_pct = 0.20 if level >= 5 else (0.10 if level == 4 else 0.0)
+
+    draws: list[ManiacDraw] = []
+    for round_no in range(1, run_level + 1):
+        target = rng.choice(names)
+        if level >= 3:
+            # Cross-locus: any spec-wide fault may land on any pool container.
+            fault = rng.choice(all_faults)
+        elif level == 2:
+            fault = rng.choice(pool[target].faults)
+        else:
+            fault = pool[target].faults[0]
+        if jitter_pct > 0:
+            fault = _jitter_duration(fault, jitter_pct, rng)
+        draws.append(ManiacDraw(container=target, fault=fault, round=round_no))
+    return tuple(draws)

mayhem/domain/observability.py

@@ -0,0 +1,146 @@
+"""Declarative observability/metrics source DSL (ADR-M4-4 §22).
+
+An ``observability`` section on a drill spec collects *evidence* into the
+outcome record — the values an evaluator later needs — without driving the
+run's own verdict. Every source is:
+
+- **named** (``source_id``) so observations can be cross-referenced,
+- **bounded** (per-source ``timeout`` and an overall ``total_timeout``), and
+- **best-effort** (a failing source is recorded as a skip note, never fatal).
+
+Source kinds: container logs tail, container inspect, an external probe
+(HTTP/exec/TCP/process/metric/file — the same closed union as checks) and a
+Prometheus text-format metrics scrape.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from mayhem.domain.checks import Probe
+from mayhem.domain.common import Duration, parse_duration
+from mayhem.domain.errors import SchemaValidationError
+
+
+def duration_seconds(value: object) -> float:
+    """Coerce a ``Duration`` to seconds, tolerating un-validated string defaults.
+
+    Pydantic v2 returns a class-level string default (e.g. ``"10s"``) *without*
+    running the duration validators, so naive ``float()`` calls would raise.
+    """
+    if isinstance(value, str):
+        return parse_duration(value)
+    if isinstance(value, (int, float)):
+        return float(value)
+    raise TypeError(f"cannot interpret {value!r} as a duration")
+
+
+class ObservabilitySourceKind(StrEnum):
+    """Kinds of observability evidences a drill can collect (ADR-M4-4)."""
+
+    LOGS = "logs"
+    INSPECTION = "inspection"
+    PROBE = "probe"
+    METRICS = "metrics"
+
+
+class LogsSource(BaseModel):
+    """Container log tail (Docker/Podman ``logs``) captured as evidence."""
+
+    model_config = ConfigDict(frozen=True)
+
+    kind: Literal[ObservabilitySourceKind.LOGS] = ObservabilitySourceKind.LOGS
+    source_id: str
+    container: str  # container name from the drill's containers:
+    tail: int = Field(default=100, ge=1, le=10_000)  # bounded tail, never unbounded
+    since: str = ""  # optional engine `--since` filter; empty = latest tail
+    timeout: Duration = 10.0
+
+
+class InspectionSource(BaseModel):
+    """Container runtime inspect JSON captured as key/value evidence."""
+
+    model_config = ConfigDict(frozen=True)
+
+    kind: Literal[ObservabilitySourceKind.INSPECTION] = ObservabilitySourceKind.INSPECTION
+    source_id: str
+    container: str
+    timeout: Duration = 10.0
+
+
+class ProbeSource(BaseModel):
+    """An external probe observed on a cadence inside the run window."""
+
+    model_config = ConfigDict(frozen=True)
+
+    kind: Literal[ObservabilitySourceKind.PROBE] = ObservabilitySourceKind.PROBE
+    source_id: str
+    probe: Probe
+    cadence: Duration = 0.0  # >0 polls repeatedly; 0 = collect once
+    timeout: Duration = 10.0
+
+
+class MetricsSource(BaseModel):
+    """Prometheus text-format scrape of one metric, sampled on a cadence."""
+
+    model_config = ConfigDict(frozen=True)
+
+    kind: Literal[ObservabilitySourceKind.METRICS] = ObservabilitySourceKind.METRICS
+    source_id: str
+    endpoint: str  # e.g. http://127.0.0.1:9100/metrics
+    metric: str  # Prometheus metric name to resolve to a sample value
+    cadence: Duration = 0.0
+    timeout: Duration = 10.0
+
+
+ObservabilitySource = Annotated[
+    LogsSource | InspectionSource | ProbeSource | MetricsSource,
+    Field(discriminator="kind"),
+]
+
+
+class ObservabilityConfig(BaseModel):
+    """One bounding ``observability`` section on a drill spec."""
+
+    model_config = ConfigDict(frozen=True)
+
+    sources: tuple[ObservabilitySource, ...] = ()
+    cadence: Duration = 5.0  # default polling cadence where cadence > 0
+    total_timeout: Duration = 30.0  # hard bound across the whole pass
+
+    @model_validator(mode="after")
+    def _bounds_hold(self) -> ObservabilityConfig:
+        total = duration_seconds(self.total_timeout)
+        default_cadence = duration_seconds(self.cadence)
+        if total <= 0:
+            raise SchemaValidationError("total_timeout", "must be > 0")
+        if default_cadence < 0:
+            raise SchemaValidationError("cadence", "must be >= 0")
+        visited: set[str] = set()
+        for source in self.sources:
+            if source.source_id in visited:
+                raise SchemaValidationError(
+                    "sources",
+                    f"duplicate source_id {source.source_id!r}",
+                )
+            visited.add(source.source_id)
+            if duration_seconds(source.timeout) > total:
+                raise SchemaValidationError(
+                    f"sources[{source.source_id}]",
+                    f"timeout {duration_seconds(source.timeout)}s exceeds total_timeout {total}s",
+                )
+            cadence = getattr(source, "cadence", "0s")
+            if duration_seconds(cadence) > 0 and duration_seconds(cadence) > total:
+                raise SchemaValidationError(
+                    f"sources[{source.source_id}]",
+                    f"cadence {duration_seconds(cadence)}s exceeds total_timeout {total}s; "
+                    "no sampling window remains",
+                )
+        return self
+
+    @property
+    def empty(self) -> bool:
+        return not self.sources