petfishframework 0.1.0__py3-none-any.whl

petfishframework/reliability/pass_at_k.py

@@ -0,0 +1,232 @@
+"""Pass^k — structural reliability metric (decision 4, v0.2).
+
+Absorbed from contract-driven-harness-study: Pass^k is NOT simple k-repetition.
+It is freeze-TaskSpec + k-repetition + perturbation-suite, separating
+provider variance from contract failure.
+
+Empirical baseline: contract-driven-harness Stage B v5.4 achieved 40/40
+across 5 perturbation types. This module implements the same methodology.
+"""
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+from typing import Any, Callable, Protocol
+
+from petfishframework.core.types import Result, Task
+
+# ---------------------------------------------------------------------------
+# Agreement functions
+# ---------------------------------------------------------------------------
+
+class AgreementFn(Protocol):
+    """Checks whether a set of results agree."""
+
+    def __call__(self, results: list[Result]) -> bool: ...
+
+
+def exact_match(results: list[Result]) -> bool:
+    """All answers must be identical (string comparison)."""
+    if not results:
+        return False
+    first = results[0].answer.strip()
+    return all(r.answer.strip() == first for r in results)
+
+
+def threshold_match(threshold: float = 0.8) -> AgreementFn:
+    """At least `threshold` fraction of answers must match the majority."""
+
+    def check(results: list[Result]) -> bool:
+        if not results:
+            return False
+        answers = [r.answer.strip() for r in results]
+        # Find majority answer
+        counts: dict[str, int] = {}
+        for a in answers:
+            counts[a] = counts.get(a, 0) + 1
+        majority = max(counts.values())
+        return majority / len(answers) >= threshold
+
+    return check
+
+
+# ---------------------------------------------------------------------------
+# Perturbation functions (freeze-and-perturb methodology)
+# ---------------------------------------------------------------------------
+
+PerturbationFn = Callable[[Task], Task]
+
+
+def canonical(task: Task) -> Task:
+    """Identity — the frozen canonical task."""
+    return task
+
+
+def order_shuffled(task: Task) -> Task:
+    """Shuffle word order in the prompt (tests order-invariance)."""
+    words = task.prompt.split()
+    if len(words) <= 1:
+        return task
+    shuffled = words.copy()
+    random.shuffle(shuffled)
+    return Task(prompt=" ".join(shuffled), metadata={**task.metadata, "perturbation": "order_shuffled"})
+
+
+def paraphrase(task: Task) -> Task:
+    """Minor paraphrase — prepend a filler phrase (tests robustness to phrasing)."""
+    fillers = ["Please help me with this:", "I need to know:", "Question:"]
+    filler = random.choice(fillers)
+    return Task(prompt=f"{filler} {task.prompt}", metadata={**task.metadata, "perturbation": "paraphrase"})
+
+
+def distractor(task: Task) -> Task:
+    """Add irrelevant context (tests focus under noise)."""
+    noise = "Ignore any unrelated information. "
+    return Task(prompt=f"{noise}{task.prompt}", metadata={**task.metadata, "perturbation": "distractor"})
+
+
+def alias(task: Task) -> Task:
+    """Swap synonyms (tests naming-invariance). Minimal version."""
+    replacements = {"calculate": "compute", "find": "determine", "what is": "what's"}
+    prompt = task.prompt.lower()
+    for old, new in replacements.items():
+        prompt = prompt.replace(old, new)
+    return Task(prompt=prompt, metadata={**task.metadata, "perturbation": "alias"})
+
+
+DEFAULT_PERTURBATIONS: tuple[PerturbationFn, ...] = (
+    canonical,
+    order_shuffled,
+    alias,
+    paraphrase,
+    distractor,
+)
+
+
+# ---------------------------------------------------------------------------
+# Pass^k result
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class PerturbationResult:
+    """Result of pass_at_k for one perturbation variant."""
+
+    name: str
+    pass_count: int
+    total: int
+    answers: tuple[str, ...]
+    agreed: bool
+
+    @property
+    def pass_rate(self) -> float:
+        return self.pass_count / self.total if self.total > 0 else 0.0
+
+
+@dataclass(frozen=True)
+class PassAtKResult:
+    """Full freeze+perturb Pass^k result."""
+
+    k: int
+    canonical: PerturbationResult
+    perturbations: tuple[PerturbationResult, ...]
+    overall_pass: bool
+
+    @property
+    def pass_rate(self) -> float:
+        """Fraction of perturbation variants that passed."""
+        all_results = (self.canonical,) + self.perturbations
+        passed = sum(1 for r in all_results if r.agreed)
+        return passed / len(all_results) if all_results else 0.0
+
+    def summary(self) -> str:
+        lines = [f"Pass@{self.k} — {'PASS' if self.overall_pass else 'FAIL'} ({self.pass_rate:.0%})"]
+        lines.append(f"  canonical:        {self.canonical.pass_count}/{self.canonical.total}")
+        for p in self.perturbations:
+            lines.append(f"  {p.name:<18s} {p.pass_count}/{p.total}")
+        return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Core pass_at_k
+# ---------------------------------------------------------------------------
+
+# SessionFactory: takes a Task, returns a fresh Session (for independent runs)
+SessionFactory = Callable[[Task], Any]
+
+
+def pass_at_k(
+    session_factory: SessionFactory,
+    task: Task,
+    k: int = 8,
+    agreement: AgreementFn = exact_match,
+) -> PerturbationResult:
+    """Run k independent sessions on the task, measure agreement.
+
+    Each call to session_factory must create a FRESH session (independent
+    model calls). This measures provider variance + strategy determinism.
+    """
+    results: list[Result] = []
+    for _i in range(k):
+        session = session_factory(task)
+        result = session.run()
+        results.append(result)
+
+    answers = tuple(r.answer for r in results)
+    agreed = agreement(results)
+    # pass_count = k if all agreed, else 0 (binary at the k level)
+    pass_count = k if agreed else 0
+    return PerturbationResult(
+        name="canonical",
+        pass_count=pass_count,
+        total=k,
+        answers=answers,
+        agreed=agreed,
+    )
+
+
+def pass_at_k_with_perturbations(
+    session_factory: SessionFactory,
+    task: Task,
+    k: int = 8,
+    agreement: AgreementFn = exact_match,
+    perturbations: tuple[PerturbationFn, ...] = DEFAULT_PERTURBATIONS,
+) -> PassAtKResult:
+    """Freeze-and-perturb Pass^k (contract-driven-harness methodology).
+
+    1. Freeze the canonical task.
+    2. Run k repetitions on canonical.
+    3. For each perturbation variant, run k repetitions.
+    4. Overall pass = ALL variants agree.
+
+    This separates provider variance (intra-variant) from contract robustness
+    (inter-variant). A strategy that passes canonical but fails perturbations
+    has a fragile contract, not a reliable one.
+    """
+    canonical_result = pass_at_k(session_factory, task, k, agreement)
+
+    perturbation_results: list[PerturbationResult] = []
+    for perturb_fn in perturbations:
+        if perturb_fn is canonical:
+            continue  # canonical already done
+        perturbed_task = perturb_fn(task)
+        name = perturbed_task.metadata.get("perturbation", perturb_fn.__name__)
+        result = pass_at_k(session_factory, perturbed_task, k, agreement)
+        perturbation_results.append(
+            PerturbationResult(
+                name=name,
+                pass_count=result.pass_count,
+                total=result.total,
+                answers=result.answers,
+                agreed=result.agreed,
+            )
+        )
+
+    all_results = (canonical_result,) + tuple(perturbation_results)
+    overall_pass = all(r.agreed for r in all_results)
+
+    return PassAtKResult(
+        k=k,
+        canonical=canonical_result,
+        perturbations=tuple(perturbation_results),
+        overall_pass=overall_pass,
+    )

petfishframework/reliability/replay.py

@@ -0,0 +1,190 @@
+"""ReplayMode — deterministic replay for model non-determinism (open question 3).
+
+Three modes handle the fundamental tension:
+  AUDIT  — re-inject ALL recorded outputs (deterministic, for debugging/audit)
+  RESUME — re-inject to checkpoint, then fresh calls (for failure recovery)
+  RERUN  — fresh from start (for Pass^k, non-determinism IS the metric)
+
+Design: RecordingEnvironment wraps any Environment during run(), capturing
+ModelResponse and ToolResult objects. ReplayEnvironment serves them back
+without calling the real model/tools. No core/ modifications needed.
+
+Resolves architecture open question 3.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+from petfishframework.core.contracts import Environment, Tool
+from petfishframework.core.types import (
+    ModelRequest,
+    ModelResponse,
+    Snippet,
+    ToolRef,
+    ToolResult,
+)
+
+
+class ReplayMode(Enum):
+    """Three replay strategies for handling model non-determinism."""
+
+    AUDIT = "audit"  # re-inject all recorded outputs (deterministic)
+    RESUME = "resume"  # re-inject to checkpoint, then fresh calls
+    RERUN = "rerun"  # fresh from start (non-determinism expected)
+
+
+# ---------------------------------------------------------------------------
+# RecordingEnvironment — captures responses during a real run
+# ---------------------------------------------------------------------------
+
+class RecordingEnvironment:
+    """Wraps a real Environment, capturing all model/tool responses for replay.
+
+    Use during Session.run() to capture a recording. The recording can then
+    be replayed via ReplayEnvironment (AUDIT) or ResumableEnvironment (RESUME).
+    """
+
+    def __init__(self, env: Environment) -> None:
+        self._env = env
+        self.model_responses: list[ModelResponse] = []
+        self.tool_calls: list[tuple[str, dict[str, Any], ToolResult]] = []
+        self.retrievals: list[tuple[str, list[Snippet]]] = []
+
+    def tools(self) -> list[Tool]:
+        return self._env.tools()
+
+    def call(self, ref: ToolRef, args: dict[str, Any]) -> ToolResult:
+        result = self._env.call(ref, args)
+        self.tool_calls.append((ref.name, args, result))
+        return result
+
+    def retrieve(self, query: str, top_k: int = 5) -> list[Snippet]:
+        snippets = self._env.retrieve(query, top_k)
+        self.retrievals.append((query, snippets))
+        return snippets
+
+    def query_model(self, request: ModelRequest) -> ModelResponse:
+        response = self._env.query_model(request)
+        self.model_responses.append(response)
+        return response
+
+
+# ---------------------------------------------------------------------------
+# ReplayEnvironment — serves recorded responses (AUDIT mode)
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ReplayEnvironment:
+    """Serves recorded responses without calling the real model/tools.
+
+    For AUDIT replay: deterministic re-execution of the same trajectory.
+    If the recording is exhausted, raises RuntimeError (safety: detects
+    divergence between original and replay execution).
+    """
+
+    model_responses: list[ModelResponse]
+    tool_results: list[tuple[str, dict[str, Any], ToolResult]]
+    retrievals: list[tuple[str, list[Snippet]]] = field(default_factory=list)
+    _tools: list[Tool] = field(default_factory=list)
+    _model_idx: int = field(default=0, repr=False)
+    _tool_idx: int = field(default=0, repr=False)
+    _retrieval_idx: int = field(default=0, repr=False)
+
+    def tools(self) -> list[Tool]:
+        return self._tools
+
+    def call(self, ref: ToolRef, args: dict[str, Any]) -> ToolResult:
+        if self._tool_idx >= len(self.tool_results):
+            raise RuntimeError(
+                f"AUDIT replay divergence: tool call #{self._tool_idx} "
+                f"({ref.name}) not in recording "
+                f"(only {len(self.tool_results)} recorded)"
+            )
+        _name, _args, result = self.tool_results[self._tool_idx]
+        self._tool_idx += 1
+        return result
+
+    def retrieve(self, query: str, top_k: int = 5) -> list[Snippet]:
+        if self._retrieval_idx >= len(self.retrievals):
+            return []
+        _query, snippets = self.retrievals[self._retrieval_idx]
+        self._retrieval_idx += 1
+        return snippets
+
+    def query_model(self, request: ModelRequest) -> ModelResponse:
+        if self._model_idx >= len(self.model_responses):
+            raise RuntimeError(
+                f"AUDIT replay divergence: model call #{self._model_idx} "
+                f"not in recording "
+                f"(only {len(self.model_responses)} recorded)"
+            )
+        response = self.model_responses[self._model_idx]
+        self._model_idx += 1
+        return response
+
+
+# ---------------------------------------------------------------------------
+# ResumableEnvironment — hybrid: recorded up to checkpoint, then fresh
+# ---------------------------------------------------------------------------
+
+class ResumableEnvironment:
+    """Serves recorded responses up to a checkpoint, then delegates to live env.
+
+    For RESUME replay: re-execute the deterministic prefix, then continue
+    with fresh model calls from the checkpoint onward. This enables
+    failure recovery — the agent re-does the checkpoint-to-failure segment
+    with potentially different (and hopefully successful) model outputs.
+    """
+
+    def __init__(
+        self,
+        recording: RecordingEnvironment,
+        live_env: Environment,
+        checkpoint_model_idx: int,
+        checkpoint_tool_idx: int,
+    ) -> None:
+        self._recording = recording
+        self._live = live_env
+        self._cp_model = checkpoint_model_idx
+        self._cp_tool = checkpoint_tool_idx
+        self._model_idx = 0
+        self._tool_idx = 0
+        self._switched = False
+
+    def tools(self) -> list[Tool]:
+        return self._live.tools()
+
+    def call(self, ref: ToolRef, args: dict[str, Any]) -> ToolResult:
+        if not self._switched and self._tool_idx < self._cp_tool:
+            _name, _args, result = self._recording.tool_calls[self._tool_idx]
+            self._tool_idx += 1
+            return result
+        self._switched = True
+        return self._live.call(ref, args)
+
+    def retrieve(self, query: str, top_k: int = 5) -> list[Snippet]:
+        return self._live.retrieve(query, top_k)
+
+    def query_model(self, request: ModelRequest) -> ModelResponse:
+        if not self._switched and self._model_idx < self._cp_model:
+            response = self._recording.model_responses[self._model_idx]
+            self._model_idx += 1
+            return response
+        self._switched = True
+        return self._live.query_model(request)
+
+
+# ---------------------------------------------------------------------------
+# Convenience: build ReplayEnvironment from a RecordingEnvironment
+# ---------------------------------------------------------------------------
+
+def replay_environment_from_recording(recording: RecordingEnvironment) -> ReplayEnvironment:
+    """Create a ReplayEnvironment (AUDIT mode) from a recording."""
+    return ReplayEnvironment(
+        model_responses=list(recording.model_responses),
+        tool_results=list(recording.tool_calls),
+        retrievals=list(recording.retrievals),
+        _tools=recording.tools(),
+    )

petfishframework/reliability/retry.py

@@ -0,0 +1,224 @@
+"""Retry infrastructure for transient-failure resilience.
+
+Wraps any ModelAdapter with configurable exponential-backoff retry.
+Designed as a pure wrapper so no existing core/ source needs modification.
+"""
+from __future__ import annotations
+
+import asyncio
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Awaitable, Callable, TypeVar
+
+from petfishframework.core.contracts import ModelAdapter
+from petfishframework.core.types import ModelRequest, ModelResponse
+
+
+class RetryableError(Exception):
+    """Raised when all retry attempts are exhausted.
+
+    Carries the original exception, the number of attempts made, and the
+    total elapsed time so callers can diagnose the failure.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        original: Exception,
+        attempts: int,
+        elapsed_s: float,
+    ):
+        super().__init__(message)
+        self.original = original
+        self.attempts = attempts
+        self.elapsed_s = elapsed_s
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    """Configuration for retry/backoff behavior."""
+
+    max_retries: int = 3
+    initial_delay: float = 1.0
+    backoff_factor: float = 2.0
+    retryable_exceptions: tuple[type[Exception], ...] = (Exception,)
+    jitter: bool = True
+    max_delay: float = 60.0
+
+    def delay_for_attempt(self, attempt: int) -> float:
+        """Compute the backoff delay before attempt number ``attempt``.
+
+        ``attempt`` is 0-indexed: the first retry uses ``attempt=0``.
+        """
+        base = self.initial_delay * (self.backoff_factor**attempt)
+        capped = min(base, self.max_delay)
+        if not self.jitter:
+            return capped
+        # Add/subtract up to 25% jitter, but never go below zero.
+        jitter_amount = capped * 0.25
+        return max(0.0, capped + random.uniform(-jitter_amount, jitter_amount))
+
+
+T = TypeVar("T")
+
+
+def with_retry(
+    fn: Callable[[], T],
+    policy: RetryPolicy,
+    attempts_log: list[Exception] | None = None,
+) -> Callable[[], T]:
+    """Wrap a synchronous callable with retry logic.
+
+    The wrapper re-invokes ``fn`` up to ``policy.max_retries`` times when
+    ``policy.retryable_exceptions`` are raised. Between attempts it sleeps
+    using ``time.sleep`` with exponential backoff and optional jitter.
+
+    If ``attempts_log`` is supplied, every retryable exception that triggered
+    a backoff is appended to it, enabling callers to count retries.
+    """
+
+    def wrapper() -> T:
+        start = time.monotonic()
+        last_error: Exception | None = None
+        for attempt in range(policy.max_retries + 1):
+            try:
+                return fn()
+            except Exception as exc:  # noqa: BLE001
+                last_error = exc
+                if not isinstance(exc, policy.retryable_exceptions):
+                    raise
+                if attempt >= policy.max_retries:
+                    break
+                if attempts_log is not None:
+                    attempts_log.append(exc)
+                time.sleep(policy.delay_for_attempt(attempt))
+        elapsed = time.monotonic() - start
+        assert last_error is not None
+        raise RetryableError(
+            f"Failed after {policy.max_retries + 1} attempt(s): {last_error}",
+            original=last_error,
+            attempts=policy.max_retries + 1,
+            elapsed_s=elapsed,
+        )
+
+    return wrapper
+
+
+def with_retry_async(
+    fn: Callable[[], Awaitable[T]],
+    policy: RetryPolicy,
+    attempts_log: list[Exception] | None = None,
+) -> Callable[[], Awaitable[T]]:
+    """Wrap an asynchronous callable with retry logic.
+
+    Identical semantics to ``with_retry`` but uses ``asyncio.sleep``.
+    """
+
+    async def wrapper() -> T:
+        start = time.monotonic()
+        last_error: Exception | None = None
+        for attempt in range(policy.max_retries + 1):
+            try:
+                return await fn()
+            except Exception as exc:  # noqa: BLE001
+                last_error = exc
+                if not isinstance(exc, policy.retryable_exceptions):
+                    raise
+                if attempt >= policy.max_retries:
+                    break
+                if attempts_log is not None:
+                    attempts_log.append(exc)
+                await asyncio.sleep(policy.delay_for_attempt(attempt))
+        elapsed = time.monotonic() - start
+        assert last_error is not None
+        raise RetryableError(
+            f"Failed after {policy.max_retries + 1} attempt(s): {last_error}",
+            original=last_error,
+            attempts=policy.max_retries + 1,
+            elapsed_s=elapsed,
+        )
+
+    return wrapper
+
+
+@dataclass
+class RetryModelAdapter(ModelAdapter):
+    """A ModelAdapter wrapper that retries transient failures.
+
+    Delegates ``name`` to the inner adapter and wraps ``query`` (and
+    ``query_async`` when the inner adapter supports it) with retry logic.
+    """
+
+    inner: ModelAdapter
+    policy: RetryPolicy = field(default_factory=RetryPolicy)
+
+    retry_count: int = field(default=0, init=False, repr=False)
+    last_error: Exception | None = field(default=None, init=False, repr=False)
+    name: str = field(default="", init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        self.name = self.inner.name
+
+    def query(self, request: ModelRequest) -> ModelResponse:
+        """Query the inner model, retrying according to policy."""
+        self.retry_count = 0
+        self.last_error = None
+        attempts: list[Exception] = []
+
+        def _call() -> ModelResponse:
+            return self.inner.query(request)
+
+        try:
+            response = with_retry(_call, self.policy, attempts)()
+        except RetryableError as exc:
+            self.retry_count = exc.attempts - 1
+            self.last_error = exc.original
+            raise
+
+        self.retry_count = len(attempts)
+        if attempts:
+            self.last_error = attempts[-1]
+        return response
+
+    def query_async(self, request: ModelRequest) -> Awaitable[ModelResponse]:
+        """Async query the inner model, retrying according to policy.
+
+        Detects an async inner ``query`` via ``asyncio.iscoroutinefunction``
+        (consistent with RuntimeEnvironment) and awaits it; otherwise the
+        synchronous ``query`` is run inside the async retry loop.
+        """
+        self.retry_count = 0
+        self.last_error = None
+        attempts: list[Exception] = []
+
+        async def _async_call() -> ModelResponse:
+            if asyncio.iscoroutinefunction(self.inner.query):
+                return await self.inner.query(request)
+            return self.inner.query(request)
+
+        async def _run() -> ModelResponse:
+            try:
+                response = await with_retry_async(_async_call, self.policy, attempts)()
+            except RetryableError as exc:
+                self.retry_count = exc.attempts - 1
+                self.last_error = exc.original
+                raise
+
+            self.retry_count = len(attempts)
+            if attempts:
+                self.last_error = attempts[-1]
+            return response
+
+        return _run()
+
+    def reset_stats(self) -> None:
+        """Reset retry statistics."""
+        self.retry_count = 0
+        self.last_error = None
+
+
+def retry_model_adapter(model: ModelAdapter, policy: RetryPolicy | None = None) -> RetryModelAdapter:
+    """Convenience factory for wrapping a model adapter with retries."""
+    return RetryModelAdapter(inner=model, policy=policy or RetryPolicy())

petfishframework/reliability/timeout.py

@@ -0,0 +1,55 @@
+"""Operation timeout utilities (M4 gap).
+
+Provides a small timeout policy object and a thread-pool based wrapper for
+applying a hard wall-clock timeout to synchronous callables. Intended as a
+building block for model-call, tool-call, and retrieval timeouts without
+modifying callers' interfaces.
+"""
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import Callable, ParamSpec, TypeVar
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+
+class OperationTimedOut(Exception):
+    """Raised when an operation exceeds its configured timeout."""
+
+    def __init__(self, operation: str, timeout_s: float):
+        self.operation = operation
+        self.timeout_s = timeout_s
+        super().__init__(f"Operation '{operation}' timed out after {timeout_s}s")
+
+
+@dataclass(frozen=True)
+class TimeoutPolicy:
+    """Timeout defaults for different operation categories."""
+
+    model_call_timeout_s: float = 60.0
+    tool_call_timeout_s: float = 30.0
+    retrieval_timeout_s: float = 10.0
+
+
+def with_timeout(fn: Callable[P, T], timeout_s: float) -> Callable[P, T]:
+    """Wrap a synchronous callable with a hard timeout.
+
+    The callable runs in a single worker thread from a temporary
+    ``ThreadPoolExecutor``. If it does not complete within ``timeout_s``
+    seconds, a ``concurrent.futures.TimeoutError`` is translated into a
+    framework-specific ``OperationTimedOut`` error carrying the operation name
+    and timeout value.
+    """
+
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+        with ThreadPoolExecutor(max_workers=1) as executor:
+            future = executor.submit(fn, *args, **kwargs)
+            try:
+                return future.result(timeout=timeout_s)
+            except TimeoutError as exc:
+                operation = getattr(fn, "__name__", repr(fn))
+                raise OperationTimedOut(operation=operation, timeout_s=timeout_s) from exc
+
+    return wrapper