hypermind 0.11.0__py3-none-any.whl

hypermind/errors.py

@@ -0,0 +1,218 @@
+"""Exception hierarchy + structured `BlockReason`.
+
+Each exception carries a `telemetry` dict matching the spec/13-observability.md
+log schema, so errors emitted from the SDK boundary are immediately
+operationally legible without re-wrapping.
+
+`BlockReason` (this module) is the structured-refusal contract surfaced by
+`HyperMindAgent.why_blocked(...)` and (eventually, post-WS-F) attached to every
+`HyperMindError` via the `reason=` keyword argument. The `[rule_id] spec_ref:
+hint` `__str__` format is a stability contract: log-parsing regexes
+`r"\\[([A-Z0-9-]+)\\]"` MUST keep working across minor versions.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import timedelta
+from typing import Any, Literal
+
+# -----------------------------------------------------------------------
+# BlockReason — structured refusal carrying rule_id + spec_ref
+# -----------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class BlockReason:
+    """Structured refusal reason — every Blocked exception carries one.
+
+    Per draft-hypermind-scitt-contested-claims-profile §16 (Error Taxonomy).
+    The bracketed ``[rule_id]`` prefix in ``__str__`` is part of the stable
+    contract; log-parsing regexes ``r"\\[([A-Z0-9-]+)\\]"`` MUST keep working
+    across minor versions.
+
+    The legacy fields (``human_explanation``, ``current_value``,
+    ``required_value``, ``retry_after_seconds``) are retained for backward
+    compatibility with `HyperMindAgent.why_blocked(...)` consumers shipped in
+    v0.1.0a0; new code should use ``hint`` and ``telemetry`` instead.
+    """
+
+    rule_id: str
+    spec_ref: str
+    docs_url: str | None = None
+    hint: str | None = None
+    telemetry: Mapping[str, Any] = field(default_factory=dict)
+    # Legacy (v0.1.0a0) compatibility fields — retained so existing callers of
+    # `why_blocked()` keep working unchanged. New code SHOULD prefer `hint`.
+    human_explanation: str | None = None
+    current_value: Any | None = None
+    required_value: Any | None = None
+    retry_after_seconds: float | None = None
+
+    def __str__(self) -> str:
+        # Prefer the new `hint` field; fall back to legacy `human_explanation`
+        # so v0.1.0a0 call sites that still populate the old field render a
+        # useful message instead of "(no hint)".
+        message = self.hint or self.human_explanation or "(no hint)"
+        return f"[{self.rule_id}] {self.spec_ref}: {message}"
+
+
+# -----------------------------------------------------------------------
+# Exception hierarchy
+# -----------------------------------------------------------------------
+
+
+class HyperMindError(Exception):
+    """Base class for all SDK errors.
+
+    Optionally carries a structured ``BlockReason`` so callers can branch on
+    ``e.rule_id`` instead of scraping the message. Either pass the reason as
+    the first positional argument (``raise HyperMindError(reason)``) or via
+    the ``reason=`` keyword.
+    """
+
+    reason: BlockReason | None = None
+    rule_id: str | None = None
+
+    def __init__(
+        self,
+        message: str | BlockReason,
+        *,
+        reason: BlockReason | None = None,
+        telemetry: dict[str, Any] | None = None,
+    ) -> None:
+        if isinstance(message, BlockReason):
+            self.reason = message
+            self.rule_id = message.rule_id
+            super().__init__(str(message))
+        else:
+            self.reason = reason
+            self.rule_id = reason.rule_id if reason is not None else None
+            super().__init__(message)
+        self.telemetry: dict[str, Any] = telemetry or {}
+        self.telemetry.setdefault("event_type", type(self).__name__)
+        if self.rule_id is not None:
+            self.telemetry.setdefault("rule_id", self.rule_id)
+
+
+class NetworkError(HyperMindError):
+    """libp2p dial failure, gossip drop, partition suspected."""
+
+
+class AuthError(HyperMindError):
+    """Capability-token / caveat-chain failure."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        failed_caveat: str | None = None,
+        reason: BlockReason | None = None,
+        telemetry: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, reason=reason, telemetry=telemetry)
+        self.failed_caveat: str | None = failed_caveat
+        if failed_caveat is not None:
+            self.telemetry["failed_caveat"] = failed_caveat
+
+
+class DisputeError(HyperMindError):
+    """Malformed dispute, bond-scope mismatch, or frivolous-window violation."""
+
+
+class RateLimitError(HyperMindError):
+    """Capability rate-caveat exceeded. Raised by ``_require_cap`` when a
+    ``rate`` caveat on the agent's ``cap_token`` is exhausted for this action
+    (P1-03 / HM-RATE-CAP-001).
+    """
+
+
+class RatePauseError(HyperMindError):
+    """Gossip rate-cap hit. Carries `retry_after`."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        retry_after: timedelta,
+        reason: BlockReason | None = None,
+        telemetry: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, reason=reason, telemetry=telemetry)
+        self.retry_after: timedelta = retry_after
+        self.telemetry["retry_after_seconds"] = retry_after.total_seconds()
+
+
+class RampError(HyperMindError):
+    """30-day ramp not yet complete or paused."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        state: Literal["pending", "paused", "frozen"],
+        reason: BlockReason | None = None,
+        telemetry: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, reason=reason, telemetry=telemetry)
+        self.state = state
+        self.telemetry["ramp_state"] = state
+
+
+class KeyCompromiseError(HyperMindError):
+    """Signing key has a `compromise_window_start` declared."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        compromise_window_start: str,
+        reason: BlockReason | None = None,
+        telemetry: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, reason=reason, telemetry=telemetry)
+        self.compromise_window_start = compromise_window_start
+        self.telemetry["compromise_window_start"] = compromise_window_start
+
+
+class UnresolvedPartitionError(HyperMindError):
+    """Dispute hit UNRESOLVED_PARTITION terminal — heal-merge produced contradictory closes."""
+
+
+class ConformanceError(HyperMindError):
+    """Wire-level conformance violation — typically thrown by verifier."""
+
+
+class ProtocolVersionError(HyperMindError):
+    """Statement's `protocol_version` is below the namespace's `min_protocol_version`.
+
+    Carries `rule_id="HM-PROTO-VER-001"` per ROADMAP §0.5.1–0.5.3
+    (versioning bridge milestone). Raised pre-signature-verify so the
+    refusal is auditable as a policy decision, not a crypto failure.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        statement_version: int,
+        min_required: int,
+        reason: BlockReason | None = None,
+        telemetry: dict[str, Any] | None = None,
+    ) -> None:
+        if reason is None:
+            reason = BlockReason(
+                rule_id="HM-PROTO-VER-001",
+                spec_ref="spec/17-upgrade-v1.md §protocol-version-negotiation",
+                hint=message,
+                telemetry={
+                    "statement_version": statement_version,
+                    "min_required": min_required,
+                },
+            )
+        super().__init__(message, reason=reason, telemetry=telemetry)
+        self.statement_version = statement_version
+        self.min_required = min_required
+        self.telemetry["statement_version"] = statement_version
+        self.telemetry["min_required"] = min_required

hypermind/eval/__init__.py

@@ -0,0 +1,49 @@
+"""Evaluation primitives — calibration, comparison, replay.
+
+This package provides:
+  * :class:`BrierScoreTracker` — per-(agent, topic) Brier score history
+    and reliability diagram, fed by ``oracle_resolve()``.
+  * :func:`compare_responders` (Phase 5) — A/B model comparison harness.
+  * :func:`replay_audit_log` (Phase 5) — re-derive verdicts from a
+    signed audit log to verify determinism.
+
+The Brier tracker is wired into :class:`ReputationKernel` so every
+oracle resolution automatically records the calibration delta. Operators
+read the data via the management API at ``GET /v1/calibration``.
+"""
+
+from __future__ import annotations
+
+from hypermind.eval.calibration import (
+    BrierEntry,
+    BrierScoreTracker,
+    ReliabilityBin,
+    brier_score,
+    calibration_adjustment,
+    reliability_diagram,
+)
+from hypermind.eval.comparison import (
+    ComparisonReport,
+    ResponderResult,
+    compare_responders,
+)
+from hypermind.eval.replay import (
+    ReplayResult,
+    replay_audit_log,
+    replay_lines,
+)
+
+__all__ = [
+    "BrierEntry",
+    "BrierScoreTracker",
+    "ComparisonReport",
+    "ReliabilityBin",
+    "ReplayResult",
+    "ResponderResult",
+    "brier_score",
+    "calibration_adjustment",
+    "compare_responders",
+    "reliability_diagram",
+    "replay_audit_log",
+    "replay_lines",
+]

hypermind/eval/calibration.py

@@ -0,0 +1,307 @@
+"""Closed-loop calibration tracking.
+
+When an oracle resolves a claim, the agent that made the prediction
+should be measured: did its stated 70% confidence match observed
+frequencies of YES outcomes? :class:`BrierScoreTracker` records the
+(predicted, outcome) pair, computes per-(agent, topic) Brier scores,
+and produces reliability diagrams.
+
+The tracker also derives a :func:`calibration_adjustment` shrinkage
+multiplier ∈ [0.5, 2.0] that callers pass to
+:func:`hypermind.responders.confidence_to_uncertainty` so that
+well-calibrated agents have *sharper* posteriors and overconfident
+agents have *softer* ones — reputation is no longer just "were you
+right?", it's "was your confidence calibrated?".
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+# ---------------------------------------------------------------------------
+# Brier-score primitive
+# ---------------------------------------------------------------------------
+
+
+def brier_score(predicted: float, outcome: float) -> float:
+    """Squared error between predicted probability and binary outcome.
+
+    Lower is better. ``brier_score(0.9, 1.0) == 0.01`` (well-calibrated),
+    ``brier_score(0.9, 0.0) == 0.81`` (overconfident).
+    """
+    p = max(0.0, min(1.0, float(predicted)))
+    o = max(0.0, min(1.0, float(outcome)))
+    return (p - o) ** 2
+
+
+# ---------------------------------------------------------------------------
+# History entries
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class BrierEntry:
+    """One (predicted, outcome, Brier) record."""
+
+    ts_wall_ms: int
+    predicted_confidence: float
+    outcome: float
+    brier_score: float
+
+    def to_dict(self) -> dict:
+        return {
+            "ts_wall_ms": self.ts_wall_ms,
+            "predicted_confidence": self.predicted_confidence,
+            "outcome": self.outcome,
+            "brier_score": self.brier_score,
+        }
+
+
+@dataclass(frozen=True)
+class ReliabilityBin:
+    """One bin of a reliability diagram."""
+
+    lower: float
+    upper: float
+    mean_confidence: float
+    mean_outcome: float
+    count: int
+
+    @property
+    def gap(self) -> float:
+        """Calibration gap for this bin (signed: positive = overconfident)."""
+        return self.mean_confidence - self.mean_outcome
+
+    def to_dict(self) -> dict:
+        return {
+            "lower": self.lower,
+            "upper": self.upper,
+            "mean_confidence": self.mean_confidence,
+            "mean_outcome": self.mean_outcome,
+            "count": self.count,
+            "gap": self.gap,
+        }
+
+
+def reliability_diagram(
+    entries: Iterable[BrierEntry],
+    *,
+    n_bins: int = 10,
+) -> list[ReliabilityBin]:
+    """Build a reliability diagram from BrierEntry history.
+
+    Returns a list of ``n_bins`` :class:`ReliabilityBin` records. Empty
+    bins are omitted from the output. A perfectly-calibrated agent has
+    every bin's ``gap`` near zero.
+    """
+    if n_bins < 1:
+        raise ValueError("n_bins must be >= 1")
+
+    rows = list(entries)
+    if not rows:
+        return []
+
+    edges = [i / n_bins for i in range(n_bins + 1)]
+    bins: list[list[BrierEntry]] = [[] for _ in range(n_bins)]
+    for r in rows:
+        # Find which bucket the predicted confidence falls into.
+        # Edge case: confidence == 1.0 → last bin.
+        c = max(0.0, min(0.9999999, r.predicted_confidence))
+        idx = int(c * n_bins)
+        bins[idx].append(r)
+
+    out: list[ReliabilityBin] = []
+    for i, b in enumerate(bins):
+        if not b:
+            continue
+        mean_conf = sum(e.predicted_confidence for e in b) / len(b)
+        mean_outcome = sum(e.outcome for e in b) / len(b)
+        out.append(
+            ReliabilityBin(
+                lower=edges[i],
+                upper=edges[i + 1],
+                mean_confidence=mean_conf,
+                mean_outcome=mean_outcome,
+                count=len(b),
+            )
+        )
+    return out
+
+
+# ---------------------------------------------------------------------------
+# calibration_adjustment — drives the shrinkage multiplier
+# ---------------------------------------------------------------------------
+
+#: Brier value at which we treat the agent as "well-calibrated" — boost above this.
+WELL_CALIBRATED_BRIER = 0.10
+#: Brier value above which we treat the agent as "overconfident" — shrink hard.
+OVERCONFIDENT_BRIER = 0.25
+#: Multiplier bounds — never zero (pure shrinkage) or unbounded (pure boost).
+ADJ_MIN = 0.5
+ADJ_MAX = 2.0
+#: Minimum observations before we adjust at all.
+MIN_OBSERVATIONS = 5
+
+
+def calibration_adjustment(mean_brier: float | None, n_observations: int) -> float:
+    """Map a mean Brier score to a multiplicative effective_n adjustment.
+
+    Returns 1.0 (no adjustment) if there are too few observations. Otherwise:
+
+      mean_brier ≤ 0.10 → boost (multiplier > 1.0, up to 2.0)
+      mean_brier ≥ 0.25 → shrink (multiplier < 1.0, down to 0.5)
+      between → linear interpolation
+
+    Tuning rationale:
+      - Brier ≤ 0.10 is what well-calibrated forecasters achieve on
+        binary questions in academic studies (e.g. Tetlock's superforecasters).
+      - Brier ≥ 0.25 is roughly worse than random for a 50/50 question
+        (Brier of a fair coin on a 50/50 → 0.25), so the agent's
+        confidence is not informative and should be down-weighted.
+    """
+    if mean_brier is None or n_observations < MIN_OBSERVATIONS:
+        return 1.0
+
+    b = max(0.0, float(mean_brier))
+    if b <= WELL_CALIBRATED_BRIER:
+        # Map [0.0, 0.10] → [ADJ_MAX, 1.0]
+        frac = b / WELL_CALIBRATED_BRIER
+        return ADJ_MAX - frac * (ADJ_MAX - 1.0)
+    if b >= OVERCONFIDENT_BRIER:
+        return ADJ_MIN
+    # Linear from 1.0 (at b=0.10) to ADJ_MIN (at b=0.25)
+    span = OVERCONFIDENT_BRIER - WELL_CALIBRATED_BRIER
+    frac = (b - WELL_CALIBRATED_BRIER) / span
+    return 1.0 - frac * (1.0 - ADJ_MIN)
+
+
+# ---------------------------------------------------------------------------
+# BrierScoreTracker
+# ---------------------------------------------------------------------------
+
+#: Cap on per-(agent, topic) history length to bound memory.
+MAX_HISTORY_PER_KEY = 200
+
+
+class BrierScoreTracker:
+    """Per-(agent, topic) Brier-score history and aggregates.
+
+    Lives on :class:`ReputationKernel` (one tracker per kernel) so
+    every oracle resolution can call :meth:`record` without plumbing.
+    """
+
+    def __init__(self, *, max_history_per_key: int = MAX_HISTORY_PER_KEY) -> None:
+        self.max_history_per_key = max_history_per_key
+        self._history: dict[tuple[bytes, str], list[BrierEntry]] = {}
+
+    # ------------------------------------------------------------------
+    # Write path
+    # ------------------------------------------------------------------
+
+    def record(
+        self,
+        agent_kid: bytes,
+        topic: str,
+        predicted_confidence: float,
+        outcome: float,
+        *,
+        ts_wall_ms: int | None = None,
+    ) -> BrierEntry:
+        """Append one observation. Returns the newly-built BrierEntry."""
+        key = (bytes(agent_kid), str(topic))
+        ts = int(time.time() * 1000) if ts_wall_ms is None else int(ts_wall_ms)
+        entry = BrierEntry(
+            ts_wall_ms=ts,
+            predicted_confidence=float(predicted_confidence),
+            outcome=float(outcome),
+            brier_score=brier_score(predicted_confidence, outcome),
+        )
+        bucket = self._history.setdefault(key, [])
+        bucket.append(entry)
+        if len(bucket) > self.max_history_per_key:
+            # Keep the most recent N
+            self._history[key] = bucket[-self.max_history_per_key :]
+        return entry
+
+    # ------------------------------------------------------------------
+    # Read path
+    # ------------------------------------------------------------------
+
+    def history(
+        self,
+        agent_kid: bytes,
+        topic: str,
+        *,
+        since_ms: int | None = None,
+    ) -> list[BrierEntry]:
+        rows = self._history.get((bytes(agent_kid), str(topic)), [])
+        if since_ms is None:
+            return list(rows)
+        return [r for r in rows if r.ts_wall_ms >= since_ms]
+
+    def mean_brier(
+        self,
+        agent_kid: bytes,
+        topic: str,
+        *,
+        since_ms: int | None = None,
+    ) -> float | None:
+        rows = self.history(agent_kid, topic, since_ms=since_ms)
+        if not rows:
+            return None
+        return sum(r.brier_score for r in rows) / len(rows)
+
+    def reliability_diagram(
+        self,
+        agent_kid: bytes,
+        topic: str,
+        *,
+        n_bins: int = 10,
+    ) -> list[ReliabilityBin]:
+        return reliability_diagram(
+            self.history(agent_kid, topic),
+            n_bins=n_bins,
+        )
+
+    def adjustment(self, agent_kid: bytes, topic: str) -> float:
+        """The shrinkage multiplier for this agent/topic — applied at consult time.
+
+        Returns 1.0 when there is insufficient history to adjust.
+        """
+        rows = self.history(agent_kid, topic)
+        if len(rows) < MIN_OBSERVATIONS:
+            return 1.0
+        m = sum(r.brier_score for r in rows) / len(rows)
+        return calibration_adjustment(m, len(rows))
+
+    def to_dict(self, agent_kid: bytes, topic: str) -> dict:
+        rows = self.history(agent_kid, topic)
+        return {
+            "agent_kid": bytes(agent_kid).hex(),
+            "topic": topic,
+            "n_observations": len(rows),
+            "mean_brier": self.mean_brier(agent_kid, topic),
+            "calibration_adjustment": self.adjustment(agent_kid, topic),
+            "reliability_diagram": [
+                b.to_dict() for b in self.reliability_diagram(agent_kid, topic)
+            ],
+            "history": [r.to_dict() for r in rows[-50:]],  # tail
+        }
+
+
+__all__ = [
+    "ADJ_MAX",
+    "ADJ_MIN",
+    "MAX_HISTORY_PER_KEY",
+    "MIN_OBSERVATIONS",
+    "OVERCONFIDENT_BRIER",
+    "WELL_CALIBRATED_BRIER",
+    "BrierEntry",
+    "BrierScoreTracker",
+    "ReliabilityBin",
+    "brier_score",
+    "calibration_adjustment",
+    "reliability_diagram",
+]