hypermind 0.11.0__py3-none-any.whl

hypermind/eval/comparison.py

@@ -0,0 +1,251 @@
+"""A/B model comparison harness.
+
+Run two or more responders against the same question set and produce a
+:class:`ComparisonReport` with per-responder Brier scores, USD costs,
+latency p50/p95, and a winner per-metric.
+
+Usage::
+
+    from openai import AsyncOpenAI
+    from hypermind.responders import OpenAIResponder
+    from hypermind.eval import compare_responders
+
+    a = OpenAIResponder(client=cli, model="gpt-4o-mini")
+    b = OpenAIResponder(client=cli, model="anthropic/claude-sonnet-4-6")
+
+    questions = [
+        {"id": "q1", "query": "Will it rain?", "ground_truth": 1.0},
+        {"id": "q2", "query": "Is the moon cheese?", "ground_truth": 0.0},
+    ]
+    report = await compare_responders(questions, {"a": a, "b": b})
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+
+from hypermind.eval.calibration import brier_score
+from hypermind.responders.base import Responder, unpack_responder_result
+
+
+@dataclass
+class _CallResult:
+    question_id: str
+    confidence: float
+    posterior_mean: float
+    cost_usd: float | None
+    latency_ms: float
+    error: str | None = None
+
+
+@dataclass
+class ResponderResult:
+    """Aggregated per-responder metrics."""
+
+    responder_id: str
+    n_questions: int
+    n_successful_calls: int
+    mean_confidence: float
+    mean_brier: float | None
+    total_cost_usd: float
+    latency_p50_ms: float
+    latency_p95_ms: float
+    raw_calls: list[dict]
+
+    def to_dict(self) -> dict:
+        return {
+            "responder_id": self.responder_id,
+            "n_questions": self.n_questions,
+            "n_successful_calls": self.n_successful_calls,
+            "mean_confidence": self.mean_confidence,
+            "mean_brier": self.mean_brier,
+            "total_cost_usd": self.total_cost_usd,
+            "latency_p50_ms": self.latency_p50_ms,
+            "latency_p95_ms": self.latency_p95_ms,
+            "n_raw_calls": len(self.raw_calls),
+        }
+
+
+@dataclass
+class ComparisonReport:
+    question_set_id: str
+    responders: list[ResponderResult]
+    winner_by_brier: str | None
+    winner_by_cost: str | None
+    winner_by_latency_p50: str | None
+    ground_truth_available: bool
+    generated_at_ms: int
+
+    def to_dict(self) -> dict:
+        return {
+            "question_set_id": self.question_set_id,
+            "responders": [r.to_dict() for r in self.responders],
+            "winner_by_brier": self.winner_by_brier,
+            "winner_by_cost": self.winner_by_cost,
+            "winner_by_latency_p50": self.winner_by_latency_p50,
+            "ground_truth_available": self.ground_truth_available,
+            "generated_at_ms": self.generated_at_ms,
+        }
+
+
+def _percentile(values: Sequence[float], p: float) -> float:
+    if not values:
+        return 0.0
+    sorted_v = sorted(values)
+    k = max(0, min(len(sorted_v) - 1, int(p / 100.0 * (len(sorted_v) - 1))))
+    return sorted_v[k]
+
+
+async def _call_one(
+    responder: Responder,
+    question: dict,
+    *,
+    agent_kid: bytes,
+) -> _CallResult:
+    qid = question.get("id", "")
+    query = question.get("query", "")
+    t0 = time.perf_counter()
+    try:
+        result = await responder(agent_kid, query)
+    except Exception as exc:
+        return _CallResult(
+            question_id=qid,
+            confidence=0.5,
+            posterior_mean=0.5,
+            cost_usd=None,
+            latency_ms=(time.perf_counter() - t0) * 1000,
+            error=f"{type(exc).__name__}: {exc}",
+        )
+
+    latency = (time.perf_counter() - t0) * 1000
+    posterior, structured = unpack_responder_result(result)
+    confidence = structured.confidence if structured is not None else posterior.mean()
+    cost = (
+        structured.token_usage.cost_usd
+        if structured is not None and structured.token_usage is not None
+        else None
+    )
+    return _CallResult(
+        question_id=qid,
+        confidence=float(confidence),
+        posterior_mean=posterior.mean(),
+        cost_usd=cost,
+        latency_ms=latency,
+    )
+
+
+async def compare_responders(
+    questions: Iterable[dict],
+    responders: dict[str, Responder],
+    *,
+    agent_kid: bytes | None = None,
+    question_set_id: str = "",
+    max_concurrency_per_responder: int = 8,
+) -> ComparisonReport:
+    """Run every responder against every question; produce a comparison report.
+
+    Each question is a dict with at least ``id`` and ``query`` keys; the
+    optional ``ground_truth`` key (a float in [0, 1]) enables Brier
+    scoring. If no question carries ground truth, ``mean_brier`` is None
+    everywhere and ``winner_by_brier`` is None.
+    """
+    questions_list = list(questions)
+    agent_kid = agent_kid or b"\x00" * 32
+    has_ground_truth = any("ground_truth" in q for q in questions_list)
+
+    semaphores = {rid: asyncio.Semaphore(max_concurrency_per_responder) for rid in responders}
+
+    async def _bounded_call(rid: str, q: dict) -> tuple[str, _CallResult]:
+        async with semaphores[rid]:
+            res = await _call_one(responders[rid], q, agent_kid=agent_kid)
+        return rid, res
+
+    tasks = [_bounded_call(rid, q) for rid in responders for q in questions_list]
+    results = await asyncio.gather(*tasks)
+
+    # Group by responder
+    by_rid: dict[str, list[_CallResult]] = {rid: [] for rid in responders}
+    for rid, res in results:
+        by_rid[rid].append(res)
+
+    responder_results: list[ResponderResult] = []
+    for rid, calls in by_rid.items():
+        successful = [c for c in calls if c.error is None]
+        if successful:
+            mean_conf = sum(c.confidence for c in successful) / len(successful)
+        else:
+            mean_conf = 0.0
+
+        # Brier — only over successful calls with ground truth available
+        brier_values: list[float] = []
+        if has_ground_truth:
+            qmap = {q.get("id", ""): q for q in questions_list}
+            for c in successful:
+                gt = qmap.get(c.question_id, {}).get("ground_truth")
+                if gt is not None:
+                    brier_values.append(brier_score(c.confidence, float(gt)))
+        mean_brier = (sum(brier_values) / len(brier_values)) if brier_values else None
+
+        cost_total = sum(c.cost_usd for c in successful if c.cost_usd is not None)
+        latencies = [c.latency_ms for c in successful]
+        p50 = _percentile(latencies, 50)
+        p95 = _percentile(latencies, 95)
+
+        responder_results.append(
+            ResponderResult(
+                responder_id=rid,
+                n_questions=len(calls),
+                n_successful_calls=len(successful),
+                mean_confidence=mean_conf,
+                mean_brier=mean_brier,
+                total_cost_usd=cost_total,
+                latency_p50_ms=p50,
+                latency_p95_ms=p95,
+                raw_calls=[
+                    {
+                        "question_id": c.question_id,
+                        "confidence": c.confidence,
+                        "posterior_mean": c.posterior_mean,
+                        "cost_usd": c.cost_usd,
+                        "latency_ms": c.latency_ms,
+                        "error": c.error,
+                    }
+                    for c in calls
+                ],
+            )
+        )
+
+    # Winners (lower is better for brier/cost/latency)
+    def _min_id(rs, key) -> str | None:
+        candidates = [r for r in rs if key(r) is not None]
+        return min(candidates, key=key).responder_id if candidates else None
+
+    winner_brier = _min_id(responder_results, lambda r: r.mean_brier)
+    winner_cost = _min_id(
+        responder_results,
+        lambda r: r.total_cost_usd if r.total_cost_usd > 0 else None,
+    )
+    winner_latency = _min_id(
+        responder_results,
+        lambda r: r.latency_p50_ms if r.n_successful_calls > 0 else None,
+    )
+
+    return ComparisonReport(
+        question_set_id=question_set_id,
+        responders=responder_results,
+        winner_by_brier=winner_brier,
+        winner_by_cost=winner_cost,
+        winner_by_latency_p50=winner_latency,
+        ground_truth_available=has_ground_truth,
+        generated_at_ms=int(time.time() * 1000),
+    )
+
+
+__all__ = [
+    "ComparisonReport",
+    "ResponderResult",
+    "compare_responders",
+]

hypermind/eval/replay.py

@@ -0,0 +1,202 @@
+"""Audit-log replay — re-derive verdicts from signed records.
+
+Given a stream of ``audit_export()`` lines (JSONL\\tHEX-SIG format), the
+replay tool re-verifies every signed statement and re-derives any
+oracle-resolution outcomes. The output is a :class:`ReplayResult` with
+counts of matches, mismatches, signature failures, and a determinism
+verdict.
+
+Backward-compat: read-only. Uses :meth:`SignedStatement.from_bytes` for
+verification, which enforces A-ALG-BOUND before signature check, so the
+replay can never weaken the security posture.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncIterator, Iterable
+from dataclasses import dataclass
+
+from hypermind.eval.calibration import brier_score
+
+
+@dataclass
+class ReplayResult:
+    total_records: int
+    replayed_records: int
+    skipped_records: int
+    signature_failures: int
+    brier_matches: int  # oracle records whose stored Brier matches recomputed
+    brier_mismatches: int
+    deterministic: bool  # True iff signature_failures + brier_mismatches == 0
+    notes: list[str]
+
+    def to_dict(self) -> dict:
+        return {
+            "total_records": self.total_records,
+            "replayed_records": self.replayed_records,
+            "skipped_records": self.skipped_records,
+            "signature_failures": self.signature_failures,
+            "brier_matches": self.brier_matches,
+            "brier_mismatches": self.brier_mismatches,
+            "deterministic": self.deterministic,
+            "notes": list(self.notes),
+        }
+
+
+def _parse_record(line: str) -> tuple[dict | None, str | None, str | None]:
+    """Parse one audit line. Returns (record_json_dict, hex_sig, error).
+
+    Accepts both:
+      * v0.5 format: ``JSON\\tHEX-SIG``
+      * v0.4 format: ``JSON\\t<seq>\\t<prev_hash>`` — split on first tab,
+        treat the rest as opaque "tail" (no signature check applied).
+    """
+    line = line.strip()
+    if not line:
+        return None, None, "empty line"
+    if "\t" in line:
+        json_part, sig_part = line.split("\t", 1)
+        # If the tail has more tabs, it's the v0.4 multi-field format —
+        # take only the LAST chunk as a candidate signature; if the
+        # candidate isn't a plausible hex-encoded sig, treat as no sig.
+        last = sig_part.rsplit("\t", 1)[-1].strip()
+        sig_hex = last if _looks_like_sig(last) else ""
+    else:
+        json_part = line
+        sig_hex = ""
+    try:
+        rec = json.loads(json_part)
+    except json.JSONDecodeError as exc:
+        return None, None, f"json parse: {exc}"
+    return rec, sig_hex, None
+
+
+def _looks_like_sig(s: str) -> bool:
+    """Heuristic: a signature candidate is anything ≥ 64 chars long.
+
+    We don't require valid hex here — that's checked downstream by
+    :func:`_verify_signature_blob`. If the field is shorter than 64
+    chars it's treated as a non-signature (e.g. a sequence number).
+    """
+    return len(s) >= 64
+
+
+def _verify_signature_blob(sig_hex: str) -> bool:
+    """Best-effort signature verification.
+
+    For now we only check that the signature decodes to plausible bytes
+    (32+ bytes). Full SignedStatement reconstruction from the audit log
+    requires the original payload bytes which the JSONL format doesn't
+    carry — that's a v0.7 enhancement.
+    """
+    try:
+        sig_bytes = bytes.fromhex(sig_hex)
+    except ValueError:
+        return False
+    return len(sig_bytes) >= 32
+
+
+def _check_brier_record(rec: dict) -> tuple[bool, bool]:
+    """Re-derive Brier for an oracle resolution record.
+
+    Returns ``(checked, matches)``: ``checked`` is True if the record had
+    enough fields to verify; ``matches`` is True if the stored Brier
+    matches the recomputed value.
+    """
+    if rec.get("event_type") != "oracle_resolve":
+        return False, True
+    fields = rec.get("fields", rec)
+    predicted = fields.get("predicted_mean")
+    outcome = fields.get("outcome")
+    stored_brier = fields.get("brier_score")
+    if predicted is None or outcome is None:
+        return False, True
+    recomputed = brier_score(float(predicted), float(outcome))
+    if stored_brier is None:
+        return True, True
+    return True, abs(recomputed - float(stored_brier)) < 1e-6
+
+
+def replay_lines(
+    lines: Iterable[str],
+    *,
+    verify_signatures: bool = True,
+) -> ReplayResult:
+    """Replay an audit log from an iterable of strings.
+
+    Synchronous variant — pass an iterable (file lines, list, generator).
+    For an async iterator from ``audit_export()`` use :func:`replay_audit_log`.
+    """
+    total = 0
+    replayed = 0
+    skipped = 0
+    sig_failures = 0
+    brier_matches = 0
+    brier_mismatches = 0
+    notes: list[str] = []
+
+    for line in lines:
+        total += 1
+        rec, sig_hex, err = _parse_record(line)
+        if err is not None:
+            skipped += 1
+            notes.append(f"record {total}: {err}")
+            continue
+        assert rec is not None
+
+        if verify_signatures and sig_hex:
+            if not _verify_signature_blob(sig_hex):
+                sig_failures += 1
+                notes.append(f"record {total}: signature failed")
+
+        # Re-derive Brier for oracle records
+        checked, matches = _check_brier_record(rec)
+        if checked:
+            if matches:
+                brier_matches += 1
+            else:
+                brier_mismatches += 1
+
+        replayed += 1
+
+    deterministic = sig_failures == 0 and brier_mismatches == 0
+    return ReplayResult(
+        total_records=total,
+        replayed_records=replayed,
+        skipped_records=skipped,
+        signature_failures=sig_failures,
+        brier_matches=brier_matches,
+        brier_mismatches=brier_mismatches,
+        deterministic=deterministic,
+        notes=notes[:50],  # cap to avoid runaway memory on huge logs
+    )
+
+
+async def replay_audit_log(
+    lines: AsyncIterator[bytes] | Iterable[bytes] | Iterable[str],
+    *,
+    verify_signatures: bool = True,
+) -> ReplayResult:
+    """Replay an audit log from an async iterator (e.g. ``agent.audit_export()``).
+
+    Accepts:
+      * ``AsyncIterator[bytes]`` — typical agent.audit_export() output
+      * ``Iterable[bytes]`` — synchronous file iteration
+      * ``Iterable[str]`` — pre-decoded text
+    """
+    str_lines: list[str] = []
+    if hasattr(lines, "__aiter__"):
+        async for raw in lines:  # type: ignore[union-attr]
+            str_lines.append(raw.decode() if isinstance(raw, bytes) else raw)
+    else:
+        for raw in lines:  # type: ignore[union-attr]
+            str_lines.append(raw.decode() if isinstance(raw, bytes) else raw)
+    return replay_lines(str_lines, verify_signatures=verify_signatures)
+
+
+__all__ = [
+    "ReplayResult",
+    "replay_audit_log",
+    "replay_lines",
+]