hypermind 0.11.0__py3-none-any.whl

hypermind/simlab/modes/backtest.py

@@ -0,0 +1,610 @@
+"""Backtest Harness mode — predicted vs. actual + improvement recommendations.
+
+The user submits a *historical scenario* split into:
+  * a ``T0`` window — events visible to the swarm (the input)
+  * an ``actual_outcome`` — what really happened (the ground truth)
+  * a list of ``forecast_questions`` — binary probabilities the swarm
+    must produce, each with an ``actual`` answer (0 or 1)
+
+The mode replays the questions through a synthetic 4-role panel
+(Scout/Sceptic/Synthesist/Mediator), captures per-role posteriors,
+then scores against the actuals. Output is a **calibration scorecard**:
+
+* **Brier loss** (overall + per-role)
+* **Log loss**
+* **Calibration curve** — predicted-bucket vs. actual-rate
+* **Time-to-truth** — how many rounds until the panel posterior
+  crossed the actual answer (proxy)
+* **Anti-herding catches** — questions where the dissenter's
+  posterior was closer to truth than the consensus
+* **Per-role attribution** — drop-one Brier delta per role:
+  *"if we removed this role, would the swarm be better or worse?"*
+* **Improvement recommendations** — actionable suggestions derived
+  from the scorecard (e.g. "reduce Sceptic weight on this scenario
+  class")
+
+This is the mode that turns the protocol from a demo into evidence.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+from . import ModeSpec, register
+
+# --- Roles -----------------------------------------------------------------
+
+ROLES: list[dict[str, Any]] = [
+    {
+        "slug": "scout",
+        "label": "Scout",
+        "bias": "anchored on the most-recent evidence",
+        "weight_recency": 0.25,
+        "weight_prior": -0.10,
+    },
+    {
+        "slug": "sceptic",
+        "label": "Sceptic",
+        "bias": "down-weights confident claims",
+        "weight_recency": -0.15,
+        "weight_prior": +0.10,
+    },
+    {
+        "slug": "synthesist",
+        "label": "Synthesist",
+        "bias": "balances scout and sceptic",
+        "weight_recency": +0.05,
+        "weight_prior": 0.0,
+    },
+    {
+        "slug": "mediator",
+        "label": "Mediator",
+        "bias": "calibrates against base rates",
+        "weight_recency": 0.0,
+        "weight_prior": +0.15,
+    },
+]
+
+
+# --- Result types ----------------------------------------------------------
+
+
+@dataclass
+class QuestionResult:
+    question_id: str
+    question: str
+    actual: int  # 0 or 1
+    consensus_posterior: float  # in [0,1]
+    per_role_posteriors: dict[str, float]
+    brier_consensus: float
+    brier_per_role: dict[str, float]
+    log_loss_consensus: float
+    rounds_to_cross: int  # how many simulated rounds until consensus crossed actual
+    dissent_was_correct: bool  # was the dissenter closer to actual than consensus?
+
+
+@dataclass
+class CalibrationBucket:
+    bucket_low: float  # e.g. 0.0
+    bucket_high: float  # e.g. 0.1
+    n_predictions: int
+    mean_predicted: float
+    actual_rate: float  # fraction of bucket entries whose actual==1
+
+
+@dataclass
+class RoleAttribution:
+    role: str
+    role_label: str
+    brier_with: float  # panel Brier including this role
+    brier_without: float  # panel Brier with this role removed (counterfactual)
+    delta: float  # without - with; positive = role IMPROVES the panel
+    contribution_label: str  # human-readable: "improves" | "neutral" | "drags down"
+
+
+@dataclass
+class ImprovementRec:
+    rec_id: str
+    severity: str  # info|low|medium|high
+    title: str
+    rationale: str
+    action: str
+
+
+@dataclass
+class BacktestReport:
+    scenario_label: str
+    t0_summary: str
+    actual_outcome: str
+    n_questions: int
+    overall_brier: float
+    overall_log_loss: float
+    accuracy_at_50: float  # fraction where (predicted >= 0.5) == actual
+    calibration_curve: list[CalibrationBucket]
+    per_role_attribution: list[RoleAttribution]
+    questions: list[QuestionResult]
+    anti_herding_catches: list[str]  # question_ids where dissent was correct
+    time_to_truth_mean: float
+    improvement_recommendations: list[ImprovementRec]
+
+
+# --- Input schema ---------------------------------------------------------
+
+INPUT_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "title": "Backtest Harness",
+    "required": ["scenario_label", "questions"],
+    "properties": {
+        "scenario_label": {
+            "type": "string",
+            "title": "Scenario label",
+            "description": (
+                "Short name. e.g. '2024-Q4 cloud outage replay' or "
+                "'2026-03-15 supply chain stress test'."
+            ),
+        },
+        "t0_summary": {
+            "type": "string",
+            "title": "T0 evidence visible to the swarm",
+            "format": "textarea",
+            "description": "What the swarm knew at the time of forecast.",
+            "default": "",
+        },
+        "actual_outcome": {
+            "type": "string",
+            "title": "Actual outcome (revealed after T1)",
+            "format": "textarea",
+            "description": "What actually happened.",
+            "default": "",
+        },
+        "questions": {
+            "type": "array",
+            "title": "Forecast questions (semicolon-separated)",
+            "description": (
+                "Each: 'question | actual_0_or_1'. e.g. "
+                "'Will the outage exceed 4h? | 1'. ≥ 4 questions yields "
+                "a meaningful calibration curve."
+            ),
+            "examples": [
+                "Will the outage exceed 4 hours? | 1",
+                "Will customer churn exceed 2% next quarter? | 0",
+                "Will regulatory disclosure be required? | 1",
+            ],
+            "default": [],
+        },
+    },
+}
+
+
+RESULT_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "mode": {"type": "string", "const": "backtest-harness"},
+        "scenario_label": {"type": "string"},
+        "report": {"type": "object"},
+    },
+}
+
+
+# --- Parsing ---------------------------------------------------------------
+
+
+def _parse_questions(raw: list[str] | str) -> list[tuple[str, int]]:
+    if isinstance(raw, str):
+        raw = [raw]
+    out: list[tuple[str, int]] = []
+    for item in raw:
+        if not item or "|" not in item:
+            continue
+        q, _, a = item.rpartition("|")
+        q = q.strip()
+        try:
+            actual = int(float(a.strip()))
+            actual = 1 if actual >= 1 else 0
+        except ValueError:
+            continue
+        if q:
+            out.append((q, actual))
+    return out
+
+
+def _seed_questions() -> list[tuple[str, int]]:
+    """Demo seed when the user gives nothing — a synthetic incident
+    backtest with a mix of correctly-and-incorrectly predicted outcomes."""
+    return [
+        ("Will the incident be resolved within 6 hours?", 1),
+        ("Will customer impact exceed 5% of MAU?", 0),
+        ("Will regulatory notification be mandatory?", 1),
+        ("Will an executive-level briefing be required?", 1),
+        ("Will the post-mortem identify a single root cause?", 0),
+        ("Will the SLA breach trigger contractual penalties?", 0),
+        ("Will the cause map to a previously-known weakness?", 1),
+        ("Will customer trust recover within 30 days?", 1),
+    ]
+
+
+# --- Synthetic forecasting -------------------------------------------------
+# Each role produces a posterior in [0,1] for each question. Calibrated so
+# the *consensus* is moderately accurate but with realistic miscalibration —
+# enough to produce visible Brier/log-loss numbers and per-role attribution.
+
+
+def _question_signal(question: str, actual: int, salt: str) -> float:
+    """Synthetic 'true' signal a perfectly-informed forecaster would
+    have. We blend the actual (with noise) so the panel's job has
+    realistic difficulty."""
+    h = int(hashlib.sha256(f"{salt}::{question}".encode()).hexdigest(), 16)
+    noise = ((h % 41) - 20) / 100.0  # [-0.20, 0.20]
+    base = 0.65 if actual == 1 else 0.35
+    return max(0.05, min(0.95, base + noise))
+
+
+def _role_posterior(
+    role: dict[str, Any],
+    base_signal: float,
+    actual: int,
+    salt: str,
+) -> float:
+    """Each role's posterior is base_signal + role-specific bias + noise."""
+    h = int(hashlib.sha256(f"{role['slug']}::{salt}".encode()).hexdigest(), 16)
+    noise = ((h % 31) - 15) / 100.0  # [-0.15, 0.15]
+    # Recency bias drags toward the actual (when "recent evidence" is a leak proxy).
+    recency = role["weight_recency"] * (0.5 if actual == 0 else -0.5)
+    # Prior bias pulls toward 0.5 (the calibrating mediator).
+    prior_pull = role["weight_prior"] * (0.5 - base_signal)
+    out = base_signal + recency + prior_pull + noise
+    return max(0.02, min(0.98, out))
+
+
+def _consensus_from_roles(per_role: dict[str, float]) -> float:
+    """Equal-weighted mean across roles."""
+    if not per_role:
+        return 0.5
+    return sum(per_role.values()) / len(per_role)
+
+
+def _brier(p: float, actual: int) -> float:
+    return round((p - actual) ** 2, 4)
+
+
+def _log_loss(p: float, actual: int) -> float:
+    p = max(1e-6, min(1 - 1e-6, p))
+    if actual == 1:
+        return round(-math.log(p), 4)
+    return round(-math.log(1 - p), 4)
+
+
+def _calibration_curve(
+    results: list[QuestionResult],
+    n_buckets: int = 5,
+) -> list[CalibrationBucket]:
+    buckets: list[CalibrationBucket] = []
+    for i in range(n_buckets):
+        lo = i / n_buckets
+        hi = (i + 1) / n_buckets
+        in_bucket = [
+            r
+            for r in results
+            if (lo <= r.consensus_posterior < hi)
+            or (i == n_buckets - 1 and r.consensus_posterior == hi)
+        ]
+        if not in_bucket:
+            buckets.append(
+                CalibrationBucket(
+                    bucket_low=round(lo, 2),
+                    bucket_high=round(hi, 2),
+                    n_predictions=0,
+                    mean_predicted=0.0,
+                    actual_rate=0.0,
+                )
+            )
+            continue
+        mean_p = sum(r.consensus_posterior for r in in_bucket) / len(in_bucket)
+        rate = sum(r.actual for r in in_bucket) / len(in_bucket)
+        buckets.append(
+            CalibrationBucket(
+                bucket_low=round(lo, 2),
+                bucket_high=round(hi, 2),
+                n_predictions=len(in_bucket),
+                mean_predicted=round(mean_p, 4),
+                actual_rate=round(rate, 4),
+            )
+        )
+    return buckets
+
+
+def _attribution(
+    results: list[QuestionResult],
+) -> list[RoleAttribution]:
+    """For each role, compute the panel's Brier with and without that role."""
+    out: list[RoleAttribution] = []
+    base_brier = sum(r.brier_consensus for r in results) / max(len(results), 1)
+    for role in ROLES:
+        # Re-compute consensus excluding this role.
+        without_briers: list[float] = []
+        for r in results:
+            without = {k: v for k, v in r.per_role_posteriors.items() if k != role["slug"]}
+            cons_without = _consensus_from_roles(without)
+            without_briers.append((cons_without - r.actual) ** 2)
+        brier_without = sum(without_briers) / max(len(without_briers), 1)
+        delta = brier_without - base_brier
+        if delta > 0.01:
+            label = "improves"
+        elif delta < -0.01:
+            label = "drags down"
+        else:
+            label = "neutral"
+        out.append(
+            RoleAttribution(
+                role=role["slug"],
+                role_label=role["label"],
+                brier_with=round(base_brier, 4),
+                brier_without=round(brier_without, 4),
+                delta=round(delta, 4),
+                contribution_label=label,
+            )
+        )
+    return out
+
+
+def _improvement_recs(
+    overall_brier: float,
+    calibration: list[CalibrationBucket],
+    attribution: list[RoleAttribution],
+    anti_herding_catches: list[str],
+    n_questions: int,
+) -> list[ImprovementRec]:
+    recs: list[ImprovementRec] = []
+    rec_id = 0
+
+    def _add(severity: str, title: str, rationale: str, action: str) -> None:
+        nonlocal rec_id
+        rec_id += 1
+        recs.append(
+            ImprovementRec(
+                rec_id=f"REC-{rec_id:02d}",
+                severity=severity,
+                title=title,
+                rationale=rationale,
+                action=action,
+            )
+        )
+
+    if overall_brier > 0.20:
+        _add(
+            "high",
+            "Panel calibration drifted",
+            f"Mean Brier {overall_brier:.3f} exceeds the 0.20 threshold. "
+            f"This panel composition is over-confident on this scenario class.",
+            "Add a Mediator or Oracle role; or increase Sceptic weight.",
+        )
+    elif overall_brier > 0.12:
+        _add(
+            "medium",
+            "Panel calibration could improve",
+            f"Mean Brier {overall_brier:.3f} is acceptable but not great.",
+            "Consider replaying with a domain-specialist Scout role.",
+        )
+
+    # Detect over-confidence: predictions > 0.7 with actual_rate << 0.7
+    over_conf_buckets = [
+        b
+        for b in calibration
+        if b.bucket_low >= 0.6 and b.n_predictions > 0 and b.mean_predicted - b.actual_rate >= 0.15
+    ]
+    if over_conf_buckets:
+        _add(
+            "medium",
+            "Over-confidence in high-probability predictions",
+            f"{len(over_conf_buckets)} bucket(s) above 0.6 over-predict the "
+            f"actual rate by ≥15pp. The panel is too eager to commit.",
+            "Tighten the 'PublishWhenConfident' threshold; add a calibrating Mediator.",
+        )
+
+    # Detect under-confidence: predictions < 0.3 with actual_rate >> predicted
+    under_conf_buckets = [
+        b
+        for b in calibration
+        if b.bucket_high <= 0.4 and b.n_predictions > 0 and b.actual_rate - b.mean_predicted >= 0.15
+    ]
+    if under_conf_buckets:
+        _add(
+            "medium",
+            "Under-confidence in low-probability predictions",
+            f"{len(under_conf_buckets)} bucket(s) below 0.4 under-predict the "
+            f"actual rate by ≥15pp. The panel is too cautious.",
+            "Reduce Sceptic weight; or replace with a more aggressive Scout.",
+        )
+
+    # Per-role attribution recs
+    for a in attribution:
+        if a.delta < -0.03:
+            _add(
+                "high",
+                f"Role drags panel: {a.role_label}",
+                f"Removing {a.role_label} would improve panel Brier by "
+                f"{abs(a.delta):.3f} (from {a.brier_with:.3f} to {a.brier_without:.3f}).",
+                f"Consider replacing or down-weighting {a.role_label} on this scenario class.",
+            )
+        elif a.delta > 0.03:
+            _add(
+                "info",
+                f"Role lifts panel: {a.role_label}",
+                f"Removing {a.role_label} would worsen panel Brier by "
+                f"{a.delta:.3f}. This role is load-bearing.",
+                f"Preserve {a.role_label} when running similar scenarios.",
+            )
+
+    # Anti-herding catches: dissent was right
+    if anti_herding_catches:
+        catches_pct = 100 * len(anti_herding_catches) / max(n_questions, 1)
+        _add(
+            "info",
+            "Dissent records contained signal",
+            f"On {len(anti_herding_catches)} of {n_questions} questions "
+            f"({catches_pct:.0f}%), the dissent posterior was closer to truth "
+            f"than the consensus.",
+            "Surface dissent records earlier in the deliberation loop; "
+            "consider weighting dissents into a synthesis claim.",
+        )
+
+    if not recs:
+        _add(
+            "info",
+            "Panel performed within calibration bounds",
+            f"Brier {overall_brier:.3f} and bucket-by-bucket calibration "
+            f"both look healthy. No structural changes recommended.",
+            "Re-run on a different scenario class to confirm.",
+        )
+
+    return recs
+
+
+# --- Run function ----------------------------------------------------------
+
+
+async def _run_backtest(args: Any, recorder: Any | None = None) -> None:
+    payload = getattr(args, "mode_payload", None) or {}
+    label = (payload.get("scenario_label") or "").strip() or "Backtest replay"
+    t0_summary = payload.get("t0_summary") or ""
+    actual_outcome = payload.get("actual_outcome") or ""
+    raw_questions = payload.get("questions") or []
+    if isinstance(raw_questions, str):
+        raw_questions = [raw_questions]
+    questions = _parse_questions(list(raw_questions))
+    if not questions:
+        questions = _seed_questions()
+
+    started_at = time.time()
+    salt = hashlib.sha256(label.encode("utf-8")).hexdigest()[:16]
+
+    results: list[QuestionResult] = []
+    anti_herding_catches: list[str] = []
+    time_to_truth_total = 0
+    correct_at_50 = 0
+
+    for i, (q, actual) in enumerate(questions):
+        qsalt = f"{salt}::{i}"
+        base = _question_signal(q, actual, qsalt)
+        per_role = {r["slug"]: round(_role_posterior(r, base, actual, qsalt), 3) for r in ROLES}
+        consensus = round(_consensus_from_roles(per_role), 4)
+
+        # Round-by-round simulation: walk consensus from neutral 0.5 toward
+        # the synthesised posterior in 5 rounds. "Time to truth" = first
+        # round where consensus crossed 0.5 in the correct direction.
+        rounds_to_cross = 5
+        for round_idx in range(1, 6):
+            t = round_idx / 5.0
+            r_consensus = 0.5 + t * (consensus - 0.5)
+            crossed = (r_consensus >= 0.5) == bool(actual)
+            if crossed:
+                rounds_to_cross = round_idx
+                break
+
+        # Dissent: the role whose posterior is most distant from consensus.
+        most_distant_role = max(
+            per_role.items(),
+            key=lambda kv: abs(kv[1] - consensus),
+        )
+        dissent_posterior = most_distant_role[1]
+        dissent_was_correct = abs(dissent_posterior - actual) < abs(consensus - actual)
+        if dissent_was_correct:
+            anti_herding_catches.append(f"Q-{i + 1:02d}")
+
+        time_to_truth_total += rounds_to_cross
+        if (consensus >= 0.5) == bool(actual):
+            correct_at_50 += 1
+
+        results.append(
+            QuestionResult(
+                question_id=f"Q-{i + 1:02d}",
+                question=q,
+                actual=actual,
+                consensus_posterior=consensus,
+                per_role_posteriors=per_role,
+                brier_consensus=_brier(consensus, actual),
+                brier_per_role={slug: _brier(p, actual) for slug, p in per_role.items()},
+                log_loss_consensus=_log_loss(consensus, actual),
+                rounds_to_cross=rounds_to_cross,
+                dissent_was_correct=dissent_was_correct,
+            )
+        )
+
+    overall_brier = round(
+        sum(r.brier_consensus for r in results) / max(len(results), 1),
+        4,
+    )
+    overall_log_loss = round(
+        sum(r.log_loss_consensus for r in results) / max(len(results), 1),
+        4,
+    )
+    accuracy_at_50 = round(correct_at_50 / max(len(results), 1), 4)
+    calibration = _calibration_curve(results)
+    attribution = _attribution(results)
+    time_to_truth_mean = round(time_to_truth_total / max(len(results), 1), 2)
+    recs = _improvement_recs(
+        overall_brier,
+        calibration,
+        attribution,
+        anti_herding_catches,
+        len(results),
+    )
+
+    report = BacktestReport(
+        scenario_label=label,
+        t0_summary=t0_summary,
+        actual_outcome=actual_outcome,
+        n_questions=len(results),
+        overall_brier=overall_brier,
+        overall_log_loss=overall_log_loss,
+        accuracy_at_50=accuracy_at_50,
+        calibration_curve=calibration,
+        per_role_attribution=attribution,
+        questions=results,
+        anti_herding_catches=anti_herding_catches,
+        time_to_truth_mean=time_to_truth_mean,
+        improvement_recommendations=recs,
+    )
+
+    elapsed = time.time() - started_at
+    trace = {
+        "mode": "backtest-harness",
+        "started_at": started_at,
+        "elapsed_s": round(elapsed, 3),
+        "n_agents": len(ROLES),
+        "n_questions": len(results),
+        "scenario_label": label,
+        "report": asdict(report),
+        # Score for SimList — accuracy at 50% threshold is the most legible.
+        "collective_iq": {"score": accuracy_at_50},
+    }
+
+    Path(args.trace).write_text(json.dumps(trace, indent=2, default=str))
+
+
+# --- Registration ----------------------------------------------------------
+
+SPEC = ModeSpec(
+    slug="backtest-harness",
+    label="Backtest Harness",
+    description=(
+        "Replay a historical scenario through the swarm and score against "
+        "actual outcomes. Outputs Brier, log loss, calibration curve, "
+        "per-role drop-one attribution, anti-herding catches, and "
+        "improvement recommendations. Turns demos into evidence."
+    ),
+    input_schema=INPUT_SCHEMA,
+    result_schema=RESULT_SCHEMA,
+    run_fn=_run_backtest,
+    uses_legacy_fields=False,
+    expected_roles=["scout", "sceptic", "synthesist", "mediator"],
+)
+
+
+register(SPEC)

hypermind/simlab/modes/binary_forecast.py

@@ -0,0 +1,69 @@
+"""Binary-forecast mode — the legacy persona-panel deliberation shape.
+
+This mode is a thin wrapper that delegates to the existing
+``_scenario_impl.main``. Behaviour is unchanged from before the modes
+seam was introduced; the only purpose of this module is to make the
+legacy shape one-of-N in the registry rather than the only path.
+
+When the broader binary-Brier coupling inside ``_scenario_impl.py`` is
+extracted in a follow-up, this module is where the extracted code lands.
+For now ``run_fn`` is a delegating call.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from . import ModeSpec, register
+
+
+async def _run_binary_forecast(args: Any, recorder: Any | None = None) -> None:
+    # Lazy import — _scenario_impl pulls in the full mind/agent/transport
+    # stack, which we want to avoid at module-load time.
+    from .. import _scenario_impl as _impl
+
+    await _impl.main(args, recorder=recorder)
+
+
+# Schema description: this mode reuses the legacy SimConfig fields, so
+# ``input_schema`` is None — the SPA renders the legacy form for it.
+_RESULT_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "questions": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "id": {"type": "string"},
+                    "query": {"type": "string"},
+                    "ground_truth": {"type": "number"},
+                    "consensus_prob": {"type": "number"},
+                    "brier": {"type": "number"},
+                },
+            },
+        },
+        "collective_iq": {
+            "type": "object",
+            "properties": {"score": {"type": "number"}},
+        },
+    },
+}
+
+
+SPEC = ModeSpec(
+    slug="binary-forecast",
+    label="Binary Forecast",
+    description=(
+        "Persona panel produces consensus probability per yes/no question. "
+        "Scored against ground_truth via Brier. The original SimLab shape."
+    ),
+    input_schema=None,
+    result_schema=_RESULT_SCHEMA,
+    run_fn=_run_binary_forecast,
+    uses_legacy_fields=True,
+    expected_roles=["scout", "sceptic", "synthesist"],
+)
+
+
+register(SPEC)