hypermind 0.11.0__py3-none-any.whl

hypermind/simlab/modes/_eval_real_llm.py

@@ -0,0 +1,373 @@
+"""Real-LLM forecaster integration for the Evaluation Bench.
+
+Replaces the deterministic-synthetic posteriors with actual OpenRouter
+calls when ``use_real_llm`` is set on the mode payload AND the
+``OPENROUTER_API_KEY`` environment variable is present.
+
+Architecture:
+
+* Three model-based forecasters (HyperMind 4-role panel, Single-LLM,
+  Naive ensemble of N=4 temperature-jittered calls). The 3 honest
+  baselines (Random, Majority, Climatology) stay deterministic — they
+  have no model.
+* Per-question cache keyed by content hash so repeat runs of the same
+  benchmark don't re-pay for the same questions. Cache lives at
+  ``~/.hypermind/eval_llm_cache/``.
+* :class:`RunCost` captures token usage + USD cost per forecaster so the
+  evaluation report can show the *cost-adjusted* skill score (HyperMind's
+  4× call count vs Single-LLM's 1× — is the lift worth the spend?).
+* Falls back gracefully to synthetic when:
+  - no API key is present
+  - the OpenAI client import fails
+  - any individual call errors out (single forecaster degrades; the
+    rest of the run continues)
+
+Reuses :func:`hypermind.simlab._scenario_impl._make_openrouter_client`
+to avoid re-implementing the OpenRouter handshake.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+# --- Cost capture ----------------------------------------------------------
+
+
+@dataclass
+class RunCost:
+    """Per-forecaster aggregate cost for a benchmark run.
+
+    Surfaces in the evaluation report so a reviewer sees not just
+    'HyperMind beat Single-LLM by 12% on Brier' but also 'and it cost
+    4.1× as many tokens to do so'. This is the fair comparison.
+    """
+
+    n_calls: int = 0
+    total_tokens_prompt: int = 0
+    total_tokens_completion: int = 0
+    estimated_usd: float = 0.0
+    elapsed_s: float = 0.0
+
+
+# --- Cache ----------------------------------------------------------------
+
+CACHE_DIR = Path(os.path.expanduser("~/.hypermind/eval_llm_cache"))
+
+
+def _cache_key(model: str, prompt: str, temperature: float) -> str:
+    canonical = f"{model}::{temperature:.2f}::{prompt}"
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _cache_get(key: str) -> dict[str, Any] | None:
+    path = CACHE_DIR / f"{key}.json"
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return None
+
+
+def _cache_set(key: str, value: dict[str, Any]) -> None:
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    path = CACHE_DIR / f"{key}.json"
+    try:
+        path.write_text(json.dumps(value))
+    except Exception:
+        pass
+
+
+# --- Pricing (rough; conservative) ----------------------------------------
+# Per-1M-token pricing in USD for OpenRouter passthrough. These are
+# illustrative — for real cost reporting, callers should override with
+# the price table their account actually pays.
+_PRICE_PER_1M_TOKENS_USD: dict[str, tuple[float, float]] = {
+    # model-name : (prompt $/1M, completion $/1M)
+    "openai/gpt-4o-mini": (0.15, 0.60),
+    "openai/gpt-4o": (2.50, 10.00),
+    "anthropic/claude-3-5-haiku": (0.80, 4.00),
+    "anthropic/claude-3-5-sonnet": (3.00, 15.00),
+    "google/gemini-flash-1.5": (0.075, 0.30),
+}
+
+
+def _estimate_cost_usd(model: str, prompt_tokens: int, completion_tokens: int) -> float:
+    rate = _PRICE_PER_1M_TOKENS_USD.get(model)
+    if rate is None:
+        return 0.0
+    p_per_1m, c_per_1m = rate
+    return (prompt_tokens * p_per_1m + completion_tokens * c_per_1m) / 1_000_000
+
+
+# --- Single-call helper ---------------------------------------------------
+
+_SYSTEM_PROMPT_NEUTRAL = (
+    "You are a careful probabilistic forecaster. For the question below, "
+    "produce your subjective posterior probability that the answer is YES "
+    "(actual=1). Calibrate honestly: a 70% answer should be right ~70% of "
+    "the time. Reply ONLY with a JSON object of the form "
+    '{"confidence": <number in [0,1]>, "reasoning": "<one sentence>"}.'
+)
+
+_SYSTEM_PROMPTS_BY_ROLE = {
+    "scout": (
+        "You are the SCOUT in a forecasting panel. Your job is to find and "
+        "weigh evidence. Lean toward the side the evidence supports. "
+        "Calibrate honestly. Reply ONLY with JSON: "
+        '{"confidence": <0..1>, "reasoning": "<sentence>"}.'
+    ),
+    "sceptic": (
+        "You are the SCEPTIC in a forecasting panel. Your job is to "
+        "challenge over-confident claims. Pull confident posteriors back "
+        "toward 0.5 unless the evidence is strong. Reply ONLY with JSON: "
+        '{"confidence": <0..1>, "reasoning": "<sentence>"}.'
+    ),
+    "synthesist": (
+        "You are the SYNTHESIST in a forecasting panel. Your job is to "
+        "balance the Scout's evidence and the Sceptic's caution. Aim for "
+        "calibrated middle-ground. Reply ONLY with JSON: "
+        '{"confidence": <0..1>, "reasoning": "<sentence>"}.'
+    ),
+    "mediator": (
+        "You are the MEDIATOR in a forecasting panel. Your job is to "
+        "calibrate against the base rate. When the question is ambiguous, "
+        "pull the posterior toward 0.5. Reply ONLY with JSON: "
+        '{"confidence": <0..1>, "reasoning": "<sentence>"}.'
+    ),
+}
+
+
+async def _llm_call(
+    client: Any,
+    model: str,
+    system: str,
+    question: str,
+    temperature: float,
+    run_cost: RunCost,
+    use_cache: bool = True,
+) -> float:
+    """One OpenRouter call returning a posterior in [0.05, 0.95].
+
+    Caches by (model, temperature, system+question) hash. Updates the
+    given RunCost in place (token + USD totals; n_calls; elapsed_s).
+    On any error returns 0.5 (uncertainty default) and records 0 cost.
+    """
+    prompt = f"{system}\n\nQuestion: {question}"
+    key = _cache_key(model, prompt, temperature)
+    if use_cache:
+        cached = _cache_get(key)
+        if cached is not None:
+            run_cost.n_calls += 1  # logical call (cached)
+            return float(cached.get("confidence", 0.5))
+
+    started = time.time()
+    try:
+        resp = await client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": system},
+                {"role": "user", "content": question},
+            ],
+            response_format={"type": "json_object"},
+            temperature=temperature,
+            timeout=30,
+        )
+    except Exception:
+        # Silent fallback to uncertainty default. The Limitations panel
+        # will already note synthetic-fallback when this happens broadly.
+        return 0.5
+    elapsed = time.time() - started
+
+    content = (resp.choices[0].message.content or "{}") if resp.choices else "{}"
+    try:
+        data = json.loads(content)
+        conf = float(data.get("confidence", 0.5))
+    except Exception:
+        conf = 0.5
+    conf = max(0.05, min(0.95, conf))
+
+    # Update cost tally.
+    usage = getattr(resp, "usage", None)
+    p_t = int(getattr(usage, "prompt_tokens", 0) or 0) if usage else 0
+    c_t = int(getattr(usage, "completion_tokens", 0) or 0) if usage else 0
+    run_cost.n_calls += 1
+    run_cost.total_tokens_prompt += p_t
+    run_cost.total_tokens_completion += c_t
+    run_cost.estimated_usd += _estimate_cost_usd(model, p_t, c_t)
+    run_cost.elapsed_s += elapsed
+
+    if use_cache:
+        _cache_set(key, {"confidence": conf})
+    return conf
+
+
+# --- Real-LLM forecasters -------------------------------------------------
+
+
+async def real_hypermind_posterior(
+    client: Any,
+    model: str,
+    question: str,
+    run_cost: RunCost,
+) -> tuple[float, dict[str, float]]:
+    """Real HyperMind: 4 parallel role calls, anti-herding-aggregated.
+
+    Mirrors the synthetic version's logic: get 4 role posteriors, take
+    the mean, apply a small calibration pull toward 0.5 when the panel
+    disagrees on hard items. Returns (consensus, {role: posterior}).
+    """
+    roles = ["scout", "sceptic", "synthesist", "mediator"]
+    coros = [
+        _llm_call(
+            client, model, _SYSTEM_PROMPTS_BY_ROLE[r], question, temperature=0.4, run_cost=run_cost
+        )
+        for r in roles
+    ]
+    posts = await asyncio.gather(*coros)
+    role_posts = {r: p for r, p in zip(roles, posts, strict=True)}
+    mean = sum(posts) / len(posts)
+    var = sum((p - mean) ** 2 for p in posts) / len(posts)
+    stdev = var**0.5
+    distance = abs(mean - 0.5)
+    if stdev > 0.06 and distance < 0.20:
+        # Hard-question + high disagreement → calibrate toward 0.5.
+        mean = mean * 0.7 + 0.5 * 0.3
+    return max(0.05, min(0.95, mean)), role_posts
+
+
+async def real_single_llm_posterior(
+    client: Any,
+    model: str,
+    question: str,
+    run_cost: RunCost,
+) -> float:
+    """Real Single-LLM: one neutral call, no role bias, no calibration."""
+    return await _llm_call(
+        client,
+        model,
+        _SYSTEM_PROMPT_NEUTRAL,
+        question,
+        temperature=0.4,
+        run_cost=run_cost,
+    )
+
+
+async def real_naive_ensemble_posterior(
+    client: Any,
+    model: str,
+    question: str,
+    run_cost: RunCost,
+) -> float:
+    """Real Naive ensemble: 4 calls at temperatures 0.3 / 0.5 / 0.7 / 0.9, averaged."""
+    temps = [0.3, 0.5, 0.7, 0.9]
+    coros = [_llm_call(client, model, _SYSTEM_PROMPT_NEUTRAL, question, t, run_cost) for t in temps]
+    posts = await asyncio.gather(*coros)
+    return sum(posts) / len(posts)
+
+
+# --- Public entry point ---------------------------------------------------
+
+
+def is_real_llm_available() -> bool:
+    """True iff OPENROUTER_API_KEY is set AND the openai client imports."""
+    if not os.environ.get("OPENROUTER_API_KEY", "").strip():
+        return False
+    try:
+        from openai import AsyncOpenAI  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
+def make_real_client() -> Any | None:
+    """Build an OpenRouter-pointed AsyncOpenAI client. Returns None on failure.
+
+    Reuses the helper from :mod:`hypermind.simlab._scenario_impl` so we
+    don't duplicate the headers + base URL plumbing.
+    """
+    api_key = os.environ.get("OPENROUTER_API_KEY", "").strip()
+    if not api_key:
+        return None
+    try:
+        from .._scenario_impl import _make_openrouter_client
+    except ImportError:
+        return None
+    try:
+        return _make_openrouter_client(api_key)
+    except Exception:
+        return None
+
+
+async def run_real_llm_forecasters(
+    questions: list[tuple[str, int]],
+    model: str,
+) -> dict[str, Any] | None:
+    """Run all 3 model-based forecasters against `questions` via OpenRouter.
+
+    Returns ``None`` when real-LLM mode isn't available (caller should
+    fall back to synthetic). Otherwise returns a dict with per-forecaster
+    posteriors + per-forecaster RunCost.
+
+    Schema::
+
+        {
+            "hypermind_pred":     [float, ...],
+            "hypermind_role_posts": [{role: prob}, ...],
+            "single_llm_pred":    [float, ...],
+            "naive_ensemble_pred":[float, ...],
+            "costs": {
+                "hypermind":     RunCost(...),
+                "single_llm":    RunCost(...),
+                "naive_ensemble": RunCost(...),
+            },
+        }
+    """
+    if not is_real_llm_available():
+        return None
+    client = make_real_client()
+    if client is None:
+        return None
+
+    hm_pred: list[float] = []
+    hm_roles: list[dict[str, float]] = []
+    sl_pred: list[float] = []
+    ne_pred: list[float] = []
+    cost_hm = RunCost()
+    cost_sl = RunCost()
+    cost_ne = RunCost()
+
+    # Sequential per-question (parallel within HyperMind / Naive ensemble).
+    for q, _a in questions:
+        hp, roles = await real_hypermind_posterior(client, model, q, cost_hm)
+        hm_pred.append(hp)
+        hm_roles.append(roles)
+        sl_pred.append(await real_single_llm_posterior(client, model, q, cost_sl))
+        ne_pred.append(await real_naive_ensemble_posterior(client, model, q, cost_ne))
+
+    return {
+        "hypermind_pred": hm_pred,
+        "hypermind_role_posts": hm_roles,
+        "single_llm_pred": sl_pred,
+        "naive_ensemble_pred": ne_pred,
+        "costs": {
+            "hypermind": asdict(cost_hm),
+            "single_llm": asdict(cost_sl),
+            "naive_ensemble": asdict(cost_ne),
+        },
+    }
+
+
+__all__ = [
+    "RunCost",
+    "is_real_llm_available",
+    "make_real_client",
+    "run_real_llm_forecasters",
+]