hypermind 0.11.0__py3-none-any.whl

hypermind/simlab/learning.py

@@ -0,0 +1,295 @@
+"""Cross-sim learning metrics for the SimLab.
+
+Exposes three pure functions that power the "Learning" tab in SimDetail:
+
+    find_related_sims(topic)          — sims on a related topic, ranked by Brier
+    compute_knowledge_reuse(sim_id)   — how much prior knowledge this sim reused
+    get_persona_trajectory(persona)   — Brier trajectory across sims for one persona
+
+These are read-only; they never mutate any registry record.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from .registry import list_sims, load_status, load_trace, sim_dir
+
+# ---------------------------------------------------------------------------
+# Text-similarity helpers
+# ---------------------------------------------------------------------------
+
+_STOPWORDS = frozenset(
+    {
+        "a", "an", "the", "in", "on", "at", "to", "for", "of", "and", "or",
+        "but", "is", "are", "was", "were", "be", "been", "will", "would",
+        "could", "should", "may", "might", "do", "does", "did", "have",
+        "has", "had", "that", "this", "these", "those", "with", "by", "from",
+        "than", "more", "less", "very", "also", "as", "up", "about", "into",
+        "through", "between", "during", "can", "not", "if",
+    }
+)
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+
+def _tokenize(text: str) -> set[str]:
+    tokens = _TOKEN_RE.findall(text.lower())
+    return {t for t in tokens if t not in _STOPWORDS and len(t) > 1}
+
+
+def _jaccard(a: set[str], b: set[str]) -> float:
+    if not a and not b:
+        return 1.0
+    union = a | b
+    if not union:
+        return 0.0
+    return len(a & b) / len(union)
+
+
+def _extract_persona_slug(agent_name: str) -> str:
+    """Strip trailing replica suffix ('-01', '-02', ...) from an agent name."""
+    return re.sub(r"-\d+$", "", agent_name)
+
+
+# ---------------------------------------------------------------------------
+# find_related_sims
+# ---------------------------------------------------------------------------
+
+
+def find_related_sims(topic: str, *, limit: int = 5) -> list[dict]:
+    """Return up to *limit* completed sims on a related topic.
+
+    Exact-match sims are ranked before partial-match. Within each group
+    results are ordered by mean_brier ascending (best calibrated first).
+    Sims with no oracle resolutions are included but ranked last.
+    """
+    if not topic or not topic.strip():
+        return []
+
+    topic_norm = topic.strip()
+    topic_tokens = _tokenize(topic_norm)
+
+    results: list[dict] = []
+    for sim in list_sims(limit=2000):
+        if sim.state != "done":
+            continue
+        sim_topic = (sim.topic or "").strip()
+        if not sim_topic:
+            # Fall back to the topic stored in trace.json
+            trace = load_trace(sim.sim_id)
+            if trace:
+                sim_topic = (trace.get("topic") or "").strip()
+        if not sim_topic:
+            continue
+
+        # Jaccard similarity
+        sim_tokens = _tokenize(sim_topic)
+        j = _jaccard(topic_tokens, sim_tokens)
+        if j < 0.15:
+            continue
+
+        status = load_status(sim.sim_id)
+        mean_brier = status.get("mean_brier")  # set by _handle_resolve_outcomes
+        n_resolved = status.get("n_resolved", 0)
+
+        results.append(
+            {
+                "sim_id": sim.sim_id,
+                "topic": sim_topic,
+                "similarity": round(j, 4),
+                "exact_match": sim_topic.lower() == topic_norm.lower(),
+                "mean_brier": mean_brier,
+                "n_resolved": n_resolved,
+                "n_questions": sim.n_questions,
+                "n_agents": sim.n_agents,
+                "finished_at": sim.finished_at,
+                "model": sim.model,
+            }
+        )
+
+    # Sort: exact match first, then by similarity desc, then by brier asc
+    # (None brier → rank after real Brier scores)
+    def _sort_key(r: dict) -> tuple:
+        exact = 0 if r["exact_match"] else 1
+        brier = r["mean_brier"] if r["mean_brier"] is not None else 9999.0
+        return (exact, -r["similarity"], brier)
+
+    results.sort(key=_sort_key)
+    return results[:limit]
+
+
+# ---------------------------------------------------------------------------
+# compute_knowledge_reuse
+# ---------------------------------------------------------------------------
+
+_TOP_K = 5  # number of top-Brier personas selected when reuse kicks in
+
+
+def compute_knowledge_reuse(sim_id: str) -> dict | None:
+    """Compute cross-sim learning stats for a completed sim.
+
+    Returns None if the sim directory doesn't exist or is not done.
+    Returns a dict with:
+        prior_sims_used         int   — number of related prior sims found
+        claims_available_from_prior  int   — total claims in those sims
+        n_personas              int   — persona replicas in this sim
+        n_questions             int   — questions in this sim
+        delib_rounds            int   — deliberation rounds (>=1)
+        actual_llm_calls        int   — estimated actual LLM calls
+        optimized_llm_calls     int   — estimated calls if TOP_K selection used
+        calls_saved_potential   int   — difference
+        pct_reduction           float — fraction saved (0-1)
+        related_sims            list  — from find_related_sims (up to 5)
+    """
+    if not sim_dir(sim_id).exists():
+        return None
+    status = load_status(sim_id)
+    if status.get("state") != "done":
+        return None
+    trace = load_trace(sim_id)
+    if trace is None:
+        return None
+
+    topic = trace.get("topic", "")
+    n_agents: int = trace.get("n_agents") or 0
+    n_questions: int = trace.get("n_questions") or len(trace.get("panel_records") or [])
+    delib_rounds: int = trace.get("n_deliberation_rounds") or 1
+
+    # Actual LLM calls: each agent answers each question once per round
+    actual_llm_calls = n_agents * n_questions * max(1, delib_rounds)
+
+    # If we had selected only TOP_K agents based on prior Brier, how many calls?
+    top_k = min(_TOP_K, n_agents)
+    optimized_llm_calls = top_k * n_questions * max(1, delib_rounds)
+    calls_saved_potential = actual_llm_calls - optimized_llm_calls
+    pct_reduction = calls_saved_potential / actual_llm_calls if actual_llm_calls > 0 else 0.0
+
+    # Find related sims (exclude self)
+    related = [r for r in find_related_sims(topic, limit=6) if r["sim_id"] != sim_id][:5]
+    claims_available = 0
+    for rel in related:
+        rel_trace = load_trace(rel["sim_id"])
+        if rel_trace:
+            for pr in rel_trace.get("panel_records") or []:
+                claims_available += len(pr.get("claims") or [])
+
+    return {
+        "prior_sims_used": len(related),
+        "claims_available_from_prior": claims_available,
+        "n_personas": n_agents,
+        "n_questions": n_questions,
+        "delib_rounds": delib_rounds,
+        "actual_llm_calls": actual_llm_calls,
+        "optimized_llm_calls": optimized_llm_calls,
+        "calls_saved_potential": calls_saved_potential,
+        "pct_reduction": round(pct_reduction, 4),
+        "related_sims": related,
+    }
+
+
+# ---------------------------------------------------------------------------
+# get_persona_trajectory
+# ---------------------------------------------------------------------------
+
+
+def get_persona_trajectory(persona: str) -> dict:
+    """Return the Brier trajectory for a persona slug across all completed sims.
+
+    The trajectory is a list of per-sim observations, sorted by sim finish time.
+    Each observation includes the mean Brier for that persona across all agents
+    whose name matches the slug (after stripping the replica suffix).
+
+    Returns:
+        {
+            "persona": str,
+            "n_sims": int,
+            "mean_brier_all": float | None,   # lifetime mean across observations
+            "trajectory": [
+                {
+                    "sim_id": str,
+                    "topic": str,
+                    "sim_n": int,              # sequential run number (1-based)
+                    "mean_brier": float,
+                    "n_resolved": int,
+                    "finished_at": int | None,
+                }
+            ]
+        }
+    """
+    observations: list[dict] = []
+    for sim in list_sims(limit=2000):
+        if sim.state != "done":
+            continue
+        trace = load_trace(sim.sim_id)
+        if trace is None:
+            continue
+
+        # Look at agent_brier_scores stored on trace
+        abs_scores: dict[str, Any] = trace.get("agent_brier_scores") or {}
+        persona_briers: list[float] = []
+        persona_n: int = 0
+
+        for agent_name, scores in abs_scores.items():
+            slug = _extract_persona_slug(agent_name)
+            if slug != persona:
+                continue
+            mb = scores.get("mean_brier")
+            n = scores.get("n", 0)
+            if mb is not None and n > 0:
+                persona_briers.append(mb)
+                persona_n += n
+
+        # Also scan per_agent columns in panel_records if no abs_scores yet
+        if not persona_briers:
+            panel_records = trace.get("panel_records") or []
+            resolved_panels = [pr for pr in panel_records if pr.get("ground_truth") is not None]
+            for pr in resolved_panels:
+                gt = float(pr.get("ground_truth", 0))
+                for claim in pr.get("claims") or []:
+                    slug = _extract_persona_slug(claim.get("agent", ""))
+                    if slug != persona:
+                        continue
+                    conf = claim.get("confidence")
+                    if conf is not None:
+                        try:
+                            brier = (float(conf) - gt) ** 2
+                            persona_briers.append(brier)
+                            persona_n += 1
+                        except (TypeError, ValueError):
+                            pass
+
+        if not persona_briers:
+            continue
+
+        mean_brier = sum(persona_briers) / len(persona_briers)
+        observations.append(
+            {
+                "sim_id": sim.sim_id,
+                "topic": sim.topic or "",
+                "mean_brier": round(mean_brier, 6),
+                "n_resolved": persona_n,
+                "finished_at": sim.finished_at,
+            }
+        )
+
+    # Sort chronologically
+    observations.sort(key=lambda o: (o["finished_at"] or 0))
+
+    # Assign sequential run numbers
+    for i, obs in enumerate(observations, 1):
+        obs["sim_n"] = i
+
+    # Lifetime mean
+    if observations:
+        mean_brier_all = sum(o["mean_brier"] for o in observations) / len(observations)
+    else:
+        mean_brier_all = None
+
+    return {
+        "persona": persona,
+        "n_sims": len(observations),
+        "mean_brier_all": round(mean_brier_all, 6) if mean_brier_all is not None else None,
+        "trajectory": observations,
+    }

hypermind/simlab/modes/__init__.py

@@ -0,0 +1,115 @@
+"""Deliberation-mode registry for SimLab.
+
+A *mode* selects the deliberation shape and result schema for a sim run.
+The default mode is ``binary-forecast`` — the legacy persona-panel shape
+(consensus probability vs. ``ground_truth ∈ [0,1]``). Additional modes
+(``governance-committee``, ``what-if-tree``, ...) plug in here without
+touching the engine entry point in :mod:`hypermind.simlab.scenario`.
+
+Each mode is a :class:`ModeSpec` with:
+
+* ``slug``         — stable identifier used by ``SimConfig.mode``.
+* ``label``        — human-readable name for the SPA dropdown.
+* ``input_schema`` — JSON schema (draft-2020-12 subset) describing the
+  fields the user submits via ``SimConfig.mode_payload``. ``None`` means
+  the mode reuses the legacy SimConfig fields directly.
+* ``result_schema`` — JSON schema for the trace's mode-specific output.
+* ``run_fn``       — async callable ``(args, recorder) -> None`` that
+  matches the signature of the legacy ``_scenario_impl.main``. The mode
+  is responsible for writing ``trace.json`` to ``args.trace``.
+
+Public API:
+
+    register(spec)                — register a mode (idempotent on slug)
+    get_mode(slug) -> ModeSpec    — lookup; raises KeyError if unknown
+    list_modes() -> list[ModeSpec]
+    is_registered_mode(slug) -> bool
+
+The default ``binary-forecast`` mode is registered at import time.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+# Run signature: (args: SimpleNamespace-like, recorder: Recorder | None) -> None.
+# Kept loose so modes can accept additional kwargs without touching this type.
+ModeRunFn = Callable[..., Awaitable[None]]
+
+
+@dataclass(frozen=True)
+class ModeSpec:
+    slug: str
+    label: str
+    description: str
+    input_schema: dict[str, Any] | None
+    result_schema: dict[str, Any] | None
+    run_fn: ModeRunFn
+    # Whether this mode reuses the legacy SimConfig fields (personas,
+    # questions, n_questions, ...) for its inputs. ``False`` means the
+    # mode reads only ``mode_payload``. The schema-driven SimNew form
+    # uses this to decide whether to render legacy controls.
+    uses_legacy_fields: bool = True
+    # Optional list of role slugs the mode expects to be assignable.
+    expected_roles: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "slug": self.slug,
+            "label": self.label,
+            "description": self.description,
+            "input_schema": self.input_schema,
+            "result_schema": self.result_schema,
+            "uses_legacy_fields": self.uses_legacy_fields,
+            "expected_roles": list(self.expected_roles),
+        }
+
+
+_REGISTRY: dict[str, ModeSpec] = {}
+
+
+def register(spec: ModeSpec) -> None:
+    """Register a mode. Re-registering the same slug overwrites — useful
+    in tests but should not happen in production code paths."""
+    _REGISTRY[spec.slug] = spec
+
+
+def get_mode(slug: str) -> ModeSpec:
+    if slug not in _REGISTRY:
+        raise KeyError(f"unknown mode {slug!r}; registered: {sorted(_REGISTRY.keys())}")
+    return _REGISTRY[slug]
+
+
+def list_modes() -> list[ModeSpec]:
+    return sorted(_REGISTRY.values(), key=lambda s: s.slug)
+
+
+def is_registered_mode(slug: str) -> bool:
+    return slug in _REGISTRY
+
+
+# --- Default registrations -------------------------------------------------
+# Imported for side effect; each module calls ``register(...)`` at import.
+from . import (  # noqa: E402
+    aar,  # noqa: F401
+    backtest,  # noqa: F401
+    binary_forecast,  # noqa: F401
+    conformance,  # noqa: F401
+    evaluation,  # noqa: F401
+    governance,  # noqa: F401
+    redteam,  # noqa: F401
+    tabletop,  # noqa: F401
+    tournament,  # noqa: F401
+    whatif,  # noqa: F401
+)
+
+__all__ = [
+    "ModeRunFn",
+    "ModeSpec",
+    "get_mode",
+    "is_registered_mode",
+    "list_modes",
+    "register",
+]