hypermind 0.11.0__py3-none-any.whl

hypermind/simlab/runner.py

@@ -0,0 +1,239 @@
+"""Single-slot worker that runs persona-panel sims sequentially.
+
+Why single-slot? Concurrent runs would (a) collide on OpenRouter rate
+limits, (b) make the live UI confusing (whose claim is whose?), (c)
+fight over shared in-process state (the global recorder, FROST seen-set).
+
+Lifecycle:
+    submit(config) → returns sim_id
+        → registry: create dir, write config.json + status.json (queued)
+        → enqueue
+    worker loop:
+        → pop config
+        → status: running
+        → run_persona_sim(config, output_dir, recorder=Recorder.scoped(sim_id))
+        → status: done | failed (with stderr.log captured)
+        → loop
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import io
+import time
+import traceback
+from dataclasses import dataclass
+
+from hypermind.observability.recorder import Recorder
+
+from .config import SimConfig
+from .registry import (
+    append_stderr,
+    new_sim_dir,
+    sim_dir,
+    update_status,
+    write_config,
+    write_status,
+)
+from .scenario import run_persona_sim
+
+QUEUE_CAP = 10
+
+
+@dataclass
+class _QueuedJob:
+    sim_id: str
+    config: SimConfig
+
+
+class SimRunner:
+    """In-process single-slot sim runner. One per app instance."""
+
+    def __init__(self, *, queue_cap: int = QUEUE_CAP) -> None:
+        self._queue: asyncio.Queue[_QueuedJob] = asyncio.Queue(maxsize=queue_cap)
+        self._worker_task: asyncio.Task | None = None
+        self._current_sim_id: str | None = None
+        self._stopped = False
+        self._reaped = False  # orphan reaper runs once per process
+
+    def start(self) -> None:
+        if self._worker_task is None or self._worker_task.done():
+            self._worker_task = asyncio.create_task(self._worker_loop())
+
+    def reap_orphans_at_boot(self) -> None:
+        """Mark any sim still in 'running' or 'queued' state on disk
+        as failed. Call this exactly once, on process startup, before
+        any submit() — otherwise it would clobber freshly-queued sims.
+        """
+        if self._reaped:
+            return
+        self._reap_orphans()
+        self._reaped = True
+
+    def _reap_orphans(self) -> None:
+        """At startup, mark stale `running` / `queued` sims as failed."""
+        from .registry import append_stderr, sims_dir, update_status
+
+        for d in sims_dir().iterdir():
+            if not d.is_dir():
+                continue
+            status_path = d / "status.json"
+            if not status_path.exists():
+                continue
+            try:
+                import json as _json
+
+                status = _json.loads(status_path.read_text())
+            except (OSError, ValueError):
+                continue
+            state = status.get("state")
+            if state in ("running", "queued"):
+                update_status(
+                    d.name,
+                    state="failed",
+                    finished_at=time.time(),
+                    error="orphaned: simlab process restarted before this run finished",
+                )
+                append_stderr(d.name, "[reaper] marked as failed at simlab startup\n")
+
+    async def stop(self) -> None:
+        self._stopped = True
+        if self._worker_task and not self._worker_task.done():
+            self._worker_task.cancel()
+            try:
+                await self._worker_task
+            except asyncio.CancelledError:
+                pass
+
+    @property
+    def current_sim_id(self) -> str | None:
+        return self._current_sim_id
+
+    @property
+    def queue_size(self) -> int:
+        return self._queue.qsize()
+
+    async def submit(self, config: SimConfig) -> str:
+        """Allocate sim_id + persist config + enqueue. Returns sim_id."""
+        config.validate()
+        sim_id, _path = new_sim_dir()
+        write_config(sim_id, config.to_dict())
+        write_status(
+            sim_id,
+            {
+                "sim_id": sim_id,
+                "state": "queued",
+                "queued_at": time.time(),
+                "progress": {"q_done": 0, "q_total": 0, "phase": "queued"},
+            },
+        )
+        try:
+            self._queue.put_nowait(_QueuedJob(sim_id=sim_id, config=config))
+        except asyncio.QueueFull:
+            update_status(sim_id, state="failed", error="queue full", finished_at=time.time())
+            raise
+        # Lazy-start worker on first submission.
+        self.start()
+        return sim_id
+
+    async def _worker_loop(self) -> None:
+        # Reap orphans on the worker loop's first tick — runs in the
+        # asyncio event loop, after lifespan.startup has completed,
+        # so it can never block the boot path.
+        if not self._reaped:
+            try:
+                self._reap_orphans()
+            except Exception:
+                pass
+            self._reaped = True
+        while not self._stopped:
+            try:
+                job = await self._queue.get()
+            except asyncio.CancelledError:
+                return
+            await self._run_one(job)
+
+    async def _run_one(self, job: _QueuedJob) -> None:
+        self._current_sim_id = job.sim_id
+        output_dir = sim_dir(job.sim_id)
+        recorder = Recorder.scoped(job.sim_id)
+        update_status(
+            job.sim_id,
+            state="running",
+            started_at=time.time(),
+            progress={"q_done": 0, "q_total": 0, "phase": "running"},
+        )
+
+        # Capture stderr to stderr.log so failures are diagnosable from the UI.
+        stderr_buf = io.StringIO()
+        try:
+            with contextlib.redirect_stderr(stderr_buf):
+                result = await run_persona_sim(
+                    job.config,
+                    output_dir,
+                    sim_id=job.sim_id,
+                    recorder=recorder,
+                    status_cb=lambda upd: update_status(job.sim_id, progress=upd),
+                )
+            update_status(
+                job.sim_id,
+                state="done",
+                finished_at=time.time(),
+                n_agents=result.n_agents,
+                n_questions=result.n_questions,
+                ciq_score=result.ciq_score,
+                eig_mean=result.eig_mean,
+                progress={
+                    "q_done": result.n_questions,
+                    "q_total": result.n_questions,
+                    "phase": "done",
+                },
+            )
+            # Invalidate the knowledge index so the next /v1/knowledge call
+            # rebuilds with this run's claims/citations/disputes folded in.
+            try:
+                from .knowledge import invalidate as _invalidate_knowledge
+
+                _invalidate_knowledge()
+            except Exception:
+                pass
+        except asyncio.CancelledError:
+            update_status(
+                job.sim_id,
+                state="failed",
+                finished_at=time.time(),
+                error="CancelledError: sim worker cancelled",
+            )
+            raise
+        except Exception as exc:
+            update_status(
+                job.sim_id,
+                state="failed",
+                finished_at=time.time(),
+                error=f"{type(exc).__name__}: {exc}",
+            )
+            append_stderr(job.sim_id, traceback.format_exc())
+        finally:
+            captured = stderr_buf.getvalue()
+            if captured:
+                append_stderr(job.sim_id, captured)
+            self._current_sim_id = None
+            self._queue.task_done()
+
+
+# Process-singleton runner (one per ASGI app instance).
+_RUNNER: SimRunner | None = None
+
+
+def get_runner() -> SimRunner:
+    global _RUNNER
+    if _RUNNER is None:
+        _RUNNER = SimRunner()
+    return _RUNNER
+
+
+def reset_runner_for_tests() -> None:
+    """Recreate the singleton — used by tests that share an event loop."""
+    global _RUNNER
+    _RUNNER = None

hypermind/simlab/scenario.py

@@ -0,0 +1,147 @@
+"""Public entry point for the persona-panel simulator.
+
+`run_persona_sim(config, output_dir, recorder=None, status_cb=None)` is the
+single async function callable by:
+- the FastAPI/ASGI runner in `src/hypermind/simlab/runner.py`
+- tests in `tests/simlab/`
+- the legacy shim at `examples/scenario_persona_panel.py`
+
+It writes:
+  output_dir/trace.json     full Trace dataclass-as-dict
+  output_dir/persona_audit.jsonl  audit log captured for replay determinism
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+from hypermind.observability.recorder import Recorder
+
+from .config import SimConfig
+from .modes import get_mode
+
+
+@dataclass
+class SimResult:
+    sim_id: str
+    output_dir: Path
+    trace_path: Path
+    audit_path: Path
+    n_agents: int
+    n_questions: int
+    elapsed_s: float
+    ciq_score: float | None
+    eig_mean: float | None = None
+
+
+async def run_persona_sim(
+    config: SimConfig,
+    output_dir: Path,
+    *,
+    sim_id: str,
+    recorder: Recorder | None = None,
+    status_cb: Callable[[dict[str, Any]], None] | None = None,
+) -> SimResult:
+    """Run a persona-panel simulation, writing outputs into `output_dir`.
+
+    Parameters
+    ----------
+    config
+        Validated SimConfig.
+    output_dir
+        Existing directory where trace.json, persona_audit.jsonl, and any
+        other artifacts are written.
+    sim_id
+        Stable identifier (used for log fields; recorder scoping is the
+        runner's job).
+    recorder
+        Per-sim scoped recorder. If None, the global recorder is used.
+        Today the global recorder receives all events via Recorder._parent
+        fan-out; threading the recorder through the agent layer directly
+        is a future refinement.
+    status_cb
+        Optional callback invoked with dict updates during the run, e.g.
+        ``{"phase": "running", "q_done": 3, "q_total": 10}``. The runner
+        uses this to write status.json without coupling scenario.py to
+        the registry layer.
+    """
+    config.validate()
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    trace_path = output_dir / "trace.json"
+    audit_path = output_dir / "persona_audit.jsonl"
+    html_path = output_dir / "viewer.html"  # legacy compatibility
+
+    # Build a SimpleNamespace proxy so we can call the existing main()
+    # without copying its 200-line body. Field names mirror the legacy
+    # argparse Namespace exactly (--enable-tools → enable_tools, etc.).
+    args = SimpleNamespace(
+        personas=config.personas,
+        replicas=config.replicas,
+        questions=config.questions,
+        topic=config.topic,
+        n_questions=config.n_questions,
+        trace=str(trace_path),
+        html=str(html_path),
+        debug=config.debug,
+        enable_tools=config.enable_tools,
+        llm_model=config.model,
+        deliberation_rounds=config.deliberation_rounds,
+        no_llm=config.no_llm,
+        # Swarm composition — see SimConfig docstring
+        swarm=config.swarm,
+        personas_override=config.personas_override,
+        # Pass sim_id through so _scenario_impl can publish persona
+        # metadata to the SSE-enrichment registry under this key.
+        sim_id=sim_id,
+        # Namespace slug — determines which HyperMind namespace the agents
+        # publish claims into. Defaults to "default" for new sims.
+        namespace=config.namespace,
+        # Mode + mode_payload — surfaced on args so mode run_fns that
+        # accept the legacy SimpleNamespace shape can read both. The
+        # binary-forecast mode ignores these.
+        mode=config.mode,
+        mode_payload=config.mode_payload or {},
+        # Pre-generated questions — skip the in-worker LLM question-gen call.
+        _pregenerated_questions=config._pregenerated_questions,
+    )
+
+    if status_cb:
+        status_cb({"phase": "running", "q_done": 0})
+
+    # Dispatch to the registered mode. ``binary-forecast`` (default)
+    # delegates to the legacy ``_scenario_impl.main``; other modes own
+    # their own writers but still drop a ``trace.json`` at args.trace.
+    mode_spec = get_mode(config.mode)
+    await mode_spec.run_fn(args, recorder=recorder)
+
+    # Read back the trace to surface KPIs to the runner.
+    summary = json.loads(trace_path.read_text())
+    # Stamp the trace with the mode that produced it so the SPA can
+    # branch the result view. Modes that already write ``mode`` (e.g. a
+    # future governance mode that emits its own trace shape) are left
+    # untouched.
+    if "mode" not in summary:
+        summary["mode"] = config.mode
+        trace_path.write_text(json.dumps(summary, indent=2, default=str))
+    ciq = (summary.get("collective_iq") or {}).get("score")
+    eig_mean = (summary.get("collective_iq") or {}).get("eig_mean")
+    return SimResult(
+        sim_id=sim_id,
+        output_dir=output_dir,
+        trace_path=trace_path,
+        audit_path=audit_path,
+        n_agents=summary.get("n_agents", config.n_agents),
+        n_questions=summary.get("n_questions", 0),
+        elapsed_s=summary.get("elapsed_s", 0.0),
+        ciq_score=ciq,
+        eig_mean=eig_mean,
+    )
+
+
+__all__ = ["SimConfig", "SimResult", "run_persona_sim"]

hypermind/simlab/swarms.py

@@ -0,0 +1,180 @@
+"""Swarm group library — saved recipes that compose personas into a panel.
+
+A Swarm is a deployable unit. It references personas by slug (built-in
+or user-custom from ``personas.py``), assigns per-member replica counts,
+selects which tools the swarm has access to, and optionally appends a
+shared bias prompt to every member's system message.
+
+Disk: ``~/.hypermind/swarms/{slug}.json``.
+
+Public API:
+    Swarm, SwarmMember — dataclasses
+    load_swarms()      — list[Swarm], sorted by created_at desc
+    get_swarm(slug)
+    save_swarm(s)      — atomic write; validates persona refs exist
+    delete_swarm(slug)
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .personas import get_persona
+from .registry import _write_json_atomic, home
+
+VALID_TOOLS = frozenset({"web_search", "retrieve_document", "query_knowledge_base"})
+
+
+@dataclass
+class SwarmMember:
+    persona_slug: str
+    replicas: int = 1
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> SwarmMember:
+        return cls(
+            persona_slug=str(d.get("persona_slug", "")),
+            replicas=int(d.get("replicas", 1)),
+        )
+
+
+@dataclass
+class Swarm:
+    slug: str
+    name: str
+    description: str = ""
+    members: list[SwarmMember] = field(default_factory=list)
+    tools: list[str] = field(default_factory=list)
+    bias_prompt: str = ""
+    created_at_ms: int | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "slug": self.slug,
+            "name": self.name,
+            "description": self.description,
+            "members": [m.to_dict() for m in self.members],
+            "tools": list(self.tools),
+            "bias_prompt": self.bias_prompt,
+            "created_at_ms": self.created_at_ms,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> Swarm:
+        return cls(
+            slug=str(d.get("slug", "")),
+            name=str(d.get("name", "")),
+            description=str(d.get("description", "")),
+            members=[SwarmMember.from_dict(m) for m in d.get("members", [])],
+            tools=list(d.get("tools", []) or []),
+            bias_prompt=str(d.get("bias_prompt", "")),
+            created_at_ms=d.get("created_at_ms"),
+        )
+
+    @property
+    def total_agents(self) -> int:
+        return sum(max(1, m.replicas) for m in self.members)
+
+    def validate(self, *, check_persona_refs: bool = True) -> None:
+        if not self.slug or not all(c.isalnum() or c in "-_" for c in self.slug):
+            raise ValueError(
+                f"slug must be alphanumeric/dash/underscore: {self.slug!r}",
+            )
+        if not self.name.strip():
+            raise ValueError("name is required")
+        if not self.members:
+            raise ValueError("swarm must have at least one member")
+        if self.total_agents > 60:
+            raise ValueError(
+                f"total agents must be ≤60, got {self.total_agents} "
+                "(reduce replicas or remove members)",
+            )
+        for m in self.members:
+            if m.replicas < 1 or m.replicas > 10:
+                raise ValueError(
+                    f"replicas must be 1..10, got {m.replicas} for persona {m.persona_slug!r}",
+                )
+            if check_persona_refs and get_persona(m.persona_slug) is None:
+                raise ValueError(
+                    f"unknown persona slug {m.persona_slug!r} in swarm {self.slug!r}",
+                )
+        try:
+            from .tool_registry import list_catalog as _list_catalog
+
+            catalog_slugs = {td.slug for td in _list_catalog()}
+        except Exception:
+            catalog_slugs = set()
+        valid_tools = catalog_slugs | VALID_TOOLS
+        for t in self.tools:
+            if t not in valid_tools:
+                raise ValueError(
+                    f"tool {t!r} not in catalog; valid options: {sorted(valid_tools)}",
+                )
+
+
+# ---------------------------------------------------------------------------
+# Disk layout: ~/.hypermind/swarms/{slug}.json
+# ---------------------------------------------------------------------------
+
+
+def swarms_dir() -> Path:
+    d = home() / "swarms"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def load_swarms() -> list[Swarm]:
+    """All saved swarms, newest-first."""
+    out: list[Swarm] = []
+    for path in swarms_dir().glob("*.json"):
+        try:
+            data = json.loads(path.read_text())
+            s = Swarm.from_dict(data)
+        except (OSError, json.JSONDecodeError, TypeError, ValueError):
+            continue
+        out.append(s)
+    out.sort(key=lambda s: -(s.created_at_ms or 0))
+    return out
+
+
+def get_swarm(slug: str) -> Swarm | None:
+    path = swarms_dir() / f"{slug}.json"
+    if not path.exists():
+        return None
+    try:
+        return Swarm.from_dict(json.loads(path.read_text()))
+    except (OSError, json.JSONDecodeError, TypeError, ValueError):
+        return None
+
+
+def save_swarm(s: Swarm) -> None:
+    """Persist a swarm. Validates persona references exist on save."""
+    if s.created_at_ms is None:
+        s.created_at_ms = int(time.time() * 1000)
+    s.validate()
+    _write_json_atomic(swarms_dir() / f"{s.slug}.json", s.to_dict())
+
+
+def delete_swarm(slug: str) -> None:
+    path = swarms_dir() / f"{slug}.json"
+    if path.exists():
+        path.unlink()
+
+
+__all__ = [
+    "VALID_TOOLS",
+    "Swarm",
+    "SwarmMember",
+    "delete_swarm",
+    "get_swarm",
+    "load_swarms",
+    "save_swarm",
+    "swarms_dir",
+]

hypermind/simlab/tool_handlers/__init__.py

@@ -0,0 +1,4 @@
+# Tool handler package — one module per catalog entry.
+# Each module exports:
+#   async def handle(args: dict, credentials: dict) -> str
+#   async def test_connection(credentials: dict) -> tuple[bool, str]

hypermind/simlab/tool_handlers/alpha_vantage.py

@@ -0,0 +1,39 @@
+"""Alpha Vantage financial data handler."""
+
+from __future__ import annotations
+
+import json
+import urllib.parse
+import urllib.request
+from typing import Any
+
+
+async def handle(args: dict[str, Any], credentials: dict[str, str]) -> str:
+    api_key = credentials.get("api_key", "")
+    if not api_key:
+        return json.dumps({"error": "alpha_vantage: no api_key configured"})
+    symbol = args.get("symbol", "").upper()
+    function = args.get("function", "GLOBAL_QUOTE")
+    qs = urllib.parse.urlencode({"function": function, "symbol": symbol, "apikey": api_key})
+    try:
+        with urllib.request.urlopen(f"https://www.alphavantage.co/query?{qs}", timeout=10) as resp:
+            data = json.loads(resp.read())
+        if "Note" in data:
+            return json.dumps({"error": "Rate limit reached", "symbol": symbol})
+        if "Error Message" in data:
+            return json.dumps({"error": data["Error Message"], "symbol": symbol})
+        return json.dumps({"symbol": symbol, "function": function, "data": data})
+    except Exception as e:
+        return json.dumps({"error": str(e), "symbol": symbol})
+
+
+async def test_connection(credentials: dict[str, str]) -> tuple[bool, str]:
+    try:
+        result = json.loads(
+            await handle({"symbol": "IBM", "function": "GLOBAL_QUOTE"}, credentials)
+        )
+        if "error" in result:
+            return False, result["error"]
+        return True, "Connected — Alpha Vantage responding"
+    except Exception as e:
+        return False, str(e)

hypermind/simlab/tool_handlers/brave.py

@@ -0,0 +1,46 @@
+"""Brave Web Search handler."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+async def handle(args: dict[str, Any], credentials: dict[str, str]) -> str:
+    import urllib.parse
+    import urllib.request
+
+    api_key = credentials.get("api_key", "")
+    if not api_key:
+        return json.dumps({"error": "brave_web_search: no api_key configured"})
+    query = args.get("query", "")
+    max_results = int(args.get("max_results", 5))
+    qs = urllib.parse.urlencode({"q": query, "count": max_results})
+    req = urllib.request.Request(
+        f"https://api.search.brave.com/res/v1/web/search?{qs}",
+        headers={"Accept": "application/json", "X-Subscription-Token": api_key},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            data = json.loads(resp.read())
+        results = [
+            {
+                "title": w.get("title", ""),
+                "url": w.get("url", ""),
+                "description": w.get("description", ""),
+            }
+            for w in data.get("web", {}).get("results", [])[:max_results]
+        ]
+        return json.dumps({"query": query, "results": results})
+    except Exception as e:
+        return json.dumps({"error": str(e), "query": query, "results": []})
+
+
+async def test_connection(credentials: dict[str, str]) -> tuple[bool, str]:
+    try:
+        result = json.loads(await handle({"query": "test", "max_results": 1}, credentials))
+        if "error" in result:
+            return False, result["error"]
+        return True, f"Connected — {len(result.get('results', []))} result(s) returned"
+    except Exception as e:
+        return False, str(e)