hypermind 0.12.0__tar.gz → 0.13.0__tar.gz

{hypermind-0.12.0 → hypermind-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypermind
-Version: 0.12.0
+Version: 0.13.0
 Summary: Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile
 Project-URL: Homepage, https://github.com/ZySec-AI/hypermind
 Project-URL: Documentation, https://github.com/ZySec-AI/hypermind/tree/main/docs

{hypermind-0.12.0 → hypermind-0.13.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hypermind"
-version = "0.12.0"
+version = "0.13.0"
 description = "Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile"
 readme = "README.md"
 requires-python = ">=3.11,<3.14"

{hypermind-0.12.0 → hypermind-0.13.0}/src/hypermind/__init__.py

@@ -73,7 +73,7 @@ open = HyperMindAgent.open
 #: is held until the §External & human gates clear (IETF I-D, IANA,
 #: third-party audit, OpenSSF acceptance, ≥3 pilots) — at that point we
 #: cut ``1.0.0`` GA. See ROADMAP.md §Status snapshot.
-__version__ = "0.12.0"
+__version__ = "0.13.0"
 
 #: SLSA build-provenance level committed for every release artefact.
 #: Cosign keyless signing + slsa-github-generator + PEP 740 attestations

{hypermind-0.12.0 → hypermind-0.13.0}/src/hypermind/orgs/__init__.py

@@ -24,6 +24,12 @@ worked examples of the mapping.
 
 from __future__ import annotations
 
+from hypermind.orgs.groundtruth import (
+    CallableGroundTruth,
+    GroundTruth,
+    LLMJudgeGroundTruth,
+    StaticGroundTruth,
+)
 from hypermind.orgs.interaction import (
     Adversarial,
     Cooperative,
@@ -31,6 +37,7 @@ from hypermind.orgs.interaction import (
     Interaction,
     get_interaction,
 )
+from hypermind.orgs.metrics import ScoreCard, compute_scorecard
 from hypermind.orgs.model import (
     DynamicsConfig,
     OrgModel,
@@ -51,18 +58,24 @@ from hypermind.orgs.tools import (
 
 __all__ = [
     "Adversarial",
+    "CallableGroundTruth",
     "Cooperative",
     "Crowd",
     "DynamicsConfig",
+    "GroundTruth",
     "Interaction",
+    "LLMJudgeGroundTruth",
     "MemberStanding",
     "OrgModel",
     "OrgReport",
     "OrgSim",
+    "ScoreCard",
     "Seat",
     "Slot",
+    "StaticGroundTruth",
     "Unit",
     "build_tool",
+    "compute_scorecard",
     "declarative_grants",
     "get_interaction",
     "list_tools",

hypermind-0.13.0/src/hypermind/orgs/groundtruth.py

@@ -0,0 +1,126 @@
+"""Pluggable ground-truth oracles (#157).
+
+Reputation should reflect **real accuracy**, not a `confidence >= 0.5`
+heuristic. A ``GroundTruth`` resolves a published claim to a boolean outcome
+(or ``None`` = unknown → fall back to the interaction default). Feeding real
+outcomes into `oracle_resolve` makes reputation and the scorecard's Brier
+score meaningful, and enables backtesting ("did the crowd actually predict
+right?").
+
+Built-ins:
+  * ``StaticGroundTruth`` — known outcomes keyed by member label or topic.
+  * ``CallableGroundTruth`` — wrap any sync/async ``(topic, claim) -> bool|None``.
+  * ``LLMJudgeGroundTruth`` — an independent LLM judges each claim vs a rubric.
+
+All are injectable; the LLM judge takes a client (fake in tests).
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+from collections.abc import Awaitable, Callable
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class GroundTruth(Protocol):
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        """Return True/False for a resolved claim, or None if unknown."""
+        ...
+
+
+class StaticGroundTruth:
+    """Fixed answers keyed by member label first, then topic.
+
+    ``answers`` maps a claim's ``by`` label (or the topic string) to a bool.
+    """
+
+    def __init__(self, answers: dict[str, bool]) -> None:
+        self._answers = dict(answers)
+
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        by = claim.get("by")
+        if by in self._answers:
+            return self._answers[by]
+        return self._answers.get(topic)
+
+
+class CallableGroundTruth:
+    """Wrap any ``(topic, claim) -> bool | None`` (sync or async)."""
+
+    def __init__(
+        self,
+        fn: Callable[[str, dict[str, Any]], bool | None | Awaitable[bool | None]],
+    ) -> None:
+        self._fn = fn
+
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        out = self._fn(topic, claim)
+        if inspect.isawaitable(out):
+            return await out
+        return out
+
+
+_JUDGE_SYSTEM = (
+    "You are an impartial evaluator. Given a question and a claim, decide "
+    "whether the claim is correct. Respond with a JSON object "
+    '{"correct": true|false} and nothing else.'
+)
+
+
+class LLMJudgeGroundTruth:
+    """An independent LLM scores each claim against an optional rubric.
+
+    ``client`` is an OpenAI-compatible async client; injectable so tests pass a
+    fake. Returns None on any parse/transport failure (→ interaction default).
+    """
+
+    def __init__(
+        self,
+        *,
+        client: Any,
+        model: str = "gpt-4o-mini",
+        rubric: str = "",
+        timeout_s: float = 30.0,
+    ) -> None:
+        self._client = client
+        self._model = model
+        self._rubric = rubric
+        self._timeout = timeout_s
+
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        finding = claim.get("finding") or f"(confidence {claim.get('confidence')})"
+        rubric = f"\nRubric: {self._rubric}" if self._rubric else ""
+        user = (
+            f"Question: {topic}\nClaim: {finding}{rubric}\n"
+            'Is the claim correct? Reply as {"correct": true|false}.'
+        )
+        try:
+            resp = await self._client.chat.completions.create(
+                model=self._model,
+                messages=[
+                    {"role": "system", "content": _JUDGE_SYSTEM},
+                    {"role": "user", "content": user},
+                ],
+                response_format={"type": "json_object"},
+                timeout=self._timeout,
+            )
+            content = resp.choices[0].message.content or "{}"
+            verdict = json.loads(content).get("correct")
+            return bool(verdict) if verdict is not None else None
+        except Exception:
+            return None
+
+
+def as_ground_truth(
+    obj: GroundTruth | Callable[[str, dict[str, Any]], Any] | None,
+) -> GroundTruth | None:
+    """Normalise a GroundTruth, a plain callable, or None into a GroundTruth."""
+    if obj is None:
+        return None
+    if isinstance(obj, GroundTruth):
+        return obj
+    if callable(obj):
+        return CallableGroundTruth(obj)
+    raise TypeError(f"ground_truth must be a GroundTruth or callable, got {type(obj).__name__}")

hypermind-0.13.0/src/hypermind/orgs/metrics.py

@@ -0,0 +1,148 @@
+"""Quantitative scorecard for a simulation run (#151).
+
+Everything here is a pure function over data ``OrgSim`` already tracks — no LLM
+or Relata needed — so runs become **comparable across experiments** and the
+whole module is unit-testable with hand-built inputs.
+
+Metrics:
+  * **calibration (Brier)** — mean and per-member, over oracle-resolved claims.
+  * **consensus / convergence** — posterior variance per round, time-to-consensus.
+  * **polarization** — final belief spread and cross-unit gap (faction disagreement).
+  * **influence** — citation in-degree weighted by the *citer's* reputation.
+  * **activity** — claims / disputes / citations and participation rate.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+from statistics import mean, pvariance
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ScoreCard:
+    rounds: int
+    posterior_variance_by_round: list[float]
+    converged: bool
+    time_to_consensus: int | None  # 0-based round index, or None
+    final_spread: float  # max-min member posterior, last round
+    unit_polarization: float  # max-min of per-unit mean posterior, last round
+    brier_mean: float | None
+    brier_by_member: dict[str, float]
+    influence: dict[str, float]
+    top_influencer: str | None
+    claims: int
+    disputes: int
+    citations: int
+    participation_rate: float
+    _extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        d.pop("_extra", None)
+        return d
+
+    def summary_lines(self) -> list[str]:
+        """Compact human-readable lines for OrgReport.print()."""
+        cal = "n/a" if self.brier_mean is None else f"{self.brier_mean:.3f}"
+        conv = (
+            f"converged@round{self.time_to_consensus}"
+            if self.converged and self.time_to_consensus is not None
+            else ("polarized" if self.unit_polarization >= 0.25 else "no consensus")
+        )
+        return [
+            f"consensus: {conv} (final spread {self.final_spread:.3f}, "
+            f"unit polarization {self.unit_polarization:.3f})",
+            f"calibration (Brier, lower=better): {cal}",
+            f"top influencer: {self.top_influencer or '—'}"
+            + (f" ({self.influence[self.top_influencer]:.2f})" if self.top_influencer else ""),
+            f"activity: {self.claims} claims · {self.citations} citations · "
+            f"{self.disputes} disputes · participation {self.participation_rate:.0%}",
+        ]
+
+
+def _brier(confidence: float, outcome: bool) -> float:
+    return (float(confidence) - (1.0 if outcome else 0.0)) ** 2
+
+
+def compute_scorecard(
+    *,
+    per_round_posteriors: list[dict[str, float]],
+    member_units: dict[str, str],
+    claims: list[dict[str, Any]],
+    citations: list[tuple[str, str]],
+    reputations: dict[str, float],
+    disputes_count: int,
+    convergence_threshold: float = 0.02,
+) -> ScoreCard:
+    """Compute the scorecard from a run's tracked data.
+
+    ``per_round_posteriors`` — one ``{member_label: posterior_mean}`` dict per round.
+    ``member_units`` — every member's unit (denominator for participation).
+    ``claims`` — dicts with keys ``kid_hex, label, unit, confidence, outcome``.
+    ``citations`` — ``(citer_label, target_kid_hex)`` pairs.
+    ``reputations`` — final ``{member_label: score}``.
+    """
+    # --- consensus / convergence ---
+    var_by_round = [
+        pvariance(list(p.values())) if len(p) > 1 else 0.0 for p in per_round_posteriors
+    ]
+    time_to_consensus = next(
+        (i for i, v in enumerate(var_by_round) if v <= convergence_threshold), None
+    )
+    converged = bool(var_by_round) and var_by_round[-1] <= convergence_threshold
+
+    last = per_round_posteriors[-1] if per_round_posteriors else {}
+    final_spread = (max(last.values()) - min(last.values())) if len(last) > 1 else 0.0
+
+    unit_vals: dict[str, list[float]] = defaultdict(list)
+    for label, p in last.items():
+        unit_vals[member_units.get(label, "?")].append(p)
+    unit_means = [mean(v) for v in unit_vals.values()]
+    unit_polarization = (max(unit_means) - min(unit_means)) if len(unit_means) > 1 else 0.0
+
+    # --- calibration (Brier) over resolved claims ---
+    resolved = [
+        (c["label"], float(c.get("confidence", 0.0)), c["outcome"])
+        for c in claims
+        if c.get("outcome") is not None
+    ]
+    brier_all = [_brier(conf, out) for _, conf, out in resolved]
+    brier_mean = mean(brier_all) if brier_all else None
+    by_member: dict[str, list[float]] = defaultdict(list)
+    for label, conf, out in resolved:
+        by_member[label].append(_brier(conf, out))
+    brier_by_member = {k: mean(v) for k, v in by_member.items()}
+
+    # --- influence: reputation-weighted citation in-degree ---
+    kid2label = {c["kid_hex"]: c["label"] for c in claims if "kid_hex" in c}
+    influence: dict[str, float] = defaultdict(float)
+    for citer, target_hex in citations:
+        tgt = kid2label.get(target_hex)
+        if tgt is not None:
+            influence[tgt] += reputations.get(citer, 0.5)
+    influence = dict(influence)
+    top_influencer = max(influence, key=lambda k: influence[k]) if influence else None
+
+    # --- activity ---
+    publishers = {c["label"] for c in claims}
+    total_members = len(member_units)
+    participation = len(publishers) / total_members if total_members else 0.0
+
+    return ScoreCard(
+        rounds=len(per_round_posteriors),
+        posterior_variance_by_round=var_by_round,
+        converged=converged,
+        time_to_consensus=time_to_consensus,
+        final_spread=final_spread,
+        unit_polarization=unit_polarization,
+        brier_mean=brier_mean,
+        brier_by_member=brier_by_member,
+        influence=influence,
+        top_influencer=top_influencer,
+        claims=len(claims),
+        disputes=disputes_count,
+        citations=len(citations),
+        participation_rate=participation,
+    )

{hypermind-0.12.0 → hypermind-0.13.0}/src/hypermind/orgs/model.py

@@ -109,10 +109,19 @@ class DynamicsConfig:
     vouch_juniors: bool = True
     evolve: bool = True
     interaction: str = "cooperative"
+    # Convergence-driven stopping (used when run() is called without rounds=).
+    convergence_threshold: float = 0.02  # posterior variance <= this = consensus
+    convergence_patience: int = 2  # consecutive rounds required to declare a stop
+    polarization_threshold: float = 0.06  # stable variance >= this = entrenched split
+    max_rounds: int = 12  # hard cap for convergence-driven runs
 
     def __post_init__(self) -> None:
         if self.rounds < 1:
             raise ValueError(f"rounds must be >= 1, got {self.rounds}")
+        if self.max_rounds < 1:
+            raise ValueError(f"max_rounds must be >= 1, got {self.max_rounds}")
+        if self.convergence_patience < 1:
+            raise ValueError(f"convergence_patience must be >= 1, got {self.convergence_patience}")
         for name in ("publish_floor", "dispute_gap", "oracle_resolve_frac"):
             v = getattr(self, name)
             if not 0.0 <= v <= 1.0:

{hypermind-0.12.0 → hypermind-0.13.0}/src/hypermind/orgs/report.py

@@ -38,6 +38,8 @@ class OrgReport:
     citations: list[tuple[str, str]]
     disputes: list[dict[str, Any]]
     tool_grants: dict[str, list[str]]
+    scorecard: dict[str, Any] = field(default_factory=dict)
+    stop_reason: str = "fixed"  # fixed | converged | polarized | max_rounds
 
     # ---- derived views -------------------------------------------------
 
@@ -74,9 +76,15 @@ class OrgReport:
         s = self.summary()
         print(f"\n=== {self.org} — “{self.topic}” ({self.interaction}) ===")
         print(
-            f"{self.rounds} rounds · {s['members']} members · {s['claims']} claims · "
-            f"{s['citations']} citations · {s['disputes']} disputes"
+            f"{self.rounds} rounds ({self.stop_reason}) · {s['members']} members · "
+            f"{s['claims']} claims · {s['citations']} citations · {s['disputes']} disputes"
         )
+        if self.scorecard:
+            from hypermind.orgs.metrics import ScoreCard
+
+            print("\nScorecard:")
+            for line in ScoreCard(**self.scorecard).summary_lines():
+                print(f"  {line}")
         print("\nTop standing:")
         for st in self.top(10):
             tools = f"  tools={st.tools}" if st.tools else ""

{hypermind-0.12.0 → hypermind-0.13.0}/src/hypermind/orgs/sim.py

@@ -21,11 +21,14 @@ from __future__ import annotations
 
 from collections.abc import Callable
 from dataclasses import dataclass
+from statistics import pvariance
 from typing import Any
 
 from hypermind.agent import HyperMindAgent
 from hypermind.mind.memory_store import RelataMemoryStore
+from hypermind.orgs.groundtruth import GroundTruth, as_ground_truth
 from hypermind.orgs.interaction import ClaimRec, Interaction, get_interaction
+from hypermind.orgs.metrics import compute_scorecard
 from hypermind.orgs.model import OrgModel, Slot
 from hypermind.orgs.report import MemberStanding, OrgReport
 from hypermind.orgs.tools import (
@@ -99,7 +102,7 @@ class OrgSim:
         model_router: Any = None,
         tool_context: dict[str, Any] | None = None,
         extra_tool_grants: dict[str, frozenset[str]] | None = None,
-        ground_truth: Callable[[str, dict[str, Any]], bool] | None = None,
+        ground_truth: GroundTruth | Callable[[str, dict[str, Any]], Any] | None = None,
         relata_client_factory: Callable[[], tuple[Any, Any, Any]] | None = None,
         relata_memory_factory: Callable[[], Any] | None = None,
         members: int | None = None,
@@ -116,7 +119,7 @@ class OrgSim:
         self._model_router = model_router
         self._tool_context = dict(tool_context or {})
         self._extra_grants = extra_tool_grants or {}
-        self._ground_truth = ground_truth
+        self._ground_truth = as_ground_truth(ground_truth)
         self._relata_client_factory = relata_client_factory
         self._relata_memory_factory = relata_memory_factory
 
@@ -133,6 +136,8 @@ class OrgSim:
         self._citations: list[tuple[str, str]] = []
         self._disputes: list[dict[str, Any]] = []
         self._grants: dict[str, frozenset[str]] = {}
+        self._round_posteriors: list[dict[str, float]] = []  # per-round {label: mean}
+        self._stop_reason: str = "fixed"
 
     # ---- lifecycle -----------------------------------------------------
 
@@ -235,10 +240,47 @@ class OrgSim:
     # ---- the loop ------------------------------------------------------
 
     async def run(self, topic: str, *, rounds: int | None = None) -> OrgReport:
-        n_rounds = rounds if rounds is not None else self.model.dynamics.rounds
-        for r in range(n_rounds):
+        """Run the loop.
+
+        With an explicit ``rounds`` the count is fixed (backward compatible).
+        Otherwise the loop runs until beliefs **converge** (posterior variance
+        stays below ``convergence_threshold`` for ``convergence_patience``
+        rounds), lock into stable **polarization**, or hit ``max_rounds``.
+        """
+        dyn = self.model.dynamics
+        if rounds is not None:
+            for r in range(rounds):
+                await self._round(topic, r)
+            self._stop_reason = "fixed"
+            return self._build_report(topic, rounds)
+
+        cap = dyn.max_rounds
+        self._stop_reason = "max_rounds"
+        for r in range(cap):
             await self._round(topic, r)
-        return self._build_report(topic, n_rounds)
+            reason = self._convergence_reason()
+            if reason is not None:
+                self._stop_reason = reason
+                break
+        return self._build_report(topic, len(self._round_posteriors))
+
+    def _convergence_reason(self) -> str | None:
+        """Return 'converged' / 'polarized' once stable, else None."""
+        dyn = self.model.dynamics
+        variances = [
+            pvariance(list(p.values())) if len(p) > 1 else 0.0 for p in self._round_posteriors
+        ]
+        if len(variances) < dyn.convergence_patience:
+            return None
+        recent = variances[-dyn.convergence_patience :]
+        if all(v <= dyn.convergence_threshold for v in recent):
+            return "converged"
+        # High and *stable* variance for the patience window → entrenched split.
+        if all(v >= dyn.polarization_threshold for v in recent) and (
+            max(recent) - min(recent) <= dyn.convergence_threshold
+        ):
+            return "polarized"
+        return None
 
     async def _round(self, topic: str, r: int) -> None:
         dyn = self.model.dynamics
@@ -253,6 +295,7 @@ class OrgSim:
             if post.mean() >= floor:
                 kid = await self._publish(m, topic, r, post, reasoning)
                 published.append((m, kid, post))
+        self._round_posteriors.append(dict(posteriors))
 
         claim_recs = [
             ClaimRec(
@@ -392,11 +435,17 @@ class OrgSim:
             resolvable = [r for r in recs if r.member_kid != oracle.kid]
             n = max(1, round(len(recs) * frac))
             for rec in resolvable[:n]:
-                gt = (
-                    self._ground_truth(topic, {"confidence": rec.confidence})
-                    if self._ground_truth is not None
-                    else None
-                )
+                gt = None
+                if self._ground_truth is not None:
+                    info = self._claims.get(rec.kid.hex())
+                    claim_view = {
+                        "confidence": rec.confidence,
+                        "by": rec.member_label,
+                        "role": rec.role,
+                        "unit": rec.unit,
+                        "finding": info.finding if info else "",
+                    }
+                    gt = await self._ground_truth.resolve(topic, claim_view)
                 outcome = self.interaction.resolve_outcome(claim=rec, ground_truth=gt)
                 await oracle.agent.oracle_resolve(rec.kid, outcome=outcome, backprop=True)
                 info = self._claims.get(rec.kid.hex())
@@ -450,6 +499,14 @@ class OrgSim:
                 )
             )
         standings.sort(key=lambda s: s.reputation, reverse=True)
+        scorecard = compute_scorecard(
+            per_round_posteriors=self._round_posteriors,
+            member_units={m.label: m.unit for m in self._members},
+            claims=[c.__dict__ for c in self._claims.values()],
+            citations=list(self._citations),
+            reputations={s.label: s.reputation for s in standings},
+            disputes_count=len(self._disputes),
+        )
         return OrgReport(
             org=self.model.name,
             topic=topic,
@@ -460,4 +517,6 @@ class OrgSim:
             citations=list(self._citations),
             disputes=list(self._disputes),
             tool_grants={label: sorted(t) for label, t in self._grants.items()},
+            scorecard=scorecard.to_dict(),
+            stop_reason=self._stop_reason,
         )

hypermind-0.13.0/tests/orgs/_fakes.py

@@ -0,0 +1,93 @@
+"""Shared in-memory Relata fakes for orgs tests (no server, no LLM).
+
+Not a test module (no ``test_`` prefix) — imported by test files. pytest's
+prepend import mode puts this directory on sys.path, so ``from _fakes import ...``
+resolves without making the tests dir a package.
+"""
+
+from __future__ import annotations
+
+
+class FakeStore:
+    def __init__(self):
+        self.objects, self.log = {}, []
+
+
+class FakeResult:
+    def __init__(self, rows):
+        self.rows = rows
+
+
+class FakeObject:
+    def __init__(self, s):
+        self._s = s
+
+    def upsert(self, t, oid, f):
+        self._s.objects.setdefault(t, {})[oid] = {**f, "id": oid}
+        return {"object_id": oid}
+
+    def batch_upsert(self, t, rows):
+        for r in rows:
+            self._s.objects.setdefault(t, {})[r["id"]] = r
+        return {"accepted": len(rows)}
+
+    def close(self):
+        pass
+
+
+class FakeLog:
+    def __init__(self, s):
+        self._s = s
+
+    def append(self, d):
+        self._s.log.append(d)
+        return len(self._s.log), 0
+
+    def load_leaves(self, *, since=0, limit=1000):
+        return []
+
+    def head(self):
+        return len(self._s.log)
+
+    def close(self):
+        pass
+
+
+class FakeClient:
+    def __init__(self, s):
+        self._s = s
+
+    def query(self, sql, *, purpose=None):
+        objs = self._s.objects
+        if "ORDER BY seq DESC" in sql:
+            seqs = [f["seq"] for f in objs.get("HmClaim", {}).values()]
+            return FakeResult([{"seq": max(seqs)}] if seqs else [])
+        return FakeResult([])
+
+    def close(self):
+        pass
+
+
+class FakeMemory:
+    def __init__(self):
+        self._n = 0
+
+    def add(self, content, *, confidence=1.0, memory_class="semantic"):
+        self._n += 1
+        return f"m{self._n}"
+
+    def add_batch(self, items, *, session_id=None):
+        return [self.add(i) for i in items]
+
+    def search(self, q, *, top_k=5, as_of=None):
+        return []
+
+    def justify(self, mid):
+        return {"memory_id": mid}
+
+    def close(self):
+        pass
+
+
+def client_factory(store):
+    return lambda: (FakeClient(store), FakeLog(store), FakeObject(store))