hypermind 0.12.0__tar.gz → 0.14.0__tar.gz

{hypermind-0.12.0 → hypermind-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypermind
-Version: 0.12.0
+Version: 0.14.0
 Summary: Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile
 Project-URL: Homepage, https://github.com/ZySec-AI/hypermind
 Project-URL: Documentation, https://github.com/ZySec-AI/hypermind/tree/main/docs

{hypermind-0.12.0 → hypermind-0.14.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hypermind"
-version = "0.12.0"
+version = "0.14.0"
 description = "Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile"
 readme = "README.md"
 requires-python = ">=3.11,<3.14"

{hypermind-0.12.0 → hypermind-0.14.0}/src/hypermind/__init__.py

@@ -73,7 +73,7 @@ open = HyperMindAgent.open
 #: is held until the §External & human gates clear (IETF I-D, IANA,
 #: third-party audit, OpenSSF acceptance, ≥3 pilots) — at that point we
 #: cut ``1.0.0`` GA. See ROADMAP.md §Status snapshot.
-__version__ = "0.12.0"
+__version__ = "0.14.0"
 
 #: SLSA build-provenance level committed for every release artefact.
 #: Cosign keyless signing + slsa-github-generator + PEP 740 attestations

{hypermind-0.12.0 → hypermind-0.14.0}/src/hypermind/orgs/__init__.py

@@ -24,6 +24,18 @@ worked examples of the mapping.
 
 from __future__ import annotations
 
+from hypermind.orgs.experiment import (
+    Experiment,
+    ExperimentResult,
+    Variant,
+    VariantResult,
+)
+from hypermind.orgs.groundtruth import (
+    CallableGroundTruth,
+    GroundTruth,
+    LLMJudgeGroundTruth,
+    StaticGroundTruth,
+)
 from hypermind.orgs.interaction import (
     Adversarial,
     Cooperative,
@@ -31,6 +43,13 @@ from hypermind.orgs.interaction import (
     Interaction,
     get_interaction,
 )
+from hypermind.orgs.intervention import (
+    FlipPersona,
+    InjectEvent,
+    Intervention,
+    RemoveMember,
+)
+from hypermind.orgs.metrics import ScoreCard, compute_scorecard
 from hypermind.orgs.model import (
     DynamicsConfig,
     OrgModel,
@@ -51,18 +70,32 @@ from hypermind.orgs.tools import (
 
 __all__ = [
     "Adversarial",
+    "CallableGroundTruth",
     "Cooperative",
     "Crowd",
     "DynamicsConfig",
+    "Experiment",
+    "ExperimentResult",
+    "FlipPersona",
+    "GroundTruth",
+    "InjectEvent",
     "Interaction",
+    "Intervention",
+    "LLMJudgeGroundTruth",
     "MemberStanding",
     "OrgModel",
     "OrgReport",
     "OrgSim",
+    "RemoveMember",
+    "ScoreCard",
     "Seat",
     "Slot",
+    "StaticGroundTruth",
     "Unit",
+    "Variant",
+    "VariantResult",
     "build_tool",
+    "compute_scorecard",
     "declarative_grants",
     "get_interaction",
     "list_tools",

hypermind-0.14.0/src/hypermind/orgs/experiment.py

@@ -0,0 +1,159 @@
+"""Experiment harness (#158) — sweeps, replicas, A/B, run diffing.
+
+Researchers need reproducibility and comparison, not one-shot demos. An
+``Experiment`` runs a base :class:`OrgModel` under several named **variants**
+(dynamics overrides, member count, structural transforms, or sim-kwarg
+overrides), N replicas each, and aggregates the per-run scorecards into
+mean±stdev per metric — with a ``diff`` to see what moved between variants.
+
+Pure orchestration over ``OrgSim`` + the scorecard (#151); with a deterministic
+(fake) responder the whole thing is unit-testable with no live LLM/Relata.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from dataclasses import dataclass, field, replace
+from statistics import mean, pstdev
+from typing import Any
+
+from hypermind.orgs.model import OrgModel
+from hypermind.orgs.sim import OrgSim
+
+# Scalar scorecard fields we aggregate (bool coerced to 0/1; None skipped).
+_SCALAR_METRICS = (
+    "rounds",
+    "converged",
+    "time_to_consensus",
+    "final_spread",
+    "unit_polarization",
+    "brier_mean",
+    "claims",
+    "disputes",
+    "citations",
+    "participation_rate",
+)
+
+
+@dataclass(frozen=True)
+class Variant:
+    """A named experimental condition applied to the base model + sim kwargs."""
+
+    name: str
+    dynamics: dict[str, Any] | None = None  # DynamicsConfig field overrides
+    members: int | None = None
+    model_transform: Callable[[OrgModel], OrgModel] | None = None
+    sim_overrides: dict[str, Any] = field(default_factory=dict)
+
+    def build_model(self, base: OrgModel) -> OrgModel:
+        model = self.model_transform(base) if self.model_transform else base
+        if self.dynamics:
+            model = replace(model, dynamics=replace(model.dynamics, **self.dynamics))
+        return model
+
+
+@dataclass(frozen=True)
+class VariantResult:
+    name: str
+    replicas: int
+    metrics: dict[str, tuple[float, float]]  # metric -> (mean, stdev)
+    raw: list[dict[str, Any]]  # per-replica scorecards
+
+    def value(self, metric: str) -> float | None:
+        m = self.metrics.get(metric)
+        return m[0] if m else None
+
+
+@dataclass(frozen=True)
+class ExperimentResult:
+    topic: str
+    variants: dict[str, VariantResult]
+
+    def diff(self, a: str, b: str) -> dict[str, float]:
+        """Metric deltas (variant ``b`` minus variant ``a``) for shared metrics."""
+        va, vb = self.variants[a], self.variants[b]
+        out: dict[str, float] = {}
+        for metric, (mean_b, _) in vb.metrics.items():
+            if metric in va.metrics:
+                out[metric] = mean_b - va.metrics[metric][0]
+        return out
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "topic": self.topic,
+            "variants": {
+                name: {"replicas": v.replicas, "metrics": v.metrics}
+                for name, v in self.variants.items()
+            },
+        }
+
+
+def _aggregate(scorecards: list[dict[str, Any]]) -> dict[str, tuple[float, float]]:
+    out: dict[str, tuple[float, float]] = {}
+    for metric in _SCALAR_METRICS:
+        vals: list[float] = []
+        for sc in scorecards:
+            v = sc.get(metric)
+            if v is None:
+                continue
+            vals.append(1.0 if v is True else 0.0 if v is False else float(v))
+        if vals:
+            out[metric] = (mean(vals), pstdev(vals) if len(vals) > 1 else 0.0)
+    return out
+
+
+class Experiment:
+    """Run a base model under several variants and compare the scorecards."""
+
+    def __init__(
+        self,
+        base_model: OrgModel,
+        *,
+        relata_url: str,
+        max_concurrency: int = 4,
+        **sim_kwargs: Any,
+    ) -> None:
+        self.base_model = base_model
+        self.relata_url = relata_url
+        self._sim_kwargs = sim_kwargs
+        self._variants: list[Variant] = []
+        self._sem = asyncio.Semaphore(max(1, max_concurrency))
+
+    def variant(self, name: str, **overrides: Any) -> Experiment:
+        """Register a variant. ``overrides`` may include ``dynamics``,
+        ``members``, ``model_transform``, ``sim_overrides``."""
+        if any(v.name == name for v in self._variants):
+            raise ValueError(f"duplicate variant name {name!r}")
+        self._variants.append(Variant(name=name, **overrides))
+        return self
+
+    async def _run_one(self, variant: Variant, topic: str, rounds: int | None) -> dict[str, Any]:
+        model = variant.build_model(self.base_model)
+        kwargs = {**self._sim_kwargs, **variant.sim_overrides}
+        if variant.members is not None:
+            kwargs["members"] = variant.members
+        async with self._sem:
+            sim = OrgSim(model, relata_url=self.relata_url, **kwargs)
+            report = await sim.simulate(topic, rounds=rounds)
+        return report.scorecard
+
+    async def run(
+        self, topic: str, *, replicas: int = 1, rounds: int | None = None
+    ) -> ExperimentResult:
+        if not self._variants:
+            raise ValueError("no variants registered; call .variant(...) first")
+        if replicas < 1:
+            raise ValueError(f"replicas must be >= 1, got {replicas}")
+        results: dict[str, VariantResult] = {}
+        for variant in self._variants:
+            scorecards = await asyncio.gather(
+                *(self._run_one(variant, topic, rounds) for _ in range(replicas))
+            )
+            results[variant.name] = VariantResult(
+                name=variant.name,
+                replicas=replicas,
+                metrics=_aggregate(list(scorecards)),
+                raw=list(scorecards),
+            )
+        return ExperimentResult(topic=topic, variants=results)

hypermind-0.14.0/src/hypermind/orgs/groundtruth.py

@@ -0,0 +1,126 @@
+"""Pluggable ground-truth oracles (#157).
+
+Reputation should reflect **real accuracy**, not a `confidence >= 0.5`
+heuristic. A ``GroundTruth`` resolves a published claim to a boolean outcome
+(or ``None`` = unknown → fall back to the interaction default). Feeding real
+outcomes into `oracle_resolve` makes reputation and the scorecard's Brier
+score meaningful, and enables backtesting ("did the crowd actually predict
+right?").
+
+Built-ins:
+  * ``StaticGroundTruth`` — known outcomes keyed by member label or topic.
+  * ``CallableGroundTruth`` — wrap any sync/async ``(topic, claim) -> bool|None``.
+  * ``LLMJudgeGroundTruth`` — an independent LLM judges each claim vs a rubric.
+
+All are injectable; the LLM judge takes a client (fake in tests).
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+from collections.abc import Awaitable, Callable
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class GroundTruth(Protocol):
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        """Return True/False for a resolved claim, or None if unknown."""
+        ...
+
+
+class StaticGroundTruth:
+    """Fixed answers keyed by member label first, then topic.
+
+    ``answers`` maps a claim's ``by`` label (or the topic string) to a bool.
+    """
+
+    def __init__(self, answers: dict[str, bool]) -> None:
+        self._answers = dict(answers)
+
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        by = claim.get("by")
+        if by in self._answers:
+            return self._answers[by]
+        return self._answers.get(topic)
+
+
+class CallableGroundTruth:
+    """Wrap any ``(topic, claim) -> bool | None`` (sync or async)."""
+
+    def __init__(
+        self,
+        fn: Callable[[str, dict[str, Any]], bool | None | Awaitable[bool | None]],
+    ) -> None:
+        self._fn = fn
+
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        out = self._fn(topic, claim)
+        if inspect.isawaitable(out):
+            return await out
+        return out
+
+
+_JUDGE_SYSTEM = (
+    "You are an impartial evaluator. Given a question and a claim, decide "
+    "whether the claim is correct. Respond with a JSON object "
+    '{"correct": true|false} and nothing else.'
+)
+
+
+class LLMJudgeGroundTruth:
+    """An independent LLM scores each claim against an optional rubric.
+
+    ``client`` is an OpenAI-compatible async client; injectable so tests pass a
+    fake. Returns None on any parse/transport failure (→ interaction default).
+    """
+
+    def __init__(
+        self,
+        *,
+        client: Any,
+        model: str = "gpt-4o-mini",
+        rubric: str = "",
+        timeout_s: float = 30.0,
+    ) -> None:
+        self._client = client
+        self._model = model
+        self._rubric = rubric
+        self._timeout = timeout_s
+
+    async def resolve(self, topic: str, claim: dict[str, Any]) -> bool | None:
+        finding = claim.get("finding") or f"(confidence {claim.get('confidence')})"
+        rubric = f"\nRubric: {self._rubric}" if self._rubric else ""
+        user = (
+            f"Question: {topic}\nClaim: {finding}{rubric}\n"
+            'Is the claim correct? Reply as {"correct": true|false}.'
+        )
+        try:
+            resp = await self._client.chat.completions.create(
+                model=self._model,
+                messages=[
+                    {"role": "system", "content": _JUDGE_SYSTEM},
+                    {"role": "user", "content": user},
+                ],
+                response_format={"type": "json_object"},
+                timeout=self._timeout,
+            )
+            content = resp.choices[0].message.content or "{}"
+            verdict = json.loads(content).get("correct")
+            return bool(verdict) if verdict is not None else None
+        except Exception:
+            return None
+
+
+def as_ground_truth(
+    obj: GroundTruth | Callable[[str, dict[str, Any]], Any] | None,
+) -> GroundTruth | None:
+    """Normalise a GroundTruth, a plain callable, or None into a GroundTruth."""
+    if obj is None:
+        return None
+    if isinstance(obj, GroundTruth):
+        return obj
+    if callable(obj):
+        return CallableGroundTruth(obj)
+    raise TypeError(f"ground_truth must be a GroundTruth or callable, got {type(obj).__name__}")

hypermind-0.14.0/src/hypermind/orgs/intervention.py

@@ -0,0 +1,72 @@
+"""Counterfactual interventions (#159) — perturb a run mid-flight.
+
+Controlled what-if is the point of a simulator: inject a shock, ablate an
+agent, or flip a persona at a chosen round, then compare the perturbed run to
+the baseline (pair two runs with the same seed via
+:class:`~hypermind.orgs.experiment.Experiment` and read ``diff``).
+
+An ``Intervention`` fires at the *start* of its ``round`` (before members
+think), so its effect propagates through that round's reasoning.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from hypermind.orgs.sim import OrgSim
+
+
+@runtime_checkable
+class Intervention(Protocol):
+    round: int
+
+    async def apply(self, sim: OrgSim) -> None: ...
+
+
+@dataclass
+class InjectEvent:
+    """A shock every member observes — written to each agent's memory so it
+    surfaces in the next round's recall and colours reasoning."""
+
+    round: int
+    text: str
+
+    async def apply(self, sim: OrgSim) -> None:
+        for m in sim._members:
+            await m.agent.remember(f"[EVENT] {self.text}", confidence=1.0)
+        sim._events.append({"round": self.round, "text": self.text})
+
+
+@dataclass
+class RemoveMember:
+    """Ablate a member (by label) from the simulation."""
+
+    round: int
+    label: str
+
+    async def apply(self, sim: OrgSim) -> None:
+        before = len(sim._members)
+        sim._members = [m for m in sim._members if m.label != self.label]
+        if len(sim._members) != before:
+            sim._events.append({"round": self.round, "removed": self.label})
+
+
+@dataclass
+class FlipPersona:
+    """Swap a member's responder mid-run (e.g. change its stance/model).
+
+    ``responder`` is any ``async (agent_kid, query) -> ResponderResult`` — build
+    it with a new system prompt/model, or inject a fake in tests.
+    """
+
+    round: int
+    label: str
+    responder: Any
+
+    async def apply(self, sim: OrgSim) -> None:
+        for m in sim._members:
+            if m.label == self.label:
+                m.responder = self.responder
+                sim._events.append({"round": self.round, "flipped": self.label})

hypermind-0.14.0/src/hypermind/orgs/metrics.py

@@ -0,0 +1,148 @@
+"""Quantitative scorecard for a simulation run (#151).
+
+Everything here is a pure function over data ``OrgSim`` already tracks — no LLM
+or Relata needed — so runs become **comparable across experiments** and the
+whole module is unit-testable with hand-built inputs.
+
+Metrics:
+  * **calibration (Brier)** — mean and per-member, over oracle-resolved claims.
+  * **consensus / convergence** — posterior variance per round, time-to-consensus.
+  * **polarization** — final belief spread and cross-unit gap (faction disagreement).
+  * **influence** — citation in-degree weighted by the *citer's* reputation.
+  * **activity** — claims / disputes / citations and participation rate.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+from statistics import mean, pvariance
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ScoreCard:
+    rounds: int
+    posterior_variance_by_round: list[float]
+    converged: bool
+    time_to_consensus: int | None  # 0-based round index, or None
+    final_spread: float  # max-min member posterior, last round
+    unit_polarization: float  # max-min of per-unit mean posterior, last round
+    brier_mean: float | None
+    brier_by_member: dict[str, float]
+    influence: dict[str, float]
+    top_influencer: str | None
+    claims: int
+    disputes: int
+    citations: int
+    participation_rate: float
+    _extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        d.pop("_extra", None)
+        return d
+
+    def summary_lines(self) -> list[str]:
+        """Compact human-readable lines for OrgReport.print()."""
+        cal = "n/a" if self.brier_mean is None else f"{self.brier_mean:.3f}"
+        conv = (
+            f"converged@round{self.time_to_consensus}"
+            if self.converged and self.time_to_consensus is not None
+            else ("polarized" if self.unit_polarization >= 0.25 else "no consensus")
+        )
+        return [
+            f"consensus: {conv} (final spread {self.final_spread:.3f}, "
+            f"unit polarization {self.unit_polarization:.3f})",
+            f"calibration (Brier, lower=better): {cal}",
+            f"top influencer: {self.top_influencer or '—'}"
+            + (f" ({self.influence[self.top_influencer]:.2f})" if self.top_influencer else ""),
+            f"activity: {self.claims} claims · {self.citations} citations · "
+            f"{self.disputes} disputes · participation {self.participation_rate:.0%}",
+        ]
+
+
+def _brier(confidence: float, outcome: bool) -> float:
+    return (float(confidence) - (1.0 if outcome else 0.0)) ** 2
+
+
+def compute_scorecard(
+    *,
+    per_round_posteriors: list[dict[str, float]],
+    member_units: dict[str, str],
+    claims: list[dict[str, Any]],
+    citations: list[tuple[str, str]],
+    reputations: dict[str, float],
+    disputes_count: int,
+    convergence_threshold: float = 0.02,
+) -> ScoreCard:
+    """Compute the scorecard from a run's tracked data.
+
+    ``per_round_posteriors`` — one ``{member_label: posterior_mean}`` dict per round.
+    ``member_units`` — every member's unit (denominator for participation).
+    ``claims`` — dicts with keys ``kid_hex, label, unit, confidence, outcome``.
+    ``citations`` — ``(citer_label, target_kid_hex)`` pairs.
+    ``reputations`` — final ``{member_label: score}``.
+    """
+    # --- consensus / convergence ---
+    var_by_round = [
+        pvariance(list(p.values())) if len(p) > 1 else 0.0 for p in per_round_posteriors
+    ]
+    time_to_consensus = next(
+        (i for i, v in enumerate(var_by_round) if v <= convergence_threshold), None
+    )
+    converged = bool(var_by_round) and var_by_round[-1] <= convergence_threshold
+
+    last = per_round_posteriors[-1] if per_round_posteriors else {}
+    final_spread = (max(last.values()) - min(last.values())) if len(last) > 1 else 0.0
+
+    unit_vals: dict[str, list[float]] = defaultdict(list)
+    for label, p in last.items():
+        unit_vals[member_units.get(label, "?")].append(p)
+    unit_means = [mean(v) for v in unit_vals.values()]
+    unit_polarization = (max(unit_means) - min(unit_means)) if len(unit_means) > 1 else 0.0
+
+    # --- calibration (Brier) over resolved claims ---
+    resolved = [
+        (c["label"], float(c.get("confidence", 0.0)), c["outcome"])
+        for c in claims
+        if c.get("outcome") is not None
+    ]
+    brier_all = [_brier(conf, out) for _, conf, out in resolved]
+    brier_mean = mean(brier_all) if brier_all else None
+    by_member: dict[str, list[float]] = defaultdict(list)
+    for label, conf, out in resolved:
+        by_member[label].append(_brier(conf, out))
+    brier_by_member = {k: mean(v) for k, v in by_member.items()}
+
+    # --- influence: reputation-weighted citation in-degree ---
+    kid2label = {c["kid_hex"]: c["label"] for c in claims if "kid_hex" in c}
+    influence: dict[str, float] = defaultdict(float)
+    for citer, target_hex in citations:
+        tgt = kid2label.get(target_hex)
+        if tgt is not None:
+            influence[tgt] += reputations.get(citer, 0.5)
+    influence = dict(influence)
+    top_influencer = max(influence, key=lambda k: influence[k]) if influence else None
+
+    # --- activity ---
+    publishers = {c["label"] for c in claims}
+    total_members = len(member_units)
+    participation = len(publishers) / total_members if total_members else 0.0
+
+    return ScoreCard(
+        rounds=len(per_round_posteriors),
+        posterior_variance_by_round=var_by_round,
+        converged=converged,
+        time_to_consensus=time_to_consensus,
+        final_spread=final_spread,
+        unit_polarization=unit_polarization,
+        brier_mean=brier_mean,
+        brier_by_member=brier_by_member,
+        influence=influence,
+        top_influencer=top_influencer,
+        claims=len(claims),
+        disputes=disputes_count,
+        citations=len(citations),
+        participation_rate=participation,
+    )

{hypermind-0.12.0 → hypermind-0.14.0}/src/hypermind/orgs/model.py

@@ -109,10 +109,19 @@ class DynamicsConfig:
     vouch_juniors: bool = True
     evolve: bool = True
     interaction: str = "cooperative"
+    # Convergence-driven stopping (used when run() is called without rounds=).
+    convergence_threshold: float = 0.02  # posterior variance <= this = consensus
+    convergence_patience: int = 2  # consecutive rounds required to declare a stop
+    polarization_threshold: float = 0.06  # stable variance >= this = entrenched split
+    max_rounds: int = 12  # hard cap for convergence-driven runs
 
     def __post_init__(self) -> None:
         if self.rounds < 1:
             raise ValueError(f"rounds must be >= 1, got {self.rounds}")
+        if self.max_rounds < 1:
+            raise ValueError(f"max_rounds must be >= 1, got {self.max_rounds}")
+        if self.convergence_patience < 1:
+            raise ValueError(f"convergence_patience must be >= 1, got {self.convergence_patience}")
         for name in ("publish_floor", "dispute_gap", "oracle_resolve_frac"):
             v = getattr(self, name)
             if not 0.0 <= v <= 1.0:

{hypermind-0.12.0 → hypermind-0.14.0}/src/hypermind/orgs/report.py

@@ -38,6 +38,9 @@ class OrgReport:
     citations: list[tuple[str, str]]
     disputes: list[dict[str, Any]]
     tool_grants: dict[str, list[str]]
+    scorecard: dict[str, Any] = field(default_factory=dict)
+    stop_reason: str = "fixed"  # fixed | converged | polarized | max_rounds
+    events: list[dict[str, Any]] = field(default_factory=list)  # applied interventions
 
     # ---- derived views -------------------------------------------------
 
@@ -74,9 +77,15 @@ class OrgReport:
         s = self.summary()
         print(f"\n=== {self.org} — “{self.topic}” ({self.interaction}) ===")
         print(
-            f"{self.rounds} rounds · {s['members']} members · {s['claims']} claims · "
-            f"{s['citations']} citations · {s['disputes']} disputes"
+            f"{self.rounds} rounds ({self.stop_reason}) · {s['members']} members · "
+            f"{s['claims']} claims · {s['citations']} citations · {s['disputes']} disputes"
         )
+        if self.scorecard:
+            from hypermind.orgs.metrics import ScoreCard
+
+            print("\nScorecard:")
+            for line in ScoreCard(**self.scorecard).summary_lines():
+                print(f"  {line}")
         print("\nTop standing:")
         for st in self.top(10):
             tools = f"  tools={st.tools}" if st.tools else ""