hypermind 0.20.0__tar.gz → 0.22.0__tar.gz

{hypermind-0.20.0 → hypermind-0.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypermind
-Version: 0.20.0
+Version: 0.22.0
 Summary: Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile
 Project-URL: Homepage, https://github.com/ZySec-AI/hypermind
 Project-URL: Documentation, https://github.com/ZySec-AI/hypermind/tree/main/docs

{hypermind-0.20.0 → hypermind-0.22.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hypermind"
-version = "0.20.0"
+version = "0.22.0"
 description = "Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile"
 readme = "README.md"
 requires-python = ">=3.11,<3.14"

{hypermind-0.20.0 → hypermind-0.22.0}/src/hypermind/__init__.py

@@ -73,7 +73,7 @@ open = HyperMindAgent.open
 #: is held until the §External & human gates clear (IETF I-D, IANA,
 #: third-party audit, OpenSSF acceptance, ≥3 pilots) — at that point we
 #: cut ``1.0.0`` GA. See ROADMAP.md §Status snapshot.
-__version__ = "0.20.0"
+__version__ = "0.22.0"
 
 #: SLSA build-provenance level committed for every release artefact.
 #: Cosign keyless signing + slsa-github-generator + PEP 740 attestations

{hypermind-0.20.0 → hypermind-0.22.0}/src/hypermind/agent.py

@@ -18,9 +18,11 @@ from __future__ import annotations
 import asyncio
 import hashlib
 import json
+import os
 import secrets
 import time
 from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import timedelta
 from typing import Any, Literal
@@ -853,9 +855,19 @@ class HyperMindAgent:
         world.ts.witness_keypairs = [Keypair.generate(alg=kp_alg) for _ in range(3)]
         world.ts.min_witness_quorum = 1  # sandbox quorum
 
+        # #177 (spin-up ceiling): keygen is CPU-bound but the crypto releases
+        # the GIL, so generating N keypairs in a thread pool cuts spin-up of
+        # large swarms (measured ~3× Ed25519, ~1.5× ML-DSA). World mutations
+        # below stay single-threaded; ex.map preserves order so the roster is
+        # unchanged. Small swarms skip the pool (thread setup isn't worth it).
+        if agents >= 8:
+            with ThreadPoolExecutor(max_workers=min(32, (os.cpu_count() or 4))) as ex:
+                keypairs = list(ex.map(lambda _: Keypair.generate(alg=kp_alg), range(agents)))
+        else:
+            keypairs = [Keypair.generate(alg=kp_alg) for _ in range(agents)]
+
         agents_list: list[HyperMindAgent] = []
-        for _ in range(agents):
-            kp = Keypair.generate(alg=kp_alg)
+        for kp in keypairs:
             agent = HyperMindAgent(
                 keypair=kp,
                 namespace=namespace,

{hypermind-0.20.0 → hypermind-0.22.0}/src/hypermind/orgs/__init__.py

@@ -59,6 +59,12 @@ from hypermind.orgs.model import (
     Slot,
     Unit,
 )
+from hypermind.orgs.quality import (
+    ALL_RUBRICS,
+    QualityReport,
+    assess_answer,
+    llm_judge,
+)
 from hypermind.orgs.query import Answer, Contributor, aggregate_answer
 from hypermind.orgs.receipt import VerifyResult, build_receipt, verify_receipt
 from hypermind.orgs.report import MemberStanding, OrgReport
@@ -87,6 +93,7 @@ from hypermind.orgs.tools import (
 )
 
 __all__ = [
+    "ALL_RUBRICS",
     "Adversarial",
     "Answer",
     "CallableGroundTruth",
@@ -108,6 +115,7 @@ __all__ = [
     "OrgModel",
     "OrgReport",
     "OrgSim",
+    "QualityReport",
     "RemoveMember",
     "ScoreCard",
     "Seat",
@@ -118,6 +126,7 @@ __all__ = [
     "VariantResult",
     "VerifyResult",
     "aggregate_answer",
+    "assess_answer",
     "build_deliberator",
     "build_receipt",
     "build_tool",
@@ -132,6 +141,7 @@ __all__ = [
     "graph_snapshot",
     "hybrid_retriever",
     "list_tools",
+    "llm_judge",
     "llm_ner_extractor",
     "openai_embedder",
     "pioneer_ner_extractor",

{hypermind-0.20.0 → hypermind-0.22.0}/src/hypermind/orgs/metrics.py

@@ -66,25 +66,27 @@ def _brier(confidence: float, outcome: bool) -> float:
     return (float(confidence) - (1.0 if outcome else 0.0)) ** 2
 
 
-def compute_scorecard(
+def scorecard_from_aggregates(
     *,
     per_round_posteriors: list[dict[str, float]],
     member_units: dict[str, str],
-    claims: list[dict[str, Any]],
-    citations: list[tuple[str, str]],
-    reputations: dict[str, float],
+    brier_sums: dict[str, tuple[float, int]],
+    influence: dict[str, float],
+    publishers: set[str],
+    claims_count: int,
+    citations_count: int,
     disputes_count: int,
     convergence_threshold: float = 0.02,
 ) -> ScoreCard:
-    """Compute the scorecard from a run's tracked data.
+    """Build a ScoreCard from **streaming aggregates** (bounded by member count,
+    not record count). Lets ``OrgSim(lightweight=True)`` produce the same
+    scorecard without holding every claim/citation in RAM (#176).
 
-    ``per_round_posteriors`` — one ``{member_label: posterior_mean}`` dict per round.
-    ``member_units`` — every member's unit (denominator for participation).
-    ``claims`` — dicts with keys ``kid_hex, label, unit, confidence, outcome``.
-    ``citations`` — ``(citer_label, target_kid_hex)`` pairs.
-    ``reputations`` — final ``{member_label: score}``.
+    ``brier_sums`` — per member ``{label: (sum_of_brier, n_resolved)}``.
+    ``influence`` — per member the reputation-weighted citation in-degree.
+    ``publishers`` — labels that published at least one claim.
     """
-    # --- consensus / convergence ---
+    # --- consensus / convergence (per-round, already bounded) ---
     var_by_round = [
         pvariance(list(p.values())) if len(p) > 1 else 0.0 for p in per_round_posteriors
     ]
@@ -102,31 +104,15 @@ def compute_scorecard(
     unit_means = [mean(v) for v in unit_vals.values()]
     unit_polarization = (max(unit_means) - min(unit_means)) if len(unit_means) > 1 else 0.0
 
-    # --- calibration (Brier) over resolved claims ---
-    resolved = [
-        (c["label"], float(c.get("confidence", 0.0)), c["outcome"])
-        for c in claims
-        if c.get("outcome") is not None
-    ]
-    brier_all = [_brier(conf, out) for _, conf, out in resolved]
-    brier_mean = mean(brier_all) if brier_all else None
-    by_member: dict[str, list[float]] = defaultdict(list)
-    for label, conf, out in resolved:
-        by_member[label].append(_brier(conf, out))
-    brier_by_member = {k: mean(v) for k, v in by_member.items()}
-
-    # --- influence: reputation-weighted citation in-degree ---
-    kid2label = {c["kid_hex"]: c["label"] for c in claims if "kid_hex" in c}
-    influence: dict[str, float] = defaultdict(float)
-    for citer, target_hex in citations:
-        tgt = kid2label.get(target_hex)
-        if tgt is not None:
-            influence[tgt] += reputations.get(citer, 0.5)
-    influence = dict(influence)
-    top_influencer = max(influence, key=lambda k: influence[k]) if influence else None
+    # --- calibration (Brier) from per-member sums ---
+    total_sum = sum(s for s, _n in brier_sums.values())
+    total_n = sum(n for _s, n in brier_sums.values())
+    brier_mean = (total_sum / total_n) if total_n else None
+    brier_by_member = {k: s / n for k, (s, n) in brier_sums.items() if n}
+
+    infl = dict(influence)
+    top_influencer = max(infl, key=lambda k: infl[k]) if infl else None
 
-    # --- activity ---
-    publishers = {c["label"] for c in claims}
     total_members = len(member_units)
     participation = len(publishers) / total_members if total_members else 0.0
 
@@ -139,10 +125,57 @@ def compute_scorecard(
         unit_polarization=unit_polarization,
         brier_mean=brier_mean,
         brier_by_member=brier_by_member,
-        influence=influence,
+        influence=infl,
         top_influencer=top_influencer,
-        claims=len(claims),
+        claims=claims_count,
         disputes=disputes_count,
-        citations=len(citations),
+        citations=citations_count,
         participation_rate=participation,
     )
+
+
+def compute_scorecard(
+    *,
+    per_round_posteriors: list[dict[str, float]],
+    member_units: dict[str, str],
+    claims: list[dict[str, Any]],
+    citations: list[tuple[str, str]],
+    reputations: dict[str, float],
+    disputes_count: int,
+    convergence_threshold: float = 0.02,
+) -> ScoreCard:
+    """Compute the scorecard from a run's fully-retained data (default mode).
+
+    ``per_round_posteriors`` — one ``{member_label: posterior_mean}`` dict per round.
+    ``member_units`` — every member's unit (denominator for participation).
+    ``claims`` — dicts with keys ``kid_hex, label, unit, confidence, outcome``.
+    ``citations`` — ``(citer_label, target_kid_hex)`` pairs.
+    ``reputations`` — final ``{member_label: score}``.
+    """
+    # Aggregate the full lists, then delegate to the streaming builder so both
+    # paths share one scorecard definition.
+    brier_sums: dict[str, tuple[float, int]] = {}
+    for c in claims:
+        if c.get("outcome") is None:
+            continue
+        s, n = brier_sums.get(c["label"], (0.0, 0))
+        brier_sums[c["label"]] = (s + _brier(float(c.get("confidence", 0.0)), c["outcome"]), n + 1)
+
+    kid2label = {c["kid_hex"]: c["label"] for c in claims if "kid_hex" in c}
+    influence: dict[str, float] = defaultdict(float)
+    for citer, target_hex in citations:
+        tgt = kid2label.get(target_hex)
+        if tgt is not None:
+            influence[tgt] += reputations.get(citer, 0.5)
+
+    return scorecard_from_aggregates(
+        per_round_posteriors=per_round_posteriors,
+        member_units=member_units,
+        brier_sums=brier_sums,
+        influence=dict(influence),
+        publishers={c["label"] for c in claims},
+        claims_count=len(claims),
+        citations_count=len(citations),
+        disputes_count=disputes_count,
+        convergence_threshold=convergence_threshold,
+    )

hypermind-0.22.0/src/hypermind/orgs/quality.py

@@ -0,0 +1,251 @@
+"""Native response-quality rubrics for the study→ask path.
+
+Generic RAG-eval frameworks judge text against text. HyperMind can do more:
+every finding is a *signed* claim on a Merkle-logged ledger, carries earned
+reputation, and may be disputed. So a class of grounding rubrics is here
+**deterministic and verifiable**, not estimated. This module scores an
+:class:`~hypermind.orgs.query.Answer` across four rubric groups:
+
+    native       (A) — provenance_coverage, dispute_exposure, reputation_support,
+                        consensus, abstention_ok. No LLM; computed from the ledger.
+    judge        (B) — grounding, hallucination_rate, answer_relevance,
+                        context_precision. Needs a ``judge`` callable.
+    calibration  (C) — calibration_brier (needs ``ground_truth``),
+                        self_consistency (needs ``samples>=2``, via sim.assess).
+    recall       (D) — context_recall. Needs ``relevant_labels``.
+
+Every rubric is **on by default**; a missing dependency degrades to an
+informational note (how to enable), never a crash. Turn any rubric or whole
+group off via ``disable`` (a metric name, or a group token above).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+import statistics
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+_A = ("provenance_coverage", "dispute_exposure", "reputation_support", "consensus", "abstention_ok")
+_B = ("grounding", "hallucination_rate", "answer_relevance", "context_precision")
+_C = ("calibration_brier", "self_consistency")
+_D = ("context_recall",)
+ALL_RUBRICS: tuple[str, ...] = _A + _B + _C + _D
+GROUPS: dict[str, tuple[str, ...]] = {"native": _A, "judge": _B, "calibration": _C, "recall": _D}
+
+# Metrics where a *higher* value is *worse* (inverted when forming a composite).
+_HIGHER_WORSE = frozenset({"dispute_exposure", "hallucination_rate", "calibration_brier"})
+
+
+def _expand_disable(disable: frozenset[str] | set[str]) -> set[str]:
+    """Expand group tokens (native/judge/calibration/recall) to their metrics."""
+    out: set[str] = set()
+    for tok in disable:
+        out |= set(GROUPS.get(tok, (tok,)))
+    return out
+
+
+@dataclass
+class QualityReport:
+    """Scores for one answer. ``metrics`` holds only the rubrics that ran."""
+
+    metrics: dict[str, float] = field(default_factory=dict)
+    hallucinated_claims: list[str] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+    _expanded_disable: set[str] = field(default_factory=set)
+
+    def overall(self) -> float | None:
+        """A single defensible composite in [0,1]: mean of available rubrics,
+        inverting the ones where higher is worse. ``None`` if nothing ran."""
+        vals = [
+            (1.0 - v) if k in _HIGHER_WORSE else v
+            for k, v in self.metrics.items()
+            if isinstance(v, (int, float))
+        ]
+        return round(sum(vals) / len(vals), 4) if vals else None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "metrics": dict(self.metrics),
+            "overall": self.overall(),
+            "hallucinated_claims": list(self.hallucinated_claims),
+            "notes": list(self.notes),
+        }
+
+    def print(self) -> None:
+        ov = self.overall()
+        print(f"\nQuality {'—' if ov is None else f'{ov:.2f}'} overall")
+        for k in ALL_RUBRICS:
+            if k in self.metrics:
+                arrow = "↓" if k in _HIGHER_WORSE else "↑"
+                print(f"  {k:22s} {self.metrics[k]:.3f} {arrow}")
+        for claim in self.hallucinated_claims:
+            print(f"  ⚠ unsupported: {claim}")
+        for note in self.notes:
+            print(f"  · {note}")
+
+
+# --- judge factory --------------------------------------------------------
+_JUDGE_SYS = (
+    "You are a strict grader detecting hallucination. Given a QUESTION, the "
+    "CONTEXT the responder could use, and its ANSWER, judge ONLY against the "
+    "context (closed-book). Respond with JSON only: "
+    '{"grounding": 0.0-1.0, "hallucinated": true|false, "relevance": 0.0-1.0, '
+    '"context_precision": 0.0-1.0, "unsupported": ["claim", ...], "note": "one sentence"}. '
+    "grounding = fraction of the answer's factual claims supported by context. "
+    "relevance = does it answer the question. "
+    "context_precision = fraction of the context that was actually relevant."
+)
+
+
+def llm_judge(
+    client: Any, *, model: str = "openai/gpt-4o", timeout_s: float = 30.0
+) -> Callable[[str, list[str], str], dict[str, Any]]:
+    """Build a sync judge from an OpenAI-compatible **sync** client.
+
+    Returns ``(question, context_texts, answer) -> verdict dict``. Plug into
+    ``sim.assess(judge=llm_judge(client))``. Parse/transport failures return an
+    empty verdict (the rubric group is then reported as unavailable)."""
+
+    def judge(question: str, context: list[str], answer: str) -> dict[str, Any]:
+        ctx = "\n".join(f"- {c}" for c in context) or "(no context retrieved)"
+        user = f"QUESTION:\n{question}\n\nCONTEXT:\n{ctx}\n\nANSWER:\n{answer}"
+        try:
+            resp = client.chat.completions.create(
+                model=model,
+                messages=[
+                    {"role": "system", "content": _JUDGE_SYS},
+                    {"role": "user", "content": user},
+                ],
+                temperature=0,
+                timeout=timeout_s,
+            )
+            content = resp.choices[0].message.content or ""
+            m = re.search(r"\{.*\}", content, re.S)
+            return json.loads(m.group(0)) if m else {}
+        except Exception:
+            return {}
+
+    return judge
+
+
+# --- the rubrics ----------------------------------------------------------
+async def assess_answer(
+    answer: Any,
+    context: list[tuple[str, str]],
+    *,
+    sim: Any = None,
+    judge: Callable[[str, list[str], str], dict[str, Any]] | None = None,
+    ground_truth: Any = None,
+    relevant_labels: set[str] | None = None,
+    disable: frozenset[str] | set[str] = frozenset(),
+) -> QualityReport:
+    """Score ``answer`` (a query.Answer) grounded on ``context`` (retrieved
+    ``(label, text)`` pairs). See the module docstring for the rubric groups."""
+    off = _expand_disable(disable)
+    rep = QualityReport(_expanded_disable=off)
+    m = rep.metrics
+    ctx_labels = {lbl for lbl, _t in context}
+    ctx_texts = [t for _l, t in context]
+    contribs = list(getattr(answer, "contributors", []) or [])
+
+    def on(name: str) -> bool:
+        return name not in off
+
+    # --- A. native / deterministic (from the ledger) ----------------------
+    if sim is None:
+        if any(on(k) for k in _A):
+            rep.notes.append("native rubrics: SKIPPED — pass sim= to score against the ledger.")
+    else:
+        claims = list(getattr(sim, "_claims", {}).values())
+        authored = {c.label for c in claims if getattr(c, "finding", None)}
+        if on("provenance_coverage"):
+            # Fraction of contributors whose stance is backed by a signed claim.
+            if contribs:
+                backed = sum(1 for c in contribs if c.label in authored)
+                m["provenance_coverage"] = round(backed / len(contribs), 4)
+            else:
+                m["provenance_coverage"] = 0.0
+        if on("dispute_exposure"):
+            disputed = {d.get("target_by") for d in getattr(sim, "_disputes", [])}
+            if ctx_labels:
+                exposed = sum(1 for lbl in ctx_labels if lbl in disputed)
+                m["dispute_exposure"] = round(exposed / len(ctx_labels), 4)
+            else:
+                m["dispute_exposure"] = 0.0
+        if on("reputation_support"):
+            # Reputation mass agreeing with the answer's direction / total mass.
+            lead = 1 if answer.confidence >= 0.5 else 0
+            tot = sum(c.reputation for c in contribs)
+            if tot > 0:
+                agree = sum(c.reputation for c in contribs if (c.confidence >= 0.5) == bool(lead))
+                m["reputation_support"] = round(agree / tot, 4)
+            else:
+                m["reputation_support"] = 0.0
+        if on("consensus") and contribs:
+            spread = (
+                statistics.pstdev([c.confidence for c in contribs]) if len(contribs) > 1 else 0.0
+            )
+            m["consensus"] = round(1.0 - min(1.0, spread * 2.0), 4)
+        if on("abstention_ok"):
+            # Thin evidence should yield an 'uncertain' stance, not overconfidence.
+            k = getattr(sim, "_retrieval_k", 4)
+            thin = len(context) < max(1, k // 2)
+            m["abstention_ok"] = 1.0 if (not thin or answer.stance == "uncertain") else 0.0
+
+    # --- B. judge-based faithfulness --------------------------------------
+    if any(on(k) for k in _B):
+        if judge is None:
+            rep.notes.append(
+                "judge rubrics (grounding/hallucination/relevance/precision): SKIPPED — "
+                "pass judge=llm_judge(client) to enable."
+            )
+        else:
+            v = await asyncio.to_thread(judge, answer.question, ctx_texts, answer.reasoning)
+            if not v:
+                rep.notes.append("judge rubrics: SKIPPED — judge returned no verdict.")
+            else:
+                if on("grounding") and v.get("grounding") is not None:
+                    m["grounding"] = float(v["grounding"])
+                if on("answer_relevance") and v.get("relevance") is not None:
+                    m["answer_relevance"] = float(v["relevance"])
+                if on("context_precision") and v.get("context_precision") is not None:
+                    m["context_precision"] = float(v["context_precision"])
+                if on("hallucination_rate") and v.get("hallucinated") is not None:
+                    m["hallucination_rate"] = 1.0 if v["hallucinated"] else 0.0
+                rep.hallucinated_claims = [str(u) for u in v.get("unsupported", [])]
+
+    # --- C. calibration ---------------------------------------------------
+    if on("calibration_brier"):
+        if ground_truth is None:
+            rep.notes.append(
+                "calibration_brier: SKIPPED — pass ground_truth= (a GroundTruth) to enable."
+            )
+        else:
+            claim = {"finding": answer.reasoning, "by": contribs[0].label if contribs else None}
+            correct = await ground_truth.resolve(answer.question, claim)
+            if correct is None:
+                rep.notes.append("calibration_brier: SKIPPED — ground_truth returned unknown.")
+            else:
+                m["calibration_brier"] = round(
+                    (answer.confidence - (1.0 if correct else 0.0)) ** 2, 4
+                )
+
+    # --- D. context recall ------------------------------------------------
+    if on("context_recall"):
+        if relevant_labels is None:
+            rep.notes.append(
+                "context_recall: SKIPPED — pass relevant_labels={..} (the gold-relevant "
+                "finding authors) to enable."
+            )
+        elif relevant_labels:
+            m["context_recall"] = round(
+                len(ctx_labels & set(relevant_labels)) / len(relevant_labels), 4
+            )
+        else:
+            m["context_recall"] = 1.0
+
+    return rep