hypermind 0.11.0__py3-none-any.whl

hypermind/knowledge/__init__.py

@@ -0,0 +1,17 @@
+"""Knowledge / reputation / receipt layer."""
+
+from hypermind.knowledge.correlated_agreement import Report, score_reports
+from hypermind.knowledge.reputation import ReputationKernel, ReputationSnapshot
+from hypermind.knowledge.transparency import (
+    TransparencyService,
+    WitnessCosignature,
+)
+
+__all__ = [
+    "Report",
+    "ReputationKernel",
+    "ReputationSnapshot",
+    "TransparencyService",
+    "WitnessCosignature",
+    "score_reports",
+]

hypermind/knowledge/citation_graph.py

@@ -0,0 +1,141 @@
+"""Citation-graph queries (ROADMAP §0.3.4).
+
+A :class:`CitationGraph` builds a directed acyclic graph (DAG) from
+the ``cites`` field of each claim. Each ClaimEvent's payload may
+include::
+
+    {"cites": [b"<parent_kid>", ...], ...}
+
+The graph supports ancestor / descendant queries with bounded depth
+plus a cycle check (claims may not transitively cite themselves).
+
+Cycle detection uses iterative depth-first search with a recursion
+stack; it short-circuits on the first back-edge so the worst-case is
+``O(V + E)``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+import cbor2
+
+
+@dataclass
+class CitationGraph:
+    """In-memory citation DAG."""
+
+    #: Forward edges: ``claim_kid → set(parent_kids)``.
+    _cites: dict[bytes, set[bytes]] = field(default_factory=dict)
+    #: Reverse edges for descendant queries.
+    _cited_by: dict[bytes, set[bytes]] = field(default_factory=dict)
+
+    # ---- ingest ------------------------------------------------------
+
+    def add(self, claim_kid: bytes, cites: Iterable[bytes] = ()) -> None:
+        """Add a claim and its outgoing citations.
+
+        Idempotent: re-adding the same claim with the same edges is a
+        no-op. Re-adding with *new* edges merges them.
+        """
+        targets = {bytes(c) for c in cites}
+        self._cites.setdefault(claim_kid, set()).update(targets)
+        for t in targets:
+            self._cited_by.setdefault(t, set()).add(claim_kid)
+            # Ensure target has a forward-edge entry so it's a known node.
+            self._cites.setdefault(t, set())
+
+    def add_from_payload(self, claim_kid: bytes, payload: bytes) -> None:
+        """Convenience: parse `cites` from a CBOR-encoded claim payload."""
+        try:
+            d = cbor2.loads(payload)
+        except Exception:
+            return
+        cites = d.get("cites", ()) if isinstance(d, dict) else ()
+        if isinstance(cites, (list, tuple)):
+            self.add(claim_kid, [bytes(c) for c in cites if isinstance(c, (bytes, bytearray))])
+        else:
+            self.add(claim_kid, ())
+
+    # ---- queries -----------------------------------------------------
+
+    def ancestors(self, claim_kid: bytes, *, depth: int = 1) -> set[bytes]:
+        """Return ancestors (transitively cited claims) up to ``depth``."""
+        if depth <= 0:
+            return set()
+        seen: set[bytes] = set()
+        frontier = {claim_kid}
+        for _ in range(depth):
+            next_frontier: set[bytes] = set()
+            for node in frontier:
+                for parent in self._cites.get(node, ()):  # cited-by-this
+                    if parent not in seen:
+                        seen.add(parent)
+                        next_frontier.add(parent)
+            frontier = next_frontier
+            if not frontier:
+                break
+        seen.discard(claim_kid)
+        return seen
+
+    def descendants(self, claim_kid: bytes, *, depth: int = 1) -> set[bytes]:
+        """Return descendants (claims that cite this) up to ``depth``."""
+        if depth <= 0:
+            return set()
+        seen: set[bytes] = set()
+        frontier = {claim_kid}
+        for _ in range(depth):
+            next_frontier: set[bytes] = set()
+            for node in frontier:
+                for child in self._cited_by.get(node, ()):
+                    if child not in seen:
+                        seen.add(child)
+                        next_frontier.add(child)
+            frontier = next_frontier
+            if not frontier:
+                break
+        seen.discard(claim_kid)
+        return seen
+
+    def cycle_check(self) -> list[bytes] | None:
+        """Return a list of nodes forming a cycle, or ``None`` if DAG.
+
+        Iterative DFS with recursion-stack tracking. Short-circuits at
+        the first back-edge.
+        """
+        WHITE, GRAY, BLACK = 0, 1, 2
+        color: dict[bytes, int] = {n: WHITE for n in self._cites}
+        for start in list(color.keys()):
+            if color[start] != WHITE:
+                continue
+            stack: list[tuple[bytes, list[bytes]]] = [(start, list(self._cites.get(start, ())))]
+            path = [start]
+            color[start] = GRAY
+            while stack:
+                node, children = stack[-1]
+                if not children:
+                    color[node] = BLACK
+                    stack.pop()
+                    if path and path[-1] == node:
+                        path.pop()
+                    continue
+                child = children.pop()
+                cstate = color.get(child, WHITE)
+                if cstate == GRAY:
+                    # cycle found; reconstruct from path
+                    if child in path:
+                        idx = path.index(child)
+                        return [*path[idx:], child]
+                    return [child]
+                if cstate == WHITE:
+                    color[child] = GRAY
+                    stack.append((child, list(self._cites.get(child, ()))))
+                    path.append(child)
+        return None
+
+    def __len__(self) -> int:
+        return len(self._cites)
+
+    def nodes(self) -> set[bytes]:
+        return set(self._cites.keys())

hypermind/knowledge/correlated_agreement.py

@@ -0,0 +1,167 @@
+"""Correlated-Agreement peer prediction (ROADMAP §0.2.2).
+
+Implements a Prelec / Krishnamurthy-Mittendorff-style Bayesian Truth
+Serum scoring rule. Each reporter provides:
+
+  * an ``answer`` (categorical), and
+  * a ``prediction`` — a probability distribution over what other
+    reporters will answer.
+
+The score for reporter *i* is::
+
+    score_i = log(actual_freq[answer_i] / predicted_freq[answer_i])
+              + ALPHA * log(predicted_correctness_i)
+
+where ``predicted_correctness_i`` is the geometric-mean agreement
+between reporter *i*'s prediction and the empirical answer
+distribution. The first term rewards "surprisingly common" answers;
+the second term rewards calibrated predictions of the population.
+
+The scoring rule is *incentive-compatible* in the limit: truth-telling
+maximises expected score for any prior the reporter may hold. See
+Prelec (Science, 2004) and the Krishnamurthy-Mittendorff
+generalisation for the multi-answer / continuous-prediction case.
+
+Wire into :class:`~hypermind.knowledge.reputation.ReputationKernel`
+via :meth:`ReputationKernel.update_with_correlated_agreement`.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass, field
+
+ALPHA_DEFAULT = 1.0
+EPS = 1e-9
+
+
+@dataclass(frozen=True)
+class Report:
+    """A single peer report.
+
+    Attributes:
+        reporter_id: Opaque identifier (kid bytes are typical).
+        answer: The reporter's chosen answer (categorical).
+        prediction: Reporter's estimate of the population's answer
+            distribution. Keys MUST be a superset of every answer that
+            actually appears among reporters; values must be
+            non-negative and sum to ~1.0 (normalised internally).
+    """
+
+    reporter_id: bytes
+    answer: str
+    prediction: Mapping[str, float] = field(default_factory=dict)
+
+
+def _normalise(d: Mapping[str, float]) -> dict[str, float]:
+    total = sum(max(0.0, v) for v in d.values())
+    if total <= 0:
+        return {k: 0.0 for k in d}
+    return {k: max(0.0, v) / total for k, v in d.items()}
+
+
+def _empirical_freq(reports: Iterable[Report]) -> dict[str, float]:
+    counts: dict[str, float] = {}
+    n = 0
+    for r in reports:
+        counts[r.answer] = counts.get(r.answer, 0.0) + 1.0
+        n += 1
+    if n == 0:
+        return {}
+    return {k: v / n for k, v in counts.items()}
+
+
+def _avg_prediction(reports: list[Report]) -> dict[str, float]:
+    """Geometric-mean average of reporters' predictions (BTS canonical).
+
+    Aggregating predictions in log-space matches Prelec's BTS derivation
+    and is well-behaved when individual predictions assign probability 0
+    to an answer that another reporter actually picks (the EPS floor).
+    """
+    if not reports:
+        return {}
+    keys = set()
+    for r in reports:
+        keys.update(r.prediction.keys())
+    log_acc: dict[str, float] = {k: 0.0 for k in keys}
+    for r in reports:
+        norm = _normalise({k: r.prediction.get(k, 0.0) for k in keys})
+        for k, v in norm.items():
+            log_acc[k] += math.log(max(v, EPS))
+    n = len(reports)
+    log_acc = {k: v / n for k, v in log_acc.items()}
+    raw = {k: math.exp(v) for k, v in log_acc.items()}
+    return _normalise(raw)
+
+
+def score_reports(
+    reports: list[Report],
+    *,
+    alpha: float = ALPHA_DEFAULT,
+) -> dict[bytes, float]:
+    """Score each reporter under correlated-agreement / BTS.
+
+    Returns a mapping from reporter_id (bytes) to score (float). Higher
+    is better; calibrated truthful reporters obtain positive scores.
+
+    The ``alpha`` weight controls the trade-off between "surprisingly
+    common" answer reward (information score) and prediction
+    calibration. ``alpha=1.0`` is the canonical Prelec choice.
+    """
+    if not reports:
+        return {}
+    if len(reports) == 1:
+        # BTS is undefined with a single reporter; return a sentinel
+        # zero so callers can still feed the result into the reputation
+        # kernel without special-casing.
+        return {reports[0].reporter_id: 0.0}
+
+    actual = _empirical_freq(reports)
+    predicted = _avg_prediction(reports)
+
+    # Universe of answers: union of every answer that appears AND every
+    # answer any reporter mentioned in its prediction.
+    universe = set(actual) | set(predicted)
+    for r in reports:
+        universe.update(r.prediction.keys())
+    # Floor predicted to EPS for any answer that was actually given.
+    predicted = {k: max(predicted.get(k, 0.0), EPS) for k in universe}
+    # Renormalise after flooring so probabilities still sum to 1.
+    predicted = _normalise(predicted)
+
+    scores: dict[bytes, float] = {}
+    for r in reports:
+        info_term = math.log(
+            max(actual.get(r.answer, 0.0), EPS) / max(predicted.get(r.answer, EPS), EPS)
+        )
+        # Reporter's predicted-correctness: geometric agreement with
+        # the empirical distribution.
+        pred_norm = _normalise({k: r.prediction.get(k, 0.0) for k in universe})
+        agreement = 0.0
+        for k in universe:
+            agreement += actual.get(k, 0.0) * math.log(max(pred_norm.get(k, 0.0), EPS))
+        # `agreement` is the cross-entropy of actual relative to
+        # reporter's prediction — already a log-probability.
+        scores[r.reporter_id] = info_term + alpha * agreement
+    return scores
+
+
+def update_kernel_with_reports(
+    kernel,
+    reports: list[Report],
+    *,
+    topic: str,
+    alpha: float = ALPHA_DEFAULT,
+    threshold: float = 0.0,
+) -> dict[bytes, float]:
+    """Apply correlated-agreement scores to a ReputationKernel.
+
+    Reporters scoring above ``threshold`` are recorded as ``correct``;
+    those below as ``incorrect``. The raw score map is returned for
+    inspection / tests.
+    """
+    raw = score_reports(reports, alpha=alpha)
+    for kid, score in raw.items():
+        kernel.record_outcome(kid, topic, correct=score >= threshold)
+    return raw

hypermind/knowledge/embedding_registry.py

@@ -0,0 +1,115 @@
+"""Namespace-scoped embedding-model registry (ROADMAP §0.4.2).
+
+Each namespace registers the set of embedding models its claims may
+reference. A claim that includes ``embedding_model_id`` and
+``embedding_vector`` fields MUST refer to a registered model; the
+verifier rejects anything else.
+
+This closes a class of model-substitution attacks where an attacker
+publishes claims with vectors derived from an unauthenticated model
+(or a different model whose vectors are not comparable to the
+namespace's canonical model).
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class EmbeddingModelEntry:
+    """A registered embedding model.
+
+    Attributes:
+        model_id: Caller-chosen identifier (typically a string like
+            ``"openai/text-embedding-3-large"``).
+        model_hash: 32-byte hash binding the model artifact (e.g.
+            SHA-256 of the canonical weights archive). Verifiers MAY
+            require this to match a published artifact.
+        dim: Vector dimension. Vectors with the wrong length are
+            rejected.
+        source_url: Optional provenance URL for human auditors.
+    """
+
+    model_id: str
+    model_hash: bytes
+    dim: int
+    source_url: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.model_id:
+            raise ValueError("model_id must be non-empty")
+        if len(self.model_hash) != 32:
+            raise ValueError(f"model_hash must be 32 bytes, got {len(self.model_hash)}")
+        if self.dim <= 0:
+            raise ValueError("dim must be positive")
+
+
+@dataclass
+class EmbeddingModelRegistry:
+    """Per-namespace embedding-model registry."""
+
+    namespace_id: bytes = b"\x00" * 32
+    _entries: dict[str, EmbeddingModelEntry] = field(default_factory=dict)
+
+    def register(
+        self,
+        model_id: str,
+        *,
+        model_hash: bytes,
+        dim: int,
+        source_url: str = "",
+    ) -> EmbeddingModelEntry:
+        """Register a new model. Re-registering an existing ``model_id``
+        with different parameters raises :class:`ValueError`."""
+        new = EmbeddingModelEntry(
+            model_id=model_id,
+            model_hash=model_hash,
+            dim=dim,
+            source_url=source_url,
+        )
+        existing = self._entries.get(model_id)
+        if existing is not None and existing != new:
+            raise ValueError(
+                f"model_id {model_id!r} already registered with different "
+                f"parameters; refusing silent overwrite"
+            )
+        self._entries[model_id] = new
+        return new
+
+    def get(self, model_id: str) -> EmbeddingModelEntry | None:
+        return self._entries.get(model_id)
+
+    def __contains__(self, model_id: object) -> bool:
+        return isinstance(model_id, str) and model_id in self._entries
+
+    def __len__(self) -> int:
+        return len(self._entries)
+
+    def verify(
+        self,
+        *,
+        model_id: str,
+        vector: list[float] | tuple[float, ...] | None,
+    ) -> None:
+        """Raise :class:`ValueError` if the vector / model ref is invalid.
+
+        Returns ``None`` on success — convention matches other verifier
+        primitives in the SDK.
+        """
+        entry = self._entries.get(model_id)
+        if entry is None:
+            raise ValueError(f"embedding model {model_id!r} not registered in this namespace")
+        if vector is None:
+            raise ValueError("embedding_vector required when embedding_model_id set")
+        if len(vector) != entry.dim:
+            raise ValueError(
+                f"vector dim {len(vector)} mismatches registered dim {entry.dim} "
+                f"for model {model_id!r}"
+            )
+
+    @staticmethod
+    def hash_artifact(blob: bytes) -> bytes:
+        """SHA-256 hash helper for callers building model_hash values."""
+        return hashlib.sha256(blob).digest()

hypermind/knowledge/kernel_store.py

@@ -0,0 +1,197 @@
+"""S4 — Cross-sim posterior persistence.
+
+Goal: the swarm gets smarter *across* sims, not just within a run.
+
+Today every ``run_persona_sim`` builds a fresh testbed cohort, and the
+ReputationKernel + BrierFeedback state lives only for that process. That
+makes "the swarm learns" an unverifiable demo claim: each sim is a cold
+start.
+
+KernelStore persists per-(persona_slug, topic) posteriors and a compact
+Brier-tracker summary to ``HYPERMIND_HOME/swarm_memory/<persona>.json``.
+The simlab profile builder hydrates ``working_memory["posteriors"]`` from
+disk at sim start, and BrierFeedback writes back at apply-time. Schema-
+versioned for forward compatibility.
+
+NOT used for cryptographic state — Beta-posterior summaries are
+operationally useful but not authoritative. The transparency log
+remains the only source of truth for claims.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from threading import RLock
+from typing import Any
+
+# v1 schema. Bump only on breaking changes; readers handle older versions.
+SCHEMA_VERSION = 1
+
+
+def _home_dir() -> Path:
+    """Resolve HYPERMIND_HOME/swarm_memory, defaulting to ~/.hypermind."""
+    env = os.environ.get("HYPERMIND_HOME")
+    base = Path(env).expanduser() if env else (Path.home() / ".hypermind")
+    return base / "swarm_memory"
+
+
+@dataclass
+class PersonaKernelState:
+    """A single persona's per-topic memory.
+
+    Persisted as a JSON object keyed by topic. Each topic carries the
+    last posterior probability (``post``), a running Brier mean, the
+    number of resolved oracle outcomes contributing, and ``last_ms``.
+    """
+
+    persona: str
+    posteriors: dict[str, float] = field(default_factory=dict)
+    brier_mean: dict[str, float] = field(default_factory=dict)
+    n_observations: dict[str, int] = field(default_factory=dict)
+    last_ms: int = 0
+    n_sims: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "version": SCHEMA_VERSION,
+            "persona": self.persona,
+            "posteriors": dict(self.posteriors),
+            "brier_mean": dict(self.brier_mean),
+            "n_observations": dict(self.n_observations),
+            "last_ms": int(self.last_ms),
+            "n_sims": int(self.n_sims),
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> PersonaKernelState:
+        # Tolerate older versions: only refuse on completely unknown shapes.
+        v = int(d.get("version", 0) or 0)
+        if v > SCHEMA_VERSION:
+            # Future schema — refuse rather than silently downgrade.
+            raise ValueError(f"unknown KernelStore schema version {v}")
+        return cls(
+            persona=str(d.get("persona", "")),
+            posteriors={str(k): float(v) for k, v in (d.get("posteriors") or {}).items()},
+            brier_mean={str(k): float(v) for k, v in (d.get("brier_mean") or {}).items()},
+            n_observations={str(k): int(v) for k, v in (d.get("n_observations") or {}).items()},
+            last_ms=int(d.get("last_ms") or 0),
+            n_sims=int(d.get("n_sims") or 0),
+        )
+
+
+class KernelStore:
+    """Threadsafe singleton: load/save persona kernel state per persona slug.
+
+    All reads/writes are atomic via tempfile + os.replace. The on-disk
+    location is ``HYPERMIND_HOME/swarm_memory/<persona>.json``. Tests
+    overriding ``HYPERMIND_HOME`` get full isolation for free.
+    """
+
+    _LOCK = RLock()
+
+    @staticmethod
+    def _path_for(persona: str) -> Path:
+        # Persona slugs come from a closed set in personas.py and pass a
+        # filename safety check anyway, but we still sanitise hard.
+        safe = "".join(ch for ch in persona if ch.isalnum() or ch in "-_") or "default"
+        return _home_dir() / f"{safe}.json"
+
+    @classmethod
+    def load(cls, persona: str) -> PersonaKernelState:
+        """Read this persona's state from disk. Returns empty state if no
+        prior file exists or if the file is unreadable."""
+        if not persona:
+            return PersonaKernelState(persona="")
+        p = cls._path_for(persona)
+        with cls._LOCK:
+            if not p.exists():
+                return PersonaKernelState(persona=persona)
+            try:
+                raw = json.loads(p.read_text())
+                state = PersonaKernelState.from_dict(raw)
+                if not state.persona:
+                    state.persona = persona
+                return state
+            except (OSError, ValueError, json.JSONDecodeError):
+                # Corrupted file — log via stderr but never break the sim.
+                return PersonaKernelState(persona=persona)
+
+    @classmethod
+    def save(cls, state: PersonaKernelState) -> None:
+        """Atomically persist state. No-ops on empty persona name."""
+        if not state.persona:
+            return
+        p = cls._path_for(state.persona)
+        with cls._LOCK:
+            p.parent.mkdir(parents=True, exist_ok=True)
+            payload = json.dumps(state.to_dict(), indent=2, sort_keys=True)
+            # Atomic write so a crash mid-write never corrupts the file.
+            fd, tmp = tempfile.mkstemp(prefix=f".{p.name}.", dir=str(p.parent))
+            try:
+                with os.fdopen(fd, "w") as f:
+                    f.write(payload)
+                os.replace(tmp, p)
+            except OSError:
+                try:
+                    os.unlink(tmp)
+                except OSError:
+                    pass
+                raise
+
+    @classmethod
+    def update(
+        cls,
+        persona: str,
+        topic: str,
+        *,
+        posterior: float | None = None,
+        brier_observation: float | None = None,
+        n_sims_increment: int = 0,
+    ) -> PersonaKernelState:
+        """Read-modify-write convenience. Returns the new state."""
+        state = cls.load(persona)
+        if posterior is not None:
+            state.posteriors[topic] = float(posterior)
+        if brier_observation is not None:
+            n = state.n_observations.get(topic, 0)
+            mean = state.brier_mean.get(topic, 0.0)
+            new_n = n + 1
+            new_mean = (mean * n + float(brier_observation)) / new_n
+            state.n_observations[topic] = new_n
+            state.brier_mean[topic] = new_mean
+        if n_sims_increment:
+            state.n_sims += int(n_sims_increment)
+        state.last_ms = int(time.time() * 1000)
+        cls.save(state)
+        return state
+
+    @classmethod
+    def lifetime_summary(cls, persona: str) -> dict[str, Any]:
+        """Read-only summary used by SimList / Home / Detail Lineage UIs."""
+        state = cls.load(persona)
+        n_topics = len(state.brier_mean)
+        if n_topics == 0:
+            mean_brier: float | None = None
+        else:
+            mean_brier = sum(state.brier_mean.values()) / n_topics
+        return {
+            "persona": state.persona,
+            "n_sims": state.n_sims,
+            "n_topics": n_topics,
+            "mean_brier": mean_brier,
+            "last_ms": state.last_ms,
+            "topics": dict(state.brier_mean),
+        }
+
+    @classmethod
+    def list_personas(cls) -> list[str]:
+        """Personas with persisted state in HYPERMIND_HOME."""
+        d = _home_dir()
+        if not d.exists():
+            return []
+        return sorted(p.stem for p in d.glob("*.json"))