hypermind 0.11.0__py3-none-any.whl

hypermind/knowledge/swarm_memory.py

@@ -0,0 +1,523 @@
+"""SwarmMemory — CRDT-backed shared scratchpad for the collective.
+
+Three sub-stores per spec §20.4:
+
+  * **consensus_state**     — per-topic compressed posterior + last HLC.
+  * **lessons_learned**     — append-only (claim_kid, oracle_outcome, rationale).
+  * **swarm_state_vector**  — fixed-dim embedding summarising recent consensus.
+
+CRDT semantics: last-write-wins-with-evidence. Every write carries the
+issuer kid + HLC; reads are local. Conflicts resolve by HLC (then by
+kid as a tiebreaker).
+
+Wire-format-clean: when serialised for gossip, each record rides inside
+a SEAL envelope as ``payload.swarm_record`` — no envelope-schema-hash
+bump, no v1.0 freeze risk.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import warnings
+from collections import deque
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+import cbor2
+
+_log = logging.getLogger(__name__)
+
+SWARM_RECORD_VERSION = 1
+SWARM_VECTOR_MAX_DIM = 1024
+
+
+class SwarmVectorOverflowWarning(UserWarning):
+    """Emitted when topic count exceeds SWARM_VECTOR_MAX_DIM and extra topics are truncated."""
+
+
+# DoS gate: oversize CBOR payloads are rejected before parsing. Swarm
+# records carry vectors up to SWARM_VECTOR_MAX_DIM floats; 1 MiB is a
+# generous ceiling that still bounds parser-amplification attacks.
+MAX_SWARM_RECORD_BYTES = 1024 * 1024
+MAX_LESSONS = 10_000
+
+
+# ---------------------------------------------------------------------------
+# Records
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class ConsensusEntry:
+    topic: str
+    posterior: float
+    n_contributors: int
+    last_hlc_ms: int
+    issuer_kid: bytes
+
+
+@dataclass(frozen=True)
+class LessonEntry:
+    claim_kid: bytes
+    topic: str
+    oracle_outcome: float
+    predicted_consensus: float
+    rationale: str
+    hlc_ms: int
+    issuer_kid: bytes
+
+
+@dataclass(frozen=True)
+class SwarmStateVector:
+    topic: str
+    vector: tuple[float, ...]
+    last_hlc_ms: int
+    issuer_kid: bytes
+
+    def __post_init__(self) -> None:
+        if len(self.vector) > SWARM_VECTOR_MAX_DIM:
+            raise ValueError(
+                f"swarm_state_vector dim {len(self.vector)} > max {SWARM_VECTOR_MAX_DIM}"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Memory
+# ---------------------------------------------------------------------------
+
+
+class SwarmMemory:
+    """Per-namespace shared memory. In-process LWW CRDT for v0.9b.
+
+    T4-04 (cross-namespace federation): a SwarmMemory may :meth:`federate`
+    with a remote SwarmMemory through a named bridge_kid. Bridge-mediated
+    merges record ``provenance_namespace`` on every fork-log entry so
+    auditors can trace where a conflict originated.
+
+    Per the §20 invariant (also enforced by ``mind/federation.assert_never_merge``)
+    foreign reputation/state is NEVER silently merged into the home
+    kernel — it is exposed via :meth:`foreign_consensus` for display only.
+    """
+
+    def __init__(self, namespace_id: bytes | None = None) -> None:
+        self.namespace_id: bytes | None = bytes(namespace_id) if namespace_id else None
+        self._consensus: dict[str, ConsensusEntry] = {}
+        self._lessons: list[LessonEntry] = []
+        self._vectors: dict[str, SwarmStateVector] = {}
+        # Fork log: records (hlc, kid, winner_hash) for each conflict that
+        # was resolved by deterministic tiebreak rather than crashing.
+        # Bounded at 100 entries — older conflicts are evicted.
+        self.fork_log: deque[dict] = deque(maxlen=100)
+        # Cross-namespace bridges: remote_namespace_id_hex -> bridge_kid.
+        self._bridges: dict[str, bytes] = {}
+        # Foreign read-through cache populated on bridge merge; never
+        # writes into self._consensus to preserve never-merge invariant.
+        self._foreign_consensus: dict[tuple[str, str], ConsensusEntry] = {}
+
+    # ---- consensus_state ---------------------------------------------------
+
+    def upsert_consensus(self, entry: ConsensusEntry) -> bool:
+        """Apply LWW write. Returns True if accepted.
+
+        Tiebreak rules at identical HLC:
+          * smaller ``issuer_kid`` loses (lex compare, strict ``<``);
+          * equal ``issuer_kid`` requires byte-identical payload — a
+            divergent payload at the same (hlc, kid) is a CRDT protocol
+            violation and raises ``ValueError``. Same-(hlc, kid) with
+            identical payload is an idempotent no-op (returns True).
+        """
+        existing = self._consensus.get(entry.topic)
+        if existing is not None:
+            if entry.last_hlc_ms < existing.last_hlc_ms:
+                return False
+            if entry.last_hlc_ms == existing.last_hlc_ms:
+                if entry.issuer_kid == existing.issuer_kid:
+                    if entry == existing:
+                        return True
+                    # Conflict: same (hlc, kid) with different payload.
+                    # Log the fork and apply deterministic tiebreak: keep
+                    # the entry whose CBOR encoding has the lexicographically
+                    # smaller SHA-256 hash (stable, reproducible on every peer).
+                    import cbor2 as _cbor2
+
+                    h_existing = hashlib.sha256(
+                        _cbor2.dumps(
+                            existing.issuer_kid
+                            + existing.last_hlc_ms.to_bytes(8, "big")
+                            + str(existing.posterior).encode(),
+                            canonical=True,
+                        )
+                    ).digest()
+                    h_entry = hashlib.sha256(
+                        _cbor2.dumps(
+                            entry.issuer_kid
+                            + entry.last_hlc_ms.to_bytes(8, "big")
+                            + str(entry.posterior).encode(),
+                            canonical=True,
+                        )
+                    ).digest()
+                    self.fork_log.append(
+                        {
+                            "store": "consensus",
+                            "topic": entry.topic,
+                            "hlc": entry.last_hlc_ms,
+                            "kid": entry.issuer_kid,
+                            "winner_hash": min(h_existing, h_entry).hex(),
+                        }
+                    )
+                    _log.warning(
+                        "SwarmMemory fork: consensus (hlc=%s, kid=%s) has conflicting payload; "
+                        "applying deterministic tiebreak",
+                        entry.last_hlc_ms,
+                        entry.issuer_kid[:4].hex(),
+                    )
+                    if h_entry < h_existing:
+                        self._consensus[entry.topic] = entry
+                        return True
+                    return False
+                if entry.issuer_kid < existing.issuer_kid:
+                    return False
+        self._consensus[entry.topic] = entry
+        return True
+
+    def consensus(self, topic: str) -> ConsensusEntry | None:
+        return self._consensus.get(topic)
+
+    def all_consensus(self) -> list[ConsensusEntry]:
+        return sorted(self._consensus.values(), key=lambda c: c.topic)
+
+    # ---- lessons_learned ---------------------------------------------------
+
+    def append_lesson(self, lesson: LessonEntry) -> None:
+        self._lessons.append(lesson)
+        if len(self._lessons) > MAX_LESSONS:
+            self._lessons = self._lessons[-MAX_LESSONS:]
+
+    def lessons_for(self, topic: str | None = None) -> list[LessonEntry]:
+        if topic is None:
+            return list(self._lessons)
+        return [lesson for lesson in self._lessons if lesson.topic == topic]
+
+    def lesson_count(self) -> int:
+        return len(self._lessons)
+
+    # ---- swarm_state_vector ------------------------------------------------
+
+    def upsert_vector(self, vec: SwarmStateVector) -> bool:
+        existing = self._vectors.get(vec.topic)
+        if existing is not None:
+            if vec.last_hlc_ms < existing.last_hlc_ms:
+                return False
+            if vec.last_hlc_ms == existing.last_hlc_ms:
+                if vec.issuer_kid == existing.issuer_kid:
+                    if vec == existing:
+                        return True
+                    # Conflict: same (hlc, kid) with different payload.
+                    import cbor2 as _cbor2
+
+                    h_existing = hashlib.sha256(
+                        _cbor2.dumps(
+                            existing.issuer_kid
+                            + existing.last_hlc_ms.to_bytes(8, "big")
+                            + str(existing.vector).encode(),
+                            canonical=True,
+                        )
+                    ).digest()
+                    h_vec = hashlib.sha256(
+                        _cbor2.dumps(
+                            vec.issuer_kid
+                            + vec.last_hlc_ms.to_bytes(8, "big")
+                            + str(vec.vector).encode(),
+                            canonical=True,
+                        )
+                    ).digest()
+                    self.fork_log.append(
+                        {
+                            "store": "vectors",
+                            "topic": vec.topic,
+                            "hlc": vec.last_hlc_ms,
+                            "kid": vec.issuer_kid,
+                            "winner_hash": min(h_existing, h_vec).hex(),
+                        }
+                    )
+                    _log.warning(
+                        "SwarmMemory fork: vector (hlc=%s, kid=%s) has conflicting payload; "
+                        "applying deterministic tiebreak",
+                        vec.last_hlc_ms,
+                        vec.issuer_kid[:4].hex(),
+                    )
+                    if h_vec < h_existing:
+                        self._vectors[vec.topic] = vec
+                        return True
+                    return False
+                if vec.issuer_kid < existing.issuer_kid:
+                    return False
+        self._vectors[vec.topic] = vec
+        return True
+
+    def vector(self, topic: str) -> SwarmStateVector | None:
+        return self._vectors.get(topic)
+
+    # ---- bulk ops ----------------------------------------------------------
+
+    # ---- T4-04: cross-namespace federation --------------------------------
+
+    def federate(self, remote_namespace_id: bytes, bridge_kid: bytes) -> None:
+        """Establish a bridge to another namespace's SwarmMemory.
+
+        ``bridge_kid`` is the kid of the bridge agent authorised to gossip
+        across the boundary. Only one bridge per remote namespace; calling
+        :meth:`federate` again replaces the existing bridge (rotation).
+        """
+        ns_hex = bytes(remote_namespace_id).hex()
+        self._bridges[ns_hex] = bytes(bridge_kid)
+
+    def bridges(self) -> dict[str, bytes]:
+        return dict(self._bridges)
+
+    def merge_from_bridge(
+        self,
+        other: SwarmMemory,
+        *,
+        remote_namespace_id: bytes,
+        bridge_kid: bytes,
+    ) -> dict[str, int]:
+        """Bridge-mediated cross-namespace merge.
+
+        Foreign consensus entries are written into a *separate*
+        ``_foreign_consensus`` map (read-only via :meth:`foreign_consensus`)
+        so the home reputation/state remains untouched. Conflicts produce
+        fork-log entries enriched with ``provenance_namespace``.
+        """
+        ns_hex = bytes(remote_namespace_id).hex()
+        if self._bridges.get(ns_hex) != bytes(bridge_kid):
+            raise ValueError(
+                f"merge_from_bridge: bridge_kid does not match registered "
+                f"bridge for namespace {ns_hex[:16]}…"
+            )
+        applied = {"foreign_consensus": 0, "lessons": 0}
+        # Foreign consensus: never merged into home; always displayed.
+        for c in other.all_consensus():
+            key = (ns_hex, c.topic)
+            existing = self._foreign_consensus.get(key)
+            if existing is None or existing.last_hlc_ms < c.last_hlc_ms:
+                self._foreign_consensus[key] = c
+                applied["foreign_consensus"] += 1
+            elif existing.last_hlc_ms == c.last_hlc_ms and existing != c:
+                # Cross-namespace conflict at identical HLC — fork-log it.
+                self.fork_log.append(
+                    {
+                        "store": "foreign_consensus",
+                        "topic": c.topic,
+                        "hlc": c.last_hlc_ms,
+                        "kid": c.issuer_kid,
+                        "provenance_namespace": ns_hex,
+                        "reason": "CROSS-NS-CONFLICT",
+                    }
+                )
+        # Lessons may cross — they are append-only and globally addressed
+        # by claim_kid; no risk of silent reputation merge.
+        existing_kids = {lesson.claim_kid for lesson in self._lessons}
+        for lesson in other.lessons_for():
+            if lesson.claim_kid not in existing_kids:
+                self._lessons.append(lesson)
+                existing_kids.add(lesson.claim_kid)
+                applied["lessons"] += 1
+        return applied
+
+    def foreign_consensus(
+        self, *, remote_namespace_id: bytes | None = None, topic: str | None = None
+    ) -> list[tuple[str, ConsensusEntry]]:
+        """Return foreign consensus entries; ``[(ns_hex, entry), …]``.
+
+        Filtered by namespace and/or topic when provided. This is the
+        ONLY read path for foreign state — it is intentionally separate
+        from :meth:`consensus` to make the never-merge invariant
+        structural rather than aspirational.
+        """
+        ns_filter = bytes(remote_namespace_id).hex() if remote_namespace_id else None
+        out: list[tuple[str, ConsensusEntry]] = []
+        for (ns_hex, tpc), entry in self._foreign_consensus.items():
+            if ns_filter is not None and ns_hex != ns_filter:
+                continue
+            if topic is not None and tpc != topic:
+                continue
+            out.append((ns_hex, entry))
+        return out
+
+    def merge(self, other: SwarmMemory) -> dict[str, int]:
+        """LWW-merge another memory's records. Returns counters per sub-store.
+
+        Counts only *state-changing* applications — an idempotent
+        re-application of a byte-identical entry at the same (hlc, kid)
+        does not increment the counter.
+        """
+        applied = {"consensus": 0, "lessons": 0, "vectors": 0}
+        for c in other.all_consensus():
+            before = self._consensus.get(c.topic)
+            if self.upsert_consensus(c) and self._consensus.get(c.topic) is not before:
+                applied["consensus"] += 1
+        # Lessons are append-only with idempotent dedup by claim_kid only.
+        # claim_kid is a 32-byte SHA-256 hash, globally unique per claim per spec,
+        # so deduping by claim_kid alone correctly drops re-gossip duplicates while
+        # still accepting valid lessons from different issuers on the same topic.
+        existing = {lesson.claim_kid for lesson in self._lessons}
+        for lesson in other.lessons_for():
+            if lesson.claim_kid not in existing:
+                self._lessons.append(lesson)
+                existing.add(lesson.claim_kid)
+                applied["lessons"] += 1
+        for v in other._vectors.values():
+            before_v = self._vectors.get(v.topic)
+            if self.upsert_vector(v) and self._vectors.get(v.topic) is not before_v:
+                applied["vectors"] += 1
+        return applied
+
+
+# ---------------------------------------------------------------------------
+# Wire-clean encoding (rides inside SEAL payload)
+# ---------------------------------------------------------------------------
+
+_KEY_KIND = 0
+_KEY_BODY = 1
+_KEY_VERSION = 2
+
+
+def encode_swarm_record(
+    record: ConsensusEntry | LessonEntry | SwarmStateVector,
+) -> bytes:
+    if isinstance(record, ConsensusEntry):
+        body = {
+            "topic": record.topic,
+            "posterior": record.posterior,
+            "n_contributors": record.n_contributors,
+            "last_hlc_ms": record.last_hlc_ms,
+            "issuer_kid": record.issuer_kid,
+        }
+        kind = "consensus"
+    elif isinstance(record, LessonEntry):
+        body = {
+            "claim_kid": record.claim_kid,
+            "topic": record.topic,
+            "oracle_outcome": record.oracle_outcome,
+            "predicted_consensus": record.predicted_consensus,
+            "rationale": record.rationale,
+            "hlc_ms": record.hlc_ms,
+            "issuer_kid": record.issuer_kid,
+        }
+        kind = "lesson"
+    elif isinstance(record, SwarmStateVector):
+        body = {
+            "topic": record.topic,
+            "vector": list(record.vector),
+            "last_hlc_ms": record.last_hlc_ms,
+            "issuer_kid": record.issuer_kid,
+        }
+        kind = "vector"
+    else:
+        raise TypeError(f"unknown swarm record: {type(record).__name__}")
+    return cbor2.dumps(
+        {_KEY_KIND: kind, _KEY_BODY: body, _KEY_VERSION: SWARM_RECORD_VERSION},
+        canonical=True,
+    )
+
+
+def decode_swarm_record(
+    raw: bytes,
+    *,
+    expected_signer_kid: bytes,
+) -> ConsensusEntry | LessonEntry | SwarmStateVector:
+    """Decode a swarm record envelope.
+
+    The ``expected_signer_kid`` is the kid of the outer SignedStatement
+    signer; it MUST match the embedded ``issuer_kid`` in the record body
+    to prevent a kid spoofing attack where a valid signer wraps a payload
+    that falsely attributes authorship to another agent.
+
+    Raises ``ValueError`` on mismatch, oversize input, or malformed CBOR.
+    """
+    if len(raw) > MAX_SWARM_RECORD_BYTES:
+        raise ValueError(f"swarm record too large: {len(raw)} > {MAX_SWARM_RECORD_BYTES}")
+    try:
+        env = cbor2.loads(raw)
+    except Exception as e:
+        raise ValueError(f"swarm record decode: not CBOR: {e}") from e
+    if not isinstance(env, dict):
+        raise ValueError("swarm record decode: not a map")
+    if env.get(_KEY_VERSION) != SWARM_RECORD_VERSION:
+        raise ValueError(f"swarm record version: {env.get(_KEY_VERSION)!r}")
+    kind = env.get(_KEY_KIND)
+    body = env.get(_KEY_BODY) or {}
+    embedded_kid = bytes(body.get("issuer_kid", b""))
+    if embedded_kid != bytes(expected_signer_kid):
+        raise ValueError(
+            "issuer_kid mismatch: payload claims "
+            f"{embedded_kid[:8].hex()}, signer is "
+            f"{bytes(expected_signer_kid)[:8].hex()}"
+        )
+    if kind == "consensus":
+        return ConsensusEntry(
+            topic=str(body["topic"]),
+            posterior=float(body["posterior"]),
+            n_contributors=int(body["n_contributors"]),
+            last_hlc_ms=int(body["last_hlc_ms"]),
+            issuer_kid=embedded_kid,
+        )
+    if kind == "lesson":
+        return LessonEntry(
+            claim_kid=bytes(body["claim_kid"]),
+            topic=str(body["topic"]),
+            oracle_outcome=float(body["oracle_outcome"]),
+            predicted_consensus=float(body["predicted_consensus"]),
+            rationale=str(body.get("rationale", "")),
+            hlc_ms=int(body["hlc_ms"]),
+            issuer_kid=embedded_kid,
+        )
+    if kind == "vector":
+        return SwarmStateVector(
+            topic=str(body["topic"]),
+            vector=tuple(float(x) for x in body["vector"]),
+            last_hlc_ms=int(body["last_hlc_ms"]),
+            issuer_kid=embedded_kid,
+        )
+    raise ValueError(f"swarm record decode: unknown kind {kind!r}")
+
+
+# ---------------------------------------------------------------------------
+# Embedding helper
+# ---------------------------------------------------------------------------
+
+
+def compress_consensus_to_vector(
+    entries: Iterable[ConsensusEntry],
+    *,
+    dim: int = 32,
+) -> tuple[float, ...]:
+    """Project a set of per-topic posteriors into a fixed-dim vector.
+
+    Stable ordering by topic name; unused dimensions zero-padded.
+    Bounded at SWARM_VECTOR_MAX_DIM. Pure deterministic.
+
+    When the number of topics exceeds ``dim`` (or when ``dim`` equals
+    ``SWARM_VECTOR_MAX_DIM`` and the topic count exceeds it), a
+    :class:`SwarmVectorOverflowWarning` is emitted so callers are aware
+    that extra topics are silently truncated.
+    """
+    if dim > SWARM_VECTOR_MAX_DIM:
+        raise ValueError(f"dim {dim} > SWARM_VECTOR_MAX_DIM {SWARM_VECTOR_MAX_DIM}")
+    rows = sorted(entries, key=lambda e: e.topic)
+    n = len(rows)
+    if n > dim:
+        warnings.warn(
+            f"SwarmMemory: topic count {n} exceeds SWARM_VECTOR_MAX_DIM={SWARM_VECTOR_MAX_DIM};"
+            " extra topics truncated",
+            SwarmVectorOverflowWarning,
+            stacklevel=2,
+        )
+    out = [0.0] * dim
+    for i, e in enumerate(rows):
+        if i >= dim:
+            break
+        out[i] = float(e.posterior)
+    return tuple(out)