hypermind 0.11.0__py3-none-any.whl

hypermind/storage/backend.py

@@ -0,0 +1,71 @@
+"""StorageBackend Protocol — the contract every persistence layer satisfies."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from hypermind.dispute import Dispute
+    from hypermind.wire import Kid, SignedStatement
+
+
+class StorageBackend(Protocol):
+    """Durable storage for a HyperMind world.
+
+    Every method is synchronous — backends should use threadlocal
+    connections and short transactions. Async wrapping happens at the
+    caller (``_NamespaceWorld``) when needed.
+
+    Implementations MUST be safe to call from multiple coroutines on a
+    single event loop. They are NOT required to be safe under multiple
+    OS threads, multiple processes, or concurrent writers.
+    """
+
+    def open(self) -> None:
+        """Connect, create schema if missing, prepare for I/O."""
+
+    def close(self) -> None:
+        """Flush and disconnect. Idempotent."""
+
+    # ---- Claims ---------------------------------------------------------
+
+    def append_claim(self, stmt: SignedStatement) -> None:
+        """Persist one sealed claim. Idempotent on ``stmt.kid``."""
+
+    def load_claims(self) -> list[SignedStatement]:
+        """Return every persisted claim in append order."""
+
+    # ---- Disputes -------------------------------------------------------
+
+    def append_dispute(self, kid: Kid, dispute: Dispute) -> None:
+        """Persist (or upsert) a dispute keyed by claim ``kid``."""
+
+    def load_disputes(self) -> dict[Kid, Dispute]:
+        """Return every persisted dispute as ``{claim_kid: Dispute}``."""
+
+    # ---- Reputation -----------------------------------------------------
+
+    def snapshot_reputation(self, rows: list[tuple[bytes, str, float, float, int]]) -> None:
+        """Persist reputation rows ``(agent_kid, topic, alpha, beta, last_update_ms)``."""
+
+    def load_reputation(self) -> list[tuple[bytes, str, float, float, int]]:
+        """Return every persisted reputation row."""
+
+    # ---- Merkle leaves --------------------------------------------------
+
+    def append_merkle_leaf(self, leaf: bytes) -> int:
+        """Append a Merkle leaf; return its 1-based index in the log."""
+
+    def load_merkle_leaves(self) -> list[bytes]:
+        """Return every persisted Merkle leaf in append order."""
+
+    # ---- FROST transcripts (DURABILITY for nonce-reuse defence) --------
+
+    def remember_transcript(self, transcript_id: bytes) -> bool:
+        """Record a FROST transcript_id. Return True if newly added,
+        False if the transcript was already present (nonce reuse)."""
+
+    # ---- Health ---------------------------------------------------------
+
+    def stats(self) -> dict[str, int]:
+        """Return basic metrics: ``{claims, disputes, reputation_rows, leaves, transcripts}``."""

hypermind/storage/relata.py

@@ -0,0 +1,245 @@
+"""RelataDB-backed StorageBackend — durable ledger over the Relata Python SDK.
+
+Same :class:`~hypermind.storage.backend.StorageBackend` Protocol as
+:class:`~hypermind.storage.sqlite.SqliteBackend`; swaps single-file SQLite for a
+governed, bi-temporal, multi-tenant RelataDB server reached over HTTP / pgwire.
+
+Identity mapping (the spine): **HyperMind namespace → Relata tenant**. Every
+sub-client carries ``X-Organization-Id = namespace_id`` so claims, disputes,
+reputation, and the Merkle log are tenant-isolated. Within the tenant:
+
+  * claim ``kid``          → ``object_id`` of ``HmClaim``   (wire bytes, base64)
+  * dispute (claim ``kid``)→ ``object_id`` of ``HmDispute`` (pickled, base64)
+  * (agent, topic)         → ``object_id`` of ``HmReputation``
+  * Merkle leaf            → Relata's durable WAL-backed ordered log (``LogClient``)
+  * FROST transcript id    → ``object_id`` of ``HmTranscript`` (durable test-and-set)
+
+**Why transcripts use an object row, not ``TokenClient``:** Relata's dedup-token
+store is in-memory today (resets on restart), which fails the replay-defence
+durability contract. The ``HmTranscript`` object row is WAL-durable and safe
+under the single-writer-per-namespace model.
+
+The ``relata`` SDK is imported lazily inside :meth:`open` so this module imports
+cleanly without the optional ``hypermind[relata]`` extra installed. Tests inject
+a ``client_factory`` to run the full round-trip against a fake SDK — no live
+server required.
+
+Run the server on ``RELATA_ORG_MODE=strict`` so type-ownership isolation is
+enforced (row-level ``with_tenant`` filtering applies regardless of mode).
+"""
+
+from __future__ import annotations
+
+import base64
+import pickle
+import time
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from hypermind.dispute import Dispute
+    from hypermind.wire import Kid, SignedStatement
+
+_PURPOSE = "hm_ledger"
+
+# A factory returns the three clients the backend needs, already tenant-scoped:
+#   (query_client, log_client, object_client)
+# ``query_client`` needs ``.query(sql) -> QueryResult`` (with ``.rows``) + ``.close()``.
+ClientFactory = Callable[[], "tuple[Any, Any, Any]"]
+
+
+class RelataBackend:
+    """RelataDB StorageBackend. One instance per (server, namespace)."""
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        namespace_id: str,
+        bearer_token: str | None = None,
+        purpose: str = _PURPOSE,
+        client_factory: ClientFactory | None = None,
+    ) -> None:
+        self._url = base_url
+        self._ns = namespace_id
+        self._tok = bearer_token
+        self._purpose = purpose
+        self._factory = client_factory
+        self._c: Any = None
+        self._log: Any = None
+        self._obj: Any = None
+        self._seq = 1
+        # ponytail: in-memory set gives read-your-writes consistency for the test-and-set;
+        # the durable upsert covers the restart-replay defence.
+        self._seen_transcripts: set[str] = set()
+
+    # ---- Lifecycle ------------------------------------------------------
+
+    def open(self) -> None:
+        if self._c is not None:
+            return
+        if self._factory is not None:
+            self._c, self._log, self._obj = self._factory()
+        else:  # pragma: no cover - requires the optional `relata` SDK + live server
+            from relata import RelataClient
+            from relata.log import LogClient
+            from relata.objects import ObjectClient
+
+            self._c = RelataClient(
+                self._url,
+                bearer_token=self._tok,
+                purpose=self._purpose,
+                tenant=self._ns,  # → X-Organization-Id on every request
+            )
+            # from_client forwards _extra_headers (incl. the tenant header) so
+            # the log + object doors stay tenant-scoped. Do NOT construct these
+            # bare — that would un-tenant the Merkle log.
+            self._log = LogClient.from_client(self._c)
+            self._obj = ObjectClient.from_client(self._c)
+        # Initialise next_seq from the high-water mark so append order survives
+        # restart. Relata's SQL subset rejects COALESCE(MAX(...)); use ORDER BY
+        # DESC LIMIT 1 instead (verified against the live server).
+        rows = self._rows("SELECT seq FROM HmClaim ORDER BY seq DESC LIMIT 1")
+        self._seq = (int(rows[0]["seq"]) + 1) if rows else 1
+
+    def close(self) -> None:
+        for client in (self._log, self._obj, self._c):
+            if client is not None:
+                client.close()
+        self._c = self._log = self._obj = None
+
+    def _require(self) -> Any:
+        if self._c is None:
+            raise RuntimeError("RelataBackend used before open()")
+        return self._c
+
+    def _rows(self, sql: str) -> list[dict[str, Any]]:
+        return list(self._require().query(sql).rows)
+
+    # ---- Claims (immutable, append-ordered) -----------------------------
+
+    def append_claim(self, stmt: SignedStatement) -> None:
+        self._obj.upsert(
+            "HmClaim",
+            bytes(stmt.kid).hex(),
+            {
+                "seq": self._seq,
+                "wire_b64": base64.b64encode(stmt.to_bytes()).decode("ascii"),
+                "created_ms": int(time.time() * 1000),
+            },
+        )
+        self._seq += 1
+
+    def load_claims(self) -> list[SignedStatement]:
+        from hypermind.wire import SignedStatement
+
+        rows = self._rows("SELECT wire_b64, seq FROM HmClaim ORDER BY seq ASC")
+        return [SignedStatement.from_bytes(base64.b64decode(r["wire_b64"])) for r in rows]
+
+    # ---- Disputes (upsert by claim kid) ---------------------------------
+
+    def append_dispute(self, kid: Kid, dispute: Dispute) -> None:
+        payload = pickle.dumps(dispute, protocol=pickle.HIGHEST_PROTOCOL)
+        self._obj.upsert(
+            "HmDispute",
+            bytes(kid).hex(),
+            {
+                "claim_kid": bytes(kid).hex(),
+                "payload_b64": base64.b64encode(payload).decode("ascii"),
+                "updated_ms": int(time.time() * 1000),
+            },
+        )
+
+    def load_disputes(self) -> dict[Kid, Dispute]:
+        out: dict[bytes, Dispute] = {}
+        for r in self._rows("SELECT claim_kid, payload_b64 FROM HmDispute"):
+            try:
+                out[bytes.fromhex(r["claim_kid"])] = pickle.loads(
+                    base64.b64decode(r["payload_b64"])
+                )
+            except Exception:
+                continue
+        return out
+
+    # ---- Reputation (bulk upsert, keyed on agent+topic) -----------------
+
+    def snapshot_reputation(self, rows: list[tuple[bytes, str, float, float, int]]) -> None:
+        if not rows:
+            return
+        self._obj.batch_upsert(
+            "HmReputation",
+            [
+                {
+                    "id": f"{agent.hex()}:{topic}",
+                    "agent_hex": agent.hex(),
+                    "topic": topic,
+                    "alpha": alpha,
+                    "beta": beta,
+                    "last_update_ms": last_ms,
+                }
+                for (agent, topic, alpha, beta, last_ms) in rows
+            ],
+        )
+
+    def load_reputation(self) -> list[tuple[bytes, str, float, float, int]]:
+        rows = self._rows("SELECT agent_hex, topic, alpha, beta, last_update_ms FROM HmReputation")
+        return [
+            (bytes.fromhex(r["agent_hex"]), r["topic"], r["alpha"], r["beta"], r["last_update_ms"])
+            for r in rows
+        ]
+
+    # ---- Merkle leaves → Relata's durable ordered log -------------------
+
+    def append_merkle_leaf(self, leaf: bytes) -> int:
+        index, _commit_ns = self._log.append(leaf)
+        return int(index)
+
+    def load_merkle_leaves(self) -> list[bytes]:
+        # `since` semantics vary (SDK docstring says inclusive; the live server
+        # behaves exclusive), so page defensively: dedup by index, advance to the
+        # max index seen, and stop when a page yields no new leaves. Correct
+        # under either semantics. Ordered by index at the end.
+        seen: dict[int, bytes] = {}
+        since = 0
+        while True:
+            batch = self._log.load_leaves(since=since, limit=1000)
+            new = [lf for lf in batch if lf.index not in seen]
+            if not new:
+                break
+            for lf in new:
+                seen[lf.index] = lf.data
+            since = max(lf.index for lf in batch)
+        return [seen[i] for i in sorted(seen)]
+
+    # ---- FROST transcripts (durable test-and-set) -----------------------
+
+    def remember_transcript(self, transcript_id: bytes) -> bool:
+        """Return True if newly recorded, False on duplicate.
+
+        Single-writer-per-namespace check-then-insert. ``tid`` is hex (0-9a-f)
+        so the WHERE literal is injection-safe.
+        """
+        tid = transcript_id.hex()
+        if tid in self._seen_transcripts:
+            return False
+        # Also check durable store to catch replays across restarts.
+        if self._rows(f"SELECT id FROM HmTranscript WHERE id = '{tid}' LIMIT 1"):
+            self._seen_transcripts.add(tid)
+            return False
+        self._obj.upsert("HmTranscript", tid, {"created_ms": int(time.time() * 1000)})
+        self._seen_transcripts.add(tid)
+        return True
+
+    # ---- Health ---------------------------------------------------------
+
+    def stats(self) -> dict[str, int]:
+        def count(table: str) -> int:
+            return int(self._rows(f"SELECT COUNT(*) AS n FROM {table}")[0]["n"])
+
+        return {
+            "claims": count("HmClaim"),
+            "disputes": count("HmDispute"),
+            "reputation_rows": count("HmReputation"),
+            "leaves": int(self._log.head()),
+            "transcripts": count("HmTranscript"),
+        }

hypermind/storage/sqlite.py

@@ -0,0 +1,258 @@
+"""SQLite-backed StorageBackend — single-file durability, zero extra deps.
+
+Schema (auto-created on ``open()``):
+
+    claims
+      kid BLOB PRIMARY KEY
+      seq INTEGER NOT NULL              -- monotonic append order
+      bytes BLOB NOT NULL                -- canonical CBOR-encoded SignedStatement
+      created_ms INTEGER NOT NULL
+
+    disputes
+      claim_kid BLOB PRIMARY KEY         -- the disputed claim kid
+      payload BLOB NOT NULL              -- pickled Dispute (bond+state+transitions)
+      updated_ms INTEGER NOT NULL
+
+    reputation
+      agent_kid BLOB NOT NULL
+      topic TEXT NOT NULL
+      alpha REAL NOT NULL
+      beta REAL NOT NULL
+      last_update_ms INTEGER NOT NULL
+      PRIMARY KEY (agent_kid, topic)
+
+    merkle_leaves
+      seq INTEGER PRIMARY KEY AUTOINCREMENT
+      leaf BLOB NOT NULL
+
+    frost_transcripts
+      transcript_id BLOB PRIMARY KEY
+      created_ms INTEGER NOT NULL
+
+The ``Dispute`` payload is pickled rather than columnised because the
+state-transition history is variable-length and the field set will grow
+across versions. Pickle is acceptable here — the SQLite file is local-
+process-private; no untrusted dispute payloads cross this boundary.
+"""
+
+from __future__ import annotations
+
+import pickle
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from hypermind.dispute import Dispute
+    from hypermind.wire import Kid, SignedStatement
+
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS claims (
+    kid BLOB PRIMARY KEY,
+    seq INTEGER NOT NULL,
+    bytes BLOB NOT NULL,
+    created_ms INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_claims_seq ON claims(seq);
+
+CREATE TABLE IF NOT EXISTS disputes (
+    claim_kid BLOB PRIMARY KEY,
+    payload BLOB NOT NULL,
+    updated_ms INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS reputation (
+    agent_kid BLOB NOT NULL,
+    topic TEXT NOT NULL,
+    alpha REAL NOT NULL,
+    beta REAL NOT NULL,
+    last_update_ms INTEGER NOT NULL,
+    PRIMARY KEY (agent_kid, topic)
+);
+
+CREATE TABLE IF NOT EXISTS merkle_leaves (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    leaf BLOB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS frost_transcripts (
+    transcript_id BLOB PRIMARY KEY,
+    created_ms INTEGER NOT NULL
+);
+"""
+
+
+class SqliteBackend:
+    """File-backed StorageBackend.
+
+    Construct with a path; pass ``":memory:"`` for an ephemeral test backend
+    (useful when you want the persistence code paths exercised without
+    touching disk).
+    """
+
+    def __init__(self, path: str | Path) -> None:
+        self.path = str(path)
+        self._conn: sqlite3.Connection | None = None
+        self._lock = threading.Lock()
+        self._next_seq = 1
+
+    # ---- Lifecycle ------------------------------------------------------
+
+    def open(self) -> None:
+        if self._conn is not None:
+            return
+        # ``check_same_thread=False`` because asyncio may schedule writes
+        # from different worker threads under uvicorn — we serialise via
+        # ``self._lock`` instead.
+        self._conn = sqlite3.connect(
+            self.path,
+            check_same_thread=False,
+            isolation_level=None,  # autocommit; we manage transactions explicitly
+        )
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA synchronous=NORMAL")
+        self._conn.executescript(_SCHEMA)
+        # Initialise next_seq from the high-water mark so claim ordering
+        # survives restart.
+        row = self._conn.execute("SELECT COALESCE(MAX(seq), 0) FROM claims").fetchone()
+        self._next_seq = (row[0] or 0) + 1
+
+    def close(self) -> None:
+        with self._lock:
+            if self._conn is not None:
+                self._conn.close()
+                self._conn = None
+
+    def _require(self) -> sqlite3.Connection:
+        if self._conn is None:
+            raise RuntimeError("SqliteBackend used before open()")
+        return self._conn
+
+    # ---- Claims ---------------------------------------------------------
+
+    def append_claim(self, stmt: SignedStatement) -> None:
+        kid = bytes(stmt.kid)
+        wire = stmt.to_bytes()
+        now = int(time.time() * 1000)
+        with self._lock:
+            conn = self._require()
+            conn.execute(
+                "INSERT OR IGNORE INTO claims (kid, seq, bytes, created_ms) VALUES (?, ?, ?, ?)",
+                (kid, self._next_seq, wire, now),
+            )
+            self._next_seq += 1
+
+    def load_claims(self) -> list[SignedStatement]:
+        from hypermind.wire import SignedStatement
+
+        with self._lock:
+            conn = self._require()
+            rows = conn.execute("SELECT bytes FROM claims ORDER BY seq ASC").fetchall()
+        return [SignedStatement.from_bytes(bytes(r[0])) for r in rows]
+
+    # ---- Disputes -------------------------------------------------------
+
+    def append_dispute(self, kid: Kid, dispute: Dispute) -> None:
+        payload = pickle.dumps(dispute, protocol=pickle.HIGHEST_PROTOCOL)
+        now = int(time.time() * 1000)
+        with self._lock:
+            conn = self._require()
+            conn.execute(
+                "INSERT INTO disputes (claim_kid, payload, updated_ms) "
+                "VALUES (?, ?, ?) "
+                "ON CONFLICT(claim_kid) DO UPDATE SET "
+                "payload=excluded.payload, updated_ms=excluded.updated_ms",
+                (bytes(kid), payload, now),
+            )
+
+    def load_disputes(self) -> dict[Kid, Dispute]:
+        with self._lock:
+            conn = self._require()
+            rows = conn.execute("SELECT claim_kid, payload FROM disputes").fetchall()
+        out: dict[bytes, Dispute] = {}
+        for kid, payload in rows:
+            try:
+                out[bytes(kid)] = pickle.loads(bytes(payload))
+            except Exception:
+                continue
+        return out
+
+    # ---- Reputation -----------------------------------------------------
+
+    def snapshot_reputation(self, rows: list[tuple[bytes, str, float, float, int]]) -> None:
+        if not rows:
+            return
+        with self._lock:
+            conn = self._require()
+            conn.execute("BEGIN")
+            try:
+                conn.executemany(
+                    "INSERT INTO reputation "
+                    "(agent_kid, topic, alpha, beta, last_update_ms) "
+                    "VALUES (?, ?, ?, ?, ?) "
+                    "ON CONFLICT(agent_kid, topic) DO UPDATE SET "
+                    "alpha=excluded.alpha, beta=excluded.beta, "
+                    "last_update_ms=excluded.last_update_ms",
+                    rows,
+                )
+                conn.execute("COMMIT")
+            except Exception:
+                conn.execute("ROLLBACK")
+                raise
+
+    def load_reputation(self) -> list[tuple[bytes, str, float, float, int]]:
+        with self._lock:
+            conn = self._require()
+            rows = conn.execute(
+                "SELECT agent_kid, topic, alpha, beta, last_update_ms FROM reputation"
+            ).fetchall()
+        return [(bytes(r[0]), r[1], r[2], r[3], r[4]) for r in rows]
+
+    # ---- Merkle leaves --------------------------------------------------
+
+    def append_merkle_leaf(self, leaf: bytes) -> int:
+        with self._lock:
+            conn = self._require()
+            cur = conn.execute("INSERT INTO merkle_leaves (leaf) VALUES (?)", (leaf,))
+            return int(cur.lastrowid or 0)
+
+    def load_merkle_leaves(self) -> list[bytes]:
+        with self._lock:
+            conn = self._require()
+            rows = conn.execute("SELECT leaf FROM merkle_leaves ORDER BY seq ASC").fetchall()
+        return [bytes(r[0]) for r in rows]
+
+    # ---- FROST transcripts ---------------------------------------------
+
+    def remember_transcript(self, transcript_id: bytes) -> bool:
+        """Atomic insert. Returns True if newly recorded, False on duplicate.
+
+        Uses ``INSERT OR IGNORE`` so two callers racing on the same
+        transcript_id always have one survivor — the loser sees False
+        and MUST refuse to release its z_i (FROST nonce-reuse defence).
+        """
+        now = int(time.time() * 1000)
+        with self._lock:
+            conn = self._require()
+            cur = conn.execute(
+                "INSERT OR IGNORE INTO frost_transcripts (transcript_id, created_ms) VALUES (?, ?)",
+                (transcript_id, now),
+            )
+            return cur.rowcount > 0
+
+    # ---- Health ---------------------------------------------------------
+
+    def stats(self) -> dict[str, int]:
+        with self._lock:
+            conn = self._require()
+            rows = {
+                "claims": conn.execute("SELECT COUNT(*) FROM claims").fetchone()[0],
+                "disputes": conn.execute("SELECT COUNT(*) FROM disputes").fetchone()[0],
+                "reputation_rows": conn.execute("SELECT COUNT(*) FROM reputation").fetchone()[0],
+                "leaves": conn.execute("SELECT COUNT(*) FROM merkle_leaves").fetchone()[0],
+                "transcripts": conn.execute("SELECT COUNT(*) FROM frost_transcripts").fetchone()[0],
+            }
+        return rows