hypermind 0.11.0__py3-none-any.whl

hypermind/knowledge/transparency.py

@@ -0,0 +1,409 @@
+"""Transparency service + witness cosigning — real RFC 6962 Merkle.
+
+Backed by `pymerkle.InmemoryTree` which produces standard RFC 6962
+inclusion proofs. A receipt issuer commits to the current tree head
+(STH = signed tree head); witnesses cosign the STH; verifiers can
+later re-derive the leaf hash and validate the inclusion path against
+the cosigned root.
+
+Production deployments swap this for a CCF-backed SCITT TS or a
+sigsum-tlog instance; the agent code does not change.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import warnings
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from typing import Any
+
+from pymerkle import InmemoryTree
+from pymerkle import verify_inclusion as _pm_verify_inclusion
+
+from hypermind.crypto.domain import DomainTag
+from hypermind.crypto.hashing import sha256_with_tag
+from hypermind.crypto.signing import Keypair
+from hypermind.errors import ConformanceError
+
+
+@dataclass(frozen=True)
+class WitnessCosignature:
+    witness_kid: bytes
+    sth_root: bytes
+    tree_size: int
+    signature: bytes
+
+
+@dataclass(frozen=True)
+class PortableInclusionProof:
+    """RFC 6962 §2.1.1 inclusion proof — portable to any verifier.
+
+    Unlike `InclusionProof.proof_obj` (which carries a live pymerkle
+    `MerkleProof` instance), this representation is wire-portable: a
+    plain `(audit_path, leaf_index, tree_size)` tuple that any RFC 6962
+    verifier in any language can consume.
+
+    audit_path: list of 32-byte sibling hashes from leaf toward root.
+    leaf_index: 0-based leaf position in the tree.
+    tree_size: total leaf count at proof time.
+    """
+
+    audit_path: tuple[bytes, ...]
+    leaf_index: int
+    tree_size: int
+
+    def to_cbor(self) -> bytes:
+        import cbor2
+
+        return cbor2.dumps(
+            [list(self.audit_path), self.leaf_index, self.tree_size],
+            canonical=True,
+        )
+
+    @classmethod
+    def from_cbor(cls, data: bytes) -> PortableInclusionProof:
+        import cbor2
+
+        path, idx, size = cbor2.loads(data)
+        return cls(tuple(path), int(idx), int(size))
+
+
+def _hash_internal(left: bytes, right: bytes) -> bytes:
+    """RFC 6962 §2.1 internal-node hash: H(0x01 || left || right)."""
+    return hashlib.sha256(b"\x01" + left + right).digest()
+
+
+def verify_inclusion(leaf_hash: bytes, proof: PortableInclusionProof, root: bytes) -> bool:
+    """RFC 6962 §2.1.1 inclusion-proof verification — portable.
+
+    Implements the spec's reference algorithm verbatim so any third
+    party can re-implement against the wire bytes. Returns True iff the
+    audit path lifts `leaf_hash` to `root`.
+    """
+    if proof.tree_size <= 0 or proof.leaf_index < 0:
+        return False
+    if proof.leaf_index >= proof.tree_size:
+        return False
+
+    fn = proof.leaf_index
+    sn = proof.tree_size - 1
+    r = leaf_hash
+    for sibling in proof.audit_path:
+        if sn == 0:
+            # Path longer than the tree can justify.
+            return False
+        if fn % 2 == 1 or fn == sn:
+            r = _hash_internal(sibling, r)
+            while not (fn % 2 == 1 or fn == 0):
+                fn >>= 1
+                sn >>= 1
+        else:
+            r = _hash_internal(r, sibling)
+        fn >>= 1
+        sn >>= 1
+    return sn == 0 and r == root
+
+
+@dataclass(frozen=True)
+class InclusionProof:
+    """RFC 6962-shape inclusion proof.
+
+    Carries pymerkle's MerkleProof object plus the leaf hash and STH
+    root needed for offline verification. `tree_size` is the size at
+    the time the proof was generated; verifiers MUST cross-check that
+    the cosigned STH covers that exact size to detect equivocation.
+
+    `proof_obj` is an opaque pymerkle handle and is NOT portable across
+    implementations. Use `to_portable()` to obtain a wire-portable
+    `PortableInclusionProof` (RFC 6962 audit path + leaf index + tree
+    size) that any RFC 6962 verifier can consume.
+    """
+
+    leaf_index: int
+    leaf: bytes
+    leaf_hash: bytes  # pymerkle's hashed-leaf form (hash(0x00 || leaf))
+    sth_root: bytes
+    tree_size: int
+    proof_obj: Any  # pymerkle.MerkleProof — opaque, used by verifier
+
+    def to_portable(self) -> PortableInclusionProof:
+        """Convert pymerkle's proof object to the RFC 6962 portable form.
+
+        pymerkle's `path[0]` is the leaf hash itself; the remaining
+        entries are sibling hashes from leaf toward root (which is
+        exactly the RFC 6962 audit-path order).
+        """
+        pm_path = list(self.proof_obj.path)
+        # pymerkle path = [leaf_hash, sib_0, sib_1, ...]; siblings only.
+        audit_path = tuple(pm_path[1:]) if pm_path else ()
+        return PortableInclusionProof(
+            audit_path=audit_path,
+            leaf_index=self.leaf_index,
+            tree_size=self.tree_size,
+        )
+
+
+@dataclass
+class TransparencyService:
+    """In-process SCITT TS with real RFC 6962 Merkle tree."""
+
+    name: str = "hypermind-ts"
+    witness_keypairs: list[Keypair] = field(default_factory=list)
+    min_witness_quorum: int = 1
+    _tree: InmemoryTree = field(default_factory=lambda: InmemoryTree(algorithm="sha256"))
+    _leaf_index: dict[bytes, int] = field(default_factory=dict)
+    pending_reconciliation: bool = False
+    _last_seen_remote_size: int = 0
+    _gossip_events: list[dict[str, Any]] = field(default_factory=list)
+    _background_tasks: set[asyncio.Task[Any]] = field(default_factory=set)
+
+    async def gossip_sth(self, bus: Any, namespace_id: str) -> bytes:
+        """Publish (root, tree_size, cosigs) to /hm/{ns}/sth.
+
+        Serialised as canonical CBOR with tag ``STH_CROSS_GOSSIP`` and
+        signed by every configured witness. Returns the published bytes
+        (test hook). Silently no-ops when no witnesses are configured —
+        an unsigned STH offers no security value.
+        """
+        import cbor2
+
+        if not self.witness_keypairs:
+            return b""
+        root = self.current_root()
+        size = self.size()
+        cosigs = self._do_cosign(root, tree_size=size)
+        payload = {
+            "tag": "STH_CROSS_GOSSIP",
+            "root": root,
+            "tree_size": size,
+            "cosigs": [{"witness_kid": c.witness_kid, "signature": c.signature} for c in cosigs],
+        }
+        msg = cbor2.dumps(payload, canonical=True)
+        await bus.publish(f"/hm/{namespace_id}/sth", msg)
+        return msg
+
+    def receive_sth(self, sth_msg: bytes, sender_kid: bytes) -> bool:
+        """Verify a gossiped STH; flag pending_reconciliation if larger.
+
+        Returns True iff at least one cosignature verified AND the
+        received tree_size exceeds our local size.
+        """
+        import cbor2
+
+        try:
+            payload = cbor2.loads(sth_msg)
+        except Exception:
+            return False
+        if not isinstance(payload, dict) or payload.get("tag") != "STH_CROSS_GOSSIP":
+            return False
+        root = payload.get("root")
+        size = payload.get("tree_size")
+        cosigs = payload.get("cosigs") or []
+        if not isinstance(root, (bytes, bytearray)) or not isinstance(size, int):
+            return False
+        verified_count = 0
+        for entry in cosigs:
+            cosig = WitnessCosignature(
+                witness_kid=bytes(entry["witness_kid"]),
+                sth_root=bytes(root),
+                tree_size=int(size),
+                signature=bytes(entry["signature"]),
+            )
+            if self.verify_cosignature(cosig):
+                verified_count += 1
+        if verified_count < max(1, self.min_witness_quorum):
+            return False
+        self._last_seen_remote_size = max(self._last_seen_remote_size, int(size))
+        if int(size) > self.size():
+            self.pending_reconciliation = True
+            self._gossip_events.append(
+                {
+                    "event": "sth_remote_ahead",
+                    "remote_size": int(size),
+                    "local_size": self.size(),
+                    "sender_kid": sender_kid,
+                }
+            )
+            return True
+        return False
+
+    def append(self, statement_kid: bytes) -> InclusionProof:
+        """Append a statement kid as a Merkle leaf and return inclusion proof."""
+        if len(statement_kid) != 32:
+            raise ConformanceError("statement_kid must be 32 bytes")
+        # RFC 6962 §2.1: leaves are hashed with prefix 0x00 to prevent
+        # second-preimage attacks (collision between leaf and inner-node).
+        # pymerkle's InmemoryTree applies the same prefix internally when
+        # `disable_security=False` (default).
+        self._tree.append_entry(statement_kid)
+        size = self._tree.get_size()
+        leaf_idx = size - 1
+        self._leaf_index[statement_kid] = leaf_idx
+        proof = self._tree.prove_inclusion(leaf_idx + 1, size)
+        # pymerkle hashes leaves with the RFC 6962 0x00 prefix internally.
+        leaf_hash = hashlib.sha256(b"\x00" + statement_kid).digest()
+        return InclusionProof(
+            leaf_index=leaf_idx,
+            leaf=statement_kid,
+            leaf_hash=leaf_hash,
+            sth_root=bytes(self._tree.get_state()),
+            tree_size=size,
+            proof_obj=proof,
+        )
+
+    def witness_cosign(
+        self,
+        sth_root: bytes,
+        *,
+        tree_size: int | None = None,
+        witnesses: Iterable[Keypair] | None = None,
+    ) -> list[WitnessCosignature]:
+        """k-of-n witness cosignatures over the (root, size) pair.
+
+        Binding `tree_size` into the signed payload is the equivocation
+        defence: a witness cannot sign two roots with the same size.
+
+        IMPORTANT (closes self-heal Cycle-1 Finding #6): callers MUST
+        pass the `tree_size` that pairs with `sth_root` (use
+        `proof.tree_size` from the InclusionProof returned by
+        `append()`). If `tree_size` is omitted we read the live tree
+        size, which is correct only when no concurrent appends can
+        occur — that's a hard precondition we cannot enforce, so the
+        caller is responsible.
+
+        .. deprecated::
+            Use ``append_and_cosign()`` instead. Calling this method
+            directly is a footgun if any append occurs between the
+            ``append()`` call and this call, because the tree size will
+            no longer pair with the root you're signing.
+        """
+        warnings.warn(
+            "witness_cosign() is deprecated and will be removed in v1.0. "
+            "Use append_and_cosign() to atomically append and cosign. "
+            "Calling witness_cosign() standalone is unsafe under concurrent appends.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._do_cosign(sth_root, tree_size=tree_size, witnesses=witnesses)
+
+    def _do_cosign(
+        self,
+        sth_root: bytes,
+        *,
+        tree_size: int | None = None,
+        witnesses: Iterable[Keypair] | None = None,
+    ) -> list[WitnessCosignature]:
+        """Internal cosign — same as :meth:`witness_cosign` but no
+        deprecation warning. Used by :meth:`append_and_cosign`."""
+        chosen = list(witnesses) if witnesses is not None else self.witness_keypairs
+        if len(chosen) < self.min_witness_quorum:
+            raise ConformanceError(
+                f"witness quorum {self.min_witness_quorum} not reachable (have {len(chosen)})"
+            )
+        size = tree_size if tree_size is not None else self._tree.get_size()
+        cosigs: list[WitnessCosignature] = []
+        for w in chosen:
+            preimage = _witness_signing_input(sth_root, size, w.kid)
+            cosigs.append(
+                WitnessCosignature(
+                    witness_kid=w.kid,
+                    sth_root=sth_root,
+                    tree_size=size,
+                    signature=w.sign(preimage),
+                )
+            )
+        return cosigs
+
+    def append_and_cosign(
+        self,
+        statement_kid: bytes,
+        *,
+        witnesses: Iterable[Keypair] | None = None,
+    ) -> tuple[InclusionProof, list[WitnessCosignature]]:
+        """Atomic append + cosign. Use this whenever concurrent appends
+        are possible — guarantees the cosignatures bind the size paired
+        with the appended leaf's root."""
+        proof = self.append(statement_kid)
+        cosigs = self._do_cosign(proof.sth_root, tree_size=proof.tree_size, witnesses=witnesses)
+        bus = getattr(self, "_gossip_bus", None)
+        ns = getattr(self, "_gossip_namespace", None)
+        if bus is not None and ns is not None:
+            try:
+                loop = asyncio.get_running_loop()
+                task = loop.create_task(self.gossip_sth(bus, ns))
+                self._background_tasks.add(task)
+                task.add_done_callback(self._background_tasks.discard)
+            except RuntimeError:
+                pass
+        return proof, cosigs
+
+    def attach_gossip(self, bus: Any, namespace_id: str) -> None:
+        """Wire a bus + namespace so subsequent append_and_cosign() fires
+        gossip_sth() automatically. No-op when no event loop is running."""
+        self._gossip_bus = bus
+        self._gossip_namespace = namespace_id
+
+    def current_root(self) -> bytes:
+        return bytes(self._tree.get_state())
+
+    def size(self) -> int:
+        return self._tree.get_size()
+
+    def verify_inclusion(self, proof: InclusionProof, *, expected_kid: bytes | None = None) -> bool:
+        """Verify via pymerkle's RFC 6962 inclusion-proof verifier."""
+        if expected_kid is not None and proof.leaf != expected_kid:
+            return False
+        try:
+            _pm_verify_inclusion(proof.leaf_hash, proof.sth_root, proof.proof_obj)
+            return True
+        except Exception:
+            return False
+
+    def verify_cosignature(self, cosig: WitnessCosignature) -> bool:
+        """Verify a witness cosignature over (root, size, witness_kid).
+
+        Handles both classical Ed25519 (64-byte sig) and composite
+        ML-DSA-65 + Ed25519 sigs (LAMPS §6 — first 64 bytes are
+        Ed25519 over the LAMPS preimage). witness_kid is always the
+        Ed25519 half (kept stable through alg rotation by design),
+        so Ed25519 verification of the prefix binds the witness's
+        attestation. PQ-side verify is future hardening.
+
+        Used by I6 receive-side cosig verification on subscribe.
+        """
+        from hypermind.crypto.signing import verify_ed25519
+
+        sig = cosig.signature
+        if not sig:
+            return False
+        preimage = _witness_signing_input(
+            cosig.sth_root,
+            cosig.tree_size,
+            cosig.witness_kid,
+        )
+        if len(sig) == 64:
+            return verify_ed25519(cosig.witness_kid, sig, preimage)
+        # Composite path: Ed25519 prefix signed over LAMPS preimage.
+        try:
+            from hypermind.crypto.signing import _lamps_preimage  # type: ignore
+
+            return verify_ed25519(
+                cosig.witness_kid,
+                sig[:64],
+                _lamps_preimage(preimage),
+            )
+        except Exception:
+            return False
+
+
+def _witness_signing_input(sth_root: bytes, tree_size: int, witness_kid: bytes) -> bytes:
+    """Domain-separated input for witness STH signature.
+
+    Binds `(root, size, witness_kid)` so a witness cannot equivocate by
+    signing two different roots at the same size, and cross-witness
+    forwarding cannot fool a verifier.
+    """
+    payload = sth_root + tree_size.to_bytes(8, "big") + witness_kid
+    return sha256_with_tag(DomainTag.STH_LEAF, payload)

hypermind/mcp/__init__.py

@@ -0,0 +1,60 @@
+"""MCP integration — JSON-RPC 2.0 transport, server, client, and the
+legacy verb-bridge module.
+
+v0.10.1 ships the full transport stack (T1):
+
+    MCPServer / MCPClient    — JSON-RPC 2.0 over stdio (and WebSocket
+                               when the optional `websockets` extra is
+                               installed; deferred to v0.10.2 otherwise).
+    StdioTransport           — line-delimited JSON over asyncio streams.
+    Provenance               — T1-03 CapToken-bound tool-call attestation.
+
+The earlier verb-bridge classes (``HyperMindMCPServer``,
+``sealed_mcp_client``) remain importable from ``hypermind.mcp.bridge``
+and from this package, so existing user code continues to work.
+"""
+
+from hypermind.mcp.bridge import HyperMindMCPServer, sealed_mcp_client
+from hypermind.mcp.client import MCPClient, MCPRpcError
+from hypermind.mcp.provenance import (
+    ERR_PROVENANCE_INVALID,
+    ERR_PROVENANCE_MISSING,
+    ERR_PROVENANCE_REQUEST_KID_MISMATCH,
+    ERR_PROVENANCE_UNKNOWN_CAP,
+    Provenance,
+    ProvenanceError,
+    compute_cap_token_root,
+    compute_request_kid,
+    sign_provenance,
+    verify_provenance,
+)
+from hypermind.mcp.server import MCPServer, ToolDescriptor
+from hypermind.mcp.transport import (
+    MCP_SCHEMA_VERSION,
+    MCPTransport,
+    StdioTransport,
+    WebSocketTransport,
+)
+
+__all__ = [
+    "ERR_PROVENANCE_INVALID",
+    "ERR_PROVENANCE_MISSING",
+    "ERR_PROVENANCE_REQUEST_KID_MISMATCH",
+    "ERR_PROVENANCE_UNKNOWN_CAP",
+    "MCP_SCHEMA_VERSION",
+    "HyperMindMCPServer",
+    "MCPClient",
+    "MCPRpcError",
+    "MCPServer",
+    "MCPTransport",
+    "Provenance",
+    "ProvenanceError",
+    "StdioTransport",
+    "ToolDescriptor",
+    "WebSocketTransport",
+    "compute_cap_token_root",
+    "compute_request_kid",
+    "sealed_mcp_client",
+    "sign_provenance",
+    "verify_provenance",
+]

hypermind/mcp/bridge.py

@@ -0,0 +1,151 @@
+"""MCP bridge — two patterns:
+
+1. `HyperMindMCPServer(agent, expose=[...])` — expose agent verbs as
+   MCP tools. Each invocation is gated by the agent's CapToken
+   caveat-chain (closes Cycle-1 fix #5 / MCP confused-deputy gap).
+
+2. `sealed_mcp_client(agent, "stdio://...")` — wrap an external MCP
+   server so every tool call's input + output is sealed into a
+   HyperMind signed statement, providing non-repudiable provenance for
+   tool-mediated claims.
+
+The implementation here is the contract; importing the real MCP runtime
+is deferred until the SDK is extended with the `[mcp]` extra installed.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+import cbor2
+
+from hypermind.agent import HyperMindAgent
+from hypermind.crypto.hashing import blake3_32
+from hypermind.wire import Kid
+
+ToolHandler = Callable[[dict[str, Any]], Awaitable[Any]]
+
+
+@dataclass
+class HyperMindMCPServer:
+    """Expose agent verbs as MCP tools. v0.1 ships the dispatch logic;
+    binding to the actual MCP transport is done by the consumer.
+
+    Usage:
+        server = HyperMindMCPServer(agent, expose=["publish_claim", "dispute"])
+        # then plug `server.tools` into your MCP transport (stdio_server etc).
+    """
+
+    agent: HyperMindAgent
+    expose: list[str]
+
+    def __post_init__(self) -> None:
+        # Validate that every requested tool exists.
+        unknown = [t for t in self.expose if not hasattr(self.agent, t)]
+        if unknown:
+            raise ValueError(f"unknown MCP tool(s): {unknown}")
+
+    @property
+    def tools(self) -> dict[str, ToolHandler]:
+        """Map of tool_name -> async handler.
+
+        Wire these into your MCP transport (e.g. mcp.server.stdio_server).
+        """
+        return {name: self._make_handler(name) for name in self.expose}
+
+    def _make_handler(self, name: str) -> ToolHandler:
+        method = getattr(self.agent, name)
+
+        async def _handler(args: dict[str, Any]) -> Any:
+            # Cap-token enforcement happens inside the verb itself, so
+            # an MCP caller without authority gets `AuthError` with
+            # `failed_caveat` populated.
+            return await method(**args)
+
+        _handler.__name__ = f"hypermind_mcp_{name}"
+        return _handler
+
+
+@dataclass
+class _SealedToolResult:
+    tool_name: str
+    tool_server_id: str
+    input_hash: bytes
+    output_hash: bytes
+    output: Any
+    input_kid: Kid | None = None
+
+    def as_claim(self) -> dict:
+        """Convert the wrapped result into a publish_claim payload."""
+        return {
+            "tool_name": self.tool_name,
+            "tool_server_id": self.tool_server_id,
+            "input_hash": self.input_hash.hex(),
+            "output_hash": self.output_hash.hex(),
+            "output": self.output,
+        }
+
+
+@contextlib.asynccontextmanager
+async def sealed_mcp_client(agent: HyperMindAgent, endpoint: str):
+    """Async context manager wrapping an external MCP server.
+
+    For v0.1 sandbox use this returns a stub client whose `call_tool`
+    seals input+output hashes; production wiring of the MCP transport
+    is provided by the `[mcp]` extra.
+    """
+    yield _SealedMCPClient(agent=agent, endpoint=endpoint)
+
+
+@dataclass
+class _SealedMCPClient:
+    agent: HyperMindAgent
+    endpoint: str
+
+    async def call_tool(
+        self,
+        tool_name: str,
+        args: dict[str, Any],
+        *,
+        execute: Callable[[str, dict[str, Any]], Awaitable[Any]] | None = None,
+    ) -> _SealedToolResult:
+        """Call `tool_name` on the wrapped MCP server, sealing the result.
+
+        `execute` is a hook for tests/sandboxes; production wiring will
+        invoke the actual MCP server.
+        """
+        if execute is None:
+            # Sandbox fallback: echo the args as the result.
+            output: Any = {"echo": args}
+        else:
+            output = await execute(tool_name, args)
+        # Canonical CBOR hashing — `repr()` was non-stable across
+        # Python versions and dict-insertion orders, so two equivalent
+        # calls could hash differently (closes self-heal Cycle-1 honourable
+        # mention on `repr()`-based provenance).
+        input_hash = blake3_32(_canonical_bytes(args))
+        output_hash = blake3_32(_canonical_bytes(output))
+        return _SealedToolResult(
+            tool_name=tool_name,
+            tool_server_id=self.endpoint,
+            input_hash=input_hash,
+            output_hash=output_hash,
+            output=output,
+        )
+
+
+def _canonical_bytes(obj: Any) -> bytes:
+    """Deterministic CBOR encoding of an MCP tool arg/result.
+
+    Falls back to `repr()` only for non-CBOR-encodable objects (custom
+    classes), but logs a marker so the caller knows the hash is not
+    reproducible across Python versions.
+    """
+    try:
+        return cbor2.dumps(obj, canonical=True)
+    except (TypeError, ValueError):
+        # Last-resort: stringify deterministically using sorted-repr.
+        return ("non-cbor:" + repr(obj)).encode()