hypermind 0.11.0__py3-none-any.whl

hypermind/transport/bus.py

@@ -0,0 +1,206 @@
+"""Gossip bus interface + in-process implementation.
+
+Real deployments use libp2p-gossipsub v1.2 with reputation-weighted
+peer-scoring (P5) and topic-scoped meshes. The in-process bus mimics
+the topic-fan-out semantics so agents can run end-to-end tests without
+a real network. The interface is intentionally small so swapping in
+libp2p later is a one-file change.
+
+T5-03 — credit-based backpressure: each subscriber may advertise a
+per-(peer_kid, topic_id) credit window via :meth:`InProcessBus.set_credit`.
+Publishers that overrun the window get :class:`BackpressureExceeded`
+instead of an unbounded internal queue. The token-bucket implementation
+is reused from :mod:`hypermind.capability` (the same one rate-caveats
+already use, so we avoid two parallel rate machineries).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Protocol
+
+from hypermind.capability import TokenBucket
+from hypermind.errors import HyperMindError
+
+
+class BackpressureExceeded(HyperMindError):
+    """Publisher exceeded a subscriber's advertised credit window.
+
+    The bus refuses to enqueue further messages until the subscriber's
+    bucket refills. Carries ``peer_kid`` and ``topic_id`` so callers can
+    back off per-peer rather than throttling the whole namespace.
+    """
+
+    def __init__(self, message: str, *, peer_kid: str, topic_id: str) -> None:
+        super().__init__(message)
+        self.peer_kid = peer_kid
+        self.topic_id = topic_id
+        self.telemetry["peer_kid"] = peer_kid
+        self.telemetry["topic_id"] = topic_id
+
+
+def _default_credit() -> int:
+    """Default per-(peer, topic) capacity. Override via env var.
+
+    1024 messages / 1s burst is the v0.10.1 default — enough headroom
+    for steady-state gossip but small enough that runaway publishers
+    surface immediately in tests.
+    """
+    raw = os.environ.get("HYPERMIND_BUS_CREDIT_DEFAULT", "1024")
+    try:
+        v = int(raw)
+        return v if v > 0 else 1024
+    except ValueError:
+        return 1024
+
+
+@dataclass
+class Subscription:
+    topic: str
+    _cancel: Callable[[], None]
+
+    def unsubscribe(self) -> None:
+        self._cancel()
+
+
+Handler = Callable[[bytes], Awaitable[None]]
+
+
+class GossipBus(Protocol):
+    async def publish(self, topic: str, message: bytes) -> None: ...
+    async def subscribe(self, topic: str, handler: Handler) -> Subscription: ...
+    async def close(self) -> None: ...
+    def set_credit(self, peer_kid: str, topic_id: str, capacity: int) -> None: ...
+
+
+@dataclass
+class InProcessBus:
+    """An asyncio-native fan-out bus.
+
+    All published messages are delivered to every subscriber on the
+    same topic (pure broadcast). Eventual-consistency / partition
+    behaviour is not modelled at this layer — see `partition.py`.
+
+    Backpressure: callers may publish with a ``peer_kid`` keyword to
+    pin the (peer, topic) credit window. Plain ``publish(topic, msg)``
+    treats the publisher as anonymous (peer_kid="") which still shares
+    a bucket so misconfigured callers can't escape backpressure.
+    """
+
+    _subscribers: dict[str, list[Handler]] = field(default_factory=dict)
+    _lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+    _closed: bool = False
+    # Credit windows: (peer_kid, topic_id) -> TokenBucket. Lazily created
+    # on first publish; advertised capacity overrides the default.
+    _buckets: dict[tuple[str, str], TokenBucket] = field(default_factory=dict)
+    _capacities: dict[tuple[str, str], int] = field(default_factory=dict)
+    # Internal "queue" — used only as a depth gauge for tests. The
+    # in-process bus delivers synchronously inside `publish`, so the
+    # queue length is bounded by len(subscribers) at any instant; we
+    # track it explicitly so backpressure tests can assert on it.
+    _queue: list[tuple[str, str, bytes]] = field(default_factory=list)
+
+    def set_credit(self, peer_kid: str, topic_id: str, capacity: int) -> None:
+        """Advertise this subscriber's max consumption rate.
+
+        ``capacity`` is messages/second burst. The bucket is reset so a
+        previously-throttled publisher gets a clean slate after the
+        subscriber re-advertises.
+        """
+        if capacity <= 0:
+            raise ValueError(f"capacity must be > 0, got {capacity}")
+        key = (peer_kid, topic_id)
+        self._capacities[key] = capacity
+        self._buckets[key] = TokenBucket(f"{capacity}/sec")
+
+    def _bucket_for(self, peer_kid: str, topic_id: str) -> TokenBucket:
+        key = (peer_kid, topic_id)
+        if key not in self._buckets:
+            cap = self._capacities.get(key, _default_credit())
+            self._capacities[key] = cap
+            self._buckets[key] = TokenBucket(f"{cap}/sec")
+        return self._buckets[key]
+
+    async def publish(
+        self,
+        topic: str,
+        message: bytes,
+        *,
+        peer_kid: str = "",
+    ) -> None:
+        if self._closed:
+            raise RuntimeError("bus is closed")
+
+        bucket = self._bucket_for(peer_kid, topic)
+        if not bucket.consume():
+            raise BackpressureExceeded(
+                f"backpressure exceeded for peer={peer_kid!r} topic={topic!r}",
+                peer_kid=peer_kid,
+                topic_id=topic,
+            )
+
+        async with self._lock:
+            handlers = list(self._subscribers.get(topic, ()))
+        # Track in-flight depth for test observability. Bounded by the
+        # bucket's capacity per peer; we cap the gauge at sum-of-capacities
+        # so a long-running bus does not leak memory in `_queue`.
+        self._queue.append((peer_kid, topic, message))
+        max_depth = sum(self._capacities.values()) or _default_credit()
+        if len(self._queue) > max_depth:
+            # Drop oldest gauge entries — this is just an observability
+            # buffer, not a real delivery queue.
+            del self._queue[: len(self._queue) - max_depth]
+
+        try:
+            # Run handlers concurrently; isolation prevents one bad
+            # handler from blocking the whole fan-out.
+            await asyncio.gather(
+                *(_safe_invoke(h, message) for h in handlers),
+                return_exceptions=True,
+            )
+        finally:
+            try:
+                self._queue.remove((peer_kid, topic, message))
+            except ValueError:
+                pass
+
+    async def subscribe(self, topic: str, handler: Handler) -> Subscription:
+        if self._closed:
+            raise RuntimeError("bus is closed")
+        async with self._lock:
+            self._subscribers.setdefault(topic, []).append(handler)
+
+        def _cancel() -> None:
+            try:
+                self._subscribers[topic].remove(handler)
+            except (ValueError, KeyError):
+                pass
+
+        return Subscription(topic=topic, _cancel=_cancel)
+
+    async def close(self) -> None:
+        async with self._lock:
+            self._subscribers.clear()
+            self._closed = True
+
+
+async def _safe_invoke(handler: Handler, message: bytes) -> None:
+    try:
+        await handler(message)
+    except Exception as e:
+        # Surface to the global recorder so a misbehaving handler is
+        # auditable rather than silent (closes self-heal Cycle-1
+        # Finding #9 from data review). One bad handler doesn't break
+        # the fan-out; the exception is caught.
+        try:
+            from hypermind.observability.recorder import get_recorder
+
+            get_recorder().incr(
+                "hypermind_handler_errors_total",
+                kind=type(e).__name__,
+            )
+        except Exception:  # pragma: no cover  (recorder always present)
+            pass

hypermind/transport/knows_delta.py

@@ -0,0 +1,204 @@
+"""P2-16: KNOWS_DELTA gossip message type.
+
+A lightweight reputation-delta broadcast that lets agents efficiently
+synchronise knowledge of each other's reputations without replaying the
+full claim + outcome history.  Each message is signed (domain tag
+DomainTag.KNOWS_DELTA = 0x18) and carries one or more per-topic deltas.
+
+Wire shape (deterministic CBOR):
+    {
+        "issuer_kid": <32 bytes>,
+        "namespace_id": <str>,
+        "hlc": <int>,           # issuer's HLC logical_ms at emission
+        "buckets": [
+            {
+                "topic_id": <str>,
+                "subject_kid": <32 bytes>,
+                "alpha_delta": <float>,
+                "beta_delta": <float>,
+                "evidence_kid": <bytes|None>,  # optional claim that drove the delta
+            },
+            ...
+        ],
+        "sig": <bytes>,         # Ed25519 over domain_tag || canonical_cbor(body)
+    }
+
+Publishing:  call ``publish_knows_delta(delta, bus, topic=KNOWS_DELTA_BUS_TOPIC)``
+Subscribing: call ``on_knows_delta(bus, handler, topic=KNOWS_DELTA_BUS_TOPIC)``
+
+The bus topic is a fixed string so all agents in a namespace share a
+single gossip channel for reputation deltas.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+import cbor2
+
+from hypermind.crypto.domain import DomainTag
+from hypermind.crypto.signing import Keypair, verify_ed25519
+from hypermind.transport.bus import GossipBus, Subscription
+
+# Fixed gossip bus topic for KNOWS_DELTA messages.
+KNOWS_DELTA_BUS_TOPIC = "hm/knows_delta/v1"
+
+# Domain-separation tag prefix for the KNOWS_DELTA signature preimage.
+_DOMAIN_BYTE = bytes([int(DomainTag.KNOWS_DELTA)])
+
+
+@dataclass
+class KnowsDeltaBucket:
+    """One per-topic reputation delta inside a KnowsDeltaMessage."""
+
+    topic_id: str
+    subject_kid: bytes
+    alpha_delta: float
+    beta_delta: float
+    evidence_kid: bytes | None = None
+
+    def to_cbor_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {
+            "topic_id": self.topic_id,
+            "subject_kid": bytes(self.subject_kid),
+            "alpha_delta": self.alpha_delta,
+            "beta_delta": self.beta_delta,
+        }
+        if self.evidence_kid is not None:
+            d["evidence_kid"] = bytes(self.evidence_kid)
+        return d
+
+    @classmethod
+    def from_cbor_dict(cls, d: dict[str, Any]) -> KnowsDeltaBucket:
+        return cls(
+            topic_id=str(d["topic_id"]),
+            subject_kid=bytes(d["subject_kid"]),
+            alpha_delta=float(d["alpha_delta"]),
+            beta_delta=float(d["beta_delta"]),
+            evidence_kid=bytes(d["evidence_kid"]) if d.get("evidence_kid") else None,
+        )
+
+
+@dataclass
+class KnowsDeltaMessage:
+    """A signed reputation-delta gossip message (P2-16 / DomainTag.KNOWS_DELTA).
+
+    Carries one or more ``KnowsDeltaBucket`` entries — each records the
+    raw α/β increment applied by the issuer to a specific (topic, agent).
+    Recipients MAY apply these deltas to their own reputation kernel or
+    use them as a signal for peer learning.
+
+    The message is signed with ``issuer_keypair`` at construction time
+    via :meth:`create`.  Parse with :meth:`from_bytes`.
+    """
+
+    issuer_kid: bytes
+    namespace_id: str
+    hlc: int
+    buckets: list[KnowsDeltaBucket] = field(default_factory=list)
+    sig: bytes = b""
+
+    def _body_bytes(self) -> bytes:
+        body = {
+            "issuer_kid": bytes(self.issuer_kid),
+            "namespace_id": self.namespace_id,
+            "hlc": int(self.hlc),
+            "buckets": [b.to_cbor_dict() for b in self.buckets],
+        }
+        return cbor2.dumps(body, canonical=True)
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        keypair: Keypair,
+        namespace_id: str,
+        hlc: int,
+        buckets: list[KnowsDeltaBucket],
+    ) -> KnowsDeltaMessage:
+        """Build and sign a KnowsDeltaMessage."""
+        msg = cls(
+            issuer_kid=keypair.kid,
+            namespace_id=namespace_id,
+            hlc=hlc,
+            buckets=list(buckets),
+        )
+        preimage = _DOMAIN_BYTE + msg._body_bytes()
+        msg.sig = keypair.sign(preimage)
+        return msg
+
+    def verify(self) -> bool:
+        """Verify the issuer's Ed25519 signature."""
+        if len(self.issuer_kid) != 32:
+            return False
+        preimage = _DOMAIN_BYTE + self._body_bytes()
+        return verify_ed25519(self.issuer_kid, self.sig, preimage)
+
+    def to_bytes(self) -> bytes:
+        """Serialise as deterministic CBOR for bus transport."""
+        payload = {
+            "issuer_kid": bytes(self.issuer_kid),
+            "namespace_id": self.namespace_id,
+            "hlc": int(self.hlc),
+            "buckets": [b.to_cbor_dict() for b in self.buckets],
+            "sig": bytes(self.sig),
+        }
+        return cbor2.dumps(payload, canonical=True)
+
+    @classmethod
+    def from_bytes(cls, raw: bytes) -> KnowsDeltaMessage:
+        """Parse a KnowsDeltaMessage from CBOR bytes."""
+        try:
+            d = cbor2.loads(raw)
+        except Exception as exc:
+            raise ValueError(f"KnowsDeltaMessage: invalid CBOR: {exc}") from exc
+        if not isinstance(d, dict):
+            raise ValueError("KnowsDeltaMessage: expected CBOR map")
+        try:
+            buckets = [KnowsDeltaBucket.from_cbor_dict(b) for b in d.get("buckets", [])]
+            return cls(
+                issuer_kid=bytes(d["issuer_kid"]),
+                namespace_id=str(d["namespace_id"]),
+                hlc=int(d["hlc"]),
+                buckets=buckets,
+                sig=bytes(d.get("sig", b"")),
+            )
+        except (KeyError, TypeError, ValueError) as exc:
+            raise ValueError(f"KnowsDeltaMessage: malformed fields: {exc}") from exc
+
+
+async def publish_knows_delta(
+    delta: KnowsDeltaMessage,
+    bus: GossipBus,
+    *,
+    topic: str = KNOWS_DELTA_BUS_TOPIC,
+) -> None:
+    """Publish a signed KnowsDeltaMessage to the gossip bus."""
+    await bus.publish(topic, delta.to_bytes())
+
+
+async def on_knows_delta(
+    bus: GossipBus,
+    handler: Callable[[KnowsDeltaMessage], Awaitable[None]],
+    *,
+    topic: str = KNOWS_DELTA_BUS_TOPIC,
+) -> Subscription:
+    """Subscribe to KnowsDelta messages on the bus.
+
+    ``handler`` is called with each decoded and signature-verified
+    ``KnowsDeltaMessage``.  Messages that fail signature verification
+    are silently dropped (logged via the recorder if available).
+    """
+
+    async def _decode_and_call(raw: bytes) -> None:
+        try:
+            msg = KnowsDeltaMessage.from_bytes(raw)
+        except ValueError:
+            return  # malformed — drop
+        if not msg.verify():
+            return  # bad signature — drop
+        await handler(msg)
+
+    return await bus.subscribe(topic, _decode_and_call)