hypermind 0.11.0__py3-none-any.whl

hypermind/observability/swarm_trace.py

@@ -0,0 +1,438 @@
+"""Distributed swarm-trace — W3C TraceContext propagation across SignedStatements.
+
+T6-01 (v0.10.4). The transparency log is the canonical timeline (spec §28.4),
+but operators also need OpenTelemetry-shaped span trees that span multiple
+hosts so a Grafana / Jaeger view can stitch a deliberation, a saga workflow
+and a contract-net RFP into one trace.
+
+Design:
+
+* Each :class:`SwarmTrace` carries a 16-byte ``trace_id`` (W3C). Every
+  :meth:`start_span` mints a fresh 8-byte ``span_id`` and links it to the
+  parent ``span_id`` (if any). The triple ``(trace_id, span_id, flags)`` —
+  optionally followed by an 8-byte parent_span_id — is encoded as a single
+  bytes blob and placed under the protected-header label
+  :data:`hypermind.wire.HM_OTEL_TRACE_CONTEXT` (= -65635) by
+  :meth:`inject`. :meth:`extract` is the inverse.
+
+* Because the label is in the protected header, A-ALG-BOUND covers it: a
+  tampered trace context fails canonical re-encoding before signature
+  verification.
+
+* :meth:`replay_from_log` rebuilds a span tree post-hoc by reading the
+  per-namespace transparency log. It does **not** require the trace
+  context label to be present (older envelopes simply contribute no
+  span); HLC order is the secondary discriminator that produces a stable
+  total order even when wall-clock skew is large.
+
+The class is deliberately small: it has zero hard dependency on the
+``opentelemetry`` SDK — emit live spans by chaining a regular
+``Recorder.with_otel(tracer_provider)`` after :meth:`start_span`.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any
+
+# Wire-format label. Imported lazily-via-module so this file stays cheap to
+# import even when the wire module isn't otherwise needed.
+from hypermind.wire import HM_OTEL_TRACE_CONTEXT
+
+#: Length of a W3C trace_id in bytes (RFC: 16 bytes / 32 hex chars).
+TRACE_ID_LEN = 16
+#: Length of a W3C span_id in bytes (RFC: 8 bytes / 16 hex chars).
+SPAN_ID_LEN = 8
+#: Length of the flags byte (RFC: 1 byte).
+FLAGS_LEN = 1
+#: Length of an injected blob without the parent_span_id (16 + 8 + 1).
+TRACE_CONTEXT_LEN = TRACE_ID_LEN + SPAN_ID_LEN + FLAGS_LEN
+#: Length when the optional parent_span_id is appended.
+TRACE_CONTEXT_LEN_WITH_PARENT = TRACE_CONTEXT_LEN + SPAN_ID_LEN
+
+#: Sampled flag (W3C TraceContext §2.2.4).
+TRACE_FLAG_SAMPLED = 0x01
+
+
+@dataclass(frozen=True)
+class SpanCtx:
+    """Immutable snapshot of an active span.
+
+    The active span is the parent for any subsequent :meth:`SwarmTrace.start_span`
+    call until the SpanCtx is finished. This dataclass is wire-portable —
+    callers pass it across coroutine / process boundaries by serialising
+    via :meth:`SwarmTrace.inject`.
+    """
+
+    trace_id: bytes
+    span_id: bytes
+    parent_span_id: bytes | None
+    name: str
+    flags: int = TRACE_FLAG_SAMPLED
+    attributes: dict[str, Any] = field(default_factory=dict)
+    start_hlc: str | None = None
+
+    def __post_init__(self) -> None:
+        if len(self.trace_id) != TRACE_ID_LEN:
+            raise ValueError(f"trace_id must be {TRACE_ID_LEN} bytes, got {len(self.trace_id)}")
+        if len(self.span_id) != SPAN_ID_LEN:
+            raise ValueError(f"span_id must be {SPAN_ID_LEN} bytes, got {len(self.span_id)}")
+        if self.parent_span_id is not None and len(self.parent_span_id) != SPAN_ID_LEN:
+            raise ValueError(
+                f"parent_span_id must be {SPAN_ID_LEN} bytes or None, "
+                f"got {len(self.parent_span_id)}"
+            )
+
+
+@dataclass
+class Span:
+    """Materialised span used by :meth:`SwarmTrace.replay_from_log`."""
+
+    trace_id: bytes
+    span_id: bytes
+    parent_span_id: bytes | None
+    name: str
+    hlc: str
+    kid: bytes | None = None
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+
+class SwarmTrace:
+    """W3C TraceContext propagator for SignedStatement envelopes.
+
+    Usage::
+
+        tracer = SwarmTrace()
+        ctx = tracer.start_span("deliberate")
+        envelope = build_envelope(...)
+        envelope_with_ctx = tracer.inject(envelope, ctx)
+        # ... ship envelope across the bus ...
+        peer_ctx = tracer.extract(received_envelope)  # may be None
+        child = tracer.start_span("vote", parent_context=peer_ctx)
+    """
+
+    def __init__(self, *, trace_id: bytes | None = None) -> None:
+        self._trace_id: bytes = trace_id if trace_id is not None else self._fresh_trace_id()
+        self._active: SpanCtx | None = None
+
+    # ---- public API ---------------------------------------------------
+
+    @property
+    def trace_id(self) -> bytes:
+        """The 16-byte trace_id all spans on this tracer share."""
+        return self._trace_id
+
+    @property
+    def active_span(self) -> SpanCtx | None:
+        """The most recent span started, or ``None``."""
+        return self._active
+
+    def start_span(
+        self,
+        name: str,
+        *,
+        parent_context: SpanCtx | None = None,
+        flags: int = TRACE_FLAG_SAMPLED,
+        attributes: dict[str, Any] | None = None,
+        start_hlc: str | None = None,
+    ) -> SpanCtx:
+        """Mint a fresh span. ``parent_context`` defaults to the active span."""
+        parent = parent_context if parent_context is not None else self._active
+        # If we cross into a foreign trace via parent_context, follow it.
+        trace_id = parent.trace_id if parent is not None else self._trace_id
+        span_id = self._fresh_span_id()
+        ctx = SpanCtx(
+            trace_id=trace_id,
+            span_id=span_id,
+            parent_span_id=parent.span_id if parent is not None else None,
+            name=name,
+            flags=flags,
+            attributes=dict(attributes or {}),
+            start_hlc=start_hlc,
+        )
+        self._active = ctx
+        return ctx
+
+    def inject(self, envelope: Any, ctx: SpanCtx | None = None) -> Any:
+        """Place the trace context into ``envelope.protected_header``.
+
+        ``envelope`` is duck-typed: any object exposing a mutable
+        ``protected_header`` mapping or a ``protected_header`` attribute
+        on a frozen dataclass works (we re-build via ``dataclasses.replace``
+        when needed). For raw dicts, the dict is mutated in place.
+        """
+        ctx = ctx if ctx is not None else self._active
+        if ctx is None:
+            return envelope
+        blob = encode_trace_context(ctx)
+        return _set_protected_header(envelope, HM_OTEL_TRACE_CONTEXT, blob)
+
+    def extract(self, envelope: Any) -> SpanCtx | None:
+        """Inverse of :meth:`inject`. Returns ``None`` when no context is present."""
+        blob = _get_protected_header(envelope, HM_OTEL_TRACE_CONTEXT)
+        if blob is None:
+            return None
+        try:
+            return decode_trace_context(blob, name="<extracted>")
+        except ValueError:
+            return None
+
+    def replay_from_log(self, transparency_store: Any) -> list[Span]:
+        """Reconstruct the span tree from a transparency log.
+
+        ``transparency_store`` is any object exposing one of:
+
+        * ``iter_envelopes()`` → iterable of decoded ``SignedStatement``
+        * ``envelopes`` attribute (list)
+        * ``leaves`` mapping ``leaf_kid → SignedStatement``
+
+        The output is sorted by HLC (lexical compare on the canonical
+        ``"<logical_ms>:<counter>:<kid>"`` form) so the parent → child
+        edges form a DAG even when wall-clock skew is large.
+        """
+        envelopes = list(_iter_envelopes(transparency_store))
+        envelopes.sort(key=lambda e: _envelope_hlc(e))
+        spans: list[Span] = []
+        for env in envelopes:
+            blob = _get_protected_header(env, HM_OTEL_TRACE_CONTEXT)
+            if blob is None:
+                continue
+            try:
+                ctx = decode_trace_context(blob, name=_envelope_record_type(env))
+            except ValueError:
+                continue
+            spans.append(
+                Span(
+                    trace_id=ctx.trace_id,
+                    span_id=ctx.span_id,
+                    parent_span_id=ctx.parent_span_id,
+                    name=ctx.name,
+                    hlc=_envelope_hlc(env),
+                    kid=_envelope_kid(env),
+                    attributes=dict(ctx.attributes),
+                )
+            )
+        return spans
+
+    # ---- helpers ------------------------------------------------------
+
+    @staticmethod
+    def _fresh_trace_id() -> bytes:
+        return os.urandom(TRACE_ID_LEN)
+
+    @staticmethod
+    def _fresh_span_id() -> bytes:
+        return os.urandom(SPAN_ID_LEN)
+
+
+# ---- wire encoding -------------------------------------------------------
+
+
+def encode_trace_context(ctx: SpanCtx) -> bytes:
+    """Pack a SpanCtx into the wire blob format.
+
+    Layout:
+        trace_id(16) || span_id(8) || flags(1) [|| parent_span_id(8)]
+
+    The parent_span_id is included only when present so root spans get
+    the shorter 25-byte form.
+    """
+    base = ctx.trace_id + ctx.span_id + bytes([ctx.flags & 0xFF])
+    if ctx.parent_span_id is None:
+        return base
+    return base + ctx.parent_span_id
+
+
+def decode_trace_context(blob: bytes, *, name: str = "") -> SpanCtx:
+    """Parse a trace context blob. Raises ``ValueError`` on malformed input."""
+    if len(blob) not in (TRACE_CONTEXT_LEN, TRACE_CONTEXT_LEN_WITH_PARENT):
+        raise ValueError(
+            f"trace context blob must be {TRACE_CONTEXT_LEN} or "
+            f"{TRACE_CONTEXT_LEN_WITH_PARENT} bytes, got {len(blob)}"
+        )
+    trace_id = blob[:TRACE_ID_LEN]
+    span_id = blob[TRACE_ID_LEN : TRACE_ID_LEN + SPAN_ID_LEN]
+    flags = blob[TRACE_ID_LEN + SPAN_ID_LEN]
+    parent: bytes | None = None
+    if len(blob) == TRACE_CONTEXT_LEN_WITH_PARENT:
+        parent = blob[TRACE_CONTEXT_LEN:]
+    return SpanCtx(
+        trace_id=trace_id,
+        span_id=span_id,
+        parent_span_id=parent,
+        name=name,
+        flags=flags,
+    )
+
+
+# ---- duck-typing helpers -------------------------------------------------
+
+
+def _set_protected_header(envelope: Any, label: int, value: bytes) -> Any:
+    ph = getattr(envelope, "protected_header", None)
+    if ph is None and isinstance(envelope, dict):
+        ph = envelope.setdefault("protected_header", {})
+    if ph is None:
+        # Last resort: stash on a plain attribute so tests can read it back.
+        try:
+            object.__setattr__(envelope, "_otel_trace_blob", value)
+        except (TypeError, AttributeError):
+            pass
+        return envelope
+    if isinstance(ph, dict):
+        ph[label] = value
+    else:
+        # Frozen dataclass / object with a mapping-shaped property.
+        try:
+            ph[label] = value  # type: ignore[index]
+        except Exception as e:  # pragma: no cover  - defensive
+            raise TypeError(
+                f"cannot inject trace context into protected_header of type {type(ph)!r}: {e}"
+            ) from e
+    return envelope
+
+
+def _get_protected_header(envelope: Any, label: int) -> bytes | None:
+    ph = getattr(envelope, "protected_header", None)
+    if ph is None and isinstance(envelope, dict):
+        ph = envelope.get("protected_header")
+    if ph is None:
+        # Match _set_protected_header's last-resort path.
+        return getattr(envelope, "_otel_trace_blob", None)
+    if isinstance(ph, dict):
+        v = ph.get(label)
+    else:
+        try:
+            v = ph[label]  # type: ignore[index]
+        except (KeyError, TypeError, IndexError):
+            v = None
+    if v is None or isinstance(v, (bytes, bytearray)):
+        return bytes(v) if v is not None else None
+    return None
+
+
+def _iter_envelopes(store: Any):
+    if hasattr(store, "iter_envelopes"):
+        yield from store.iter_envelopes()
+        return
+    envs = getattr(store, "envelopes", None)
+    if envs is not None:
+        yield from envs
+        return
+    leaves = getattr(store, "leaves", None)
+    if isinstance(leaves, dict):
+        yield from leaves.values()
+        return
+    if isinstance(store, list):
+        yield from store
+        return
+    raise TypeError(
+        f"transparency_store of type {type(store)!r} does not expose "
+        "iter_envelopes() / envelopes / leaves"
+    )
+
+
+def _envelope_hlc(env: Any) -> str:
+    ph = getattr(env, "protected_header", None) or {}
+    if isinstance(env, dict):
+        ph = env.get("protected_header") or {}
+    # HM_HLC = -65603
+    hlc = ph.get(-65603) if isinstance(ph, dict) else None
+    if hlc is None:
+        hlc = getattr(env, "hlc", None)
+    return str(hlc) if hlc is not None else ""
+
+
+def _envelope_kid(env: Any) -> bytes | None:
+    kid = getattr(env, "kid", None)
+    if kid is None and isinstance(env, dict):
+        kid = env.get("kid")
+    return kid if isinstance(kid, (bytes, bytearray)) else None
+
+
+def _envelope_record_type(env: Any) -> str:
+    ph = getattr(env, "protected_header", None) or {}
+    if isinstance(env, dict):
+        ph = env.get("protected_header") or {}
+    rt = ph.get(-65601) if isinstance(ph, dict) else None
+    return str(rt) if rt is not None else "<span>"
+
+
+# ---------------------------------------------------------------------------
+# Normative-trace helpers (rows 37-40)
+# ---------------------------------------------------------------------------
+
+#: COSE protected-header label for the OTel trace context (spec §4.13.2).
+#: Mirrors :data:`hypermind.wire.HM_OTEL_TRACE_CONTEXT` = -65635.
+OTEL_TRACE_CONTEXT_LABEL: int = -65635
+
+
+def new_span_id() -> bytes:
+    """Mint a fresh 8-byte ``span_id`` that is never all-zero (spec §28.3.2).
+
+    Retries on the astronomically unlikely all-zero draw, satisfying
+    the spec requirement that ``span_id != b'\\x00' * 8`` at every new
+    operation boundary.
+
+    Returns
+    -------
+    bytes
+        8 cryptographically random bytes, guaranteed non-zero.
+    """
+    while True:
+        candidate = os.urandom(SPAN_ID_LEN)
+        if candidate != b"\x00" * SPAN_ID_LEN:
+            return candidate
+
+
+def encode_trace_context_list(trace_id: bytes, span_id: bytes, flags: int) -> list:
+    """Encode the OTel trace context as a CBOR-encodable list.
+
+    Returns ``[trace_id, span_id, flags]`` — the canonical encoding
+    described in spec §4.13.2 and row 37 of the normative trace::
+
+        OTEL_TRACE_CONTEXT (-65635) = [bstr .size 16, bstr .size 8, uint]
+
+    This form is used when the protected header carries the triple as a
+    structured CBOR array rather than the concatenated-bytes blob used by
+    :func:`encode_trace_context` (which targets the existing ``SwarmTrace``
+    inject/extract path).
+
+    Parameters
+    ----------
+    trace_id:
+        16-byte W3C trace identifier.
+    span_id:
+        8-byte W3C span identifier.
+    flags:
+        Integer flags byte (e.g. ``TRACE_FLAG_SAMPLED = 0x01``).
+
+    Raises
+    ------
+    ValueError
+        If ``trace_id`` is not 16 bytes or ``span_id`` is not 8 bytes.
+    """
+    if len(trace_id) != TRACE_ID_LEN:
+        raise ValueError(f"trace_id must be {TRACE_ID_LEN} bytes, got {len(trace_id)}")
+    if len(span_id) != SPAN_ID_LEN:
+        raise ValueError(f"span_id must be {SPAN_ID_LEN} bytes, got {len(span_id)}")
+    return [trace_id, span_id, flags]
+
+
+__all__ = [
+    "FLAGS_LEN",
+    "OTEL_TRACE_CONTEXT_LABEL",
+    "SPAN_ID_LEN",
+    "TRACE_CONTEXT_LEN",
+    "TRACE_CONTEXT_LEN_WITH_PARENT",
+    "TRACE_FLAG_SAMPLED",
+    "TRACE_ID_LEN",
+    "Span",
+    "SpanCtx",
+    "SwarmTrace",
+    "decode_trace_context",
+    "encode_trace_context",
+    "encode_trace_context_list",
+    "new_span_id",
+]

hypermind/pin_policy.py

@@ -0,0 +1,116 @@
+"""PinPolicy DSL — declarative replication contract.
+
+A pinner receives a stream of sealed claims via gossipsub and decides
+whether to retain each one based on:
+  - topic match
+  - publisher reputation ≥ threshold
+  - claim_type filter
+  - replication target (e.g. "≥3 geo-disjoint, TTL=180d")
+  - erasure coding scheme
+  - disk budget + eviction policy
+
+The DSL is a Python dataclass so validation, composition, and IDE
+autocomplete all work at construction time. Late-bind YAML loaders are
+non-normative sugar.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Literal
+
+EvictionPolicy = Literal["lru", "reputation_weighted", "fifo"]
+
+
+@dataclass
+class PinPolicy:
+    """Declarative contract for what a pinner replicates."""
+
+    topics: list[str] = field(default_factory=list)
+    publishers_min_rep: float = 0.0
+    claim_types: list[str] | Literal["*"] = "*"
+    replication_target: tuple[str, str] = ("≥3 geo-disjoint", "TTL=180d")
+    erasure_coding: tuple[str, str] | None = None
+    max_disk_gb: float = 50.0
+    eviction: EvictionPolicy = "reputation_weighted"
+    cite_depth: int = 2
+
+    def __post_init__(self) -> None:
+        if self.publishers_min_rep < 0 or self.publishers_min_rep > 1:
+            raise ValueError("publishers_min_rep must be in [0, 1]")
+        if self.max_disk_gb < 0:
+            raise ValueError("max_disk_gb must be non-negative")
+        if self.cite_depth < 0:
+            raise ValueError("cite_depth must be non-negative")
+
+        # Validate replication_target shape.
+        if not isinstance(self.replication_target, tuple) or len(self.replication_target) != 2:
+            raise ValueError("replication_target must be (constraint, ttl) tuple")
+        constraint, ttl = self.replication_target
+        # accept "≥3 geo-disjoint" / ">=3 geo-disjoint" / "k=3"
+        if not re.search(r"\d+", constraint):
+            raise ValueError(f"replication_target constraint must contain a count: {constraint!r}")
+        if not re.fullmatch(r"TTL=\d+d", ttl):
+            raise ValueError(f"replication_target TTL must look like 'TTL=180d': {ttl!r}")
+
+        if self.erasure_coding is not None:
+            k, n = self.erasure_coding
+            if not (re.match(r"k=\d+", k) and re.match(r"n=\d+", n)):
+                raise ValueError(f"erasure_coding must be ('k=N', 'n=M'): {self.erasure_coding!r}")
+            k_v = int(k.split("=")[1])
+            n_v = int(n.split("=")[1])
+            if k_v >= n_v or k_v <= 0:
+                raise ValueError(f"erasure_coding requires 0 < k < n; got k={k_v}, n={n_v}")
+
+    @property
+    def replication_min_count(self) -> int:
+        """Extract the minimum replica count from the constraint string."""
+        m = re.search(r"\d+", self.replication_target[0])
+        return int(m.group()) if m else 1
+
+    @property
+    def replication_ttl_days(self) -> int:
+        m = re.match(r"TTL=(\d+)d", self.replication_target[1])
+        return int(m.group(1)) if m else 0
+
+    def matches(self, *, topic: str, claim_type: str, publisher_rep: float) -> bool:
+        """Should this pinner replicate the claim?"""
+        if publisher_rep < self.publishers_min_rep:
+            return False
+        if self.claim_types != "*" and claim_type not in self.claim_types:
+            return False
+        if not any(_topic_match(topic, t) for t in self.topics):
+            return False
+        return True
+
+    def compose(self, *others: PinPolicy) -> PinPolicy:
+        """Union semantics: a claim matches if ANY composed policy matches."""
+        topics = list(self.topics)
+        for o in others:
+            for t in o.topics:
+                if t not in topics:
+                    topics.append(t)
+        # Take the loosest min-rep across the set.
+        min_rep = min(self.publishers_min_rep, *[o.publishers_min_rep for o in others])
+        return PinPolicy(
+            topics=topics,
+            publishers_min_rep=min_rep,
+            claim_types=self.claim_types,
+            replication_target=self.replication_target,
+            erasure_coding=self.erasure_coding,
+            max_disk_gb=self.max_disk_gb + sum(o.max_disk_gb for o in others),
+            eviction=self.eviction,
+            cite_depth=max(self.cite_depth, *[o.cite_depth for o in others]),
+        )
+
+
+def _topic_match(topic: str, pattern: str) -> bool:
+    """Glob-style topic matcher: `ai-safety/*` matches `ai-safety/eval`."""
+    if pattern == topic:
+        return True
+    if pattern == "*":
+        return True
+    if pattern.endswith("/*"):
+        return topic.startswith(pattern[:-1])
+    return False

hypermind/responders/__init__.py

@@ -0,0 +1,87 @@
+"""LLM responders for ``HyperMindAgent.consult(...)``.
+
+A *responder* is a callable matching the consult signature::
+
+    Callable[[bytes, str], Awaitable[Uncertainty]]
+
+Given an agent kid and a free-form query, it returns an
+:class:`Uncertainty` summarising that agent's belief.
+
+The SDK ships:
+
+  * :class:`StaticResponder` — return a fixed Uncertainty (handy for tests
+    and deterministic simulations).
+  * :class:`SyntheticResponder` — sample a Beta(α, β) profile per agent
+    so different "personalities" answer differently. Used by the
+    geopolitical crisis simulation.
+  * :class:`OpenAIResponder` — call OpenAI Chat Completions, parse a
+    structured ``{confidence, reasoning}`` JSON response into Uncertainty.
+  * :class:`AnthropicResponder` — call Anthropic Messages, same contract.
+
+Both LLM responders use a strict JSON-mode prompt so the SDK never has
+to scrape free-text. The system prompt enforces a ``confidence`` field
+in [0, 1] which maps to a Beta posterior of effective sample size 10.
+
+Both LLM responders soft-depend on their respective SDKs — install
+them via ``pip install hypermind[openai]`` or ``pip install hypermind[anthropic]``.
+"""
+
+from __future__ import annotations
+
+from hypermind.responders.anthropic import AnthropicResponder
+from hypermind.responders.base import (
+    LLMResponderError,
+    Responder,
+    ResponderResult,
+    StaticResponder,
+    SyntheticResponder,
+    confidence_to_uncertainty,
+    unpack_responder_result,
+)
+from hypermind.responders.openai import OpenAIResponder
+from hypermind.responders.router import (
+    ArchetypeMappedRouter,
+    CostAwareRouter,
+    MultiModelPanelRouter,
+    RandomPoolRouter,
+    ResponderRouter,
+)
+from hypermind.responders.structured import (
+    STRUCTURED_PROMPT_SUFFIX,
+    StructuredResponse,
+    TokenUsage,
+    parse_structured_response,
+)
+from hypermind.responders.tools import (
+    Tool,
+    ToolCallingResponder,
+    query_knowledge_base,
+    retrieve_document,
+    web_search,
+)
+
+__all__ = [
+    "STRUCTURED_PROMPT_SUFFIX",
+    "AnthropicResponder",
+    "ArchetypeMappedRouter",
+    "CostAwareRouter",
+    "LLMResponderError",
+    "MultiModelPanelRouter",
+    "OpenAIResponder",
+    "RandomPoolRouter",
+    "Responder",
+    "ResponderResult",
+    "ResponderRouter",
+    "StaticResponder",
+    "StructuredResponse",
+    "SyntheticResponder",
+    "TokenUsage",
+    "Tool",
+    "ToolCallingResponder",
+    "confidence_to_uncertainty",
+    "parse_structured_response",
+    "query_knowledge_base",
+    "retrieve_document",
+    "unpack_responder_result",
+    "web_search",
+]