curator-data 0.2.0__py3-none-any.whl

curator_data/facts.py

@@ -0,0 +1,143 @@
+"""Constructing `Fact`s correctly, so sources don't each get it wrong.
+
+Two invariants from the frozen schema are easy to violate and expensive when
+violated:
+
+  * **`Fact.source` must be the registry key.** Provenance is the whole point;
+    a fact whose source is wrong is worse than a missing fact.
+  * **`apy_fraction` is 0.0432 for 4.32%, never 4.32.** Normalise at the source
+    adapter. Messari reports interest rates as percentages, so this is a live
+    trap, not a hypothetical one.
+
+`FactBuilder` makes both structural. A source constructs one with its own key
+and then cannot emit a fact attributed to anyone else, and the APY constructors
+are named after the unit they consume (`apy_from_percent` / `apy_from_fraction`)
+so the conversion is a decision at the call site rather than an assumption.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+
+from curator_schema.models import Fact, FactKind, FactSubject
+
+_SLUG_STRIP = re.compile(r"[^a-z0-9]+")
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _slug(value: str) -> str:
+    return _SLUG_STRIP.sub("-", value.strip().lower()).strip("-")
+
+
+def subject_slug(subject: FactSubject) -> str:
+    """Compact, human-readable identity for a fact's subject.
+
+    Appears in fact ids, which the dApp renders next to the agent's reasoning —
+    `messari:yield:aave-v3/usdc` is legible in a decision feed;
+    a hash is not.
+    """
+    parts: list[str] = []
+    if subject.protocol:
+        parts.append(_slug(subject.protocol))
+    if subject.market:
+        parts.append(_slug(subject.market))
+    if subject.token:
+        parts.append(_slug(subject.token))
+    if subject.pair:
+        parts.append("-".join(_slug(p) for p in subject.pair))
+    return "/".join(parts) or "unknown"
+
+
+class FactBuilder:
+    """Emits `Fact`s already stamped with one source's provenance.
+
+    Ids are deterministic (`source:kind:subject`) rather than sequential: the
+    same market observed in two consecutive snapshots keeps the same id, which
+    is what lets the dApp diff decisions and lets us spot a model citing a fact
+    it was never shown. The registry de-duplicates collisions at merge time.
+    """
+
+    def __init__(self, source: str, *, chain: str = "base", observed_at: datetime | None = None):
+        if not source:
+            raise ValueError("FactBuilder requires a source key — provenance is not optional")
+        self.source = source
+        self.chain = chain
+        self.observed_at = observed_at or utcnow()
+
+    # ── subject helpers ───────────────────────────────────────────────────
+
+    def subject(
+        self,
+        *,
+        protocol: str | None = None,
+        market: str | None = None,
+        token: str | None = None,
+        pair: list[str] | None = None,
+    ) -> FactSubject:
+        return FactSubject(
+            protocol=protocol, market=market, token=token, pair=pair, chain=self.chain
+        )
+
+    # ── the general form ──────────────────────────────────────────────────
+
+    def fact(
+        self,
+        kind: FactKind,
+        subject: FactSubject,
+        value: float,
+        unit: str,
+        *,
+        observed_at: datetime | None = None,
+        confidence: float | None = None,
+    ) -> Fact:
+        return Fact(
+            id=f"{self.source}:{kind}:{subject_slug(subject)}",
+            kind=kind,
+            subject=subject,
+            value=float(value),
+            unit=unit,  # type: ignore[arg-type]  # validated by pydantic
+            source=self.source,
+            observed_at=observed_at or self.observed_at,
+            confidence=confidence,
+        )
+
+    # ── unit-safe constructors ────────────────────────────────────────────
+
+    def apy_from_percent(self, subject: FactSubject, percent: float, **kw) -> Fact:
+        """APY given as `4.32` meaning 4.32%. Converts to the 0.0432 the schema wants."""
+        return self.fact("yield", subject, float(percent) / 100.0, "apy_fraction", **kw)
+
+    def apy_from_fraction(self, subject: FactSubject, fraction: float, **kw) -> Fact:
+        """APY already normalised: `0.0432` meaning 4.32%."""
+        return self.fact("yield", subject, float(fraction), "apy_fraction", **kw)
+
+    def usd(self, kind: FactKind, subject: FactSubject, value: float, **kw) -> Fact:
+        return self.fact(kind, subject, float(value), "usd", **kw)
+
+    def ratio(self, kind: FactKind, subject: FactSubject, value: float, **kw) -> Fact:
+        return self.fact(kind, subject, float(value), "ratio", **kw)
+
+
+def dedupe_ids(facts: list[Fact]) -> list[Fact]:
+    """Guarantee id uniqueness within one snapshot.
+
+    Deterministic ids can legitimately collide — two Aave markets whose names
+    slugify identically, say. `AllocationDecision.facts_used` points at these,
+    so a duplicate would make a decision's citation ambiguous. Later duplicates
+    get a `#n` suffix; the first keeps the clean id so the common case stays
+    readable.
+    """
+    seen: dict[str, int] = {}
+    out: list[Fact] = []
+    for fact in facts:
+        count = seen.get(fact.id, 0)
+        seen[fact.id] = count + 1
+        out.append(fact if count == 0 else fact.model_copy(update={"id": f"{fact.id}#{count}"}))
+    return out
+
+
+__all__ = ["FactBuilder", "dedupe_ids", "subject_slug", "utcnow"]

curator_data/graph/__init__.py

@@ -0,0 +1,11 @@
+"""Transport for The Graph's decentralised gateway.
+
+Kept separate from the sources so that *how* we reach a subgraph (API key,
+x402 payment, retries) is independent of *what* we ask it. The x402 path is a
+decorator over `GatewayClient` for exactly this reason.
+"""
+
+from .errors import GatewayAuthError, GatewayError, GatewayQueryError
+from .gateway import GatewayClient
+
+__all__ = ["GatewayClient", "GatewayError", "GatewayAuthError", "GatewayQueryError"]

curator_data/graph/errors.py

@@ -0,0 +1,42 @@
+"""Transport error taxonomy.
+
+Three cases, because callers genuinely act differently on each:
+
+  * `GatewayAuthError` — the credential is missing or rejected. Retrying is
+    pointless; the operator has to do something. `verify-live` reports this
+    distinctly so a missing key never looks like a network blip.
+  * `GatewayQueryError` — the gateway answered and rejected our GraphQL. That
+    is our bug (wrong field, wrong schema family), and the message carries the
+    subgraph id so it names the protocol to fix.
+  * `GatewayError` — everything else: transport, timeout, 5xx.
+
+All three carry a message that is safe to surface into `MarketSnapshot.errors`
+and read aloud during a demo. None of them ever contains the API key.
+"""
+
+from __future__ import annotations
+
+
+class GatewayError(RuntimeError):
+    """Base: something went wrong reaching or reading the gateway."""
+
+    def __init__(self, message: str, *, subgraph_id: str | None = None):
+        self.subgraph_id = subgraph_id
+        super().__init__(f"{message} (subgraph {subgraph_id})" if subgraph_id else message)
+
+
+class GatewayAuthError(GatewayError):
+    """Missing, malformed or rejected credential — 401/402/403."""
+
+
+class GatewayQueryError(GatewayError):
+    """The gateway returned GraphQL `errors[]`. Our query is wrong."""
+
+    def __init__(
+        self, message: str, *, subgraph_id: str | None = None, errors: list[dict] | None = None
+    ):
+        self.errors = errors or []
+        super().__init__(message, subgraph_id=subgraph_id)
+
+
+__all__ = ["GatewayError", "GatewayAuthError", "GatewayQueryError"]

curator_data/graph/factory.py

@@ -0,0 +1,48 @@
+"""Choosing a gateway transport.
+
+One decision point, so no source has to know that x402 exists. `messari.py`
+asks for a gateway and gets whichever transport the configuration warrants;
+adding a third transport later touches only this file.
+
+Defaults to the API-key client. The paid path requires the flag *and* a key —
+a flag alone would produce a confusing per-query fallback rather than an
+obvious "you forgot the key".
+"""
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+from ..config import Settings
+from .gateway import GatewayClient
+
+logger = logging.getLogger(__name__)
+
+
+def make_gateway(
+    settings: Settings, *, client: httpx.AsyncClient | None = None
+) -> GatewayClient:
+    """The gateway transport this configuration calls for."""
+    if not settings.x402_ready:
+        if settings.x402_enabled:
+            logger.warning(
+                "X402_ENABLED is set but X402_PRIVATE_KEY is not - using the API-key path"
+            )
+        return GatewayClient(settings, client=client)
+
+    # Imported lazily: eth-account is an optional dependency, and a broken or
+    # absent signing stack must degrade to the API-key path rather than break
+    # importing the package at all.
+    try:
+        from ..x402.client import X402GatewayClient
+    except ImportError as exc:  # pragma: no cover - optional dependency
+        logger.warning("x402 unavailable (%s) - using the API-key path", exc)
+        return GatewayClient(settings, client=client)
+
+    logger.info("x402 enabled - the agent will pay for its own queries")
+    return X402GatewayClient(settings, client=client)
+
+
+__all__ = ["make_gateway"]

curator_data/graph/gateway.py

@@ -0,0 +1,172 @@
+"""GraphQL client for subgraphs on The Graph's decentralised gateway.
+
+Small on purpose. A GraphQL client library would buy schema validation and a
+DSL we don't want — our queries are three fixed strings — at the cost of a
+dependency and its own error taxonomy to translate. `httpx.AsyncClient` plus a
+POST is the whole protocol.
+
+Auth travels as `Authorization: Bearer <key>`. The gateway also accepts the key
+in the path (`/api/<key>/subgraphs/id/<id>`), which we deliberately do not use:
+a URL ends up in logs, tracebacks and screen shares, and this repo is public.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+
+from ..config import Settings
+from .errors import GatewayAuthError, GatewayError, GatewayQueryError
+
+logger = logging.getLogger(__name__)
+
+#: Substrings that mark a GraphQL-level error as a credential problem rather
+#: than a bad query. The gateway answers an unauthenticated request with
+#: HTTP 200 and {"errors":[{"message":"auth error: missing authorization
+#: header"}]}, so status code alone cannot tell these apart.
+_AUTH_ERROR_MARKERS = (
+    "auth error",
+    "missing authorization",
+    "invalid api key",
+    "unauthorized",
+    "payment required",
+    "rate limit",
+)
+
+
+def _looks_like_auth_failure(message: str) -> bool:
+    lowered = message.lower()
+    return any(marker in lowered for marker in _AUTH_ERROR_MARKERS)
+
+
+class GatewayClient:
+    """Executes GraphQL documents against subgraphs by id.
+
+    One client is shared by every source that needs a subgraph, so connection
+    pooling and timeouts are configured in exactly one place.
+    """
+
+    def __init__(
+        self,
+        settings: Settings,
+        *,
+        client: httpx.AsyncClient | None = None,
+    ):
+        self.settings = settings
+        # An injected client is how tests reach this without a network or a
+        # credential — `httpx.MockTransport` needs no new dependency.
+        self._client = client
+        self._owns_client = client is None
+
+    @property
+    def client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=self.settings.request_timeout_s)
+        return self._client
+
+    @property
+    def label(self) -> str:
+        """How this transport identifies itself in logs and in the CLI."""
+        return "api-key"
+
+    def _headers(self) -> dict[str, str]:
+        headers = {"Content-Type": "application/json", "Accept": "application/json"}
+        if self.settings.graph_api_key:
+            headers["Authorization"] = f"Bearer {self.settings.graph_api_key}"
+        return headers
+
+    def _url(self, subgraph_id: str) -> str:
+        return self.settings.subgraph_url(subgraph_id)
+
+    async def query(
+        self,
+        subgraph_id: str,
+        document: str,
+        variables: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Run `document` against `subgraph_id`, returning the `data` object.
+
+        Raises `GatewayAuthError` / `GatewayQueryError` / `GatewayError`. Sources
+        let these propagate to the registry, which turns them into
+        `MarketSnapshot.errors` entries.
+        """
+        if not self.settings.graph_api_key:
+            # ASCII only in messages that reach a terminal: Windows consoles
+            # default to cp1252 and turn a stray em dash into a mojibake box
+            # mid-demo.
+            raise GatewayAuthError(
+                "GRAPH_API_KEY is not set - get one free at https://thegraph.com/studio "
+                "-> API Keys and put it in .env",
+                subgraph_id=subgraph_id,
+            )
+
+        payload = {"query": document, "variables": variables or {}}
+        return await self._post(self._url(subgraph_id), payload, subgraph_id)
+
+    async def _post(
+        self, url: str, payload: dict[str, Any], subgraph_id: str
+    ) -> dict[str, Any]:
+        try:
+            response = await self.client.post(url, json=payload, headers=self._headers())
+        except httpx.TimeoutException as exc:
+            raise GatewayError(f"gateway timed out: {exc}", subgraph_id=subgraph_id) from exc
+        except httpx.HTTPError as exc:
+            raise GatewayError(f"gateway unreachable: {exc}", subgraph_id=subgraph_id) from exc
+
+        return self._read(response, subgraph_id)
+
+    @staticmethod
+    def _read(response: httpx.Response, subgraph_id: str) -> dict[str, Any]:
+        if response.status_code in (401, 403):
+            raise GatewayAuthError(
+                f"gateway rejected the API key (HTTP {response.status_code})",
+                subgraph_id=subgraph_id,
+            )
+        if response.status_code == 402:
+            raise GatewayAuthError(
+                "gateway requires payment (HTTP 402) — this endpoint is x402-gated",
+                subgraph_id=subgraph_id,
+            )
+        if response.status_code >= 400:
+            raise GatewayError(
+                f"gateway returned HTTP {response.status_code}: {response.text[:200]}",
+                subgraph_id=subgraph_id,
+            )
+
+        try:
+            body = response.json()
+        except ValueError as exc:
+            raise GatewayError(
+                f"gateway returned non-JSON: {response.text[:200]}", subgraph_id=subgraph_id
+            ) from exc
+
+        if isinstance(body, dict) and body.get("errors"):
+            errors = body["errors"]
+            first = errors[0].get("message", "unknown") if errors else "unknown"
+            # Verified against the live gateway: a missing or rejected key comes
+            # back as HTTP *200* with {"errors":[{"message":"auth error: ..."}]},
+            # not as a 401. Classifying that as a query error would send someone
+            # to debug our GraphQL when the real fix is a credential.
+            if _looks_like_auth_failure(first):
+                raise GatewayAuthError(f"gateway rejected the request: {first}",
+                                       subgraph_id=subgraph_id)
+            raise GatewayQueryError(
+                f"GraphQL error: {first}", subgraph_id=subgraph_id, errors=errors
+            )
+
+        data = body.get("data") if isinstance(body, dict) else None
+        if data is None:
+            raise GatewayError(
+                "gateway response contained no data", subgraph_id=subgraph_id
+            )
+        return data
+
+    async def aclose(self) -> None:
+        if self._client is not None and self._owns_client:
+            await self._client.aclose()
+        self._client = None
+
+
+__all__ = ["GatewayClient"]

curator_data/ports.py

@@ -0,0 +1,106 @@
+"""The lane's view of the frozen `DataSource` seam.
+
+`packages/schema` owns the protocol — it is frozen and this module does not
+redefine it, it re-exports it. What lives here is the *convenience* layer a
+source author actually writes against, which the frozen protocol deliberately
+does not specify.
+
+Writing a new source is: subclass `BaseSource`, set `key`, implement
+`fetch`. Nothing else in the package changes except one registration line.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from curator_schema.models import Fact
+from curator_schema.ports import DataSource, DataSourceRegistry
+
+__all__ = ["DataSource", "DataSourceRegistry", "BaseSource"]
+
+
+class BaseSource(ABC):
+    """Optional base class for a `DataSource`.
+
+    The frozen protocol is structural, so a source need not inherit from this —
+    any object with `key` and `async fetch` satisfies it. This exists to carry
+    the two obligations that are easy to forget:
+
+      1. `close()` — sources own HTTP clients and the registry must be able to
+         release them without knowing what a source is made of.
+      2. `describe()` — the MCP server and the genesis UI both need to tell a
+         human what a source key means. Without this, "messari" is an opaque
+         string in a checkbox list.
+
+    Failure contract (from the frozen port, restated because it is the rule
+    most likely to be broken): `fetch` must not raise for *expected* failure —
+    a timeout, a rate limit, a missing market. Return what you have. The
+    registry does catch exceptions, but a source that raises on a missing
+    market loses the facts it could have returned alongside it.
+    """
+
+    #: Registry key. Named in `Mandate.permitted_data_sources`, shown in the
+    #: genesis UI, and stamped onto every Fact as provenance — so it is
+    #: user-visible and effectively permanent once shipped.
+    key: str = ""
+
+    #: One line, human-facing. Shown wherever a user picks data sources.
+    description: str = ""
+
+    #: Which `Fact.kind`s this source can contribute.
+    #:
+    #: Declared so callers can ask for a *capability* rather than naming a
+    #: provider: "who has prices?" resolves through the registry instead of
+    #: through a hardcoded list somewhere else in the codebase. That is what
+    #: keeps "adding a source is one line" true — a new price source starts
+    #: serving price queries the moment it is registered, with no second edit.
+    #:
+    #: An empty tuple means "unknown", and such a source is consulted for every
+    #: capability. Erring toward including it costs one call that returns
+    #: nothing; erring the other way silently drops a source the user granted.
+    provides: tuple[str, ...] = ()
+
+    def __init__(self) -> None:
+        self._notes: list[str] = []
+
+    @abstractmethod
+    async def fetch(self, assets: list[str]) -> list[Fact]:
+        """Facts relevant to `assets` (symbols from the mandate)."""
+        raise NotImplementedError
+
+    # ── partial-failure channel ───────────────────────────────────────────
+    #
+    # The frozen port models a source as all-or-nothing: return facts, or raise
+    # and be recorded in `MarketSnapshot.errors`. Real sources are not
+    # all-or-nothing — this one queries five protocols and any one of them can
+    # be down. Returning the other four silently would tell the model it saw
+    # the whole market when it did not, which is the single most dangerous
+    # failure mode we have: the agent holds a key and acts on this.
+    #
+    # So a source may record notes about what it could not fetch. The registry
+    # looks for `drain_notes` and folds each into `errors[]`. This is additive —
+    # `key` + `fetch` still satisfies the frozen protocol, and a source that
+    # ignores this mechanism behaves exactly as before.
+
+    def note(self, message: str) -> None:
+        """Record a partial failure to surface in `MarketSnapshot.errors`."""
+        self._notes.append(message)
+
+    def drain_notes(self) -> list[str]:
+        """Return and clear notes. Called by the registry after each fetch."""
+        notes, self._notes = self._notes, []
+        return notes
+
+    async def close(self) -> None:
+        """Release held resources. Safe to call more than once."""
+        return None
+
+    def describe(self) -> dict[str, str]:
+        return {
+            "key": self.key,
+            "description": self.description,
+            "provides": ", ".join(self.provides),
+        }
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging affordance
+        return f"<{type(self).__name__} key={self.key!r}>"