curator-data 0.2.0__py3-none-any.whl

curator_data/__init__.py

@@ -0,0 +1,25 @@
+"""Market data for the curator agent.
+
+The public surface is deliberately small: build a registry, ask it for a
+snapshot. Everything else is an implementation detail of some source.
+
+    from curator_data import build_registry
+
+    registry = build_registry()
+    snapshot = await registry.snapshot(["messari", "token_api"], ["USDC", "WETH"])
+
+No name in this package's public API refers to a data provider. Sources are
+addressed by registry key, and those keys come from the mandate.
+"""
+
+from .facts import FactBuilder
+from .registry import Registry, build_registry
+from .sources import SOURCE_FACTORIES, available_sources
+
+__all__ = [
+    "Registry",
+    "build_registry",
+    "available_sources",
+    "SOURCE_FACTORIES",
+    "FactBuilder",
+]

curator_data/chain/__init__.py

@@ -0,0 +1,12 @@
+"""Reading data straight off a chain.
+
+Every other source in this package speaks HTTP to somebody's API. This one
+speaks JSON-RPC to a node, which is the point: the `DataSource` port abstracts
+*kinds of provider*, not just endpoints. A source that reads a contract and a
+source that queries a subgraph merge into the same `MarketSnapshot` without
+either knowing the other exists.
+"""
+
+from .rpc import RpcClient, RpcError
+
+__all__ = ["RpcClient", "RpcError"]

curator_data/chain/rpc.py

@@ -0,0 +1,121 @@
+"""A minimal `eth_call` client.
+
+Deliberately not web3.py. This lane needs exactly one JSON-RPC method against
+two view functions with fixed, argument-free selectors — that is a POST and
+some slicing, on the `httpx` client already in the tree. Pulling in a full node
+library for it would add a large dependency to a package whose whole selling
+point is that a new provider is cheap to add. (The root `pyproject.toml` also
+records a broken global `web3` that breaks pytest collection, and Lane D
+reached the same conclusion independently for the same reasons.)
+
+ABI decoding here is limited to what static-word returns need: fixed 32-byte
+slots, signed and unsigned. Anything more would be the point to reconsider.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+WORD = 32
+
+
+class RpcError(RuntimeError):
+    """The node was unreachable, or returned an error or unusable payload."""
+
+
+def decode_word(data: bytes, index: int, *, signed: bool = False) -> int:
+    """The `index`-th 32-byte word of an ABI return, as an integer."""
+    start = index * WORD
+    end = start + WORD
+    if len(data) < end:
+        raise RpcError(f"response too short for word {index} ({len(data)} bytes)")
+    return int.from_bytes(data[start:end], "big", signed=signed)
+
+
+def decode_string(data: bytes) -> str:
+    """A single dynamically-sized `string` return.
+
+    Layout is offset, then length, then the bytes. Only used for
+    `description()`, which is how a price feed states its own identity.
+    """
+    offset = decode_word(data, 0)
+    if len(data) < offset + WORD:
+        raise RpcError("string offset points past the end of the response")
+    length = int.from_bytes(data[offset : offset + WORD], "big")
+    start = offset + WORD
+    if len(data) < start + length:
+        raise RpcError("string length runs past the end of the response")
+    return data[start : start + length].decode("utf-8", errors="replace")
+
+
+class RpcClient:
+    """`eth_call` against one JSON-RPC endpoint."""
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        timeout_s: float = 15.0,
+        client: httpx.AsyncClient | None = None,
+    ):
+        if not url:
+            raise ValueError("an RPC url is required")
+        self.url = url
+        self._timeout = timeout_s
+        self._client = client
+        self._owns_client = client is None
+
+    @property
+    def client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=self._timeout)
+        return self._client
+
+    async def call(self, to: str, selector: str) -> bytes:
+        """`eth_call` a no-argument view function. Returns raw return data."""
+        payload = {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": "eth_call",
+            "params": [{"to": to, "data": selector}, "latest"],
+        }
+        try:
+            response = await self.client.post(self.url, json=payload)
+        except httpx.HTTPError as exc:
+            raise RpcError(f"node unreachable at {self.url}: {type(exc).__name__}") from exc
+
+        if response.status_code >= 400:
+            raise RpcError(f"node returned HTTP {response.status_code}")
+
+        try:
+            body = response.json()
+        except ValueError as exc:
+            raise RpcError("node returned non-JSON") from exc
+
+        if isinstance(body, dict) and body.get("error"):
+            raise RpcError(f"eth_call failed: {body['error'].get('message', body['error'])}")
+
+        result = body.get("result") if isinstance(body, dict) else None
+        if not isinstance(result, str) or not result.startswith("0x"):
+            raise RpcError(f"unusable eth_call result: {result!r}")
+        # `0x` means the call reverted or the address holds no code. Callers
+        # must not read that as a zero value.
+        if len(result) <= 2:
+            raise RpcError(f"empty return from {to} - no contract, or the call reverted")
+
+        try:
+            return bytes.fromhex(result[2:])
+        except ValueError as exc:
+            raise RpcError("eth_call result was not valid hex") from exc
+
+    async def aclose(self) -> None:
+        if self._client is not None and self._owns_client:
+            await self._client.aclose()
+        self._client = None
+
+
+__all__ = ["RpcClient", "RpcError", "decode_word", "decode_string", "WORD"]

curator_data/cli.py

@@ -0,0 +1,204 @@
+"""`curator-data` — one parameterised tool, not a folder of one-off scripts.
+
+Four subcommands covering everything this lane needs operationally:
+
+    curator-data sources              what can be granted in a mandate
+    curator-data protocols            what is configured, and where to add more
+    curator-data snapshot             take a real snapshot and print it
+    curator-data verify-live          prove the demo path hits live data
+
+`snapshot --json` emits a schema-valid `MarketSnapshot`, so it doubles as a way
+to hand another lane real data without them running any of this code.
+
+Output is ASCII: this runs in PowerShell, WSL and macOS terminals, and a
+cp1252 console turns box-drawing characters into noise.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+
+from .config import Settings
+from .queries import MARKET_KINDS, errors_as_dicts, pivot_markets, pivot_pools, prices
+from .registry import build_registry
+from .sources.protocols import ALL
+from .verify import report, summarise, verify_live
+
+
+def _fmt_usd(value: float | None) -> str:
+    if value is None:
+        return "-"
+    for threshold, suffix in ((1e9, "B"), (1e6, "M"), (1e3, "K")):
+        if abs(value) >= threshold:
+            return f"${value / threshold:,.1f}{suffix}"
+    return f"${value:,.2f}"
+
+
+def _fmt_pct(value: float | None) -> str:
+    return "-" if value is None else f"{value * 100:.2f}%"
+
+
+# ── subcommands ───────────────────────────────────────────────────────────
+
+
+def cmd_sources(args: argparse.Namespace) -> int:
+    """What a mandate may name in `permitted_data_sources`."""
+    registry = build_registry(Settings.from_env())
+    described = registry.describe()
+    if args.json:
+        print(json.dumps(described, indent=2))
+        return 0
+
+    print("Registered data sources (name these in Mandate.permitted_data_sources):\n")
+    for entry in described:
+        print(f"  {entry['key']}")
+        print(f"      {entry.get('description', '')}")
+        if entry.get("provides"):
+            print(f"      provides: {entry['provides']}")
+    print("\nAdd one: implement BaseSource in curator_data/sources/, then add a single")
+    print("line to SOURCE_FACTORIES in curator_data/sources/__init__.py.")
+    return 0
+
+
+def cmd_protocols(args: argparse.Namespace) -> int:
+    """The protocol table — the "adding a protocol is one line" claim, printed."""
+    if args.json:
+        print(json.dumps([p.__dict__ for p in ALL], indent=2))
+        return 0
+
+    print("Configured protocols:\n")
+    print(f"  {'key':<14} {'family':<10} {'chain':<7} {'on':<4} label")
+    print(f"  {'-' * 14} {'-' * 10} {'-' * 7} {'-' * 4} {'-' * 24}")
+    for p in ALL:
+        print(
+            f"  {p.key:<14} {p.family:<10} {p.chain:<7} "
+            f"{'yes' if p.enabled else 'no':<4} {p.label}"
+        )
+    print("\nAdding a protocol is one Protocol(...) line in")
+    print("curator_data/sources/protocols.py - Messari's standardized schema means")
+    print("every lending market answers the same query, so no adapter is needed.")
+    return 0
+
+
+async def _snapshot(args: argparse.Namespace) -> int:
+    settings = Settings.from_env()
+    assets = [a.strip().upper() for a in args.assets.split(",") if a.strip()]
+    registry = build_registry(settings)
+    try:
+        if args.sources:
+            keys = [s.strip() for s in args.sources.split(",") if s.strip()]
+        else:
+            keys = registry.sources_providing(*MARKET_KINDS, "price")
+        snapshot = await registry.snapshot(keys, assets)
+    finally:
+        await registry.aclose()
+
+    if args.json:
+        # Schema-valid MarketSnapshot: pipe it straight into another lane.
+        print(snapshot.model_dump_json(indent=2))
+        return 0 if snapshot.facts else 1
+
+    print(f"Snapshot at {snapshot.taken_at.isoformat(timespec='seconds')}")
+    print(f"Sources: {', '.join(keys) or 'none'}   Assets: {', '.join(assets)}\n")
+
+    markets = pivot_markets(snapshot)
+    if markets:
+        print(f"  {'protocol':<14} {'market':<8} {'APY':>8} {'TVL':>10} {'util':>7}")
+        print(f"  {'-' * 14} {'-' * 8} {'-' * 8} {'-' * 10} {'-' * 7}")
+        for row in markets:
+            print(
+                f"  {row.protocol:<14} {row.market:<8} {_fmt_pct(row.supply_apy):>8} "
+                f"{_fmt_usd(row.tvl_usd):>10} {_fmt_pct(row.utilization):>7}"
+            )
+        print()
+
+    pools = pivot_pools(snapshot)
+    for pool in pools:
+        print(f"  pool {pool.protocol} {'/'.join(pool.pair)}: {_fmt_usd(pool.liquidity_usd)}")
+    if pools:
+        print()
+
+    for symbol, price in prices(snapshot).items():
+        via = ", ".join(price["sources"])
+        line = f"  price {symbol}: ${price['price_usd']:,.2f}  (via {via}"
+        if len(price["sources"]) > 1:
+            # Independent mechanisms agreeing is worth showing, not just
+            # asserting — and disagreeing is worth showing even more.
+            line += f", spread {price['spread_pct']:.2f}%"
+            if price["disagreement"]:
+                line += " DISAGREEMENT"
+        print(line + ")")
+
+    errors = errors_as_dicts(snapshot)
+    if errors:
+        # Printed last and labelled plainly: a partial snapshot must look
+        # partial, or the number above it gets read as the whole market.
+        print("\n  Degraded - these sources could not be read:")
+        for error in errors:
+            print(f"    [{error['source']}] {error['message']}")
+
+    print(f"\n{len(snapshot.facts)} facts, {len(errors)} errors")
+    return 0 if snapshot.facts else 1
+
+
+async def _verify(args: argparse.Namespace) -> int:
+    results = await verify_live(Settings.from_env(), only=args.protocol)
+    print(report(results))
+    _, failed, skipped = summarise(results)
+    # Skipped counts as failure here: this command exists to prove the live
+    # path works, and "we did not check" is not proof.
+    return 1 if (failed or skipped) else 0
+
+
+# ── entry point ───────────────────────────────────────────────────────────
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="curator-data",
+        description="Market data registry: inspect sources, take snapshots, verify live data.",
+    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="debug logging to stderr")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    p_sources = subparsers.add_parser("sources", help="list registered data sources")
+    p_sources.add_argument("--json", action="store_true")
+    p_sources.set_defaults(func=lambda a: cmd_sources(a))
+
+    p_protocols = subparsers.add_parser("protocols", help="list configured protocols")
+    p_protocols.add_argument("--json", action="store_true")
+    p_protocols.set_defaults(func=lambda a: cmd_protocols(a))
+
+    p_snapshot = subparsers.add_parser("snapshot", help="take a live MarketSnapshot")
+    p_snapshot.add_argument("--assets", default="USDC,WETH", help="comma-separated symbols")
+    p_snapshot.add_argument("--sources", default="", help="comma-separated keys (default: all)")
+    p_snapshot.add_argument(
+        "--json", action="store_true", help="emit a schema-valid MarketSnapshot"
+    )
+    p_snapshot.set_defaults(func=lambda a: asyncio.run(_snapshot(a)))
+
+    p_verify = subparsers.add_parser(
+        "verify-live", help="prove the demo path reaches live gateway data"
+    )
+    p_verify.add_argument("--protocol", default=None, help="check only this protocol key")
+    p_verify.set_defaults(func=lambda a: asyncio.run(_verify(a)))
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.WARNING,
+        format="%(levelname)s %(name)s: %(message)s",
+        stream=sys.stderr,
+    )
+    return args.func(args)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

curator_data/config.py

@@ -0,0 +1,175 @@
+"""Every environment-dependent value, resolved in one place.
+
+Scattered `os.getenv` calls are how a lane ends up with three different
+opinions about which gateway URL is current. One `Settings` object, loaded
+once, passed explicitly.
+
+Loading `.env` is opt-in via `Settings.from_env()` so importing this module
+never has a side effect on a caller's process environment.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+# The Graph's decentralised gateway. The API key travels as a Bearer header
+# rather than in the path (`/api/{key}/subgraphs/id/{id}`, also supported) so
+# it cannot leak into an access log, a traceback or a screen share during the
+# demo.
+DEFAULT_GATEWAY_URL = "https://gateway.thegraph.com/api"
+
+# Token API is a REST service on its own host with its own credential — it is
+# NOT the same key as the subgraph gateway. See README "Credentials".
+#
+# Host verified by probe on 2026-07-25: `token-api.thegraph.com` (the name in
+# The Graph's own docs) does not resolve at all, and the docs now redirect to
+# Pinax, a Graph core developer who operates the service. `GET /health` here
+# returns {"status":"OK"}.
+DEFAULT_TOKEN_API_URL = "https://api.pinax.network/v1"
+
+# x402: the same subgraph gateway, on the payment-gated route. Deliberately
+# needs no API key — that is the entire point of the path.
+DEFAULT_X402_GATEWAY_URL = "https://gateway.thegraph.com/api/x402"
+
+DEFAULT_CHAIN = "base"
+
+
+def _env_flag(name: str, default: bool = False) -> bool:
+    raw = os.getenv(name)
+    if raw is None or raw == "":
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = os.getenv(name)
+    if raw is None or raw.strip() == "":
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def _find_dotenv(start: Path | None = None) -> Path | None:
+    """Walk up from `start` looking for a .env.
+
+    Relative to the file, never to the process CWD — the MCP server is launched
+    from whatever directory the host client happens to be in, and a teammate on
+    macOS must get the same answer as we do on Windows.
+    """
+    here = (start or Path(__file__).resolve()).parent
+    for candidate in [here, *here.parents]:
+        dotenv = candidate / ".env"
+        if dotenv.is_file():
+            return dotenv
+    return None
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Resolved configuration. Immutable — construct a new one to change it."""
+
+    graph_api_key: str | None = None
+    gateway_url: str = DEFAULT_GATEWAY_URL
+    token_api_url: str = DEFAULT_TOKEN_API_URL
+    token_api_key: str | None = None
+    chain: str = DEFAULT_CHAIN
+    #: JSON-RPC endpoint for on-chain sources (Chainlink feeds). Prefers the
+    #: local fork so a snapshot matches the chain the vault is actually on.
+    rpc_url: str | None = None
+    #: 30s, not 15: the live Uniswap V3 Base subgraph repeatedly answered in
+    #: ~20s during testing (its indexers are slow and intermittently
+    #: unavailable). A timeout below that turns a working source into a
+    #: permanently failing one.
+    request_timeout_s: float = 30.0
+
+    #: Per-source ceiling inside `Registry.snapshot`. A source that hangs must
+    #: not hold the decision loop open; it lands in `errors[]` instead. Sits
+    #: above `request_timeout_s` so a slow *request* fails with a useful
+    #: message rather than being cut off by the outer deadline.
+    source_timeout_s: float = 45.0
+
+    # ── x402 (feature-flagged, default off) ───────────────────────────────
+    x402_enabled: bool = False
+    x402_gateway_url: str = DEFAULT_X402_GATEWAY_URL
+    x402_private_key: str | None = None
+    x402_chain: str = DEFAULT_CHAIN
+
+    #: Extra registry keys are inert unless a mandate names them, so carrying
+    #: unknown config here is harmless.
+    extra: dict[str, str] = field(default_factory=dict)
+
+    @classmethod
+    def from_env(cls, *, load_dotenv: bool = True) -> Settings:
+        """Read the environment, optionally loading the repo-root `.env` first.
+
+        Real process environment always wins over the file, so a teammate can
+        override any single value without editing a committed template.
+        """
+        if load_dotenv:
+            dotenv_path = _find_dotenv()
+            if dotenv_path is not None:
+                try:
+                    from dotenv import load_dotenv as _load
+
+                    _load(dotenv_path, override=False)
+                except ImportError:  # pragma: no cover - dotenv is a hard dep
+                    pass
+
+        return cls(
+            graph_api_key=os.getenv("GRAPH_API_KEY") or None,
+            gateway_url=os.getenv("GRAPH_GATEWAY_URL") or DEFAULT_GATEWAY_URL,
+            token_api_url=os.getenv("TOKEN_API_URL") or DEFAULT_TOKEN_API_URL,
+            # Falls back to the gateway key: The Graph Market issues one
+            # credential that works for both in the common case, and a
+            # confusing 401 is worse than an attempt.
+            token_api_key=os.getenv("TOKEN_API_KEY") or os.getenv("GRAPH_API_KEY") or None,
+            chain=os.getenv("DATA_CHAIN") or DEFAULT_CHAIN,
+            # Fork first: the agent reasons about the chain its vault lives on,
+            # and during development that is anvil, not mainnet.
+            rpc_url=(
+                os.getenv("DATA_RPC_URL")
+                or os.getenv("ANVIL_RPC_URL")
+                or os.getenv("BASE_RPC_URL")
+                or None
+            ),
+            request_timeout_s=_env_float("DATA_REQUEST_TIMEOUT_S", 15.0),
+            source_timeout_s=_env_float("DATA_SOURCE_TIMEOUT_S", 20.0),
+            x402_enabled=_env_flag("X402_ENABLED", False),
+            x402_gateway_url=os.getenv("X402_GATEWAY_URL") or DEFAULT_X402_GATEWAY_URL,
+            x402_private_key=os.getenv("X402_PRIVATE_KEY") or None,
+            x402_chain=os.getenv("X402_CHAIN") or DEFAULT_CHAIN,
+        )
+
+    # ── Capability checks, so callers report rather than guess ────────────
+
+    @property
+    def has_gateway_credential(self) -> bool:
+        return bool(self.graph_api_key)
+
+    @property
+    def has_token_api_credential(self) -> bool:
+        return bool(self.token_api_key)
+
+    @property
+    def x402_ready(self) -> bool:
+        """Flag on AND a key to sign with. Either missing means fall back."""
+        return self.x402_enabled and bool(self.x402_private_key)
+
+    def subgraph_url(self, subgraph_id: str) -> str:
+        return f"{self.gateway_url.rstrip('/')}/subgraphs/id/{subgraph_id}"
+
+    def x402_subgraph_url(self, subgraph_id: str) -> str:
+        return f"{self.x402_gateway_url.rstrip('/')}/subgraphs/id/{subgraph_id}"
+
+
+__all__ = [
+    "Settings",
+    "DEFAULT_GATEWAY_URL",
+    "DEFAULT_TOKEN_API_URL",
+    "DEFAULT_X402_GATEWAY_URL",
+    "DEFAULT_CHAIN",
+]

curator_data/default.py

@@ -0,0 +1,34 @@
+"""A ready-made registry instance, for consumers that late-bind by reference.
+
+Lane B's harness resolves its data seam from configuration —
+`AGENT_DATA_REGISTRY=<module>:<attribute>` — imports it on first use, and checks
+it against the frozen `DataSourceRegistry` Protocol. That mechanism needs an
+*instance* at a stable import path, which `build_registry()` (a function) does
+not provide.
+
+So the integration point is:
+
+    AGENT_DATA_REGISTRY=curator_data.default:registry
+
+Kept in its own module rather than on `curator_data` itself, because
+constructing this reads `.env`. Importing the package should have no side
+effect; importing *this* module is an explicit request for the configured
+default. Anyone wanting different settings calls `build_registry(settings)`
+instead.
+
+Construction is cheap and cannot fail on missing credentials: sources are
+built lazily on first use, so an absent `GRAPH_API_KEY` surfaces later as a
+`MarketSnapshot.errors` entry rather than an import-time crash. That matters
+for a late-binding consumer — a raise here would silently drop them back to
+fixture data.
+"""
+
+from __future__ import annotations
+
+from .registry import Registry, build_registry
+
+#: The configured registry. Safe to import at any time; holds no connections
+#: until a source is actually consulted.
+registry: Registry = build_registry()
+
+__all__ = ["registry"]