alloc-context 0.1.0__py3-none-any.whl

alloccontext/ingest/coinbase_client.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import secrets
+import time
+from typing import Any
+
+import jwt
+import requests
+
+from alloccontext.ingest.exchange_http import should_retry_exchange_attempt
+from cryptography.hazmat.primitives import serialization
+
+COINBASE_API = "https://api.coinbase.com"
+BROKERAGE_PREFIX = "/api/v3/brokerage"
+
+STABLE_CURRENCIES = frozenset(
+    {"USD", "USDC", "USDT", "DAI", "PYUSD", "USDE", "GUSD"}
+)
+
+PRODUCT_TO_SYMBOL = {
+    "BTC-USD": "BTC",
+    "ETH-USD": "ETH",
+}
+
+_INTERVAL_TO_GRANULARITY = {
+    1: "ONE_MINUTE",
+    5: "FIVE_MINUTE",
+    15: "FIFTEEN_MINUTE",
+    30: "THIRTY_MINUTE",
+    60: "ONE_HOUR",
+    120: "TWO_HOUR",
+    360: "SIX_HOUR",
+    1440: "ONE_DAY",
+}
+
+
+def product_to_symbol(product_id: str) -> str:
+    product = product_id.upper()
+    if product in PRODUCT_TO_SYMBOL:
+        return PRODUCT_TO_SYMBOL[product]
+    base = product.split("-", 1)[0]
+    if base in {"BTC", "ETH"}:
+        return base
+    return base
+
+
+def interval_to_granularity(interval_minutes: int) -> str:
+    granularity = _INTERVAL_TO_GRANULARITY.get(interval_minutes)
+    if granularity is None:
+        raise CoinbaseError(f"unsupported_ohlc_interval_minutes:{interval_minutes}")
+    return granularity
+
+
+def normalize_pem_secret(raw: str) -> str:
+    secret = raw.strip()
+    if "\\n" in secret:
+        secret = secret.replace("\\n", "\n")
+    return secret
+
+
+def normalize_coinbase_balances(
+    accounts: list[dict[str, Any]],
+) -> tuple[dict[str, float], dict[str, float]]:
+    balances: dict[str, float] = {"BTC": 0.0, "ETH": 0.0, "USD": 0.0}
+    cash_breakdown: dict[str, float] = {}
+    for account in accounts:
+        currency = str(account.get("currency") or "").upper()
+        if not currency:
+            continue
+        available = float((account.get("available_balance") or {}).get("value") or 0)
+        hold = float((account.get("hold") or {}).get("value") or 0)
+        total = available + hold
+        if total <= 0:
+            continue
+        if currency == "BTC":
+            balances["BTC"] += total
+        elif currency == "ETH":
+            balances["ETH"] += total
+        elif currency in STABLE_CURRENCIES:
+            balances["USD"] += total
+            cash_breakdown[currency] = cash_breakdown.get(currency, 0.0) + total
+    return balances, cash_breakdown
+
+
+class CoinbaseError(Exception):
+    pass
+
+
+class CoinbaseClient:
+    """Read-only Coinbase Advanced Trade REST client (accounts, product, candles)."""
+
+    def __init__(
+        self,
+        api_key: str = "",
+        api_secret: str = "",
+        *,
+        retry_backoff: float = 2.0,
+        max_retries: int = 3,
+        session: requests.Session | None = None,
+    ) -> None:
+        self.api_key = api_key.strip()
+        self.api_secret = normalize_pem_secret(api_secret)
+        self.retry_backoff = retry_backoff
+        self.max_retries = max_retries
+        self.session = session or requests.Session()
+
+    def get_ticker(self, product_id: str) -> dict[str, float]:
+        data = self._private("GET", f"/market/products/{product_id}")
+        price = data.get("price")
+        if price is None:
+            raise CoinbaseError(f"missing_price:{product_id}")
+        last = float(price)
+        return {"last": last, "bid": last, "ask": last}
+
+    def get_ohlc(self, product_id: str, interval_minutes: int = 1440) -> list[dict[str, float]]:
+        granularity = interval_to_granularity(interval_minutes)
+        end = int(time.time())
+        start = end - 86400 * 120
+        data = self._private(
+            "GET",
+            f"/market/products/{product_id}/candles",
+            params={
+                "start": str(start),
+                "end": str(end),
+                "granularity": granularity,
+            },
+        )
+        candles: list[dict[str, float]] = []
+        for row in data.get("candles") or []:
+            candles.append(
+                {
+                    "time": float(row["start"]),
+                    "open": float(row["open"]),
+                    "high": float(row["high"]),
+                    "low": float(row["low"]),
+                    "close": float(row["close"]),
+                }
+            )
+        candles.sort(key=lambda bar: bar["time"])
+        return candles
+
+    def get_balances_with_breakdown(
+        self,
+    ) -> tuple[dict[str, float], dict[str, float]]:
+        if not self.api_key or not self.api_secret:
+            raise CoinbaseError(
+                "COINBASE_API_KEY and COINBASE_API_SECRET required for balances"
+            )
+        accounts = self._list_accounts()
+        return normalize_coinbase_balances(accounts)
+
+    def _list_accounts(self) -> list[dict[str, Any]]:
+        accounts: list[dict[str, Any]] = []
+        cursor = ""
+        while True:
+            params: dict[str, str] = {"limit": "250"}
+            if cursor:
+                params["cursor"] = cursor
+            data = self._private("GET", "/accounts", params=params)
+            accounts.extend(data.get("accounts") or [])
+            if not data.get("has_next"):
+                break
+            cursor = str(data.get("cursor") or "")
+            if not cursor:
+                break
+        return accounts
+
+    def _build_jwt(self, method: str, path: str) -> str:
+        uri = f"{method} api.coinbase.com{path}"
+        private_key = serialization.load_pem_private_key(
+            self.api_secret.encode("utf-8"),
+            password=None,
+        )
+        payload = {
+            "sub": self.api_key,
+            "iss": "cdp",
+            "nbf": int(time.time()),
+            "exp": int(time.time()) + 120,
+            "uri": uri,
+        }
+        token = jwt.encode(
+            payload,
+            private_key,
+            algorithm="ES256",
+            headers={"kid": self.api_key, "nonce": secrets.token_hex()},
+        )
+        if isinstance(token, bytes):
+            return token.decode("utf-8")
+        return token
+
+    def _private(
+        self,
+        method: str,
+        subpath: str,
+        *,
+        params: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        path = f"{BROKERAGE_PREFIX}{subpath}"
+        return self._request(method, path, params=params, auth=True)
+
+    def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: dict[str, str] | None = None,
+        auth: bool = False,
+    ) -> dict[str, Any]:
+        url = COINBASE_API + path
+        last_error: Exception | None = None
+        for attempt in range(self.max_retries):
+            try:
+                headers: dict[str, str] = {}
+                if auth:
+                    headers["Authorization"] = f"Bearer {self._build_jwt(method, path)}"
+                resp = self.session.request(
+                    method,
+                    url,
+                    params=params,
+                    headers=headers,
+                    timeout=30,
+                )
+                resp.raise_for_status()
+                body = resp.json()
+                if not isinstance(body, dict):
+                    raise CoinbaseError("invalid_response")
+                return body
+            except Exception as exc:  # noqa: BLE001
+                last_error = exc
+                if attempt + 1 < self.max_retries and should_retry_exchange_attempt(exc):
+                    time.sleep(self.retry_backoff * (attempt + 1))
+                    continue
+                break
+        raise CoinbaseError(str(last_error)) from last_error

alloccontext/ingest/coinbase_portfolio.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import os
+import sqlite3
+from datetime import datetime, timezone
+from typing import Any
+
+from alloccontext.ingest.coinbase_client import (
+    CoinbaseClient,
+    normalize_pem_secret,
+    product_to_symbol,
+)
+from alloccontext.ingest.kraken_portfolio import (
+    PortfolioSnapshot,
+    portfolio_from_balances,
+    upsert_market_bars,
+    upsert_portfolio_snapshot,
+)
+
+
+def load_coinbase_credentials() -> tuple[str, str] | None:
+    api_key = os.environ.get("COINBASE_API_KEY", "").strip()
+    api_secret = normalize_pem_secret(os.environ.get("COINBASE_API_SECRET", ""))
+    if api_key and api_secret:
+        return api_key, api_secret
+    return None
+
+
+def build_coinbase_client(spot) -> CoinbaseClient:
+    creds = load_coinbase_credentials()
+    return CoinbaseClient(
+        api_key=creds[0] if creds else "",
+        api_secret=creds[1] if creds else "",
+        retry_backoff=spot.retry_backoff_seconds,
+        max_retries=spot.max_retries,
+    )
+
+
+def fetch_portfolio_snapshot(client: CoinbaseClient, spot) -> PortfolioSnapshot:
+    prices: dict[str, float] = {}
+    for product_id in spot.pairs:
+        symbol = product_to_symbol(product_id)
+        prices[symbol] = client.get_ticker(product_id)["last"]
+    balances, cash_breakdown = client.get_balances_with_breakdown()
+    snap = portfolio_from_balances(balances, prices, cash_breakdown=cash_breakdown)
+    snap.ts = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    return snap
+
+
+def refresh_coinbase(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    from alloccontext.ingest.exchange.registry import refresh_exchange
+
+    return refresh_exchange(conn, config, "coinbase")

alloccontext/ingest/coingecko.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+from datetime import datetime, timezone
+from typing import Any
+
+from alloccontext.ingest.env_keys import optional_env_key
+from alloccontext.ingest.parse_helpers import parse_float, parse_int
+
+COINGECKO_BASE = "https://api.coingecko.com/api/v3"
+
+
+def _fetch_json(url: str, *, headers: dict[str, str] | None = None, timeout: float = 20.0) -> Any:
+    request = urllib.request.Request(
+        url,
+        headers={"User-Agent": "alloc-context/0.1", **(headers or {})},
+        method="GET",
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        return json.loads(response.read().decode("utf-8"))
+
+
+def _headers(api_key: str | None) -> dict[str, str]:
+    if not api_key:
+        return {}
+    return {"x-cg-demo-api-key": api_key}
+
+
+def fetch_coingecko_global(*, api_key: str | None, timeout: float) -> dict[str, Any]:
+    url = f"{COINGECKO_BASE}/global"
+    payload = _fetch_json(url, headers=_headers(api_key), timeout=timeout)
+    data = payload.get("data") if isinstance(payload, dict) else None
+    if not isinstance(data, dict):
+        raise ValueError("invalid coingecko global payload")
+    return data
+
+
+def fetch_coingecko_markets(
+    *,
+    coin_ids: list[str],
+    api_key: str | None,
+    timeout: float,
+) -> list[dict[str, Any]]:
+    if not coin_ids:
+        return []
+    query = urllib.parse.urlencode(
+        {
+            "vs_currency": "usd",
+            "ids": ",".join(coin_ids),
+            "order": "market_cap_desc",
+            "sparkline": "false",
+        }
+    )
+    url = f"{COINGECKO_BASE}/coins/markets?{query}"
+    payload = _fetch_json(url, headers=_headers(api_key), timeout=timeout)
+    if not isinstance(payload, list):
+        raise ValueError("invalid coingecko markets payload")
+    return [row for row in payload if isinstance(row, dict)]
+
+
+def normalize_coingecko_snapshot(
+    *,
+    global_data: dict[str, Any],
+    markets: list[dict[str, Any]],
+) -> dict[str, Any]:
+    market_caps = global_data.get("market_cap_percentage") or {}
+    total_cap = (global_data.get("total_market_cap") or {}).get("usd")
+
+    by_id = {str(row.get("id")): row for row in markets}
+    btc = by_id.get("bitcoin") or {}
+    eth = by_id.get("ethereum") or {}
+
+    return {
+        "total_market_cap_usd": _rounded(parse_float(total_cap)),
+        "btc_dominance_pct": _rounded(parse_float(market_caps.get("btc"))),
+        "eth_dominance_pct": _rounded(parse_float(market_caps.get("eth"))),
+        "btc_rank": parse_int(btc.get("market_cap_rank")),
+        "eth_rank": parse_int(eth.get("market_cap_rank")),
+        "btc_price_usd": _rounded(parse_float(btc.get("current_price"))),
+        "eth_price_usd": _rounded(parse_float(eth.get("current_price"))),
+        "btc_market_cap_usd": _rounded(parse_float(btc.get("market_cap"))),
+        "eth_market_cap_usd": _rounded(parse_float(eth.get("market_cap"))),
+        "btc_change_pct_24h": _rounded(parse_float(btc.get("price_change_percentage_24h"))),
+        "eth_change_pct_24h": _rounded(parse_float(eth.get("price_change_percentage_24h"))),
+    }
+
+
+def _rounded(value: float | None) -> float | None:
+    return round(value, 4) if value is not None else None
+
+
+def refresh_coingecko(conn, config) -> dict[str, Any]:
+    api_key = optional_env_key("COINGECKO_API_KEY") if config.coingecko.use_demo_key else None
+
+    def _fetch_snapshot(*, key: str | None) -> dict[str, Any]:
+        global_data = fetch_coingecko_global(api_key=key, timeout=config.coingecko.timeout_seconds)
+        markets = fetch_coingecko_markets(
+            coin_ids=list(config.coingecko.coin_ids),
+            api_key=key,
+            timeout=config.coingecko.timeout_seconds,
+        )
+        return normalize_coingecko_snapshot(global_data=global_data, markets=markets)
+
+    try:
+        snapshot = _fetch_snapshot(key=api_key)
+    except urllib.error.HTTPError as exc:
+        if exc.code == 429:
+            return {
+                "ok": True,
+                "rows": 0,
+                "skipped": True,
+                "reason": "coingecko_rate_limited",
+            }
+        if exc.code in (401, 403) and api_key:
+            try:
+                snapshot = _fetch_snapshot(key=None)
+            except urllib.error.HTTPError as retry_exc:
+                if retry_exc.code in (401, 403):
+                    return {
+                        "ok": True,
+                        "rows": 0,
+                        "skipped": True,
+                        "reason": "coingecko_auth_failed",
+                    }
+                return {"ok": False, "error": str(retry_exc), "rows": 0}
+            except (urllib.error.URLError, TimeoutError, ValueError, json.JSONDecodeError, RuntimeError) as retry_exc:
+                return {"ok": False, "error": str(retry_exc), "rows": 0}
+        elif exc.code in (401, 403):
+            return {
+                "ok": True,
+                "rows": 0,
+                "skipped": True,
+                "reason": "coingecko_auth_failed",
+            }
+        else:
+            return {"ok": False, "error": str(exc), "rows": 0}
+    except (urllib.error.URLError, TimeoutError, ValueError, json.JSONDecodeError, RuntimeError) as exc:
+        return {"ok": False, "error": str(exc), "rows": 0}
+
+    from alloccontext.ingest.market_snapshots import upsert_crypto_market_snapshot
+
+    ts = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    upsert_crypto_market_snapshot(conn, source="coingecko", snapshot_ts=ts, snapshot=snapshot)
+    conn.commit()
+    return {"ok": True, "rows": 1, "snapshot_ts": ts, **snapshot}

alloccontext/ingest/coinmarketcap.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+from datetime import datetime, timezone
+from typing import Any
+
+from alloccontext.ingest.env_keys import optional_env_key
+from alloccontext.ingest.parse_helpers import parse_float, parse_int
+
+CMC_BASE = "https://pro-api.coinmarketcap.com/v1"
+
+
+def _fetch_json(url: str, *, api_key: str, timeout: float) -> Any:
+    request = urllib.request.Request(
+        url,
+        headers={
+            "User-Agent": "alloc-context/0.1",
+            "X-CMC_PRO_API_KEY": api_key,
+            "Accept": "application/json",
+        },
+        method="GET",
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        return json.loads(response.read().decode("utf-8"))
+
+
+def fetch_cmc_global(*, api_key: str, timeout: float) -> dict[str, Any]:
+    payload = _fetch_json(f"{CMC_BASE}/global-metrics/quotes/latest", api_key=api_key, timeout=timeout)
+    data = payload.get("data") if isinstance(payload, dict) else None
+    if not isinstance(data, dict):
+        raise ValueError("invalid cmc global payload")
+    return data
+
+
+def fetch_cmc_quotes(
+    *,
+    symbols: list[str],
+    api_key: str,
+    timeout: float,
+) -> dict[str, Any]:
+    if not symbols:
+        return {}
+    query = urllib.parse.urlencode({"symbol": ",".join(symbols), "convert": "USD"})
+    payload = _fetch_json(
+        f"{CMC_BASE}/cryptocurrency/quotes/latest?{query}",
+        api_key=api_key,
+        timeout=timeout,
+    )
+    data = payload.get("data") if isinstance(payload, dict) else None
+    if not isinstance(data, dict):
+        raise ValueError("invalid cmc quotes payload")
+    return data
+
+
+def _quote_usd(asset: dict[str, Any]) -> dict[str, Any]:
+    quote = (asset.get("quote") or {}).get("USD") or {}
+    return quote if isinstance(quote, dict) else {}
+
+
+def normalize_cmc_snapshot(
+    *,
+    global_data: dict[str, Any],
+    quotes: dict[str, Any],
+) -> dict[str, Any]:
+    usd = _quote_usd(global_data)
+    btc = quotes.get("BTC") or {}
+    eth = quotes.get("ETH") or {}
+    btc_q = _quote_usd(btc)
+    eth_q = _quote_usd(eth)
+
+    return {
+        "total_market_cap_usd": _rounded(parse_float(usd.get("total_market_cap"))),
+        "btc_dominance_pct": _rounded(parse_float(global_data.get("btc_dominance"))),
+        "eth_dominance_pct": _rounded(parse_float(global_data.get("eth_dominance"))),
+        "btc_rank": parse_int(btc.get("cmc_rank")),
+        "eth_rank": parse_int(eth.get("cmc_rank")),
+        "btc_price_usd": _rounded(parse_float(btc_q.get("price"))),
+        "eth_price_usd": _rounded(parse_float(eth_q.get("price"))),
+        "btc_market_cap_usd": _rounded(parse_float(btc_q.get("market_cap"))),
+        "eth_market_cap_usd": _rounded(parse_float(eth_q.get("market_cap"))),
+        "btc_change_pct_24h": _rounded(parse_float(btc_q.get("percent_change_24h"))),
+        "eth_change_pct_24h": _rounded(parse_float(eth_q.get("percent_change_24h"))),
+    }
+
+
+def _rounded(value: float | None) -> float | None:
+    return round(value, 4) if value is not None else None
+
+
+def refresh_coinmarketcap(conn, config) -> dict[str, Any]:
+    api_key = optional_env_key("COINMARKETCAP_API_KEY")
+    if not api_key:
+        return {
+            "ok": True,
+            "rows": 0,
+            "skipped": True,
+            "reason": "COINMARKETCAP_API_KEY not set",
+        }
+
+    try:
+        global_data = fetch_cmc_global(api_key=api_key, timeout=config.coinmarketcap.timeout_seconds)
+        quotes = fetch_cmc_quotes(
+            symbols=list(config.coinmarketcap.symbols),
+            api_key=api_key,
+            timeout=config.coinmarketcap.timeout_seconds,
+        )
+        snapshot = normalize_cmc_snapshot(global_data=global_data, quotes=quotes)
+    except urllib.error.HTTPError as exc:
+        if exc.code in (401, 403):
+            return {
+                "ok": True,
+                "rows": 0,
+                "skipped": True,
+                "reason": "coinmarketcap_auth_failed",
+            }
+        if exc.code == 429:
+            return {
+                "ok": True,
+                "rows": 0,
+                "skipped": True,
+                "reason": "coinmarketcap_rate_limited",
+            }
+        return {"ok": False, "error": str(exc), "rows": 0}
+    except (urllib.error.URLError, TimeoutError, ValueError, json.JSONDecodeError, RuntimeError) as exc:
+        return {"ok": False, "error": str(exc), "rows": 0}
+
+    from alloccontext.ingest.market_snapshots import upsert_crypto_market_snapshot
+
+    ts = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    upsert_crypto_market_snapshot(conn, source="coinmarketcap", snapshot_ts=ts, snapshot=snapshot)
+    conn.commit()
+    return {"ok": True, "rows": 1, "snapshot_ts": ts, **snapshot}

alloccontext/ingest/env_keys.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+import os
+
+
+def optional_env_key(name: str) -> str | None:
+    """Return env value when non-empty after strip; else None."""
+    raw = os.environ.get(name)
+    if raw is None:
+        return None
+    stripped = raw.strip()
+    return stripped or None