alloc-context 0.1.0__py3-none-any.whl

alloccontext/ingest/etf_flows.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import urllib.error
+import urllib.request
+from datetime import date, datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+SOSO_BASE = "https://openapi.sosovalue.com"
+SOSO_HISTORICAL = "/openapi/v2/etf/historicalInflowChart"
+SOSO_METRICS = "/openapi/v2/etf/currentEtfDataMetrics"
+
+ETF_PRODUCTS = {
+    "BTC": "us-btc-spot",
+    "ETH": "us-eth-spot",
+}
+
+
+def _post_json(url: str, body: dict[str, Any], headers: dict[str, str], timeout: float) -> dict:
+    payload = json.dumps(body).encode("utf-8")
+    request = urllib.request.Request(
+        url,
+        data=payload,
+        headers={**headers, "Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        parsed = json.loads(response.read().decode("utf-8"))
+    if not isinstance(parsed, dict):
+        raise ValueError("invalid JSON object response")
+    if parsed.get("code", 0) != 0:
+        raise RuntimeError(str(parsed.get("msg") or "sosovalue_api_error"))
+    return parsed
+
+
+def fetch_sosovalue_historical(
+    *,
+    product: str,
+    api_key: str,
+    timeout: float,
+) -> list[dict[str, Any]]:
+    payload = _post_json(
+        f"{SOSO_BASE}{SOSO_HISTORICAL}",
+        {"type": product},
+        {"x-soso-api-key": api_key, "User-Agent": "alloc-context/0.1"},
+        timeout,
+    )
+    rows = payload.get("data") or []
+    if not isinstance(rows, list):
+        raise ValueError("invalid sosovalue historical payload")
+    return [row for row in rows if isinstance(row, dict)]
+
+
+def fetch_sosovalue_ticker_metrics(
+    *,
+    product: str,
+    api_key: str,
+    timeout: float,
+) -> list[dict[str, Any]]:
+    payload = _post_json(
+        f"{SOSO_BASE}{SOSO_METRICS}",
+        {"type": product},
+        {"x-soso-api-key": api_key, "User-Agent": "alloc-context/0.1"},
+        timeout,
+    )
+    data = payload.get("data") or {}
+    rows = data.get("list") if isinstance(data, dict) else None
+    if not isinstance(rows, list):
+        raise ValueError("invalid sosovalue metrics payload")
+    return [row for row in rows if isinstance(row, dict)]
+
+
+def _metric_value(block: Any) -> float | None:
+    if not isinstance(block, dict):
+        return None
+    value = block.get("value")
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _metric_date(block: Any) -> str | None:
+    if not isinstance(block, dict):
+        return None
+    raw = block.get("lastUpdateDate") or block.get("date")
+    return str(raw)[:10] if raw else None
+
+
+def load_fallback_snapshot(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    raw = json.loads(path.read_text())
+    return raw if isinstance(raw, dict) else {}
+
+
+def upsert_etf_flow_days(
+    conn: sqlite3.Connection,
+    *,
+    asset: str,
+    rows: list[dict[str, Any]],
+    source: str,
+) -> int:
+    fetched_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    count = 0
+    for row in rows:
+        flow_date = str(row.get("date") or row.get("flow_date") or "")[:10]
+        if not flow_date:
+            continue
+        net_flow = row.get("totalNetInflow", row.get("net_flow_usd"))
+        conn.execute(
+            """
+            INSERT INTO etf_flow_days(
+              asset, flow_date, net_flow_usd, total_value_traded_usd,
+              total_net_assets_usd, cum_net_inflow_usd, source, fetched_at
+            )
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(asset, flow_date) DO UPDATE SET
+              net_flow_usd = excluded.net_flow_usd,
+              total_value_traded_usd = excluded.total_value_traded_usd,
+              total_net_assets_usd = excluded.total_net_assets_usd,
+              cum_net_inflow_usd = excluded.cum_net_inflow_usd,
+              source = excluded.source,
+              fetched_at = excluded.fetched_at
+            """,
+            (
+                asset,
+                flow_date,
+                _float_or_none(net_flow),
+                _float_or_none(row.get("totalValueTraded", row.get("total_value_traded_usd"))),
+                _float_or_none(row.get("totalNetAssets", row.get("total_net_assets_usd"))),
+                _float_or_none(row.get("cumNetInflow", row.get("cum_net_inflow_usd"))),
+                source,
+                fetched_at,
+            ),
+        )
+        count += 1
+    return count
+
+
+def upsert_etf_ticker_flows(
+    conn: sqlite3.Connection,
+    *,
+    asset: str,
+    rows: list[dict[str, Any]],
+    source: str,
+) -> int:
+    fetched_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    count = 0
+    for row in rows:
+        ticker = str(row.get("ticker") or "").upper()
+        if not ticker:
+            continue
+        inflow = row.get("dailyNetInflow") if "dailyNetInflow" in row else row.get("net_flow_usd")
+        flow_date = _metric_date(inflow) or _metric_date(row.get("netAssets"))
+        if not flow_date and isinstance(row.get("flow_date"), str):
+            flow_date = row["flow_date"][:10]
+        if not flow_date:
+            flow_date = datetime.now(timezone.utc).date().isoformat()
+        conn.execute(
+            """
+            INSERT INTO etf_ticker_flows(
+              asset, ticker, flow_date, net_flow_usd, net_assets_usd,
+              institute, source, fetched_at
+            )
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(asset, ticker, flow_date) DO UPDATE SET
+              net_flow_usd = excluded.net_flow_usd,
+              net_assets_usd = excluded.net_assets_usd,
+              institute = excluded.institute,
+              source = excluded.source,
+              fetched_at = excluded.fetched_at
+            """,
+            (
+                asset,
+                ticker,
+                flow_date,
+                _metric_value(inflow) if isinstance(inflow, dict) else _float_or_none(inflow),
+                _metric_value(row.get("netAssets")),
+                str(row.get("institute") or "") or None,
+                source,
+                fetched_at,
+            ),
+        )
+        count += 1
+    return count
+
+
+def _float_or_none(value: Any) -> float | None:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _ingest_asset_from_fallback(
+    conn: sqlite3.Connection,
+    *,
+    asset: str,
+    snapshot: dict[str, Any],
+) -> int:
+    daily = snapshot.get("daily") or snapshot.get("history") or []
+    tickers = snapshot.get("by_ticker") or snapshot.get("tickers") or []
+    count = upsert_etf_flow_days(conn, asset=asset, rows=list(daily), source="fallback")
+    count += upsert_etf_ticker_flows(conn, asset=asset, rows=list(tickers), source="fallback")
+    return count
+
+
+def refresh_etf_flows(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    etf = config.etf
+    assets = [str(a).upper() for a in etf.assets]
+    api_key = os.environ.get("SOSOVALUE_API_KEY")
+    rows_total = 0
+    sources: set[str] = set()
+    feed_errors: dict[str, str] = {}
+
+    if api_key and etf.sosovalue_enabled:
+        for asset in assets:
+            product = ETF_PRODUCTS.get(asset)
+            if not product:
+                continue
+            try:
+                history = fetch_sosovalue_historical(
+                    product=product,
+                    api_key=api_key,
+                    timeout=etf.timeout_seconds,
+                )
+                rows_total += upsert_etf_flow_days(
+                    conn, asset=asset, rows=history, source="sosovalue"
+                )
+                metrics = fetch_sosovalue_ticker_metrics(
+                    product=product,
+                    api_key=api_key,
+                    timeout=etf.timeout_seconds,
+                )
+                rows_total += upsert_etf_ticker_flows(
+                    conn, asset=asset, rows=metrics, source="sosovalue"
+                )
+                sources.add("sosovalue")
+            except (urllib.error.URLError, TimeoutError, ValueError, RuntimeError) as exc:
+                feed_errors[f"sosovalue_{asset.lower()}"] = str(exc)
+
+    fallback_path = etf.fallback_snapshot
+    if fallback_path and fallback_path.exists():
+        snapshot = load_fallback_snapshot(fallback_path)
+        for asset in assets:
+            block = snapshot.get(asset.lower()) or snapshot.get(asset)
+            if isinstance(block, dict):
+                rows_total += _ingest_asset_from_fallback(conn, asset=asset, snapshot=block)
+                sources.add("fallback")
+
+    if rows_total == 0:
+        conn.rollback()
+        if not api_key and not (fallback_path and fallback_path.exists()):
+            return {
+                "ok": True,
+                "rows": 0,
+                "skipped": True,
+                "reason": "no_etf_data_source",
+            }
+        return {
+            "ok": False,
+            "rows": 0,
+            "sources": sorted(sources),
+            "feed_errors": feed_errors,
+            "error": "etf_ingest_failed",
+        }
+
+    conn.commit()
+    return {
+        "ok": True,
+        "rows": rows_total,
+        "sources": sorted(sources),
+        "feed_errors": feed_errors,
+    }

alloccontext/ingest/exchange/__init__.py

@@ -0,0 +1,4 @@
+from alloccontext.ingest.exchange.registry import refresh_exchange
+from alloccontext.ingest.exchange.types import ExchangeId
+
+__all__ = ["ExchangeId", "refresh_exchange"]

alloccontext/ingest/exchange/coinbase_adapter.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import sqlite3
+from typing import Any
+
+from alloccontext.horizon import bars_within_horizon, horizon_days
+from alloccontext.ingest.exchange.portfolio import writes_portfolio_snapshot
+from alloccontext.ingest.coinbase_client import CoinbaseError
+from alloccontext.ingest.coinbase_portfolio import (
+    build_coinbase_client,
+    fetch_portfolio_snapshot,
+    load_coinbase_credentials,
+)
+from alloccontext.ingest.kraken_portfolio import upsert_market_bars, upsert_portfolio_snapshot
+
+
+def refresh_coinbase_exchange(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    spot = config.exchanges.coinbase
+    if not spot.enabled:
+        return {"ok": True, "rows": 0, "skipped": True, "reason": "exchange_disabled"}
+
+    if not load_coinbase_credentials():
+        return {
+            "ok": True,
+            "rows": 0,
+            "skipped": True,
+            "reason": "missing_coinbase_credentials",
+        }
+
+    client = build_coinbase_client(spot)
+    try:
+        snap = None
+        portfolio_rows = 0
+        if writes_portfolio_snapshot(config, "coinbase"):
+            snap = fetch_portfolio_snapshot(client, spot)
+            upsert_portfolio_snapshot(conn, snap)
+            portfolio_rows = 1
+        bar_rows = 0
+        for product_id in spot.pairs:
+            bars = client.get_ohlc(product_id, spot.ohlc_interval_minutes)
+            bars = bars_within_horizon(bars, days=horizon_days(config))
+            bar_rows += upsert_market_bars(
+                conn,
+                pair=product_id,
+                interval_minutes=spot.ohlc_interval_minutes,
+                bars=bars,
+            )
+        conn.commit()
+    except CoinbaseError as exc:
+        conn.rollback()
+        return {"ok": False, "error": str(exc), "rows": 0}
+
+    result: dict[str, Any] = {
+        "ok": True,
+        "rows": portfolio_rows + bar_rows,
+        "market_bars": bar_rows,
+    }
+    if snap is not None:
+        result["portfolio"] = {
+            "ts": snap.ts,
+            "nav_usd": snap.nav_usd,
+            "cash_usd": snap.cash_usd,
+        }
+    return result

alloccontext/ingest/exchange/kraken_adapter.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import sqlite3
+from typing import Any
+
+from alloccontext.horizon import bars_within_horizon, horizon_days
+from alloccontext.ingest.exchange.portfolio import writes_portfolio_snapshot
+from alloccontext.ingest.kraken_client import KrakenError
+from alloccontext.ingest.kraken_portfolio import (
+    build_kraken_client,
+    fetch_portfolio_snapshot,
+    load_kraken_credentials,
+    upsert_market_bars,
+    upsert_portfolio_snapshot,
+)
+
+
+def refresh_kraken_exchange(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    spot = config.exchanges.kraken
+    if not spot.enabled:
+        return {"ok": True, "rows": 0, "skipped": True, "reason": "exchange_disabled"}
+
+    creds = load_kraken_credentials()
+    if writes_portfolio_snapshot(config, "kraken") and not creds:
+        return {
+            "ok": True,
+            "rows": 0,
+            "skipped": True,
+            "reason": "missing_kraken_credentials",
+        }
+
+    client = build_kraken_client(spot)
+    try:
+        snap = None
+        portfolio_rows = 0
+        if writes_portfolio_snapshot(config, "kraken"):
+            snap = fetch_portfolio_snapshot(client, spot)
+            upsert_portfolio_snapshot(conn, snap)
+            portfolio_rows = 1
+        bar_rows = 0
+        for pair in spot.pairs:
+            bars = client.get_ohlc(pair, spot.ohlc_interval_minutes)
+            bars = bars_within_horizon(bars, days=horizon_days(config))
+            bar_rows += upsert_market_bars(
+                conn,
+                pair=pair,
+                interval_minutes=spot.ohlc_interval_minutes,
+                bars=bars,
+            )
+        conn.commit()
+    except KrakenError as exc:
+        conn.rollback()
+        return {"ok": False, "error": str(exc), "rows": 0}
+
+    result: dict[str, Any] = {
+        "ok": True,
+        "rows": portfolio_rows + bar_rows,
+        "market_bars": bar_rows,
+    }
+    if snap is not None:
+        result["portfolio"] = {
+            "ts": snap.ts,
+            "nav_usd": snap.nav_usd,
+            "cash_usd": snap.cash_usd,
+        }
+    return result

alloccontext/ingest/exchange/live.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from typing import Any
+
+from alloccontext.ingest.coinbase_client import CoinbaseClient, CoinbaseError
+from alloccontext.ingest.coinbase_portfolio import fetch_portfolio_snapshot as fetch_coinbase_snapshot
+from alloccontext.ingest.exchange.types import ExchangeId
+from alloccontext.ingest.kraken_client import KrakenClient, KrakenError
+from alloccontext.ingest.kraken_portfolio import (
+    PortfolioSnapshot,
+    fetch_portfolio_snapshot as fetch_kraken_snapshot,
+)
+
+SUPPORTED_EXCHANGES = frozenset({"kraken", "coinbase"})
+
+
+class LivePortfolioError(Exception):
+    pass
+
+
+def validate_exchange_id(exchange: str) -> ExchangeId:
+    key = exchange.strip().lower()
+    if key not in SUPPORTED_EXCHANGES:
+        raise ValueError(f"unsupported exchange: {exchange}")
+    return key  # type: ignore[return-value]
+
+
+def _spot_config(config, exchange_id: ExchangeId):
+    if exchange_id == "kraken":
+        return config.exchanges.kraken
+    return config.exchanges.coinbase
+
+
+def fetch_live_portfolio_snapshot(
+    exchange_id: ExchangeId,
+    api_key: str,
+    api_secret: str,
+    config,
+) -> PortfolioSnapshot:
+    spot = _spot_config(config, exchange_id)
+    key = api_key.strip()
+    secret = api_secret.strip()
+    if not key or not secret:
+        raise LivePortfolioError("api_key and api_secret are required")
+
+    try:
+        if exchange_id == "kraken":
+            client = KrakenClient(
+                api_key=key,
+                api_secret=secret,
+                retry_backoff=spot.retry_backoff_seconds,
+                max_retries=spot.max_retries,
+            )
+            return fetch_kraken_snapshot(client, spot)
+        client = CoinbaseClient(
+            api_key=key,
+            api_secret=secret,
+            retry_backoff=spot.retry_backoff_seconds,
+            max_retries=spot.max_retries,
+        )
+        return fetch_coinbase_snapshot(client, spot)
+    except (KrakenError, CoinbaseError) as exc:
+        raise LivePortfolioError(str(exc)) from exc
+
+
+def portfolio_state_from_snapshot(
+    snap: PortfolioSnapshot,
+    *,
+    exchange_id: ExchangeId,
+    target_pct: dict[str, float],
+    band: float,
+) -> dict[str, Any]:
+    from alloccontext.rollup.band import check_allocation_band
+
+    allocation_pct = {
+        "BTC": snap.btc_pct,
+        "ETH": snap.eth_pct,
+        "CASH": snap.cash_pct,
+    }
+    band_result = check_allocation_band(allocation_pct, target_pct, float(band))
+    return {
+        "available": True,
+        "exchange": exchange_id,
+        "source": "live",
+        "nav_usd": round(float(snap.nav_usd), 2),
+        "cash_usd": round(float(snap.cash_usd), 2),
+        "allocation_pct": band_result["allocation_pct"],
+        "target_allocation_pct": target_pct,
+        "drift": band_result["drift"],
+        "rebalance_hint": band_result["hint"],
+        "outside_band": band_result["outside_band"],
+        "prices": dict(snap.prices),
+        "cash_breakdown": dict(snap.cash_breakdown),
+        "snapshot_ts": snap.ts,
+    }

alloccontext/ingest/exchange/portfolio.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from alloccontext.ingest.exchange.types import ExchangeId
+
+
+def writes_portfolio_snapshot(config, exchange_id: ExchangeId) -> bool:
+    """Only the configured primary exchange may upsert portfolio_snapshots."""
+    return config.exchanges.primary == exchange_id

alloccontext/ingest/exchange/registry.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import sqlite3
+from collections.abc import Callable
+from typing import Any
+
+from alloccontext.ingest.exchange.coinbase_adapter import refresh_coinbase_exchange
+from alloccontext.ingest.exchange.kraken_adapter import refresh_kraken_exchange
+from alloccontext.ingest.exchange.types import ExchangeId
+
+ExchangeRefreshFn = Callable[[sqlite3.Connection, Any], dict[str, Any]]
+
+_ADAPTERS: dict[ExchangeId, ExchangeRefreshFn] = {
+    "kraken": refresh_kraken_exchange,
+    "coinbase": refresh_coinbase_exchange,
+}
+
+
+def refresh_exchange(
+    conn: sqlite3.Connection,
+    config,
+    exchange_id: ExchangeId,
+) -> dict[str, Any]:
+    adapter = _ADAPTERS.get(exchange_id)
+    if adapter is None:
+        return {"ok": False, "rows": 0, "error": f"unknown_exchange:{exchange_id}"}
+    return adapter(conn, config)

alloccontext/ingest/exchange/types.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from typing import Literal
+
+ExchangeId = Literal["kraken", "coinbase"]

alloccontext/ingest/exchange_http.py

@@ -0,0 +1,28 @@
+"""HTTP retry classification for exchange API clients."""
+
+from __future__ import annotations
+
+import json
+
+import requests
+
+TRANSIENT_HTTP_STATUSES = frozenset({429, 502, 503, 504})
+
+
+def is_transient_http_status(status_code: int) -> bool:
+    return status_code in TRANSIENT_HTTP_STATUSES
+
+
+def is_retryable_exchange_request_error(exc: Exception) -> bool:
+    if isinstance(exc, (requests.Timeout, requests.ConnectionError)):
+        return True
+    if isinstance(exc, requests.HTTPError) and exc.response is not None:
+        return is_transient_http_status(exc.response.status_code)
+    return False
+
+
+def should_retry_exchange_attempt(exc: Exception) -> bool:
+    """Return True only for transient transport/HTTP errors worth retrying."""
+    if isinstance(exc, (json.JSONDecodeError, ValueError, TypeError, KeyError)):
+        return False
+    return is_retryable_exchange_request_error(exc)

alloccontext/ingest/fear_greed.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from typing import Any
+
+FNG_API = "https://api.alternative.me/fng/"
+
+
+def classify_fear_greed(value: int) -> str:
+    if value <= 24:
+        return "Extreme Fear"
+    if value <= 44:
+        return "Fear"
+    if value <= 55:
+        return "Neutral"
+    if value <= 74:
+        return "Greed"
+    return "Extreme Greed"
+
+
+def _parse_row(row: dict[str, Any]) -> dict[str, Any]:
+    value = int(row["value"])
+    ts = int(row["timestamp"])
+    return {
+        "timestamp": ts,
+        "value": value,
+        "classification": classify_fear_greed(value),
+    }
+
+
+def fetch_fear_greed(*, limit: int = 1, timeout: float = 15.0) -> list[dict[str, Any]]:
+    """Fetch Crypto Fear & Greed Index rows from alternative.me."""
+    url = f"{FNG_API}?limit={max(1, limit)}"
+    req = urllib.request.Request(url, headers={"User-Agent": "alloc-context/0.1"})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        payload = json.loads(resp.read().decode())
+    if not isinstance(payload, dict):
+        raise ValueError("invalid fear_greed payload")
+    rows = payload.get("data") or []
+    if not isinstance(rows, list):
+        raise ValueError("invalid fear_greed data")
+    return [_parse_row(row) for row in rows if isinstance(row, dict)]
+
+
+def upsert_fear_greed_rows(conn: sqlite3.Connection, rows: list[dict[str, Any]]) -> int:
+    fetched_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    count = 0
+    for row in rows:
+        conn.execute(
+            """
+            INSERT INTO fear_greed(ts, value, classification, fetched_at)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(ts) DO UPDATE SET
+              value=excluded.value,
+              classification=excluded.classification,
+              fetched_at=excluded.fetched_at
+            """,
+            (
+                str(int(row["timestamp"])),
+                int(row["value"]),
+                str(row["classification"]),
+                fetched_at,
+            ),
+        )
+        count += 1
+    return count
+
+
+def refresh_fear_greed(
+    conn: sqlite3.Connection,
+    *,
+    history_limit: int = 90,
+    timeout: float = 15.0,
+) -> dict[str, Any]:
+    """Refresh recent F&G history into SQLite."""
+    try:
+        rows = fetch_fear_greed(limit=history_limit, timeout=timeout)
+    except (urllib.error.URLError, TimeoutError, ValueError, json.JSONDecodeError) as exc:
+        conn.rollback()
+        return {"ok": False, "error": str(exc), "rows": 0}
+    if not rows:
+        return {"ok": False, "error": "empty_response", "rows": 0}
+    upserted = upsert_fear_greed_rows(conn, rows)
+    conn.commit()
+    return {"ok": True, "rows": upserted, "latest": rows[0]}