alloc-context 0.1.0__py3-none-any.whl

alloccontext/ingest/fred.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import urllib.error
+import urllib.parse
+import urllib.request
+from datetime import date, datetime, timedelta, timezone
+from typing import Any
+
+FRED_OBSERVATIONS_URL = "https://api.stlouisfed.org/fred/series/observations"
+
+
+def fetch_series_observations(
+    *,
+    series_id: str,
+    api_key: str,
+    observation_start: date,
+    observation_end: date,
+    timeout: float,
+) -> list[dict[str, Any]]:
+    params = urllib.parse.urlencode(
+        {
+            "series_id": series_id,
+            "api_key": api_key,
+            "file_type": "json",
+            "observation_start": observation_start.isoformat(),
+            "observation_end": observation_end.isoformat(),
+            "sort_order": "asc",
+        }
+    )
+    url = f"{FRED_OBSERVATIONS_URL}?{params}"
+    request = urllib.request.Request(url, headers={"User-Agent": "alloc-context/0.1"})
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        from alloccontext.ingest.http_errors import http_error_message
+
+        raise ValueError(
+            http_error_message(exc, context=f"fred series {series_id}")
+        ) from exc
+    if not isinstance(payload, dict):
+        raise ValueError(f"invalid FRED payload for {series_id}")
+    rows = payload.get("observations") or []
+    if not isinstance(rows, list):
+        raise ValueError(f"invalid FRED observations for {series_id}")
+    return [row for row in rows if isinstance(row, dict)]
+
+
+def _parse_observation(row: dict[str, Any]) -> tuple[str, float | None] | None:
+    obs_date = str(row.get("date") or "").strip()
+    if not obs_date:
+        return None
+    raw_value = row.get("value")
+    if raw_value is None or raw_value == ".":
+        return obs_date, None
+    try:
+        return obs_date, float(raw_value)
+    except (TypeError, ValueError):
+        return obs_date, None
+
+
+def upsert_fred_observations(
+    conn: sqlite3.Connection,
+    *,
+    series_id: str,
+    observations: list[dict[str, Any]],
+) -> int:
+    fetched_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    count = 0
+    for row in observations:
+        parsed = _parse_observation(row)
+        if parsed is None:
+            continue
+        obs_date, value = parsed
+        conn.execute(
+            """
+            INSERT INTO fred_observations(series_id, obs_date, value, fetched_at)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(series_id, obs_date) DO UPDATE SET
+              value=excluded.value,
+              fetched_at=excluded.fetched_at
+            """,
+            (series_id, obs_date, value, fetched_at),
+        )
+        count += 1
+    return count
+
+
+def refresh_fred(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    api_key = os.environ.get("FRED_API_KEY")
+    if not api_key:
+        return {
+            "ok": True,
+            "rows": 0,
+            "skipped": True,
+            "reason": "FRED_API_KEY not set",
+        }
+
+    fred = config.fred
+    if not fred.series:
+        return {
+            "ok": True,
+            "rows": 0,
+            "skipped": True,
+            "reason": "no_fred_series_configured",
+        }
+
+    today = datetime.now(timezone.utc).date()
+    start = today - timedelta(days=fred.lookback_days)
+    total = 0
+    series_ids: list[str] = []
+
+    for spec in fred.series:
+        try:
+            observations = fetch_series_observations(
+                series_id=spec.id,
+                api_key=api_key,
+                observation_start=start,
+                observation_end=today,
+                timeout=fred.timeout_seconds,
+            )
+            total += upsert_fred_observations(conn, series_id=spec.id, observations=observations)
+            series_ids.append(spec.id)
+        except (
+            urllib.error.URLError,
+            TimeoutError,
+            ValueError,
+            json.JSONDecodeError,
+            RuntimeError,
+        ) as exc:
+            conn.rollback()
+            return {"ok": False, "error": f"{spec.id}: {exc}", "rows": 0}
+
+    conn.commit()
+    return {"ok": True, "rows": total, "series": series_ids}

alloccontext/ingest/http_errors.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import urllib.error
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+_REDACTED_PARAMS = frozenset(
+    {
+        "token",
+        "apikey",
+        "api_key",
+    }
+)
+
+
+def redact_url_secrets(url: str) -> str:
+    """Remove common API key query params before logging or error text."""
+    parsed = urlparse(url)
+    if not parsed.query:
+        return url
+    query = [
+        (key, "***" if key.lower() in _REDACTED_PARAMS else value)
+        for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+    ]
+    return urlunparse(parsed._replace(query=urlencode(query)))
+
+
+def http_error_message(exc: urllib.error.HTTPError, *, context: str) -> str:
+    """HTTP failure text without embedding request URLs or secrets."""
+    return f"{context} HTTP {exc.code}"

alloccontext/ingest/kalshi.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from typing import Any
+
+from alloccontext.ingest.kalshi_files import load_tactical_snapshot
+from alloccontext.ingest.kalshi_state import (
+    extract_cf_price_history,
+    extract_market_quotes_from_state,
+    load_state_json,
+    tactical_to_storage,
+)
+from alloccontext.store.meta import set_meta
+
+
+def upsert_kalshi_snapshot(conn: sqlite3.Connection, row: dict[str, Any]) -> None:
+    conn.execute(
+        """
+        INSERT INTO kalshi_snapshots(ts, tape_summary, cluster_json, raw_json)
+        VALUES (?, ?, ?, ?)
+        """,
+        (
+            str(row["ts"]),
+            row.get("tape_summary"),
+            row.get("cluster_json"),
+            row.get("raw_json"),
+        ),
+    )
+
+
+def _refresh_kalshi_files(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    tactical_path = config.kalshi.fallback_tactical_snapshot
+    if tactical_path is None or not tactical_path.is_file():
+        return {"ok": False, "error": "missing_kalshi_tactical_snapshot", "rows": 0}
+
+    raw = load_state_json(tactical_path)
+    if raw is None:
+        return {"ok": False, "error": "invalid_tactical_snapshot", "rows": 0}
+
+    snapshot = load_tactical_snapshot(tactical_path)
+    if snapshot is None:
+        return {"ok": False, "error": "invalid_tactical_snapshot", "rows": 0}
+
+    storage = tactical_to_storage(snapshot, raw)
+    upsert_kalshi_snapshot(conn, storage)
+
+    state_path = config.kalshi.fallback_state
+    if state_path is not None and state_path.is_file():
+        state = load_state_json(state_path)
+        if state:
+            cf = extract_cf_price_history(state)
+            if cf:
+                set_meta(conn, "cf_price_history", json.dumps(cf))
+            quotes = extract_market_quotes_from_state(state)
+            if quotes:
+                set_meta(conn, "kalshi_markets", json.dumps(quotes))
+
+    conn.commit()
+    return {
+        "ok": True,
+        "rows": 1,
+        "ts": storage["ts"],
+        "tape_summary": storage.get("tape_summary"),
+        "source": "kalshi_files",
+    }
+
+
+def refresh_kalshi(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    if config.kalshi.use_api:
+        from alloccontext.ingest.kalshi_api import refresh_kalshi_api
+
+        result = refresh_kalshi_api(conn, config)
+        if result.get("ok"):
+            return result
+        fallback = config.kalshi.fallback_tactical_snapshot
+        if fallback is not None and fallback.is_file():
+            file_result = _refresh_kalshi_files(conn, config)
+            if file_result.get("ok"):
+                file_result["api_error"] = result.get("error")
+                return file_result
+        return result
+
+    return _refresh_kalshi_files(conn, config)

alloccontext/ingest/kalshi_api.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import datetime, timezone
+from typing import Any
+
+from alloccontext.ingest.cf_benchmarks import CFBenchmarksPriceError, fetch_prices
+from alloccontext.ingest.cf_history import load_cf_history, record_cf_price_samples, save_cf_history
+from alloccontext.ingest.kalshi_client import KalshiAPIError, KalshiClient, no_ask_cents_from_row, price_cents_from_row
+from alloccontext.ingest.kalshi_state import tactical_to_storage
+from alloccontext.rollup.cluster import MarketQuote, sentiment_up_fraction
+from alloccontext.rollup.cluster_config import RollupConfig
+from alloccontext.rollup.tape import build_live_tape_context, format_tape_summary
+from alloccontext.store.meta import set_meta
+
+
+def _series_tickers(config) -> list[str]:
+    return [row.series for row in config.kalshi.series if row.series]
+
+
+def _cf_indices(config) -> list[str]:
+    seen: set[str] = set()
+    indices: list[str] = []
+    for row in config.kalshi.series:
+        if row.cf_index and row.cf_index not in seen:
+            seen.add(row.cf_index)
+            indices.append(row.cf_index)
+    return indices
+
+
+def fetch_series_market_quotes(
+    client: KalshiClient, series_tickers: list[str]
+) -> list[MarketQuote]:
+    quotes: list[MarketQuote] = []
+    seen: set[str] = set()
+    for series in series_tickers:
+        payload = client.get_markets(status="open", limit=100, series_ticker=series)
+        for row in payload.get("markets") or []:
+            if not isinstance(row, dict):
+                continue
+            ticker = str(row.get("ticker") or "")
+            if not ticker or ticker in seen:
+                continue
+            if not ticker.upper().startswith(series.upper()):
+                continue
+            yes_bid, yes_ask = price_cents_from_row(row)
+            no_ask = no_ask_cents_from_row(row)
+            if yes_ask is None and no_ask is None:
+                continue
+            quotes.append(
+                MarketQuote(
+                    ticker=ticker,
+                    yes_ask_cents=yes_ask if yes_ask is not None else yes_bid,
+                    no_ask_cents=no_ask,
+                )
+            )
+            seen.add(ticker)
+    return quotes
+
+
+def _markets_meta_rows(markets: list[MarketQuote]) -> list[dict[str, Any]]:
+    return [
+        {
+            "ticker": market.ticker,
+            "yes_ask_cents": market.yes_ask_cents,
+            "no_ask_cents": market.no_ask_cents,
+        }
+        for market in markets
+    ]
+
+
+def build_api_tactical_payload(
+    rollup: RollupConfig,
+    *,
+    cf_history: dict[str, list[dict[str, Any]]],
+    markets: list[MarketQuote],
+    now: datetime,
+) -> dict[str, Any]:
+    computed = build_live_tape_context(
+        rollup,
+        cf_history=cf_history or None,
+        markets=markets,
+        now=now,
+    )
+    if computed is not None:
+        tape_ctx, cluster_ctx, at = computed
+        return {
+            "at": at.astimezone(timezone.utc).replace(microsecond=0).isoformat(),
+            "tape_summary": tape_ctx.get("summary"),
+            "trend_by_asset_60m": tape_ctx.get("trend_60m_pct") or {},
+            "trend_by_asset_15m": tape_ctx.get("trend_15m_pct") or {},
+            "volatility_regime": tape_ctx.get("volatility_regime"),
+            "cluster": cluster_ctx,
+            "daily_stats": {},
+            "source": "kalshi_api",
+        }
+
+    sentiment_up_frac, sentiment_sample = sentiment_up_fraction(markets)
+    cluster: dict[str, Any] = {}
+    if sentiment_sample:
+        cluster["sentiment_sample"] = sentiment_sample
+    if sentiment_up_frac is not None:
+        cluster["sentiment_up_frac"] = round(sentiment_up_frac, 4)
+
+    return {
+        "at": now.astimezone(timezone.utc).replace(microsecond=0).isoformat(),
+        "tape_summary": format_tape_summary(
+            trend_60m={},
+            vol_regime=None,
+            sentiment_up_frac=sentiment_up_frac,
+        ),
+        "trend_by_asset_60m": {},
+        "trend_by_asset_15m": {},
+        "volatility_regime": None,
+        "cluster": cluster,
+        "daily_stats": {},
+        "source": "kalshi_api",
+    }
+
+
+def refresh_kalshi_api(conn: sqlite3.Connection, config) -> dict[str, Any]:
+    kalshi = config.kalshi
+    series = _series_tickers(config)
+    if not series:
+        return {"ok": False, "error": "no_kalshi_series_configured", "rows": 0}
+
+    now = datetime.now(timezone.utc)
+    client = KalshiClient(kalshi.base_url, timeout=kalshi.timeout_seconds)
+
+    try:
+        markets = fetch_series_market_quotes(client, series)
+    except KalshiAPIError as exc:
+        return {"ok": False, "error": str(exc), "rows": 0}
+
+    cf_history = load_cf_history(conn)
+    indices = _cf_indices(config)
+    cf_errors: dict[str, str] = {}
+    if indices:
+        try:
+            prices = fetch_prices(indices, timeout=kalshi.timeout_seconds)
+            cf_history = record_cf_price_samples(
+                cf_history,
+                prices,
+                now,
+                max_age_minutes=float(kalshi.cf_history_max_age_minutes),
+            )
+            save_cf_history(conn, cf_history)
+        except CFBenchmarksPriceError as exc:
+            cf_errors["cf_benchmarks"] = str(exc)
+
+    if markets:
+        set_meta(conn, "kalshi_markets", json.dumps(_markets_meta_rows(markets)))
+
+    payload = build_api_tactical_payload(
+        config.rollup,
+        cf_history=cf_history,
+        markets=markets,
+        now=now,
+    )
+    if not markets and not cf_history:
+        conn.rollback()
+        return {
+            "ok": False,
+            "error": "no_kalshi_markets_or_cf_history",
+            "rows": 0,
+            "feed_errors": cf_errors,
+        }
+
+    from alloccontext.ingest.kalshi_files import KalshiTacticalSnapshot
+
+    snapshot = KalshiTacticalSnapshot(
+        at=payload.get("at"),
+        tape_summary=payload.get("tape_summary"),
+        trend_by_asset_60m=dict(payload.get("trend_by_asset_60m") or {}),
+        trend_by_asset_15m=dict(payload.get("trend_by_asset_15m") or {}),
+        volatility_regime=payload.get("volatility_regime"),
+        sentiment_up_frac=(payload.get("cluster") or {}).get("sentiment_up_frac"),
+        sentiment_sample=(payload.get("cluster") or {}).get("sentiment_sample"),
+        daily_stats=dict(payload.get("daily_stats") or {}),
+    )
+    storage = tactical_to_storage(snapshot, payload)
+
+    from alloccontext.ingest.kalshi import upsert_kalshi_snapshot
+
+    upsert_kalshi_snapshot(conn, storage)
+
+    result: dict[str, Any] = {
+        "ok": True,
+        "rows": 1,
+        "ts": storage["ts"],
+        "tape_summary": storage.get("tape_summary"),
+        "markets_sampled": len(markets),
+        "source": "kalshi_api",
+    }
+    if cf_errors:
+        result["feed_errors"] = cf_errors
+    conn.commit()
+    return result

alloccontext/ingest/kalshi_client.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+
+class KalshiAPIError(RuntimeError):
+    def __init__(self, message: str, status_code: int | None = None):
+        super().__init__(message)
+        self.status_code = status_code
+
+
+class KalshiClient:
+    """Read-only Kalshi REST client (public market endpoints)."""
+
+    def __init__(self, base_url: str, *, timeout: float = 20.0):
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+
+    def _request(self, method: str, path: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]:
+        if not path.startswith("/"):
+            path = f"/{path}"
+        query = f"?{urllib.parse.urlencode(params)}" if params else ""
+        url = f"{self.base_url}{path}{query}"
+        request = urllib.request.Request(
+            url,
+            headers={"Accept": "application/json", "User-Agent": "alloc-context/0.1"},
+            method=method.upper(),
+        )
+        for attempt in range(4):
+            try:
+                with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                    body = response.read().decode("utf-8")
+            except urllib.error.HTTPError as exc:
+                if exc.code == 429 and attempt < 3:
+                    time.sleep(2**attempt)
+                    continue
+                raise KalshiAPIError(
+                    f"Kalshi API {method.upper()} {path} failed: {exc.code}",
+                    status_code=exc.code,
+                ) from exc
+            except (urllib.error.URLError, TimeoutError) as exc:
+                raise KalshiAPIError(f"Kalshi API {method.upper()} {path} failed: {exc}") from exc
+            break
+        if not body:
+            return {}
+        parsed = json.loads(body)
+        return parsed if isinstance(parsed, dict) else {}
+
+    def get_markets(
+        self,
+        *,
+        status: str | None = None,
+        limit: int = 100,
+        cursor: str | None = None,
+        series_ticker: str | None = None,
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if status:
+            params["status"] = status
+        if cursor:
+            params["cursor"] = cursor
+        if series_ticker:
+            params["series_ticker"] = series_ticker
+        return self._request("GET", "/markets", params=params)
+
+
+def dollars_to_cents(value: str | int | float | None) -> int | None:
+    if value is None or value == "":
+        return None
+    return max(1, min(99, round(float(value) * 100)))
+
+
+def price_cents_from_row(row: dict[str, Any]) -> tuple[int | None, int | None]:
+    if row.get("yes_bid") is not None or row.get("yes_ask") is not None:
+        bid = row.get("yes_bid")
+        ask = row.get("yes_ask")
+        return (
+            int(bid) if bid is not None else None,
+            int(ask) if ask is not None else None,
+        )
+    return (
+        dollars_to_cents(row.get("yes_bid_dollars")),
+        dollars_to_cents(row.get("yes_ask_dollars")),
+    )
+
+
+def no_ask_cents_from_row(row: dict[str, Any]) -> int | None:
+    if row.get("no_ask") is not None:
+        return int(row["no_ask"])
+    return dollars_to_cents(row.get("no_ask_dollars"))

alloccontext/ingest/kalshi_files.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class KalshiTacticalSnapshot:
+    at: str | None
+    tape_summary: str | None
+    trend_by_asset_60m: dict[str, float | None]
+    trend_by_asset_15m: dict[str, float | None]
+    volatility_regime: str | None
+    sentiment_up_frac: float | None
+    sentiment_sample: int | None
+    daily_stats: dict[str, Any]
+
+
+def _read_json(path: Path) -> Any | None:
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def load_tactical_snapshot(path: Path) -> KalshiTacticalSnapshot | None:
+    raw = _read_json(path)
+    if not isinstance(raw, dict):
+        return None
+    cluster = raw.get("cluster") if isinstance(raw.get("cluster"), dict) else {}
+    return KalshiTacticalSnapshot(
+        at=raw.get("at"),
+        tape_summary=raw.get("tape_summary"),
+        trend_by_asset_60m=dict(raw.get("trend_by_asset_60m") or {}),
+        trend_by_asset_15m=dict(raw.get("trend_by_asset_15m") or {}),
+        volatility_regime=raw.get("volatility_regime"),
+        sentiment_up_frac=cluster.get("sentiment_up_frac"),
+        sentiment_sample=cluster.get("sentiment_sample"),
+        daily_stats=dict(raw.get("daily_stats") or {}),
+    )

alloccontext/ingest/kalshi_state.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from alloccontext.ingest.kalshi_files import KalshiTacticalSnapshot, load_tactical_snapshot
+
+
+def extract_cf_price_history(raw: dict[str, Any]) -> dict[str, Any]:
+    history = raw.get("cf_price_history")
+    if isinstance(history, dict):
+        return history
+    return {}
+
+
+def extract_market_quotes_from_state(raw: dict[str, Any]) -> list[dict[str, Any]]:
+    by_ticker: dict[str, dict[str, Any]] = {}
+    for source in ("opportunity_log_expiry", "journal_expiry"):
+        for entry in raw.get(source) or []:
+            ticker = entry.get("ticker")
+            if not ticker:
+                continue
+            upper = str(ticker).upper()
+            if not any(tag in upper for tag in ("BTCD", "ETHD", "15M")):
+                continue
+            yes_ask = entry.get("yes_ask_cents")
+            no_ask = entry.get("no_ask_cents")
+            if yes_ask is None and no_ask is None:
+                continue
+            ts = str(entry.get("at") or "")
+            existing = by_ticker.get(str(ticker))
+            if existing is None or ts >= str(existing.get("at") or ""):
+                by_ticker[str(ticker)] = {
+                    "ticker": str(ticker),
+                    "yes_ask_cents": yes_ask,
+                    "no_ask_cents": no_ask,
+                    "at": ts,
+                }
+    return [
+        {
+            "ticker": row["ticker"],
+            "yes_ask_cents": row.get("yes_ask_cents"),
+            "no_ask_cents": row.get("no_ask_cents"),
+        }
+        for row in by_ticker.values()
+    ]
+
+
+def load_state_json(path: Path) -> dict[str, Any] | None:
+    if not path.is_file():
+        return None
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+    return raw if isinstance(raw, dict) else None
+
+
+def tactical_to_storage(snapshot: KalshiTacticalSnapshot, raw: dict[str, Any]) -> dict[str, Any]:
+    cluster = raw.get("cluster") if isinstance(raw.get("cluster"), dict) else {}
+    return {
+        "ts": snapshot.at or raw.get("at"),
+        "tape_summary": snapshot.tape_summary,
+        "cluster_json": json.dumps(cluster),
+        "raw_json": json.dumps(raw),
+    }