ml-data-access 0.1.0__py3-none-any.whl

ml_data_access/__init__.py

@@ -0,0 +1,84 @@
+"""Read side of the malatium data store.
+
+    import ml_data_access
+
+    db = ml_data_access.connect()
+    reference_df = ml_data_access.load_reference_returns(db, start, end)
+    chain_df = ml_data_access.load_option_greeks(db, "AAPL", start, end)
+
+One `load_*` per table `ml-data-pipelines` writes, every one taking the
+database and an inclusive `[start, end]` window and returning a collected
+DataFrame. Nothing is screened, joined or derived: the store already holds
+canonical tables, and the two screens a multi-year study needs are named
+functions, `usable_symbol_years` and `in_universe`.
+"""
+
+from ml_data_access.loaders import (
+    in_universe,
+    load_calendar,
+    load_corporate_actions,
+    load_earnings,
+    load_factor_covariances,
+    load_factor_loadings,
+    load_factor_returns,
+    load_forecast,
+    load_idio_vol,
+    load_index_greeks,
+    load_indices,
+    load_open_interest,
+    load_option_greeks,
+    load_rates,
+    load_realized_vol,
+    load_reference_returns,
+    load_sectors,
+    load_sessions,
+    load_signals,
+    load_stock_features,
+    load_surface,
+    load_symbology_check,
+    load_underlying,
+    load_universe,
+    load_yields,
+    usable_symbol_years,
+)
+from ml_data_access.store import (
+    available_symbols,
+    available_years,
+    connect,
+    describe,
+    scan,
+)
+
+__all__ = [
+    "available_symbols",
+    "available_years",
+    "connect",
+    "describe",
+    "in_universe",
+    "load_calendar",
+    "load_corporate_actions",
+    "load_earnings",
+    "load_factor_covariances",
+    "load_factor_loadings",
+    "load_factor_returns",
+    "load_forecast",
+    "load_idio_vol",
+    "load_index_greeks",
+    "load_indices",
+    "load_open_interest",
+    "load_option_greeks",
+    "load_rates",
+    "load_realized_vol",
+    "load_reference_returns",
+    "load_sectors",
+    "load_sessions",
+    "load_signals",
+    "load_stock_features",
+    "load_surface",
+    "load_symbology_check",
+    "load_underlying",
+    "load_universe",
+    "load_yields",
+    "scan",
+    "usable_symbol_years",
+]

ml_data_access/loaders.py

@@ -0,0 +1,239 @@
+"""One loader per table. Every loader is `load_x(db, start=None, end=None)`.
+
+The per-symbol chain loaders take the symbol first, since it selects the
+partition rather than filtering rows. `db` is the connected bear-lake
+database; it is passed for the same reason at-research passes it, so a
+study can hold two stores open.
+"""
+
+import datetime as dt
+
+import bear_lake as bl
+import polars as pl
+
+from ml_data_access.store import in_window, scan, years_in
+
+
+def load_window(
+    db: bl.Database, name: str, start: dt.date | None, end: dt.date | None, sort: list[str]
+) -> pl.DataFrame:
+    frame = scan(name, years=years_in(start, end, name))
+    return db.query(in_window(frame, start, end).sort(sort))
+
+
+# --- raw ---------------------------------------------------------------------
+
+
+def load_calendar(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """Exchange sessions, one `date` per row."""
+    return load_window(db, "calendar", start, end, ["date"])
+
+
+def load_sessions(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> list[dt.date]:
+    """The calendar as a list of dates, the shape `malatium.providers.TradingCalendar` takes."""
+    return load_calendar(db, start, end)["date"].to_list()
+
+
+def load_universe(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """Point-in-time membership: `date`, `ticker` (Wikipedia), `symbol` (option root)."""
+    return load_window(db, "universe", start, end, ["date", "ticker"])
+
+
+def load_sectors(db: bl.Database) -> pl.DataFrame:
+    """`ticker`, `symbol`, `sector`, `sub_industry`; a snapshot of today's constituents."""
+    return db.query(scan("sectors").sort("ticker"))
+
+
+def load_indices(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """EOD levels for SPX, RUT, OEX, XSP and the VIX complex, 2024 on."""
+    return load_window(db, "indices", start, end, ["date", "symbol"])
+
+
+def load_yields(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """CBOE treasury yield indices (`13w`, `5y`, `10y`, `30y`) as decimals."""
+    return load_window(db, "yields", start, end, ["date", "tenor"])
+
+
+def load_rates(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """Overnight SOFR as a decimal, 2024 on."""
+    return load_window(db, "rates", start, end, ["date", "symbol"])
+
+
+def load_corporate_actions(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """Splits (ex-date ratios) and dividends, one row per (symbol, date, action)."""
+    return load_window(db, "corporate_actions", start, end, ["symbol", "date", "action"])
+
+
+def load_earnings(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """Announcement dates with a `bmo` / `amc` / `unknown` session."""
+    return load_window(db, "earnings", start, end, ["symbol", "date"])
+
+
+def load_underlying(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """EOD stock OHLCV, 2023-06 on, `symbol` in option-root spelling."""
+    return load_window(db, "underlying", start, end, ["date", "symbol"])
+
+
+def load_chain(
+    db: bl.Database, name: str, symbol: str, start: dt.date | None, end: dt.date | None
+) -> pl.DataFrame:
+    frame = scan(name, years=years_in(start, end, name), symbols=[symbol.upper()])
+    return db.query(in_window(frame, start, end).sort("date", "expiration", "strike", "right"))
+
+
+def load_option_greeks(
+    db: bl.Database, symbol: str, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """One name's EOD chain: quotes, `iv` (null where the inversion failed), greeks with
+    `vega` per vol point, and `underlying`, the spot the greeks were struck against."""
+    return load_chain(db, "option_greeks", symbol, start, end)
+
+
+def load_index_greeks(
+    db: bl.Database, symbol: str, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """The same for an index root (SPX, SPXW, XSP, VIX); repaired sessions carry null gamma."""
+    return load_chain(db, "index_greeks", symbol, start, end)
+
+
+def load_open_interest(
+    db: bl.Database, symbol: str, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """One name's EOD open interest; stamped pre-open, so it joins the same session's chain."""
+    return load_chain(db, "open_interest", symbol, start, end)
+
+
+def load_symbology_check(db: bl.Database) -> pl.DataFrame:
+    """Per symbol-year: is the stored chain the company the universe names?"""
+    return db.query(scan("symbology_check").sort("year", "symbol"))
+
+
+# --- derived -------------------------------------------------------------------
+
+
+def load_reference_returns(
+    db: bl.Database,
+    start: dt.date | None = None,
+    end: dt.date | None = None,
+    symbols: list[str] | None = None,
+) -> pl.DataFrame:
+    """The reference straddle's per-vega P&L per (date, symbol); `SPX` is the market."""
+    frame = scan(
+        "reference_returns", years=years_in(start, end, "reference_returns"), symbols=symbols
+    )
+    return db.query(in_window(frame, start, end).sort("date", "symbol"))
+
+
+def load_factor_returns(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """`(date, factor, ret)`: `market` and one GICS sector factor each."""
+    return load_window(db, "factor_returns", start, end, ["date", "factor"])
+
+
+def load_factor_loadings(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """`(date, symbol, factor, loading)`, trailing 250 sessions."""
+    return load_window(db, "factor_loadings", start, end, ["date", "symbol", "factor"])
+
+
+def load_factor_covariances(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """`(date, factor_1, factor_2, covariance)`, Ledoit-Wolf shrunk."""
+    return load_window(db, "factor_covariances", start, end, ["date", "factor_1", "factor_2"])
+
+
+def load_idio_vol(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """`(date, symbol, idio_vol)`, daily residual std per dollar of vega."""
+    return load_window(db, "idio_vol", start, end, ["date", "symbol"])
+
+
+def load_surface(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """Constant-maturity ATM IV per (date, symbol): `iv30`, `iv60`, `iv90`."""
+    return load_window(db, "surface", start, end, ["date", "symbol"])
+
+
+def load_realized_vol(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """Close-to-close realized vol: trailing `rv_1`, `rv_5`, `rv_22` and the forward `rv_fwd`."""
+    return load_window(db, "realized_vol", start, end, ["date", "symbol"])
+
+
+def load_forecast(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """The HAR forecast of the forward 60-session vol, `rv_fcst`."""
+    return load_window(db, "forecast", start, end, ["date", "symbol"])
+
+
+def load_stock_features(
+    db: bl.Database, start: dt.date | None = None, end: dt.date | None = None
+) -> pl.DataFrame:
+    """`ret`, `beta`, `idio_vol`, `gap_freq` per (date, symbol)."""
+    return load_window(db, "stock_features", start, end, ["date", "symbol"])
+
+
+def load_signals(
+    db: bl.Database,
+    signal: str,
+    start: dt.date | None = None,
+    end: dt.date | None = None,
+) -> pl.DataFrame:
+    """`(date, symbol, score)` for one named signal (`vrp`, `iv_zscore`, `momentum`)."""
+    frame = scan("signals", years=years_in(start, end, "signals")).filter(
+        pl.col("signal") == signal
+    )
+    return db.query(
+        in_window(frame, start, end).select("date", "symbol", "score").sort("date", "symbol")
+    )
+
+
+# --- screens -------------------------------------------------------------------
+
+
+def usable_symbol_years(db: bl.Database) -> pl.DataFrame:
+    """`(symbol, year)` pairs whose chain is not known to be another company's.
+
+    Only `wrong_instrument` is excluded. `thin_overlap` means the check could
+    not run, and it falls on names later delisted or acquired, so excluding
+    it would be a survivorship filter. Semi-join a panel against this.
+    """
+    return db.query(
+        scan("symbology_check")
+        .filter(pl.col("status") != "wrong_instrument")
+        .select("symbol", "year")
+        .unique()
+        .sort("symbol", "year")
+    )
+
+
+def in_universe(db: bl.Database, panel_df: pl.DataFrame) -> pl.DataFrame:
+    """Keep the (date, symbol) rows that were index members that day."""
+    start, end = panel_df["date"].min(), panel_df["date"].max()
+    members_df = load_universe(db, start, end).select("date", "symbol").unique()
+    return panel_df.join(members_df, on=["date", "symbol"], how="semi")

ml_data_access/store.py

@@ -0,0 +1,128 @@
+"""Connecting to the store and scanning only the partitions a query needs.
+
+bear-lake lays a partitioned table out as `<store>/<table>/<year>/<symbol>.parquet`
+with a `metadata.json` beside it. `bl.table()` globs the whole table, which
+is right for the small tables and wrong for a per-symbol read of a 60 GB
+chain table, so `scan` builds the path list from the partition keys in the
+table's metadata and falls back to the glob otherwise.
+"""
+
+import datetime as dt
+import json
+import os
+from pathlib import Path
+
+import bear_lake as bl
+import polars as pl
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def connect(path: str | Path | None = None) -> bl.Database:
+    """Connect to the store at `path`, or at `ML_DATA_STORE`."""
+    path = path or os.getenv("ML_DATA_STORE")
+    if not path:
+        raise RuntimeError("set ML_DATA_STORE to the bear-lake directory, or pass a path")
+    return bl.connect(str(Path(path).expanduser()))
+
+
+def store_path() -> Path:
+    if not bl.CONNECTED:
+        raise RuntimeError("not connected: call ml_data_access.connect() first")
+    return Path(bl.DATABASE_PATH)
+
+
+def table_dir(name: str) -> Path:
+    return store_path() / name
+
+
+def partition_keys(name: str) -> list[str]:
+    metadata_path = table_dir(name) / "metadata.json"
+    if not metadata_path.exists():
+        raise FileNotFoundError(
+            f"no table {name!r} in {store_path()}; run `uv run pipelines {name.replace('_', '-')}`"
+            " in ml-data-pipelines"
+        )
+    return json.loads(metadata_path.read_text()).get("partition_keys") or []
+
+
+def available_years(name: str) -> list[int]:
+    directory = table_dir(name)
+    if not directory.is_dir():
+        return []
+    return sorted(int(child.stem) for child in directory.iterdir() if child.stem.isdigit())
+
+
+def available_symbols(name: str, year: int) -> list[str]:
+    directory = table_dir(name) / str(year)
+    if not directory.is_dir():
+        return []
+    return sorted(path.stem for path in directory.glob("*.parquet"))
+
+
+def scan(
+    name: str,
+    years: list[int] | None = None,
+    symbols: list[str] | None = None,
+) -> pl.LazyFrame:
+    """Lazy scan of `name`, opening only the partitions for `years` and `symbols`."""
+    keys = partition_keys(name)
+    if keys == ["year", "symbol"] and (years is not None or symbols is not None):
+        years = years if years is not None else available_years(name)
+        paths = []
+        for year in years:
+            names = symbols if symbols is not None else available_symbols(name, year)
+            paths.extend(
+                table_dir(name) / str(year) / f"{symbol}.parquet"
+                for symbol in names
+                if (table_dir(name) / str(year) / f"{symbol}.parquet").exists()
+            )
+    elif keys == ["year"] and years is not None:
+        paths = [
+            table_dir(name) / f"{year}.parquet"
+            for year in years
+            if (table_dir(name) / f"{year}.parquet").exists()
+        ]
+    else:
+        paths = sorted(table_dir(name).glob("**/*.parquet"))
+    if not paths:
+        raise FileNotFoundError(f"no {name} partitions for years={years} symbols={symbols}")
+    frame = pl.scan_parquet(paths)
+    if symbols is not None and "symbol" in frame.collect_schema().names():
+        frame = frame.filter(pl.col("symbol").is_in(symbols))
+    return frame
+
+
+def years_in(start: dt.date | None, end: dt.date | None, name: str) -> list[int] | None:
+    """The years a window touches, clipped to what is on disk; None means every year."""
+    if start is None and end is None:
+        return None
+    on_disk = available_years(name)
+    first = start.year if start else (on_disk[0] if on_disk else None)
+    last = end.year if end else (on_disk[-1] if on_disk else None)
+    if first is None or last is None:
+        return None
+    return [year for year in on_disk if first <= year <= last]
+
+
+def in_window(
+    frame: pl.LazyFrame, start: dt.date | None, end: dt.date | None, column: str = "date"
+) -> pl.LazyFrame:
+    if start is not None:
+        frame = frame.filter(pl.col(column) >= start)
+    if end is not None:
+        frame = frame.filter(pl.col(column) <= end)
+    return frame
+
+
+def describe() -> str:
+    """One line per table: partitions and size."""
+    lines = []
+    for directory in sorted(store_path().iterdir()):
+        if not (directory / "metadata.json").exists():
+            continue
+        files = list(directory.glob("**/*.parquet"))
+        size = sum(file.stat().st_size for file in files)
+        lines.append(f"{directory.name:<20} {len(files):>5} files  {size / 1e9:7.2f} GB")
+    return "\n".join(lines)

ml_data_access-0.1.0.dist-info/METADATA

@@ -0,0 +1,79 @@
+Metadata-Version: 2.5
+Name: ml-data-access
+Version: 0.1.0
+Summary: Read side of the malatium data store: one loader per bear-lake table.
+Project-URL: Repository, https://github.com/Atium-Research/ml-data-access
+Requires-Python: >=3.13
+Requires-Dist: bear-lake>=0.1.5
+Requires-Dist: polars>=1.30
+Requires-Dist: python-dotenv>=1.0
+Description-Content-Type: text/markdown
+
+# ml-data-access
+
+Read side of the malatium data store. One `load_*` per table that [ml-data-pipelines](https://github.com/Atium-Research/ml-data-pipelines) writes, over a [bear-lake](https://github.com/andrewhall1124/bear-lake) database, in the shape of `at-research`'s data helpers.
+
+```bash
+pip install ml-data-access
+```
+
+```python
+import datetime as dt
+
+import ml_data_access
+
+db = ml_data_access.connect()  # ML_DATA_STORE, or connect(path)
+start, end = dt.date(2018, 7, 2), dt.date(2025, 6, 30)
+
+reference_df = ml_data_access.load_reference_returns(db, start, end)
+scores_df = ml_data_access.load_signals(db, "vrp", start, end)
+loadings_df = ml_data_access.load_factor_loadings(db, start, end)
+chain_df = ml_data_access.load_option_greeks(db, "AAPL", dt.date(2025, 1, 1), dt.date(2025, 3, 31))
+```
+
+Every loader is `load_x(db, start=None, end=None)` and returns a collected DataFrame for the inclusive window; the three per-symbol chain loaders take the symbol first. Nothing is screened, joined or derived: the store holds canonical tables (`symbol` is the option root, `right` is `C`/`P`, `iv` is null where the vendor's inversion failed, `vega` is per vol point).
+
+The frames slot straight into malatium:
+
+```python
+from malatium.providers import PanelProvider, TradingCalendar
+
+calendar = TradingCalendar(ml_data_access.load_sessions(db, start, end))
+reference = PanelProvider(reference_df)
+scores = PanelProvider(scores_df)
+```
+
+## Loaders
+
+| loader | table |
+| --- | --- |
+| `load_calendar`, `load_sessions` | exchange sessions, as a frame or a list of dates |
+| `load_universe` | point-in-time S&P 500 membership, `ticker` and `symbol` |
+| `load_sectors` | GICS sector snapshot |
+| `load_indices`, `load_yields`, `load_rates` | index levels (2024 on), the CBOE yield curve, SOFR |
+| `load_corporate_actions`, `load_earnings` | splits and dividends; announcement dates with a session |
+| `load_underlying` | EOD stock OHLCV, 2023-06 on |
+| `load_option_greeks(db, symbol, ...)`, `load_index_greeks`, `load_open_interest` | one name's chain, index chain, open interest |
+| `load_symbology_check`, `usable_symbol_years` | which symbol-years are the right company |
+| `load_reference_returns` | the reference straddle's per-vega P&L, `SPX` included |
+| `load_factor_returns`, `load_factor_loadings`, `load_factor_covariances`, `load_idio_vol` | the vol risk model |
+| `load_surface`, `load_realized_vol`, `load_forecast`, `load_stock_features` | the derived panels |
+| `load_signals(db, name, ...)` | one signal's `(date, symbol, score)` |
+| `in_universe` | semi-join a `(date, symbol)` panel to membership |
+
+`ml_data_access.describe()` lists what the store holds. `ml_data_access.scan(name, years, symbols)` is the lazy scan under every loader; it opens only the partitions asked for, using bear-lake's `<table>/<year>/<symbol>.parquet` layout.
+
+## Things to know before trusting a number
+
+- **Eighteen symbol-years are another company's chain.** Semi-join against `usable_symbol_years(db)` for any multi-year study.
+- **Open interest is one day stale by construction** and joins on the same `date`.
+- **`iv` is null, not wrong**, on the ~3% of contract-days that failed to invert.
+- **Repaired index sessions** (mostly 2020-21) carry null gamma and a 15:59 underlying.
+
+## Development
+
+```bash
+uv sync
+uv run pytest        # builds a synthetic store in a temp dir
+uv run ruff check . && uv run ruff format .
+```

ml_data_access-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+ml_data_access/__init__.py,sha256=mE-N1q7pCV1M2u9s_wH3PagH-ThCZKhJIAu8lR9FPUk,2042
+ml_data_access/loaders.py,sha256=KkOYxJ7I2kxCaz87-FAqa4JudWAsfBrV5pkqqub-LAk,9059
+ml_data_access/store.py,sha256=tflQQcJ-1MTyEWSsNuEcyRa9pXtaMB4Y9ztUndttaXk,4617
+ml_data_access-0.1.0.dist-info/METADATA,sha256=NaM2Z9iugDZJ23OMs37uTkuFaIFaiQv66JiB5nx2YJ0,3795
+ml_data_access-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+ml_data_access-0.1.0.dist-info/RECORD,,

ml_data_access-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any