aether-quant 0.0.1__py3-none-any.whl

data_pipeline/dividend_backfill.py

@@ -0,0 +1,243 @@
+"""Dividend ex-date/amount history backfill + forward cadence-based
+projection - powers portfolio/options_assignment_risk.py's early-
+call-assignment signal (development/Problems.md - early-assignment
+probability/pricing modeling).
+
+No dividend $ amounts/ex-div dates/schedule exist anywhere else in this
+codebase - data_pipeline/fred_backfill.py only pulls Treasury yield/credit-
+spread series, and data_pipeline/yfinance_backfill.py's auto_adjust=True
+OHLCV downloads discard raw dividend events before they ever reach a Lean
+zip. This module fetches actual historical ex-dividend dates/amounts via
+yfinance (dev-only dependency, deferred import - mirrors
+yfinance_backfill.py's own convention) and derives a best-effort FORWARD
+projection from historical cadence - NOT yfinance's own forward
+Ticker.calendar/get_earnings_dates fields, which are scraped from Yahoo's
+UI and empirically unreliable/frequently stale or absent for less-followed
+tickers. This is a deliberate accuracy-over-completeness choice; every
+consumer must treat the projection as a best-effort estimate, never a
+confirmed date (see estimate_next_ex_dividend_date()'s own
+"confidence"/"method" fields).
+
+Same two-safety-boundary shape as fred_backfill.py/yfinance_backfill.py:
+1. Writing the local cache is gated by --apply (default: dry run only).
+2. Never touches config.json - this module's only output is
+   data/reference/dividend_schedule/*.json, read back once at main.py
+   init (Lean backtests are date-bounded and must never make a live HTTP
+   call mid-run) - refreshed by re-running this module with --apply, not
+   intraday.
+
+Usage:
+    python -m data_pipeline.dividend_backfill [--tickers AAPL MSFT] [--apply]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import statistics
+from datetime import date
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+ROOT = Path(__file__).resolve().parents[1]
+CONFIG_PATH = ROOT / "config.json"
+DIVIDEND_SCHEDULE_CACHE_DIR = ROOT / "data" / "reference" / "dividend_schedule"
+
+MIN_HISTORY_EVENTS_FOR_ESTIMATE = 2
+CADENCE_LOOKBACK_EVENTS = 8
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+
+def estimate_next_ex_dividend_date(history: list[dict], as_of: date) -> dict:
+    """Pure. Projects the next ex-div date from historical CADENCE (median
+    days between consecutive ex_dates over the trailing
+    CADENCE_LOOKBACK_EVENTS events), rolled forward from the most recent
+    ex_date <= as_of - NOT a yfinance forward-calendar call (see module
+    docstring). Returns {"estimated_next_ex_date": iso-date | None,
+    "estimated_amount": float | None, "cadence_days": int | None,
+    "confidence": "low"|"medium"|"none", "method": str}.
+
+    "none" confidence (every numeric field None) when fewer than
+    MIN_HISTORY_EVENTS_FOR_ESTIMATE events are available as-of as_of - an
+    unknown schedule must never be indistinguishable from a genuinely "no
+    dividend expected" signal, the same None-vs-neutral-default
+    distinction every other pure signal function in this codebase draws.
+    "medium" only with a reasonably regular cadence over enough lookback
+    events; "low" otherwise. Never raises."""
+    past_events = sorted(
+        (event for event in history if event.get("ex_date") is not None and event["ex_date"] <= as_of),
+        key=lambda event: event["ex_date"],
+    )
+    if len(past_events) < MIN_HISTORY_EVENTS_FOR_ESTIMATE:
+        return {
+            "estimated_next_ex_date": None,
+            "estimated_amount": None,
+            "cadence_days": None,
+            "confidence": "none",
+            "method": "insufficient_history",
+        }
+
+    lookback = past_events[-CADENCE_LOOKBACK_EVENTS:]
+    gaps = [(lookback[i]["ex_date"] - lookback[i - 1]["ex_date"]).days for i in range(1, len(lookback))]
+    cadence_days = max(round(statistics.median(gaps)), 1)
+    most_recent = lookback[-1]
+    estimated_next = most_recent["ex_date"]
+    # Roll forward past `as_of` by whole cadence steps - handles a stale
+    # cache (as_of well past the last known event) without ever guessing a
+    # date already in the past.
+    while estimated_next <= as_of:
+        estimated_next = date.fromordinal(estimated_next.toordinal() + cadence_days)
+
+    spread = (max(gaps) - min(gaps)) if len(gaps) > 1 else 0
+    confidence = "medium" if len(gaps) >= 3 and spread <= 15 else "low"
+
+    return {
+        "estimated_next_ex_date": estimated_next.isoformat(),
+        "estimated_amount": most_recent["amount"],
+        "cadence_days": cadence_days,
+        "confidence": confidence,
+        "method": "historical_cadence_projection - not a confirmed forward date",
+    }
+
+
+def dividend_schedule_payload(ticker: str, history: list[dict], as_of: date) -> dict:
+    """Composes fetch_dividend_history()'s raw history +
+    estimate_next_ex_dividend_date()'s projection into the exact shape
+    write_dividend_schedule() persists / main.py reads back."""
+    estimate = estimate_next_ex_dividend_date(history, as_of)
+    return {
+        "ticker": ticker,
+        "fetched_at": as_of.isoformat(),
+        "history": [{"ex_date": event["ex_date"].isoformat(), "amount": event["amount"]} for event in history],
+        "next_ex_dividend_estimate": estimate,
+    }
+
+
+def option_underlying_tickers(config: dict) -> list[str]:
+    """Every ticker this module is actually useful for: an option asset's
+    underlying_ticker, or an equity asset's own ticker directly. Pure,
+    de-duplicated, order-preserving, never raises on a malformed asset
+    entry (missing keys just get skipped)."""
+    tickers: list[str] = []
+    seen: set[str] = set()
+    for asset in config.get("phase1", {}).get("universe", {}).get("assets", []):
+        security_type = asset.get("security_type")
+        ticker = asset.get("underlying_ticker") if security_type == "option" else asset.get("ticker")
+        if ticker and ticker not in seen:
+            seen.add(ticker)
+            tickers.append(ticker)
+    return tickers
+
+
+# ---------------------------------------------------------------------------
+# The only function that imports yfinance - deferred, mirrors
+# yfinance_backfill.py's own convention.
+# ---------------------------------------------------------------------------
+
+
+def fetch_dividend_history(yahoo_symbol: str) -> list[dict]:
+    """Real historical ex-dividend dates + per-share amounts via
+    yfinance.Ticker(yahoo_symbol).dividends. Returns [] on any fetch
+    failure (network, delisted, no dividend history) - never raises, same
+    fail-open convention as yfinance_backfill.fetch_yahoo_ohlcv()."""
+    try:
+        import yfinance as yf  # deferred - dev-only dependency, never in requirements.txt
+
+        series = yf.Ticker(yahoo_symbol).dividends
+    except Exception as exc:
+        logger.warning("fetch_dividend_history(%s): fetch failed — %s", yahoo_symbol, exc)
+        return []
+
+    if series is None or series.empty:
+        logger.warning("fetch_dividend_history(%s): no dividend history returned", yahoo_symbol)
+        return []
+
+    rows = []
+    for index, amount in series.items():
+        row_date = index.date() if hasattr(index, "date") else index
+        rows.append({"ex_date": row_date, "amount": float(amount)})
+    return sorted(rows, key=lambda row: row["ex_date"])
+
+
+# ---------------------------------------------------------------------------
+# Orchestration
+# ---------------------------------------------------------------------------
+
+
+def write_dividend_schedule(cache_dir: Path, ticker: str, history: list[dict], as_of: date) -> dict:
+    """Writes data/reference/dividend_schedule/<ticker>.json. Returns the
+    payload written (for CLI reporting)."""
+    payload = dividend_schedule_payload(ticker, history, as_of)
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    output_path = cache_dir / f"{ticker}.json"
+    output_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    return payload
+
+
+def load_cached_dividend_schedule(ticker: str, cache_dir: Path = DIVIDEND_SCHEDULE_CACHE_DIR) -> dict:
+    """Reads back data/reference/dividend_schedule/<ticker>.json. Returns
+    {} when missing/unreadable (fresh clone, backfill never run for this
+    ticker) - callers (main.py) must treat an empty dict as "no dividend
+    schedule known", never a crash, same convention as
+    fred_backfill.load_cached_fred_series()."""
+    path = cache_dir / f"{ticker}.json"
+    if not path.exists():
+        return {}
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (ValueError, OSError) as exc:
+        logger.warning("load_cached_dividend_schedule(%s): unreadable cache file — %s", ticker, exc)
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s — %(message)s")
+    parser = argparse.ArgumentParser(
+        description="Aether Quant dividend history backfill - offline/manual only, "
+        "no API key required, never run inside Lean or a Docker worker."
+    )
+    parser.add_argument(
+        "--tickers", nargs="*", default=None, help="Restrict to these tickers (default: every option underlying/equity in phase1.universe.assets)"
+    )
+    parser.add_argument("--apply", action="store_true", help="Actually write the local cache (default: dry run, report only)")
+    parser.add_argument("--config-path", type=Path, default=CONFIG_PATH)
+    parser.add_argument("--cache-dir", type=Path, default=DIVIDEND_SCHEDULE_CACHE_DIR)
+    args = parser.parse_args()
+
+    with args.config_path.open("r", encoding="utf-8") as f:
+        config = json.load(f)
+
+    tickers = args.tickers if args.tickers else option_underlying_tickers(config)
+    as_of = date.today()
+
+    print(f"{'APPLY' if args.apply else 'DRY RUN'} — fetching dividend history for {len(tickers)} ticker(s):\n")
+    for ticker in tickers:
+        history = fetch_dividend_history(ticker)
+        if not history:
+            print(f"- {ticker}: no_data_returned, events_fetched=0")
+            continue
+        payload = dividend_schedule_payload(ticker, history, as_of)
+        if args.apply:
+            write_dividend_schedule(args.cache_dir, ticker, history, as_of)
+            action = "written"
+        else:
+            action = "dry_run"
+        print(f"- {ticker}: {action}, events_fetched={len(history)}, next_ex_estimate={payload['next_ex_dividend_estimate']}")
+
+    if not args.apply:
+        print("\nDry run only — no cache files were written. Re-run with --apply to write the cache.")
+
+
+if __name__ == "__main__":
+    main()

data_pipeline/factor_file_backfill.py

@@ -0,0 +1,276 @@
+"""Lean-format split/dividend factor-file backfill for equity tickers
+missing a local factor file (development/Problems.md #91/#97/#99).
+
+train.py::apply_split_adjustments() backward-adjusts each ticker's raw
+OHLCV using data/equity/usa/factor_files/<ticker>.csv - the same
+adjustment DataNormalizationMode.Adjusted already applies to main.py's
+live/backtest data feed at runtime (Lean's default for
+self.add_equity(ticker, self.resolution)). When that local file is
+missing, load_factor_file() returns None and apply_split_adjustments()
+silently no-ops - offline trains on RAW, unadjusted prices while live
+never has this problem. Of 104 configured assets (77 equities), only 22
+have a local factor file; the other 63 don't - this whole data/ tree is
+gitignored, so this is a local/incomplete data pull, not a deliberate
+scope decision anywhere in the code.
+
+This module derives real Lean-format factor files for the missing
+tickers from yfinance's own dividend/split history (dev-only dependency,
+deferred import - mirrors dividend_backfill.py's/yfinance_backfill.py's
+own convention), computing each row's CUMULATIVE backward-adjustment
+factor the same way Lean's own factor files do (confirmed by reading a
+real one, data/equity/usa/factor_files/aapl.csv, directly: earliest row
+carries the largest cumulative adjustment, format is
+date(YYYYMMDD),price_factor,split_factor,reference_price, no header,
+terminal sentinel row 20501231,1,1,0).
+
+No changes to train.py are needed or made here - load_factor_file()/
+apply_split_adjustments() already do the right thing the moment a file
+exists at the expected path; this module is purely a data-generation
+task.
+
+Same two-safety-boundary shape as dividend_backfill.py/fred_backfill.py/
+yfinance_backfill.py:
+1. Writing factor files is gated by --apply (default: dry run only).
+2. Never touches config.json - this module's only output is
+   data/equity/usa/factor_files/*.csv, read back by train.py's own
+   dataset-build pipeline (`aq train --dataset-only`), not at Lean
+   runtime (main.py never reads these).
+
+Usage:
+    python -m data_pipeline.factor_file_backfill [--tickers NVDA GE] [--apply]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+ROOT = Path(__file__).resolve().parents[1]
+CONFIG_PATH = ROOT / "config.json"
+FACTOR_FILES_DIR = ROOT / "data" / "equity" / "usa" / "factor_files"
+
+TERMINAL_SENTINEL_ROW = {"factor_date": "20501231", "price_factor": 1.0, "split_factor": 1.0, "reference_price": 0.0}
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+
+def configured_equity_tickers(config: dict) -> list[str]:
+    """Every phase1.universe.assets entry with security_type == "equity",
+    de-duplicated, order-preserving. Deliberately broader than
+    dividend_backfill.option_underlying_tickers() (which only resolves
+    option underlyings) - factor files are needed for every directly-
+    tradeable equity/ETF asset, including bond ETFs (SHY/IEF/TLT/AGG/...),
+    which also pay real distributions and are also security_type "equity".
+    Never raises on a malformed asset entry - missing keys are skipped."""
+    tickers: list[str] = []
+    seen: set[str] = set()
+    for asset in config.get("phase1", {}).get("universe", {}).get("assets", []):
+        if asset.get("security_type") != "equity":
+            continue
+        ticker = asset.get("ticker")
+        if ticker and ticker not in seen:
+            seen.add(ticker)
+            tickers.append(ticker)
+    return tickers
+
+
+def tickers_missing_factor_file(tickers: list[str], factor_files_dir: Path = FACTOR_FILES_DIR) -> list[str]:
+    """Filters `tickers` down to those with no local
+    data/equity/usa/factor_files/<ticker-lower>.csv yet - the default CLI
+    scope, same "no flag = sane default" pattern as dividend_backfill.py's
+    --tickers default."""
+    return [ticker for ticker in tickers if not (factor_files_dir / f"{ticker.lower()}.csv").exists()]
+
+
+def compute_lean_factor_rows(history) -> list[dict]:
+    """Pure. `history` is a pandas DataFrame with a DatetimeIndex plus
+    Close/Dividends/Stock Splits columns - fetch_corporate_actions()'s
+    exact return shape (matches yfinance's own Ticker.history(actions=True)
+    column names).
+
+    Walks real events (Dividends != 0 or Stock Splits != 0) newest-to-
+    oldest, accumulating:
+      - price_factor *= (1 - dividend_amount / close_on_prior_session)
+        per dividend event (a cash dividend doesn't change share count,
+        only the ex-date price drop needs correcting)
+      - split_factor *= 1 / split_ratio per split event
+    emitting one row per event date carrying the CUMULATIVE factor as of
+    that date - Lean's own convention (confirmed against the real
+    data/equity/usa/factor_files/aapl.csv: earliest row carries the
+    largest cumulative adjustment, converging to ~1.0 near the most
+    recent event). `reference_price` is the raw Close on that event's own
+    date (matches the real file's 4th column, an audit reference, never
+    consumed by the adjustment formula itself).
+
+    Returns [] when `history` has zero real dividend/split events - no
+    row is fabricated for a ticker that never needed adjusting.
+
+    Always appends TERMINAL_SENTINEL_ROW last (Lean's own far-future
+    sentinel, matching every real factor file's last row) - EXCEPT when
+    there are zero real events, in which case [] is returned instead of a
+    sentinel-only file (see this module's docstring / write_factor_file()
+    callers: a missing file and a sentinel-only file resolve identically
+    through train.py's merge_asof(direction="forward") + .fillna(1.0), so
+    skipping is lower-risk than writing a file that isn't needed)."""
+    events = []
+    for event_date, row in history.iterrows():
+        dividend = float(row.get("Dividends", 0.0) or 0.0)
+        split = float(row.get("Stock Splits", 0.0) or 0.0)
+        if dividend == 0.0 and split == 0.0:
+            continue
+        events.append(
+            {
+                "date": event_date,
+                "dividend": dividend,
+                "split_ratio": split,
+                "close": float(row["Close"]),
+                "prior_close": None,
+            }
+        )
+
+    if not events:
+        return []
+
+    close_series = history["Close"]
+    for event in events:
+        prior_close = close_series.loc[:event["date"]].iloc[:-1]
+        event["prior_close"] = float(prior_close.iloc[-1]) if not prior_close.empty else float(event["close"])
+
+    events.sort(key=lambda event: event["date"], reverse=True)
+
+    # Cumulative factors are computed walking newest-to-oldest (each older
+    # event's factor must already include every later event's adjustment),
+    # but the OUTPUT is written oldest-first with the terminal sentinel
+    # last - matching every real Lean factor file's own on-disk order
+    # (confirmed against data/equity/usa/factor_files/aapl.csv). Not
+    # strictly required for correctness (train.py::load_factor_file()
+    # re-sorts by date on read regardless), but keeps a generated file
+    # directly diffable against a real one.
+    rows: list[dict] = []
+    price_factor = 1.0
+    split_factor = 1.0
+    for event in events:
+        if event["dividend"] != 0.0 and event["prior_close"] > 0.0:
+            price_factor *= 1.0 - (event["dividend"] / event["prior_close"])
+        if event["split_ratio"] != 0.0:
+            split_factor *= 1.0 / event["split_ratio"]
+        event_date = event["date"]
+        rows.append(
+            {
+                "factor_date": event_date.strftime("%Y%m%d") if hasattr(event_date, "strftime") else str(event_date),
+                "price_factor": price_factor,
+                "split_factor": split_factor,
+                "reference_price": event["close"],
+            }
+        )
+
+    rows.reverse()
+    rows.append(dict(TERMINAL_SENTINEL_ROW))
+    return rows
+
+
+# ---------------------------------------------------------------------------
+# The only function that imports yfinance - deferred, mirrors
+# dividend_backfill.py's/yfinance_backfill.py's own convention.
+# ---------------------------------------------------------------------------
+
+
+def fetch_corporate_actions(yahoo_symbol: str):
+    """Real historical Close/Dividends/Stock Splits via ONE yfinance call
+    (Ticker.history(period="max", auto_adjust=False, actions=True)) - the
+    raw close series and the corporate-action events come back aligned by
+    date in a single round-trip, so compute_lean_factor_rows() never needs
+    a second network call for "close on the session before the ex-date".
+    Returns None on any fetch failure (network, delisted, no history) -
+    never raises, same fail-open convention as
+    dividend_backfill.fetch_dividend_history()."""
+    try:
+        import yfinance as yf  # deferred - dev-only dependency, never in requirements.txt
+
+        history = yf.Ticker(yahoo_symbol).history(period="max", auto_adjust=False, actions=True)
+    except Exception as exc:
+        logger.warning("fetch_corporate_actions(%s): fetch failed — %s", yahoo_symbol, exc)
+        return None
+
+    if history is None or history.empty:
+        logger.warning("fetch_corporate_actions(%s): no history returned", yahoo_symbol)
+        return None
+    return history
+
+
+# ---------------------------------------------------------------------------
+# Orchestration
+# ---------------------------------------------------------------------------
+
+
+def write_factor_file(output_dir: Path, ticker: str, rows: list[dict]) -> Path:
+    """Writes data/equity/usa/factor_files/<ticker-lower>.csv, no header,
+    Lean's exact column order (date,price_factor,split_factor,
+    reference_price) - matches train.py::load_factor_file()'s own read
+    shape (names=["factor_date", "price_factor", "split_factor",
+    "reference_price"]) column-for-column."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_path = output_dir / f"{ticker.lower()}.csv"
+    lines = [
+        f"{row['factor_date']},{row['price_factor']},{row['split_factor']},{row['reference_price']}"
+        for row in rows
+    ]
+    output_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return output_path
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s — %(message)s")
+    parser = argparse.ArgumentParser(
+        description="Aether Quant Lean factor-file backfill - offline/manual only, "
+        "no API key required, never run inside Lean or a Docker worker."
+    )
+    parser.add_argument(
+        "--tickers", nargs="*", default=None,
+        help="Restrict to these tickers (default: every configured equity ticker missing a local factor file)",
+    )
+    parser.add_argument("--apply", action="store_true", help="Actually write factor files (default: dry run, report only)")
+    parser.add_argument("--config-path", type=Path, default=CONFIG_PATH)
+    parser.add_argument("--output-dir", type=Path, default=FACTOR_FILES_DIR)
+    args = parser.parse_args()
+
+    with args.config_path.open("r", encoding="utf-8") as f:
+        config = json.load(f)
+
+    tickers = args.tickers if args.tickers else tickers_missing_factor_file(configured_equity_tickers(config), args.output_dir)
+
+    print(f"{'APPLY' if args.apply else 'DRY RUN'} — checking corporate-action history for {len(tickers)} ticker(s):\n")
+    for ticker in tickers:
+        history = fetch_corporate_actions(ticker)
+        if history is None:
+            print(f"- {ticker}: no_data_returned")
+            continue
+        rows = compute_lean_factor_rows(history)
+        if not rows:
+            print(f"- {ticker}: skipped_no_corporate_actions (identity fallback is already correct, no file needed)")
+            continue
+        if args.apply:
+            write_factor_file(args.output_dir, ticker, rows)
+            action = "written"
+        else:
+            action = "dry_run"
+        print(f"- {ticker}: {action}, factor_rows={len(rows) - 1} (+ terminal sentinel)")
+
+    if not args.apply:
+        print("\nDry run only — no factor files were written. Re-run with --apply to write them.")
+
+
+if __name__ == "__main__":
+    main()