aether-quant 0.0.1__py3-none-any.whl

data_pipeline/fetch.py

@@ -0,0 +1,206 @@
+"""Ad-hoc historical OHLCV fetch for tickers not (yet) in config.json.
+
+Distinct responsibility from yfinance_backfill.py: that module fills gaps in
+already-configured assets via a config.json "backfill" block, and
+deliberately never touches config.json (widening an *existing*, already
+-trained asset's date range is a decision a human should make explicitly).
+This module fetches an explicit date range for *any* ticker, including ones
+with no config.json entry at all, and — on --apply — *does* add a new asset
+block to config.json, because adding a brand-new ticker to the universe is
+this module's entire purpose, not an accidental side effect.
+
+Reuses yfinance_backfill.py's config.json-independent pure functions
+(fetch_yahoo_ohlcv, scale_for_lean, write_lean_zip, and - V4.10 - the forex
+sibling write_lean_forex_zip) - no duplicated logic.
+
+Never runs `train.py` itself - preparing a ticker for training (fetching
+its data, formatting it for Lean, wiring it into config.json's universe) is
+this module's job; actually training on it stays a deliberate, separate
+step (`python train.py`).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Callable
+
+from json_safety import atomic_write_json
+
+from data_pipeline.yfinance_backfill import (
+    fetch_yahoo_ohlcv,
+    scale_for_lean,
+    write_lean_forex_zip,
+    write_lean_zip,
+)
+
+ROOT = Path(__file__).resolve().parents[1]
+CONFIG_PATH = ROOT / "config.json"
+
+
+def _crypto_yahoo_symbol(ticker: str) -> str:
+    """BTCUSD -> BTC-USD, DOGEUSD -> DOGE-USD: strip a trailing USD and join
+    with a hyphen, matching Yahoo Finance's crypto pair naming convention."""
+    base = ticker[:-3] if ticker.upper().endswith("USD") else ticker
+    return f"{base}-USD"
+
+
+def _forex_yahoo_symbol(ticker: str) -> str:
+    """EURUSD -> EURUSD=X: Yahoo Finance's FX ticker convention."""
+    return f"{ticker}=X"
+
+
+# One entry per asset class - V3's "futures"/"options" entries below prove
+# the docstring's original claim (adding a class is one new dict entry, not
+# a redesign): both route through data_pipeline/ib_backfill.py instead of
+# Yahoo Finance (yahoo_symbol_fn stays identity - not a real Yahoo symbol
+# for these two, just this dict's "instrument identifier passed to
+# fetch_fn" slot, reused as-is rather than renamed) and carry an
+# extra_asset_fields block (asset_class/data_source) merged into the
+# config.json asset entry fetch_adhoc_asset() writes on --apply, since IB-
+# backed assets need to be identified by the feature/risk layer
+# (risk/asset_class_router.py) independently of Lean's own security_type.
+ASSET_CLASS_CONFIG = {
+    "crypto": {
+        "security_type": "crypto",
+        "market": "coinbase",
+        "data_path_fn": lambda ticker: ROOT / "data" / "crypto" / "coinbase" / "daily" / f"{ticker.lower()}_trade.zip",
+        "yahoo_symbol_fn": _crypto_yahoo_symbol,
+    },
+    "stock": {
+        "security_type": "equity",
+        "market": "usa",
+        "data_path_fn": lambda ticker: ROOT / "data" / "equity" / "usa" / "daily" / f"{ticker.lower()}.zip",
+        "yahoo_symbol_fn": lambda ticker: ticker,
+    },
+    "futures": {
+        "security_type": "future",
+        "market": "cme",
+        "data_path_fn": lambda ticker: ROOT / "data" / "future" / "cme" / "daily" / f"{ticker.lower()}.zip",
+        "yahoo_symbol_fn": lambda ticker: ticker,
+        "extra_asset_fields": {"asset_class": "future", "data_source": "ib"},
+    },
+    "options": {
+        "security_type": "option",
+        "market": "usa",
+        "data_path_fn": lambda ticker: ROOT / "data" / "option" / "usa" / "daily" / f"{ticker.lower()}.zip",
+        "yahoo_symbol_fn": lambda ticker: ticker,
+        "extra_asset_fields": {"asset_class": "option", "data_source": "ib"},
+    },
+    "forex": {
+        # V4.10 - yfinance-backed (default fetch_fn below, no IB closure
+        # needed - forex is a fully-wired Lean asset class already, only
+        # ever blocked by "zero live tickers configured", not IB). market
+        # standardized on "oanda" - local sample data exists identically
+        # for both oanda/fxcm today, an arbitrary/low-stakes choice.
+        # write_fn overrides write_lean_zip() below: Lean's real forex
+        # daily format is a 10-column bid/ask CSV
+        # (data/forex/readme.md), not the 5-column mid-price OHLCV shape
+        # every other asset class here uses.
+        "security_type": "forex",
+        "market": "oanda",
+        "data_path_fn": lambda ticker: ROOT / "data" / "forex" / "oanda" / "daily" / f"{ticker.lower()}.zip",
+        "yahoo_symbol_fn": _forex_yahoo_symbol,
+        "write_fn": write_lean_forex_zip,
+    },
+}
+ASSET_CLASSES = tuple(ASSET_CLASS_CONFIG.keys())
+
+
+def add_asset_to_config(config_path: Path, asset_block: dict) -> str:
+    """Appends asset_block to config.json's phase1.universe.assets[] unless
+    a block with the same ticker already exists there. Returns "added" or
+    "already_exists". Preserves the file's existing exact formatting
+    (4-space indent, trailing newline). V5.4.7 (Problems.md #123): the
+    rewrite is now atomic (temp file + os.replace) - a crash mid-write of
+    the whole system's config previously left a truncated config.json
+    behind."""
+    config = json.loads(config_path.read_text(encoding="utf-8"))
+    assets = config["phase1"]["universe"]["assets"]
+
+    if any(existing["ticker"] == asset_block["ticker"] for existing in assets):
+        return "already_exists"
+
+    assets.append(asset_block)
+    atomic_write_json(config_path, config, indent=4)
+    # Match the historical trailing-newline formatting exactly.
+    with config_path.open("a", encoding="utf-8") as handle:
+        handle.write("\n")
+    return "added"
+
+
+def fetch_adhoc_asset(
+    asset_class: str,
+    ticker: str,
+    start: str,
+    end: str,
+    *,
+    apply: bool,
+    fetch_fn: Callable = fetch_yahoo_ohlcv,
+    config_path: Path = CONFIG_PATH,
+    extra_metadata: dict | None = None,
+) -> dict:
+    """apply=False (default): dry run - fetches but writes nothing (no zip,
+    no config.json edit). apply=True: writes the Lean zip via
+    write_lean_zip(), then adds a new config.json asset block if this
+    ticker isn't already configured. fetch_fn is the injection point for
+    tests (never real yfinance in tests), matching
+    yfinance_backfill.run_backfill()'s exact convention.
+
+    extra_metadata is merged into the written config.json asset block, on
+    top of class_config["extra_asset_fields"] - the per-fetch-call
+    complement to that per-asset-class static dict. Used by futures/options
+    fetches (aq_cli.py::cmd_fetch()) to store strike/expiry/right (options)
+    and family_ticker (both - defaults to the ticker itself when the
+    caller doesn't group multiple dated contracts under one family), so
+    train.py::build_derivatives_macro_features_by_date() can later
+    recognize and group them for real historical term-structure/put-call-
+    ratio computation. Ignored (None) for crypto/stock, which need no such
+    metadata."""
+    ticker = ticker.upper()
+    class_config = ASSET_CLASS_CONFIG[asset_class]
+    yahoo_symbol = class_config["yahoo_symbol_fn"](ticker)
+    output_zip = class_config["data_path_fn"](ticker)
+    security_type = class_config["security_type"]
+
+    rows = fetch_fn(yahoo_symbol, start, end)
+    scaled_rows = scale_for_lean(rows, security_type)
+
+    dates = [row["date"] for row in rows]
+    suggested_from = min(dates) if dates else None
+    suggested_to = max(dates) if dates else None
+
+    config_status = "not_attempted"
+    if not scaled_rows:
+        action = "no_data_returned"
+    elif apply:
+        write_fn = class_config.get("write_fn", write_lean_zip)
+        write_fn(output_zip, ticker, scaled_rows, merge_with_existing=True)
+        action = "written"
+        config_status = add_asset_to_config(
+            config_path,
+            {
+                "ticker": ticker,
+                "security_type": security_type,
+                "market": class_config["market"],
+                "data_path": output_zip.relative_to(ROOT).as_posix(),
+                "available_from": suggested_from.isoformat(),
+                "available_to": suggested_to.isoformat(),
+                **class_config.get("extra_asset_fields", {}),
+                **(extra_metadata or {}),
+            },
+        )
+    else:
+        action = "dry_run"
+
+    return {
+        "ticker": ticker,
+        "asset_class": asset_class,
+        "yahoo_symbol": yahoo_symbol,
+        "data_path": str(output_zip),
+        "action": action,
+        "rows_fetched": len(rows),
+        "suggested_available_from": suggested_from.isoformat() if suggested_from else None,
+        "suggested_available_to": suggested_to.isoformat() if suggested_to else None,
+        "config_status": config_status,
+    }

data_pipeline/fred_backfill.py

@@ -0,0 +1,435 @@
+"""FRED (Federal Reserve Economic Data) historical series backfill.
+
+Powers features/bond_features.py's real yield-curve/credit-spread signals -
+distinct from macro_features.py's existing yield_curve_slope_proxy/
+credit_spread_proxy, which derive a *proxy* from bond-ETF price momentum.
+This module fetches the *actual* Treasury yield/credit-spread series FRED
+publishes, no API key required (FRED's public graph CSV endpoint).
+
+Uses `httpx` over HTTP/2, NOT stdlib urllib (V5.1 Phase 2, CODESPACE RUN 1
+diagnosis, development/Problems.md). FRED's graph-export endpoint
+(fredgraph.csv) was directly confirmed - via curl and Python urllib, from
+two independent networks - to hang/reset on plain HTTP/1.1 while
+responding in ~1-2s over HTTP/2; stdlib's urllib has no HTTP/2 support, so
+this is a real, load-bearing dependency, not a convenience one. See
+fetch_fred_series()'s own docstring for the full diagnosis, including why
+the User-Agent matters here in a non-obvious way.
+
+Same two-safety-boundary shape as yfinance_backfill.py:
+1. Writing the local cache is gated by --apply (default: dry run only).
+2. Never touches config.json - this module's only output is
+   data/reference/fred_series/*.csv, read back by
+   load_cached_fred_series() at train/backtest time (Lean backtests are
+   date-bounded and must never make a live HTTP call mid-run).
+
+Usage:
+    python -m data_pipeline.fred_backfill [--series DGS10 DGS2] [--apply]
+"""
+
+from __future__ import annotations
+
+import argparse
+import bisect
+import csv
+import json
+import logging
+from datetime import date, timedelta
+
+import httpx
+from pathlib import Path
+
+from json_safety import atomic_write_text
+
+logger = logging.getLogger(__name__)
+
+ROOT = Path(__file__).resolve().parents[1]
+CONFIG_PATH = ROOT / "config.json"
+FRED_SERIES_CACHE_DIR = ROOT / "data" / "reference" / "fred_series"
+
+FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
+
+DEFAULT_BOND_REFERENCE_SERIES = {
+    "treasury_3mo": "DGS3MO",
+    "treasury_2yr": "DGS2",
+    "treasury_5yr": "DGS5",
+    "treasury_10yr": "DGS10",
+    "treasury_30yr": "DGS30",
+    "credit_spread_baa10y": "BAA10Y",
+}
+
+# Phase 4.12 (development/Problems.md #71): the alternative-data trio
+# feeding features/alt_data_features.py. Chosen from a wider candidate list
+# (VIX/VXV/NFCI/STLFSI4/UMCSENT/TEDRATE/SOFR/BAMLH0A0HYM2/DTWEXBGS) after
+# live-fetching each one and checking (a) real coverage across
+# phase1.universe.common_window (2014-12-01..2021-03-31) and (b)
+# collinearity against the 8 existing bond_* features and against each
+# other. STLFSI4/NFCI-level/BAA10Y overlap too much (rho 0.67-0.88) to add
+# both; UMCSENT/TEDRATE/SOFR fail on coverage or lag; BAMLH0A0HYM2 has NO
+# usable history before 2023-07-25 through FRED's keyless endpoint (ICE
+# BofA series are license-restricted beyond a short trailing window) - do
+# not re-add it without a paid FRED API key. DTWEXBGS was in this same
+# candidate list and cleared both checks even then - it simply wasn't
+# needed for the original 3-series alt-data trio above (a broadcast LEVEL
+# feature; VIX/VXV/NFCI already covered vol/conditions). V5.1 Phase 2 (item
+# 8 / F2, development/Problems.md) re-adds it below as a cross-asset
+# SENSITIVITY DRIVER instead - a per-asset beta against dollar-index daily
+# CHANGE varies across the cross-section (usable by a ranker) in a way a
+# broadcast dollar-index LEVEL never would have. treasury_10yr_real
+# (DFII10) and breakeven_inflation_10y (T10YIE) are new for the same
+# purpose - real-rate and inflation-expectation sensitivity drivers,
+# neither previously evaluated as a candidate here. GUARDRAIL (plan): FRED
+# coverage for all 3 must be reverified on the Codespace before the first
+# real training run - assume nothing until `aq backfill fred` confirms it;
+# if any lacks 2014-12 -> 2021-03 coverage, drop it and its 2 sensitivity
+# columns (features/cross_asset_sensitivity.py) rather than shipping a
+# constant.
+DEFAULT_ALT_DATA_REFERENCE_SERIES = {
+    "implied_volatility_vix": "VIXCLS",
+    "implied_volatility_3m": "VXVCLS",
+    "financial_conditions_nfci": "NFCI",
+    "treasury_10yr_real": "DFII10",
+    "breakeven_inflation_10y": "T10YIE",
+    "dollar_index": "DTWEXBGS",
+}
+
+# Days to subtract from the decision date before doing the as-of lookup
+# (see series_value_asof()) - i.e. how many days after a series' own date
+# column the observation is actually PUBLISHED. VIX/VXV are same-day CBOE
+# closes (0 lag, identical convention to the Treasury/BAA10Y series
+# above). NFCI observations are Friday-dated but released the FOLLOWING
+# Wednesday (+5 calendar days); 7 is a deliberately conservative superset
+# that survives a holiday-shifted release. Verified this session: every
+# cached NFCI row falls on a Friday. DFII10/T10YIE are same-day Treasury-
+# desk marks (0 lag, same convention as the nominal DGS* series).
+# DTWEXBGS (H.10 trade-weighted dollar index) is a Federal Reserve Board
+# release published the following business day (1 lag) - unlike the
+# CBOE/Treasury series above, it is NOT a same-day mark.
+ALT_DATA_PUBLICATION_LAG_DAYS = {
+    "implied_volatility_vix": 0,
+    "implied_volatility_3m": 0,
+    "financial_conditions_nfci": 7,
+    "treasury_10yr_real": 0,
+    "breakeven_inflation_10y": 0,
+    "dollar_index": 1,
+}
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+
+def parse_fred_csv(text: str, series_id: str, start: str, end: str) -> list[dict]:
+    """Parses FRED's two-column CSV. The date column header has varied
+    across FRED API versions ("DATE" historically, "observation_date" as
+    of this module's writing) - read positionally (fieldnames[0]/[1])
+    rather than by a hardcoded name so a future FRED header rename doesn't
+    silently zero out every row. FRED marks holiday/no-observation rows
+    with either an empty string or a literal "." in the value column -
+    both are dropped, not coerced to 0.0 (a missing yield observation must
+    never look like "yield is exactly 0%"). Clipped to [start, end]
+    inclusive. Never raises on malformed rows - skips them, matching
+    fetch_yahoo_ohlcv()'s "never abort the whole fetch over one bad row"
+    convention."""
+    start_date = date.fromisoformat(start)
+    end_date = date.fromisoformat(end)
+
+    rows: list[dict] = []
+    reader = csv.DictReader(text.splitlines())
+    if not reader.fieldnames or len(reader.fieldnames) < 2:
+        return rows
+    date_column, value_column = reader.fieldnames[0], reader.fieldnames[1]
+    for record in reader:
+        raw_date = record.get(date_column)
+        raw_value = record.get(value_column)
+        if not raw_date or raw_value is None or raw_value.strip() in ("", "."):
+            continue
+        try:
+            row_date = date.fromisoformat(raw_date.strip())
+            value = float(raw_value.strip())
+        except ValueError:
+            continue
+        if row_date < start_date or row_date > end_date:
+            continue
+        rows.append({"date": row_date, "value": value})
+    return rows
+
+
+def rows_to_cache_csv(rows: list[dict]) -> str:
+    """date.isoformat(),value - one row per line, sorted by date. Mirrors
+    yfinance_backfill.rows_to_lean_csv()'s "pure formatting, sorted" shape."""
+    ordered = sorted(rows, key=lambda row: row["date"])
+    lines = [f"{row['date'].isoformat()},{row['value']}" for row in ordered]
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+def cache_csv_to_rows(text: str) -> list[dict]:
+    """Inverse of rows_to_cache_csv() - reads the local cache format back."""
+    rows: list[dict] = []
+    for line in text.splitlines():
+        if not line.strip():
+            continue
+        raw_date, raw_value = line.split(",")
+        rows.append({"date": date.fromisoformat(raw_date), "value": float(raw_value)})
+    return rows
+
+
+def series_value_asof(rows: list[dict], current_date, publication_lag_days: int = 0) -> float | None:
+    """Most recent observation whose value was PUBLISHED on or before
+    current_date - the shared, publication-lag-aware as-of lookup for
+    BOTH train.py::build_alt_data_features_by_date() and
+    main.py::_build_alt_data_payload(), so the lookahead rule below cannot
+    drift between the two callers the way build_bond_features_by_date()'s
+    inline _series_asof_lookup() and main.py::_fred_series_asof() already
+    have (both correct for a same-day series, since publication_lag_days
+    is implicitly 0 there - but that helper has no lag parameter at all,
+    so it must never be reused for a lagged series).
+
+    The lookahead this guards against: FRED dates every observation by its
+    REFERENCE period, not its release date. DGS10's 2020-03-16 row is that
+    day's close, known that evening (lag 0, safe). NFCI's 2020-03-20 row
+    is the week ENDING Friday 2020-03-20, not released until Wednesday
+    2020-03-25 (lag ~5-7 days) - a bare bisect_right(dates, current_date)
+    would hand a 2020-03-23 decision a number that did not exist for two
+    more days. effective_date = current_date - publication_lag_days
+    reproduces the exact bisect every existing 0-lag caller already does
+    when publication_lag_days=0 (see
+    test_series_value_asof_zero_lag_matches_existing_bisect_behavior).
+
+    `rows` is one series' [{"date": date, "value": float}, ...] - the same
+    per-series list load_cached_fred_series() returns. Returns None (never
+    raises) if `rows` is empty or every observation postdates
+    effective_date."""
+    if not rows:
+        return None
+    target = current_date.date() if hasattr(current_date, "date") else current_date
+    effective_date = target - timedelta(days=publication_lag_days)
+    ordered = sorted(rows, key=lambda row: row["date"])
+    dates = [row["date"] for row in ordered]
+    values = [row["value"] for row in ordered]
+    position = bisect.bisect_right(dates, effective_date)
+    if position == 0:
+        return None
+    return values[position - 1]
+
+
+def series_change_asof(
+    rows: list[dict],
+    current_date,
+    publication_lag_days: int = 0,
+    periods_back: int = 4,
+) -> float | None:
+    """value_asof(current_date) minus the observation `periods_back`
+    OBSERVATIONS (not calendar days) earlier in the same series - e.g.
+    periods_back=4 on a weekly series is a 4-week change. Both endpoints
+    are read from the same lag-adjusted, sorted index position, so the
+    change inherits series_value_asof()'s publication guard by
+    construction - there is no code path where the more-recent endpoint is
+    unpublished as of current_date.
+
+    Returns None (never 0.0 - a missing change must not look like "no
+    change") when `rows` is empty, current_date predates every
+    observation, or fewer than periods_back+1 observations precede
+    effective_date."""
+    if not rows:
+        return None
+    target = current_date.date() if hasattr(current_date, "date") else current_date
+    effective_date = target - timedelta(days=publication_lag_days)
+    ordered = sorted(rows, key=lambda row: row["date"])
+    dates = [row["date"] for row in ordered]
+    values = [row["value"] for row in ordered]
+    position = bisect.bisect_right(dates, effective_date)
+    prior_index = position - 1 - periods_back
+    if position == 0 or prior_index < 0:
+        return None
+    return values[position - 1] - values[prior_index]
+
+
+# ---------------------------------------------------------------------------
+# The only function that performs network I/O.
+# ---------------------------------------------------------------------------
+
+
+def fetch_fred_series(series_id: str, start: str, end: str) -> list[dict]:
+    """GET FRED's public graph CSV endpoint for series_id, clipped to
+    [start, end]. Returns [] on any HTTP/parse failure - never raises, so
+    one bad/renamed series id never aborts a multi-series backfill run,
+    matching fetch_yahoo_ohlcv()'s convention exactly.
+
+    MUST use HTTP/2 (http2=True below) - confirmed by direct diagnosis
+    (V5.1 Phase 2, CODESPACE RUN 1) that FRED's graph-export endpoint
+    hangs/RST_STREAMs on plain HTTP/1.1 requests (reproduced with both
+    curl --http1.1 and stdlib urllib, from two independent networks) while
+    responding in ~1-2s over HTTP/2. This is why the module now depends on
+    httpx instead of stdlib urllib, which cannot speak HTTP/2 at all.
+
+    Do NOT set a browser-spoofing User-Agent here (a prior version of this
+    function did, believing FRED required one - that belief was never
+    re-verified and turned out backwards). Under HTTP/2 specifically, a
+    "Mozilla/5.0..." User-Agent paired with httpx's own (non-browser) TLS
+    fingerprint gets an IMMEDIATE RST_STREAM from FRED's edge - a
+    mismatched UA/TLS-fingerprint pair reads as automated traffic
+    disguising itself, which is worse than not disguising it at all.
+    httpx's own honest default User-Agent (python-httpx/x.y) was verified
+    working reliably; leave headers unset."""
+    url = FRED_CSV_URL.format(series_id=series_id)
+    try:
+        with httpx.Client(http2=True, timeout=15.0) as client:
+            response = client.get(url)
+            response.raise_for_status()
+            text = response.text
+    except httpx.HTTPError as exc:
+        logger.warning("fetch_fred_series(%s): fetch failed — %s", series_id, exc)
+        return []
+
+    rows = parse_fred_csv(text, series_id, start, end)
+    if not rows:
+        logger.warning("fetch_fred_series(%s): no data returned for %s..%s", series_id, start, end)
+    return rows
+
+
+# ---------------------------------------------------------------------------
+# Orchestration
+# ---------------------------------------------------------------------------
+
+
+def bond_reference_series(config: dict) -> dict[str, str]:
+    """config["phase1"]["features"]["bond_reference_series"], falling back
+    to DEFAULT_BOND_REFERENCE_SERIES so this module works standalone even
+    against an older config.json."""
+    return {
+        **DEFAULT_BOND_REFERENCE_SERIES,
+        **config.get("phase1", {}).get("features", {}).get("bond_reference_series", {}),
+    }
+
+
+def fetch_all_bond_reference_series(config: dict, start: str, end: str) -> dict[str, list[dict]]:
+    """One independent fetch per series - a single bad/rate-limited FRED
+    series never aborts the others, same convention as
+    yfinance_backfill.run_backfill()'s per-asset loop."""
+    return {
+        series_key: fetch_fred_series(series_id, start, end)
+        for series_key, series_id in bond_reference_series(config).items()
+    }
+
+
+def alt_data_reference_series(config: dict) -> dict[str, str]:
+    """Phase 4.12 (development/Problems.md #71) sibling of
+    bond_reference_series(): config["phase1"]["features"]["alt_data_reference_series"],
+    falling back to DEFAULT_ALT_DATA_REFERENCE_SERIES."""
+    return {
+        **DEFAULT_ALT_DATA_REFERENCE_SERIES,
+        **config.get("phase1", {}).get("features", {}).get("alt_data_reference_series", {}),
+    }
+
+
+def reference_series(config: dict, group: str = "all") -> dict[str, str]:
+    """Merged view over bond_reference_series()/alt_data_reference_series()
+    for the CLI's --group flag. group="bond" or "alt" returns just that
+    one map unchanged (bond_reference_series() itself is deliberately left
+    untouched by this addition - see test_bond_reference_series_unchanged_by_alt_addition);
+    "all" (default) merges both, alt_data keys taking precedence only in
+    the pathological case of an actual name collision (none exist today -
+    the two dicts' keys are disjoint by construction)."""
+    if group == "bond":
+        return bond_reference_series(config)
+    if group == "alt":
+        return alt_data_reference_series(config)
+    return {**bond_reference_series(config), **alt_data_reference_series(config)}
+
+
+def write_fred_series_cache(cache_dir: Path, series_key: str, rows: list[dict]) -> None:
+    """Writes data/reference/fred_series/{series_key}.csv - a local,
+    offline-readable cache (never committed as real market data the way
+    Lean zips are; refreshed by re-running this module with --apply).
+
+    V5.4.7 (development/Problems.md #123): the write now MERGES with any
+    existing cache file instead of replacing it - the CLI's --start/--end
+    clip a narrow refetch (e.g. one series, one recent year), and a full
+    replacement silently destroyed every cached observation outside that
+    window, degrading bond/macro features via series_value_asof() ever
+    after. New rows win on date collisions (fresh fetch is authoritative
+    for its own window); dates only in the existing cache are preserved."""
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    output_path = cache_dir / f"{series_key}.csv"
+    merged: dict = {row["date"]: row["value"] for row in rows}
+    if output_path.exists():
+        try:
+            for existing_row in cache_csv_to_rows(output_path.read_text(encoding="utf-8")):
+                merged.setdefault(existing_row["date"], existing_row["value"])
+        except Exception as exc:
+            # An unreadable old cache must not block a fresh write - the
+            # new rows alone are still strictly better than no cache.
+            logger.warning("write_fred_series_cache(%s): unreadable old cache (%s) - replacing.", series_key, exc)
+    merged_rows = [{"date": day, "value": value} for day, value in sorted(merged.items())]
+    atomic_write_text(output_path, rows_to_cache_csv(merged_rows))
+
+
+def load_cached_fred_series(cache_dir: Path = FRED_SERIES_CACHE_DIR) -> dict[str, list[dict]]:
+    """Reads back whatever *.csv files exist under cache_dir. Returns {} if
+    the directory doesn't exist yet (fresh clone, backfill never run) -
+    every caller of this (train.py/main.py) must treat a missing/empty
+    series as "neutral-default the corresponding bond feature", never a
+    crash, same as macro_features.py's own missing-reference convention."""
+    if not cache_dir.exists():
+        return {}
+    series: dict[str, list[dict]] = {}
+    for path in sorted(cache_dir.glob("*.csv")):
+        try:
+            series[path.stem] = cache_csv_to_rows(path.read_text(encoding="utf-8"))
+        except (ValueError, OSError) as exc:
+            logger.warning("load_cached_fred_series(%s): unreadable cache file — %s", path, exc)
+    return series
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s — %(message)s")
+    parser = argparse.ArgumentParser(
+        description="Aether Quant FRED historical series backfill - offline/manual only, "
+        "no API key required, never run inside Lean or a Docker worker."
+    )
+    parser.add_argument("--series", nargs="*", default=None, help="Restrict to these series keys (default: all of --group's reference series)")
+    parser.add_argument(
+        "--group",
+        choices=["bond", "alt", "all"],
+        default="all",
+        help="Which reference-series map to fetch: bond (phase1.features.bond_reference_series), "
+        "alt (phase1.features.alt_data_reference_series, Problems.md #71), or all (default, both merged)",
+    )
+    parser.add_argument("--start", default="1990-01-01")
+    parser.add_argument("--end", default=date.today().isoformat())
+    parser.add_argument("--apply", action="store_true", help="Actually write the local cache (default: dry run, report only)")
+    parser.add_argument("--config-path", type=Path, default=CONFIG_PATH)
+    parser.add_argument("--cache-dir", type=Path, default=FRED_SERIES_CACHE_DIR)
+    args = parser.parse_args()
+
+    with args.config_path.open("r", encoding="utf-8") as f:
+        config = json.load(f)
+
+    series_map = reference_series(config, args.group)
+    if args.series:
+        series_map = {key: value for key, value in series_map.items() if key in args.series}
+
+    print(f"{'APPLY' if args.apply else 'DRY RUN'} — fetching {len(series_map)} FRED series ({args.start}..{args.end}):\n")
+    for series_key, series_id in series_map.items():
+        rows = fetch_fred_series(series_id, args.start, args.end)
+        if args.apply and rows:
+            write_fred_series_cache(args.cache_dir, series_key, rows)
+            action = "written"
+        elif not rows:
+            action = "no_data_returned"
+        else:
+            action = "dry_run"
+        print(f"- {series_key} ({series_id}): {action}, rows_fetched={len(rows)}")
+
+    if not args.apply:
+        print("\nDry run only — no cache files were written. Re-run with --apply to write the cache.")
+
+
+if __name__ == "__main__":
+    main()