aether-quant 0.0.1__py3-none-any.whl

features/technical_indicators.py

@@ -0,0 +1,185 @@
+"""Shared, pure technical-indicator implementations (Phase 6) - imported by
+BOTH train.py::engineer_features() (offline, full history via a Python
+loop) and main.py::_build_model_input() (runtime, one bar at a time from a
+rolling deque) so both sides compute every indicator identically by
+construction, not by hand-matched duplicated formulas (the train/runtime
+parity convention this codebase already uses for every other feature, see
+development/Changelog.md).
+
+Each function takes a plain list of trailing values (already sliced to the
+right window by the caller) and returns a single scalar float, matching
+how train.py::engineer_features()'s existing loop already computes
+close_to_close_return_5d/rolling_volatility_20d/etc one row at a time from
+Python lists, not vectorized pandas. Every function returns a documented
+neutral default when there isn't enough history yet, so nothing ever
+raises and no extra dropna is needed - the same "adaptive lookback"
+contract every existing base feature already has.
+
+Chosen to fit main.py's existing self.symbol_windows (deque(maxlen=25), up
+to 24 trailing bars - see train.py::CROSS_SECTIONAL_WINDOW_BARS) except
+macd_histogram_normalized()/distance_from_52w_high(), which need a longer
+history and are read from a separate, longer buffer (main.py's
+self.symbol_long_windows, deque(maxlen=260)) - main.py:264-268 explicitly
+forbids resizing symbol_windows itself, so a second buffer is the only
+option for indicators needing more than ~24 bars.
+"""
+
+from __future__ import annotations
+
+RSI_NEUTRAL = 50.0
+BOLLINGER_PCTB_NEUTRAL = 0.5
+VOLUME_ZSCORE_NEUTRAL = 0.0
+MACD_HISTOGRAM_NEUTRAL = 0.0
+DIST_52W_HIGH_NEUTRAL = 0.0
+CROSS_SECTIONAL_MOMENTUM_RANK_NEUTRAL = 0.5
+
+
+def relative_strength_index(closes: list[float], period: int = 14) -> float:
+    """Wilder's RSI over the trailing `period` closes (needs period+1
+    closes for period price changes) - a bounded [0, 100] mean-reversion
+    state, a genuinely different signal shape from any existing
+    return/volatility feature (bounded and non-linear, vs. an unbounded
+    return). Returns RSI_NEUTRAL (50.0, "neither overbought nor oversold")
+    when fewer than period+1 closes are available."""
+    if len(closes) < period + 1:
+        return RSI_NEUTRAL
+    window = closes[-(period + 1):]
+    gains = []
+    losses = []
+    for index in range(1, len(window)):
+        change = window[index] - window[index - 1]
+        gains.append(max(change, 0.0))
+        losses.append(max(-change, 0.0))
+    average_gain = sum(gains) / period
+    average_loss = sum(losses) / period
+    if average_loss == 0:
+        return 100.0 if average_gain > 0 else RSI_NEUTRAL
+    relative_strength = average_gain / average_loss
+    return 100.0 - (100.0 / (1.0 + relative_strength))
+
+
+def average_true_range_pct(highs: list[float], lows: list[float], closes: list[float], period: int = 14) -> float:
+    """ATR (Wilder's simple-average variant) as a % of the latest close -
+    a gap-aware volatility measure distinct from rolling_volatility_20d's
+    close-to-close-only stddev (a true range captures overnight
+    gaps/intraday extremes a close-only measure misses entirely). Needs
+    period+1 bars (period true-range values, each needing a previous
+    close). Returns 0.0 when insufficient history or the latest close is
+    zero."""
+    if len(closes) < period + 1:
+        return 0.0
+    true_ranges = []
+    for index in range(len(closes) - period, len(closes)):
+        high = highs[index]
+        low = lows[index]
+        previous_close = closes[index - 1]
+        true_ranges.append(max(high - low, abs(high - previous_close), abs(low - previous_close)))
+    atr = sum(true_ranges) / period
+    latest_close = closes[-1]
+    return atr / latest_close if latest_close else 0.0
+
+
+def bollinger_pctb(closes: list[float], period: int = 20, num_std: float = 2.0) -> float:
+    """%B: where the latest close sits within its trailing `period`-bar
+    Bollinger Band, normalized so 0.0 = lower band, 1.0 = upper band, 0.5 =
+    the middle (SMA) - a bounded, self-normalizing measure of price
+    extension, distinct from the raw high_low_range_pct/rolling_volatility
+    features already in this codebase. Returns BOLLINGER_PCTB_NEUTRAL
+    (0.5) when fewer than `period` closes exist yet, or the band has zero
+    width (constant price - degenerate either way, not a real signal)."""
+    if len(closes) < period:
+        return BOLLINGER_PCTB_NEUTRAL
+    window = closes[-period:]
+    mean = sum(window) / period
+    variance = sum((value - mean) ** 2 for value in window) / period
+    std = variance ** 0.5
+    if std == 0:
+        return BOLLINGER_PCTB_NEUTRAL
+    upper = mean + num_std * std
+    lower = mean - num_std * std
+    return (closes[-1] - lower) / (upper - lower)
+
+
+def volume_zscore(volumes: list[float], period: int = 20) -> float:
+    """Latest volume's z-score against its own trailing `period`-bar
+    mean/std - a self-normalizing participation measure. Robust to the
+    exact class of raw-volume-unit discontinuity that broke
+    volume_change_1d before its clamp fix (see development/Problems.md):
+    a z-score compresses a single extreme outlier's effect on the bar it
+    occurs on relative to a raw ratio, though a persisting outlier would
+    still distort the trailing window's own mean/std for subsequent bars,
+    same as any rolling statistic. Returns VOLUME_ZSCORE_NEUTRAL (0.0) when
+    fewer than `period` volumes exist yet, or the trailing window has zero
+    variance (a flat volume series)."""
+    if len(volumes) < period:
+        return VOLUME_ZSCORE_NEUTRAL
+    window = volumes[-period:]
+    mean = sum(window) / period
+    variance = sum((value - mean) ** 2 for value in window) / period
+    std = variance ** 0.5
+    return (volumes[-1] - mean) / std if std else VOLUME_ZSCORE_NEUTRAL
+
+
+def cross_sectional_momentum_rank(momentum_by_symbol: dict[str, float], symbol: str) -> float:
+    """Per-date percentile rank ([0, 1], ties averaged - matches pandas'
+    .rank(pct=True) convention, same as
+    build_cross_sectional_rank_targets()'s target-side ranking) of
+    `symbol`'s own momentum_20d among every symbol present in
+    `momentum_by_symbol` - the best-documented daily cross-sectional
+    anomaly (12-1 month momentum), reusing the exact percentile-rank
+    convention already established for the ranking targets. Returns
+    CROSS_SECTIONAL_MOMENTUM_RANK_NEUTRAL (0.5, the exact middle) when
+    `symbol` has fewer than 2 total entries to rank against (undefined
+    otherwise, e.g. a single-asset universe or a thin trading day)."""
+    if len(momentum_by_symbol) < 2 or symbol not in momentum_by_symbol:
+        return CROSS_SECTIONAL_MOMENTUM_RANK_NEUTRAL
+    own_value = momentum_by_symbol[symbol]
+    values = list(momentum_by_symbol.values())
+    count_less = sum(1 for value in values if value < own_value)
+    count_equal = sum(1 for value in values if value == own_value)
+    return (count_less + (count_equal + 1) / 2.0) / len(values)
+
+
+def macd_histogram_normalized(closes: list[float], fast: int = 12, slow: int = 26, signal: int = 9) -> float:
+    """MACD histogram (MACD line - its own signal line), normalized by the
+    latest close so its scale is comparable across assets at very
+    different price levels (a raw MACD histogram is in the asset's own
+    raw price units). Both EMAs are recomputed fresh from the trailing
+    window passed in each call (an approximation of a true "since
+    inception" EMA, which never fully forgets old data) - a deliberate
+    simplification so this is exactly reproducible from a bounded window
+    on both the offline (full history slice) and runtime
+    (self.symbol_long_windows, maxlen=260) sides, with no persistent EMA
+    state to keep in sync between them. Needs at least `slow + signal`
+    closes; returns MACD_HISTOGRAM_NEUTRAL (0.0) otherwise, or when the
+    latest close is zero."""
+    if len(closes) < slow + signal or closes[-1] == 0:
+        return MACD_HISTOGRAM_NEUTRAL
+
+    def _ema_series(values: list[float], period: int) -> list[float]:
+        multiplier = 2.0 / (period + 1)
+        ema_values = [values[0]]
+        for value in values[1:]:
+            ema_values.append((value - ema_values[-1]) * multiplier + ema_values[-1])
+        return ema_values
+
+    fast_ema = _ema_series(closes, fast)
+    slow_ema = _ema_series(closes, slow)
+    macd_line = [fast_value - slow_value for fast_value, slow_value in zip(fast_ema, slow_ema)]
+    signal_line = _ema_series(macd_line, signal)
+    histogram = macd_line[-1] - signal_line[-1]
+    return histogram / closes[-1]
+
+
+def distance_from_52w_high(closes: list[float], window: int = 252) -> float:
+    """close / trailing-`window`-bar max close - 1: how far the latest
+    close sits below its own trailing high (0.0 = at the high, negative
+    otherwise) - a genuine anchoring-effect measure distinct from
+    everything else in this codebase (every existing return/momentum
+    feature is relative to a bar N periods ago, not to a rolling extreme).
+    Returns DIST_52W_HIGH_NEUTRAL (0.0) when fewer than 2 closes exist yet,
+    or the trailing high is zero."""
+    if len(closes) < 2:
+        return DIST_52W_HIGH_NEUTRAL
+    trailing_high = max(closes[-window:])
+    return closes[-1] / trailing_high - 1.0 if trailing_high else DIST_52W_HIGH_NEUTRAL

generate_backtest_report.py

@@ -0,0 +1,311 @@
+"""Regenerates README.md's "Backtest Results" section from the most recent
+real Lean backtest run under backtests/.
+
+A static markdown file can't truly "live update," so this is the practical
+substitute: `aq backtest` calls this module after a successful `lean
+backtest .` run, and it rewrites a chart image plus a small stats table
+in-place between two HTML comment markers in README.md, so the README stays
+current with whatever backtest you last ran without any manual copy-pasting.
+
+Data source: Lean's own backtest result JSON (`backtests/<run>/<id>.json`)
+already contains everything needed, natively time-aligned, in one file - no
+cross-file join needed:
+- `statistics`: a flat dict of ready-to-use strings (Sharpe Ratio, Net
+  Profit, Drawdown, ...).
+- `charts["Strategy Equity"]["series"]["Equity"]["values"]`: the algorithm's
+  own dated equity curve.
+- `charts["Benchmark"]["series"]["Benchmark"]["values"]`: Lean's own
+  benchmark series. `main.py`/`lean.json` never call `SetBenchmark(...)`, so
+  this is Lean's documented default - SPY - already time-aligned with
+  "Strategy Equity" in the same file.
+
+Usage:
+    python generate_backtest_report.py [--backtests-dir backtests] [--readme README.md] [--chart-path development/backtest_equity_chart.png]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections.abc import Iterator
+from datetime import datetime, timezone
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+ROOT_DIR = Path(__file__).resolve().parent
+BACKTESTS_DIR = ROOT_DIR / "backtests"
+README_PATH = ROOT_DIR / "README.md"
+CHART_PATH = ROOT_DIR / "development" / "backtest_equity_chart.png"
+
+_RESULT_JSON_PATTERN = re.compile(r"^\d+\.json$")
+BACKTEST_MARKER_START = "<!-- AQ:BACKTEST_START -->"
+BACKTEST_MARKER_END = "<!-- AQ:BACKTEST_END -->"
+FULL_STATS_MARKER_START = "<!-- AQ:BACKTEST_FULL_STATS_START -->"
+FULL_STATS_MARKER_END = "<!-- AQ:BACKTEST_FULL_STATS_END -->"
+
+_STAT_KEYS_FOR_TABLE = [
+    "Sharpe Ratio",
+    "Net Profit",
+    "Compounding Annual Return",
+    "Drawdown",
+    "Total Orders",
+    "Win Rate",
+]
+
+
+def _iter_backtest_result_jsons(backtests_dir: Path) -> Iterator[Path]:
+    """Yields every backtest run's bare <digits>.json result file (never
+    *-summary.json), newest folder first. Sorted by each folder's own mtime,
+    not by name: the standard `lean backtest .` folders are named
+    YYYY-MM-DD_HH-MM-SS (lexicographic == chronological), but this
+    directory can also contain differently-named ad-hoc/manual run folders
+    (e.g. `lean-local-test-N`) that would otherwise sort ahead of every
+    real timestamped folder and get picked as "latest" by mistake."""
+    if not backtests_dir.exists():
+        return
+    run_dirs = sorted(
+        (d for d in backtests_dir.iterdir() if d.is_dir()),
+        key=lambda d: d.stat().st_mtime,
+        reverse=True,
+    )
+    for run_dir in run_dirs:
+        for candidate in sorted(run_dir.iterdir()):
+            if candidate.is_file() and _RESULT_JSON_PATTERN.match(candidate.name):
+                yield candidate
+                break  # at most one result JSON per backtest folder
+
+
+def find_latest_backtest_result_json(backtests_dir: Path = BACKTESTS_DIR) -> Path | None:
+    """Returns the newest backtest run's result JSON that actually contains
+    both a populated Strategy Equity curve AND a populated statistics
+    block - skips runs that errored out or were interrupted before
+    finishing (Lean can write a result JSON with a partially-populated
+    equity curve but an empty `statistics` dict if the process didn't reach
+    the end of the run), so an incomplete run never clobbers the README
+    with a chart but no Sharpe/drawdown/etc. numbers."""
+    for candidate in _iter_backtest_result_jsons(backtests_dir):
+        try:
+            data = json.loads(candidate.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            continue
+        values = data.get("charts", {}).get("Strategy Equity", {}).get("series", {}).get("Equity", {}).get("values", [])
+        statistics = data.get("statistics", {})
+        if values and statistics:
+            return candidate
+    return None
+
+
+def load_backtest_summary(result_json_path: Path) -> dict:
+    """Parses a Lean backtest result JSON into a clean summary: the
+    statistics dict, plus dated (timestamp, value) strategy/benchmark
+    series."""
+    data = json.loads(result_json_path.read_text(encoding="utf-8"))
+    statistics = data.get("statistics", {})
+    charts = data.get("charts", {})
+
+    strategy_points = charts.get("Strategy Equity", {}).get("series", {}).get("Equity", {}).get("values", [])
+    benchmark_points = charts.get("Benchmark", {}).get("series", {}).get("Benchmark", {}).get("values", [])
+
+    # Equity points are [timestamp, open, high, low, close] with all four
+    # OHLC values identical (an equity total, not a real price bar) -
+    # benchmark points are plain [timestamp, price]. The last element is
+    # the value we want in both cases.
+    strategy_series = [(int(point[0]), float(point[-1])) for point in strategy_points]
+    benchmark_series = [(int(point[0]), float(point[-1])) for point in benchmark_points]
+
+    start_date = datetime.fromtimestamp(strategy_series[0][0], tz=timezone.utc) if strategy_series else None
+    end_date = datetime.fromtimestamp(strategy_series[-1][0], tz=timezone.utc) if strategy_series else None
+
+    return {
+        "statistics": statistics,
+        "strategy_series": strategy_series,
+        "benchmark_series": benchmark_series,
+        "start_date": start_date,
+        "end_date": end_date,
+        "result_json_path": result_json_path,
+    }
+
+
+def _rebase_to_100(series: list[tuple[int, float]]) -> list[tuple[int, float]]:
+    """Both series need a common visual scale: strategy equity starts at
+    the algorithm's starting cash (e.g. $100,000), the benchmark is a raw
+    SPY price - neither is directly comparable to the other in absolute
+    terms, so both are indexed to start at 100."""
+    if not series:
+        return []
+    base = series[0][1]
+    if base == 0:
+        return [(timestamp, 100.0) for timestamp, _ in series]
+    return [(timestamp, value / base * 100.0) for timestamp, value in series]
+
+
+def generate_equity_chart(summary: dict, output_path: Path = CHART_PATH) -> None:
+    """Renders the strategy-vs-benchmark equity chart as a PNG, embedded in
+    README.md by update_readme_backtest_section()."""
+    strategy = _rebase_to_100(summary["strategy_series"])
+    benchmark = _rebase_to_100(summary["benchmark_series"])
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    fig, ax = plt.subplots(figsize=(9, 4.5), dpi=150)
+    if strategy:
+        strategy_dates = [datetime.fromtimestamp(ts, tz=timezone.utc) for ts, _ in strategy]
+        ax.plot(
+            strategy_dates,
+            [value for _, value in strategy],
+            label="Aether Quant Strategy",
+            color="#2E86AB",
+            linewidth=1.6,
+        )
+    if benchmark:
+        benchmark_dates = [datetime.fromtimestamp(ts, tz=timezone.utc) for ts, _ in benchmark]
+        ax.plot(
+            benchmark_dates,
+            [value for _, value in benchmark],
+            label="S&P 500 (SPY) Benchmark",
+            color="#A23B72",
+            linewidth=1.6,
+            linestyle="--",
+        )
+
+    ax.set_title("Backtest Equity: Strategy vs. S&P 500 Benchmark (indexed to 100)")
+    ax.set_ylabel("Indexed value (start = 100)")
+    ax.legend(loc="best", frameon=False)
+    ax.grid(True, alpha=0.25)
+    fig.autofmt_xdate()
+    fig.tight_layout()
+    fig.savefig(output_path)
+    plt.close(fig)
+
+
+def _build_stats_markdown(summary: dict, chart_relative_path: str) -> str:
+    statistics = summary["statistics"]
+    start_date, end_date = summary["start_date"], summary["end_date"]
+    window = (
+        f"{start_date.date().isoformat()} to {end_date.date().isoformat()}" if start_date and end_date else "unknown"
+    )
+    updated_at = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+    lines = [
+        f"![Backtest equity curve]({chart_relative_path})",
+        "",
+        "| Metric | Value |",
+        "|---|---|",
+        f"| Backtest window | {window} |",
+    ]
+    for key in _STAT_KEYS_FOR_TABLE:
+        if key in statistics:
+            lines.append(f"| {key} | {statistics[key]} |")
+    lines.append(f"| Last updated | {updated_at} (auto-generated by `aq backtest`) |")
+    return "\n".join(lines)
+
+
+def _build_full_stats_markdown(summary: dict) -> str:
+    """Every statistic Lean's own backtest result reports, not just the
+    curated highlights in the compact table above it - Sharpe/Sortino/
+    Probabilistic Sharpe, Alpha/Beta, Information/Treynor Ratio, fees,
+    estimated strategy capacity, and everything else Lean computes."""
+    statistics = summary["statistics"]
+    if not statistics:
+        return "_No statistics available in this backtest result._"
+
+    lines = ["| Metric | Value |", "|---|---|"]
+    for key, value in statistics.items():
+        lines.append(f"| {key} | {value} |")
+    return "\n".join(lines)
+
+
+def _replace_between_markers(text: str, start_marker: str, end_marker: str, body: str) -> str | None:
+    """Shared atomic marker-replace helper. Returns the updated text, or
+    None if the markers aren't present or nothing changed."""
+    if start_marker not in text or end_marker not in text:
+        return None
+    pattern = re.compile(re.escape(start_marker) + r".*?" + re.escape(end_marker), re.DOTALL)
+    replacement = f"{start_marker}\n{body}\n{end_marker}"
+    updated_text = pattern.sub(replacement, text, count=1)
+    return updated_text if updated_text != text else None
+
+
+def _atomic_write(path: Path, text: str) -> None:
+    tmp_path = path.with_suffix(path.suffix + ".tmp")
+    tmp_path.write_text(text, encoding="utf-8")
+    tmp_path.replace(path)
+
+
+def update_readme_backtest_section(
+    summary: dict, readme_path: Path = README_PATH, chart_path: Path = CHART_PATH
+) -> bool:
+    """Atomically replaces the content between the AQ:BACKTEST markers in
+    README.md with a fresh stats table + chart image reference, and the
+    content between the AQ:BACKTEST_FULL_STATS markers with every statistic
+    Lean's result reports. Returns False (never raises) if the README or
+    its markers are missing - a report-generation bug must never break
+    `aq backtest` itself."""
+    if not readme_path.exists():
+        return False
+    text = readme_path.read_text(encoding="utf-8")
+
+    try:
+        chart_relative_path = chart_path.relative_to(readme_path.resolve().parent).as_posix()
+    except ValueError:
+        chart_relative_path = chart_path.as_posix()
+
+    updated_text = _replace_between_markers(
+        text, BACKTEST_MARKER_START, BACKTEST_MARKER_END, _build_stats_markdown(summary, chart_relative_path)
+    )
+    if updated_text is None:
+        return False
+    text = updated_text
+
+    full_stats_text = _replace_between_markers(
+        text, FULL_STATS_MARKER_START, FULL_STATS_MARKER_END, _build_full_stats_markdown(summary)
+    )
+    if full_stats_text is not None:
+        text = full_stats_text
+
+    _atomic_write(readme_path, text)
+    return True
+
+
+def update_readme_from_latest_backtest(
+    backtests_dir: Path = BACKTESTS_DIR,
+    readme_path: Path = README_PATH,
+    chart_path: Path = CHART_PATH,
+) -> bool:
+    """Top-level entrypoint `aq_cli.py::cmd_backtest` calls after a
+    successful `lean backtest .` run. Callers should still wrap this in a
+    try/except - a report-generation bug must never fail the backtest
+    command itself."""
+    result_json_path = find_latest_backtest_result_json(backtests_dir)
+    if result_json_path is None:
+        return False
+    summary = load_backtest_summary(result_json_path)
+    generate_equity_chart(summary, chart_path)
+    return update_readme_backtest_section(summary, readme_path, chart_path)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Regenerate README.md's Backtest Results section from the latest Lean backtest run."
+    )
+    parser.add_argument("--backtests-dir", type=Path, default=BACKTESTS_DIR)
+    parser.add_argument("--readme", type=Path, default=README_PATH)
+    parser.add_argument("--chart-path", type=Path, default=CHART_PATH)
+    args = parser.parse_args()
+
+    updated = update_readme_from_latest_backtest(args.backtests_dir, args.readme, args.chart_path)
+    if updated:
+        print(f"Updated {args.readme} with the latest backtest results.")
+        return 0
+    print("No backtest results found (or README markers missing) - nothing updated.", file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())