aether-quant 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. aether_quant-0.0.1.dist-info/METADATA +7 -0
  2. aether_quant-0.0.1.dist-info/RECORD +99 -0
  3. aether_quant-0.0.1.dist-info/WHEEL +5 -0
  4. aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
  5. aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
  6. aether_quant-0.0.1.dist-info/top_level.txt +13 -0
  7. analyzer/README.md +69 -0
  8. analyzer/__init__.py +17 -0
  9. analyzer/market_analyzer.py +521 -0
  10. aq_cli.py +3633 -0
  11. audit/README.md +47 -0
  12. audit/__init__.py +38 -0
  13. audit/hash_chain.py +52 -0
  14. audit/postgres_audit.py +61 -0
  15. audit/postgres_worker.py +402 -0
  16. audit/redis_queue.py +124 -0
  17. audit/status_export.py +62 -0
  18. data_pipeline/README.md +257 -0
  19. data_pipeline/__init__.py +6 -0
  20. data_pipeline/bar_synthesis.py +97 -0
  21. data_pipeline/dividend_backfill.py +243 -0
  22. data_pipeline/factor_file_backfill.py +276 -0
  23. data_pipeline/fetch.py +206 -0
  24. data_pipeline/fred_backfill.py +435 -0
  25. data_pipeline/ib_backfill.py +287 -0
  26. data_pipeline/v2_manifest.py +75 -0
  27. data_pipeline/yfinance_backfill.py +501 -0
  28. evaluation/README.md +152 -0
  29. evaluation/__init__.py +73 -0
  30. evaluation/ablation.py +310 -0
  31. evaluation/benchmark_comparison.py +195 -0
  32. evaluation/confidence_threshold_calibration.py +157 -0
  33. evaluation/factor_neutralization_check.py +94 -0
  34. evaluation/feature_parity.py +321 -0
  35. evaluation/feature_reconciliation.py +286 -0
  36. evaluation/impact_model.py +77 -0
  37. evaluation/kill_switch_replay.py +186 -0
  38. evaluation/limit_fill_simulator.py +152 -0
  39. evaluation/model_predictions.py +192 -0
  40. evaluation/monte_carlo.py +161 -0
  41. evaluation/public_benchmarks.py +117 -0
  42. evaluation/rank_book_simulator.py +494 -0
  43. evaluation/rank_ic_core.py +204 -0
  44. evaluation/rank_signal_calibration.py +690 -0
  45. evaluation/rolling_ic_gate_calibration.py +190 -0
  46. evaluation/rolling_ic_gate_replay.py +138 -0
  47. execution/README.md +281 -0
  48. execution/__init__.py +80 -0
  49. execution/config_cache.py +48 -0
  50. execution/cost_model.py +213 -0
  51. execution/lean_config_render.py +131 -0
  52. execution/live_credentials.py +38 -0
  53. execution/live_credentials_io.py +52 -0
  54. execution/order_gate.py +388 -0
  55. execution/paper_readiness.py +181 -0
  56. execution/paper_readiness_io.py +54 -0
  57. execution/paper_readiness_report.py +103 -0
  58. execution/paper_readiness_scheduler.py +137 -0
  59. execution/reconciliation.py +160 -0
  60. execution/runtime_config_io.py +38 -0
  61. execution/secret_scan.py +73 -0
  62. features/README.md +77 -0
  63. features/__init__.py +137 -0
  64. features/alt_data_features.py +154 -0
  65. features/bond_features.py +275 -0
  66. features/cross_asset_sensitivity.py +79 -0
  67. features/derivatives_macro_features.py +72 -0
  68. features/macro_features.py +79 -0
  69. features/options_greeks.py +329 -0
  70. features/sector_map.py +53 -0
  71. features/technical_indicators.py +185 -0
  72. generate_backtest_report.py +311 -0
  73. generate_evaluation_report.py +712 -0
  74. inference/README.md +105 -0
  75. inference/__init__.py +36 -0
  76. inference/exported_model.py +929 -0
  77. inference/parallel_inference.py +160 -0
  78. inference/strategy_selector_inference.py +75 -0
  79. json_safety.py +130 -0
  80. portfolio/README.md +447 -0
  81. portfolio/__init__.py +41 -0
  82. portfolio/book_construction.py +786 -0
  83. portfolio/book_neutrality.py +178 -0
  84. portfolio/hrp_allocation.py +159 -0
  85. portfolio/options_arbitrage_detector.py +232 -0
  86. portfolio/options_assignment_risk.py +119 -0
  87. portfolio/options_margin_sizing.py +333 -0
  88. portfolio/options_strategy.py +1999 -0
  89. portfolio/rank_signal.py +219 -0
  90. portfolio/rolling_ic_gate.py +183 -0
  91. risk/README.md +275 -0
  92. risk/__init__.py +38 -0
  93. risk/asset_class_router.py +584 -0
  94. risk/forex_risk.py +158 -0
  95. risk/futures_risk.py +202 -0
  96. risk/kill_switch.py +261 -0
  97. risk/manual_override.py +118 -0
  98. risk/position_sizing.py +303 -0
  99. risk/rl_sizing.py +189 -0
@@ -0,0 +1,157 @@
1
+ """Calibrates `phase6.risk.min_confidence_to_trade` (and, when book
2
+ selection data is available, `phase6.risk.min_confidence_to_trade_book_selected`)
3
+ from real data (V5.2.6, development/Problems.md) - the same "derive the
4
+ number, don't guess it" discipline `aq evaluate --calibrate-edge` and
5
+ `aq evaluate --calibrate-book-spread` already established for
6
+ `phase_v2.costs.edge_bps_per_rank_unit` and
7
+ `phase_v2.portfolio_book.min_rank_confidence_spread`.
8
+
9
+ Deliberately reuses, never re-derives: `portfolio.rank_signal.cross_sectional_rank_scores()`
10
+ (the exact normalization main.py applies before book selection) and
11
+ `portfolio.book_construction.build_rank_based_book()` (the exact
12
+ top_n/bottom_n selection main.py's live book uses, for the book-selected
13
+ split) and `analyzer.market_analyzer.compute_trade_metric()` (the exact
14
+ confidence-metric formula the live gate compares against
15
+ min_confidence_to_trade). A calibrated threshold computed here can never
16
+ silently disagree with what the live gate would have done on the same
17
+ historical day, because it is the same code, not a parallel
18
+ approximation.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import numpy as np
24
+ import pandas as pd
25
+
26
+ from analyzer.market_analyzer import compute_trade_metric
27
+ from portfolio.book_construction import build_rank_based_book
28
+ from portfolio.rank_signal import cross_sectional_rank_scores
29
+
30
+ _CONFIDENCE_DISTRIBUTION_PERCENTILES = {"p10": 0.10, "p25": 0.25, "median": 0.50, "p75": 0.75}
31
+
32
+
33
+ def _empty_distribution() -> dict:
34
+ return {"min": None, "p10": None, "p25": None, "median": None, "p75": None, "max": None}
35
+
36
+
37
+ def _percentile_of_paying_confidence(confidences: list[float], percentile: float) -> tuple[float, dict]:
38
+ if not confidences:
39
+ return 0.0, _empty_distribution()
40
+ array = np.array(confidences, dtype=np.float64)
41
+ distribution = {"min": float(np.min(array)), "max": float(np.max(array))}
42
+ for name, quantile in _CONFIDENCE_DISTRIBUTION_PERCENTILES.items():
43
+ distribution[name] = float(np.quantile(array, quantile))
44
+ return float(np.quantile(array, percentile)), distribution
45
+
46
+
47
+ def calibrate_confidence_threshold(
48
+ dataset: pd.DataFrame,
49
+ *,
50
+ raw_score_column: str,
51
+ forward_return_column: str,
52
+ ticker_column: str = "ticker",
53
+ date_column: str = "date",
54
+ top_n: int,
55
+ bottom_n: int,
56
+ round_trip_cost_fraction: float = 0.001,
57
+ percentile: float = 0.10,
58
+ ) -> dict:
59
+ """Per date (sorted): normalizes that date's raw scores exactly as
60
+ main.py's Pass 1 does (`cross_sectional_rank_scores()`), computes each
61
+ symbol's book-member confidence via the SAME formula main.py uses for
62
+ a book-selected symbol (`min(1.0, abs(normalized_rank - 0.5) * 2.0)` -
63
+ see main.py's Pass 2, `confidence = min(1.0, abs(book_allocation.predicted_rank_20d - 0.5) * 2.0)`),
64
+ and separately determines whether that date's natural top_n/bottom_n
65
+ selection (`build_rank_based_book()`, no gate, no hysteresis - same
66
+ "measure each day's independent natural dispersion" reasoning
67
+ `calibrate_book_confidence_spread()` already uses) would have picked
68
+ each symbol, for the book-selected/non-book-selected split.
69
+
70
+ A row "would have been worth taking" when its DIRECTIONAL realized
71
+ edge clears round_trip_cost_fraction: `direction = 1 if
72
+ normalized_rank >= 0.5 else -1`, `realized_edge = direction *
73
+ forward_return - round_trip_cost_fraction`. The calibrated threshold
74
+ is the given percentile of confidence AMONG ONLY the rows that
75
+ actually paid - the same "percentile of a real, achieved distribution,
76
+ not a guess" discipline `calibrate_book_confidence_spread()` already
77
+ established, applied to confidence instead of spread.
78
+
79
+ Returns:
80
+ {"calibrated_min_confidence_to_trade": float,
81
+ "calibrated_min_confidence_to_trade_book_selected": float,
82
+ "percentile": float,
83
+ "num_rows_used": int,
84
+ "num_rows_used_book_selected": int,
85
+ "num_dates_skipped_thin_universe": int,
86
+ "confidence_distribution": {...},
87
+ "confidence_distribution_book_selected": {...}}
88
+
89
+ An empty/all-thin dataset returns both calibrated values as 0.0 (the
90
+ gate's own "always trade" no-op value) and all-None distributions,
91
+ never raises."""
92
+ paying_confidences: list[float] = []
93
+ paying_confidences_book_selected: list[float] = []
94
+ num_dates_skipped_thin_universe = 0
95
+
96
+ for _date, group in dataset.groupby(date_column, sort=True):
97
+ raw_scores = {
98
+ str(ticker): float(score)
99
+ for ticker, score in zip(group[ticker_column], group[raw_score_column])
100
+ if score is not None and not (isinstance(score, float) and np.isnan(score))
101
+ }
102
+ normalized = cross_sectional_rank_scores(raw_scores)
103
+ if not normalized:
104
+ num_dates_skipped_thin_universe += 1
105
+ continue
106
+
107
+ candidates = {
108
+ symbol: {"predicted_rank_20d": rank, "trading_eligible": True} for symbol, rank in normalized.items()
109
+ }
110
+ book_allocations = build_rank_based_book(
111
+ candidates,
112
+ top_n=top_n,
113
+ bottom_n=bottom_n,
114
+ min_rank_confidence_spread=0.0,
115
+ spread_check_ranks=raw_scores,
116
+ )
117
+ book_selected_symbols = set(book_allocations.keys())
118
+
119
+ forward_return_by_ticker = {
120
+ str(ticker): float(value)
121
+ for ticker, value in zip(group[ticker_column], group[forward_return_column])
122
+ if value is not None and not (isinstance(value, float) and np.isnan(value))
123
+ }
124
+
125
+ for symbol, normalized_rank in normalized.items():
126
+ forward_return = forward_return_by_ticker.get(symbol)
127
+ if forward_return is None:
128
+ continue
129
+ confidence = compute_trade_metric(
130
+ confidence=min(1.0, abs(normalized_rank - 0.5) * 2.0),
131
+ regime_confidence=0.0,
132
+ topology={},
133
+ liquidity={},
134
+ use_composite_signal_score=False,
135
+ )
136
+ direction = 1.0 if normalized_rank >= 0.5 else -1.0
137
+ realized_edge = direction * forward_return - round_trip_cost_fraction
138
+ if realized_edge > 0.0:
139
+ paying_confidences.append(confidence)
140
+ if symbol in book_selected_symbols:
141
+ paying_confidences_book_selected.append(confidence)
142
+
143
+ calibrated_general, distribution_general = _percentile_of_paying_confidence(paying_confidences, percentile)
144
+ calibrated_book_selected, distribution_book_selected = _percentile_of_paying_confidence(
145
+ paying_confidences_book_selected, percentile
146
+ )
147
+
148
+ return {
149
+ "calibrated_min_confidence_to_trade": calibrated_general,
150
+ "calibrated_min_confidence_to_trade_book_selected": calibrated_book_selected,
151
+ "percentile": percentile,
152
+ "num_rows_used": len(paying_confidences),
153
+ "num_rows_used_book_selected": len(paying_confidences_book_selected),
154
+ "num_dates_skipped_thin_universe": num_dates_skipped_thin_universe,
155
+ "confidence_distribution": distribution_general,
156
+ "confidence_distribution_book_selected": distribution_book_selected,
157
+ }
@@ -0,0 +1,94 @@
1
+ """V5.4.3 - factor-neutralization check: decomposes the offline rank book's
2
+ realized returns into systematic (market/sector/momentum) and residual
3
+ (pure alpha) components, answering "how much of the claimed Sharpe is
4
+ actually skill vs. factor exposure?"
5
+
6
+ Pure: numpy/pandas only, no I/O, no torch."""
7
+
8
+ from __future__ import annotations
9
+
10
+ import numpy as np
11
+ import pandas as pd
12
+
13
+
14
+ def compute_factor_exposure(
15
+ book_returns: pd.Series,
16
+ market_returns: pd.Series,
17
+ *,
18
+ momentum_factor: pd.Series | None = None,
19
+ sector_returns: dict[str, pd.Series] | None = None,
20
+ ) -> dict:
21
+ """Regresses book returns onto factor columns and reports loadings,
22
+ R², residual Sharpe, and % of variance explained by each factor.
23
+
24
+ All input series must be aligned by date (same index or positional).
25
+ Missing values are forward-filled then dropped.
26
+
27
+ Returns:
28
+ {"alpha_sharpe": float, "total_sharpe": float, "r_squared": float,
29
+ "loadings": {factor: beta}, "alpha_annualized_return_pct": float,
30
+ "factor_contribution_pct": {factor: pct_of_variance},
31
+ "n_obs": int}
32
+ """
33
+ book = _align(book_returns)
34
+ market = _align(market_returns)
35
+ n = min(len(book), len(market))
36
+ book, market = book[:n], market[:n]
37
+
38
+ factors = {"market": market}
39
+ if momentum_factor is not None:
40
+ mom = _align(momentum_factor)[:n]
41
+ if len(mom) == n:
42
+ factors["momentum"] = mom
43
+ if sector_returns:
44
+ for name, series in sector_returns.items():
45
+ s = _align(series)
46
+ if len(s) >= n:
47
+ factors[f"sector_{name}"] = s[:n]
48
+
49
+ # Build design matrix [1, factor_1, ..., factor_k]
50
+ names = list(factors.keys())
51
+ X = np.column_stack([np.ones(n)] + [factors[name] for name in names])
52
+ y = book
53
+
54
+ # OLS: beta = (X'X)^-1 X'y
55
+ XtX = X.T @ X
56
+ Xty = X.T @ y
57
+ try:
58
+ betas = np.linalg.solve(XtX, Xty)
59
+ except np.linalg.LinAlgError:
60
+ return {"error": "singular design matrix", "n_obs": n}
61
+
62
+ residuals = y - X @ betas
63
+ ss_total = float(np.sum((y - y.mean()) ** 2))
64
+ ss_resid = float(np.sum(residuals ** 2))
65
+ r_squared = 1.0 - ss_resid / ss_total if ss_total > 0 else 0.0
66
+
67
+ # Residual alpha Sharpe (annualized with sqrt(252))
68
+ resid_std = np.std(residuals, ddof=1) if len(residuals) > 1 else 1e-12
69
+ alpha_sharpe = float(np.mean(residuals) / max(resid_std, 1e-12) * np.sqrt(252))
70
+ total_sharpe = float(np.mean(y) / max(np.std(y, ddof=1), 1e-12) * np.sqrt(252))
71
+
72
+ # Factor contribution: variance explained by each factor alone
73
+ contributions = {}
74
+ for i, name in enumerate(names):
75
+ col = X[:, i + 1]
76
+ ss_factor = float(np.sum((col - col.mean()) ** 2))
77
+ beta_i = betas[i + 1]
78
+ contribution = beta_i ** 2 * ss_factor
79
+ contributions[name] = round(contribution / ss_total * 100, 2) if ss_total > 0 else 0.0
80
+
81
+ return {
82
+ "alpha_sharpe": round(alpha_sharpe, 4),
83
+ "total_sharpe": round(total_sharpe, 4),
84
+ "r_squared": round(r_squared, 4),
85
+ "loadings": {name: round(float(betas[i + 1]), 6) for i, name in enumerate(names)},
86
+ "alpha_annualized_return_pct": round(float(np.mean(residuals) * 252 * 100), 4),
87
+ "factor_contribution_pct": contributions,
88
+ "n_obs": n,
89
+ }
90
+
91
+
92
+ def _align(series: pd.Series) -> np.ndarray:
93
+ values = pd.Series(series).ffill().dropna().to_numpy(dtype=float)
94
+ return values
@@ -0,0 +1,321 @@
1
+ """V5.3.6 Workstream A - the train-vs-live feature-parity audit's pure core.
2
+
3
+ Background (development/Problems.md #91/#98/#100 lineage): XOM is the one
4
+ tracked ticker whose live-vs-offline book-selection mismatch never resolved,
5
+ and two prior root causes WERE feature-computation drifts between the
6
+ offline training pipeline and the live Lean path (the bond duration-beta's
7
+ rolling-vs-full-history drift, V5.2.5; missing split/dividend factor files,
8
+ #100). This module is the systematic version of those one-off checks: for
9
+ every model-input feature family that can be recomputed from data already
10
+ on disk, it re-derives the RAW value and pushes it through the shipped
11
+ scaler artifact (`ml/scaler_stats.json`) in the forward direction -
12
+ `clip((raw - mean) / scale, +-clip_sigma)` - comparing against the scaled
13
+ column actually stored in `ml/datasets/full_dataset.csv`.
14
+
15
+ Three layers are covered here:
16
+ 1. **Scaler-artifact consistency** - does `scaler_stats.json` still describe
17
+ the dataset it is supposed to scale? (A stale artifact after a dataset
18
+ rebuild would mis-scale live inference systemically - an XOM-class bug.)
19
+ 2. **Formula parity** - the per-ticker OHLCV-recomputable families (10 base
20
+ technicals + 6 shared indicators + 2 liquidity) re-derived row-faithfully
21
+ from `train.py::engineer_features()` / `add_liquidity_features()`.
22
+ 3. **Cross-sectional + macro families** - cs_momentum_rank_20 and the three
23
+ macro proxies, rebuilt from a close-price pivot using the same shared
24
+ functions both sides import.
25
+
26
+ Families deliberately NOT recomputable offline (regime payload chain, alt-
27
+ data FRED inputs, cross-asset sensitivity regressions' driver series, bond
28
+ treasury inputs, topology/peer correlation graphs) get structured verdicts
29
+ from the driver instead: "SHARED_FUNCTION_CALLSITE" where both sides call
30
+ one `features/*.py` function (verified by inspection), so the residual risk
31
+ is call-site parameters only.
32
+
33
+ Pure: no I/O, no torch, no Lean imports - mirrors the repo's established
34
+ "pure core + thin CLI driver" convention (see feature_reconciliation.py).
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ from typing import Any
40
+
41
+ import numpy as np
42
+ import pandas as pd
43
+
44
+ from features.macro_features import (
45
+ credit_spread_proxy,
46
+ crypto_risk_appetite_proxy,
47
+ yield_curve_slope_proxy,
48
+ )
49
+ from features.technical_indicators import (
50
+ average_true_range_pct,
51
+ bollinger_pctb,
52
+ cross_sectional_momentum_rank,
53
+ distance_from_52w_high,
54
+ macd_histogram_normalized,
55
+ relative_strength_index,
56
+ volume_zscore,
57
+ )
58
+ from liquidity.market_liquidity import estimate_high_low_spread
59
+
60
+ DEFAULT_LONG_LOOKBACK_WINDOW_BARS = 260
61
+ DEFAULT_CROSS_SECTIONAL_WINDOW_BARS = 25
62
+
63
+ BASE_TECHNICAL_FEATURES = [
64
+ "close_to_close_return_1d",
65
+ "close_to_close_return_5d",
66
+ "close_to_close_return_20d",
67
+ "rolling_volatility_5d",
68
+ "rolling_volatility_20d",
69
+ "momentum_5d",
70
+ "momentum_20d",
71
+ "high_low_range_pct",
72
+ "open_close_range_pct",
73
+ "volume_change_1d",
74
+ ]
75
+ INDICATOR_FEATURES = [
76
+ "rsi_14",
77
+ "atr_pct_14",
78
+ "bollinger_pctb_20",
79
+ "volume_zscore_20",
80
+ "macd_histogram_norm",
81
+ "dist_52w_high",
82
+ ]
83
+ LIQUIDITY_FEATURES = ["liquidity_log_dollar_volume", "liquidity_spread_proxy"]
84
+ MACRO_PROXIES_BY_KEY = {
85
+ "long_duration": ("macro_yield_curve_slope_proxy", yield_curve_slope_proxy),
86
+ "high_yield": ("macro_credit_spread_proxy", credit_spread_proxy),
87
+ "crypto": ("macro_crypto_risk_appetite_proxy", crypto_risk_appetite_proxy),
88
+ }
89
+ DEFAULT_MACRO_REFERENCE_TICKERS = {
90
+ "long_duration": "TLT",
91
+ "short_duration": "SHY",
92
+ "high_yield": "HYG",
93
+ "investment_grade": "LQD",
94
+ "crypto": "BTCUSD",
95
+ }
96
+
97
+
98
+ def load_scaler_mapping(scaler_stats: dict[str, Any]) -> tuple[dict[str, tuple[float, float, float]], list[str]]:
99
+ """Maps `ml/scaler_stats.json` into {feature_name: (mean, scale, clip)}
100
+ and reports which schema names it does NOT cover (by design: the binary
101
+ regime/topology/asset-class one-hots skip StandardScaler entirely)."""
102
+ names = list(scaler_stats["feature_names"])
103
+ means = scaler_stats["mean"]
104
+ scales = scaler_stats["scale"]
105
+ clip = float(scaler_stats.get("clip_sigma", 10.0))
106
+ mapping = {name: (float(means[i]), float(scales[i]), clip) for i, name in enumerate(names)}
107
+ return mapping, []
108
+
109
+
110
+ def forward_scale(raw_values: np.ndarray, mean: float, scale: float, clip: float) -> np.ndarray:
111
+ """The exact live/offline transform: clip((raw - mean) / scale, +-clip)."""
112
+ return np.clip((np.asarray(raw_values, dtype=float) - mean) / scale, -clip, clip)
113
+
114
+
115
+ def compute_base_technicals_replica(frame: pd.DataFrame, *, long_window_bars: int = DEFAULT_LONG_LOOKBACK_WINDOW_BARS) -> dict[str, np.ndarray]:
116
+ """Row-faithful replica of train.py::engineer_features()'s inline base
117
+ technicals + Phase-6 indicator block (train.py:656-748), which itself is
118
+ line-mirrored by main.py::_build_model_input() (both files carry explicit
119
+ cross-reference comments asserting parity). First row is NaN exactly like
120
+ train's `[np.nan]` seed convention; volatility degrades to 0.0 under two
121
+ observations exactly like train's guard."""
122
+ closes = frame["close"].tolist()
123
+ opens = frame["open"].tolist()
124
+ highs = frame["high"].tolist()
125
+ lows = frame["low"].tolist()
126
+ volumes = frame["volume"].tolist()
127
+ n = len(frame)
128
+
129
+ columns: dict[str, list[float]] = {
130
+ **{name: [np.nan] for name in BASE_TECHNICAL_FEATURES},
131
+ **{name: [np.nan] for name in INDICATOR_FEATURES},
132
+ }
133
+
134
+ for index in range(1, n):
135
+ previous_close = closes[index - 1]
136
+ previous_volume = volumes[index - 1]
137
+ current_close = closes[index]
138
+ current_open = opens[index]
139
+
140
+ close_5 = closes[max(0, index - 5)]
141
+ close_20 = closes[max(0, index - 20)]
142
+
143
+ all_recent_returns = [
144
+ closes[position] / closes[position - 1] - 1.0
145
+ for position in range(1, index + 1)
146
+ if closes[position - 1] != 0
147
+ ]
148
+ recent_returns_5 = all_recent_returns[-5:]
149
+ recent_returns_20 = all_recent_returns[-20:]
150
+
151
+ def _sample_std(values: list[float]) -> float:
152
+ if len(values) < 2:
153
+ return 0.0
154
+ mean_value = sum(values) / len(values)
155
+ variance = sum((value - mean_value) ** 2 for value in values) / (len(values) - 1)
156
+ return float(np.sqrt(max(variance, 0.0)))
157
+
158
+ columns["close_to_close_return_1d"].append(current_close / previous_close - 1.0 if previous_close else 0.0)
159
+ columns["close_to_close_return_5d"].append(current_close / close_5 - 1.0 if close_5 else 0.0)
160
+ columns["close_to_close_return_20d"].append(current_close / close_20 - 1.0 if close_20 else 0.0)
161
+ columns["rolling_volatility_5d"].append(_sample_std(recent_returns_5))
162
+ columns["rolling_volatility_20d"].append(_sample_std(recent_returns_20))
163
+ columns["momentum_5d"].append(current_close / close_5 - 1.0 if close_5 else 0.0)
164
+ columns["momentum_20d"].append(current_close / close_20 - 1.0 if close_20 else 0.0)
165
+ columns["high_low_range_pct"].append((highs[index] - lows[index]) / current_close if current_close else 0.0)
166
+ columns["open_close_range_pct"].append((current_close - current_open) / current_open if current_open else 0.0)
167
+ raw_volume_change = 0.0 if previous_volume == 0 else volumes[index] / previous_volume - 1.0
168
+ columns["volume_change_1d"].append(max(-1.0, min(20.0, raw_volume_change)))
169
+
170
+ trailing_closes = closes[: index + 1]
171
+ columns["rsi_14"].append(relative_strength_index(trailing_closes, period=14))
172
+ columns["atr_pct_14"].append(
173
+ average_true_range_pct(highs[: index + 1], lows[: index + 1], trailing_closes, period=14)
174
+ )
175
+ columns["bollinger_pctb_20"].append(bollinger_pctb(trailing_closes, period=20))
176
+ columns["volume_zscore_20"].append(volume_zscore(volumes[: index + 1], period=20))
177
+
178
+ long_window_start = max(0, index + 1 - long_window_bars)
179
+ long_trailing_closes = closes[long_window_start : index + 1]
180
+ columns["macd_histogram_norm"].append(macd_histogram_normalized(long_trailing_closes))
181
+ columns["dist_52w_high"].append(distance_from_52w_high(long_trailing_closes, window=long_window_bars))
182
+
183
+ return {name: np.asarray(values, dtype=float) for name, values in columns.items()}
184
+
185
+
186
+ def compute_liquidity_replica(
187
+ frame: pd.DataFrame,
188
+ *,
189
+ security_type: str = "equity",
190
+ cross_sectional_window_bars: int = DEFAULT_CROSS_SECTIONAL_WINDOW_BARS,
191
+ typical_spread_by_type: dict[str, float] | None = None,
192
+ ) -> dict[str, np.ndarray]:
193
+ """Row-faithful replica of train.py::add_liquidity_features()
194
+ (train.py:946-988): log1p dollar volume plus the Corwin-Schultz spread
195
+ proxy over a trailing window, static-spread fallback below two bars."""
196
+ from liquidity.market_liquidity import TYPICAL_SPREAD_BY_TYPE
197
+
198
+ spreads = typical_spread_by_type or TYPICAL_SPREAD_BY_TYPE
199
+ fallback = float(spreads.get(str(security_type), 0.001))
200
+ closes = frame["close"].tolist()
201
+ volumes = frame["volume"].tolist()
202
+ highs = frame["high"].tolist()
203
+ lows = frame["low"].tolist()
204
+
205
+ log_dollar_volume: list[float] = []
206
+ spread_proxy: list[float] = []
207
+ for index in range(len(frame)):
208
+ log_dollar_volume.append(float(np.log1p(max(closes[index] * volumes[index], 0.0))))
209
+ window_start = max(0, index + 1 - cross_sectional_window_bars)
210
+ window_highs = highs[window_start : index + 1]
211
+ window_lows = lows[window_start : index + 1]
212
+ estimated = estimate_high_low_spread(window_highs, window_lows) if len(window_highs) >= 2 else None
213
+ spread_proxy.append(float(estimated) if estimated is not None else fallback)
214
+
215
+ return {
216
+ "liquidity_log_dollar_volume": np.asarray(log_dollar_volume, dtype=float),
217
+ "liquidity_spread_proxy": np.asarray(spread_proxy, dtype=float),
218
+ }
219
+
220
+
221
+ def compute_momentum_long(dataset: pd.DataFrame) -> pd.DataFrame:
222
+ """Per-ticker momentum_20d computed on EACH TICKER'S OWN trading rows -
223
+ position-shift(20) within that ticker's sorted history, exactly how
224
+ engineer_features() consumed each asset's raw frame. Deliberately NOT a
225
+ union-calendar pivot shift: cross-asset calendars differ (crypto trades
226
+ weekends, bonds don't), so a pivot-row shift spans different numbers of
227
+ each ticker's REAL bars - measured drift up to 0.16 absolute on TLT."""
228
+ def _momentum(closes: pd.Series) -> pd.Series:
229
+ return closes / closes.shift(20) - 1.0
230
+
231
+ out = dataset[["date", "ticker"]].copy()
232
+ out["momentum_20d"] = dataset.groupby("ticker", sort=False)["close"].transform(_momentum)
233
+ return out
234
+
235
+
236
+ def compute_macro_proxies_replica(
237
+ all_dates: list[str],
238
+ momentum_long: pd.DataFrame,
239
+ *,
240
+ reference_tickers: dict[str, str] | None = None,
241
+ ) -> pd.DataFrame:
242
+ """Replica of train.py::build_macro_features_by_date()
243
+ (train.py:1126-1189): per-date AS-OF (last value at-or-before the date)
244
+ momentum_20d of five fixed reference tickers fed through the three
245
+ shared macro proxy functions; missing references degrade to each
246
+ function's neutral default. As-of uses each reference ticker's OWN date
247
+ series - never a union calendar."""
248
+ references = {**DEFAULT_MACRO_REFERENCE_TICKERS, **(reference_tickers or {})}
249
+
250
+ def _asof_values(ticker: str) -> list[float | None]:
251
+ rows = momentum_long[momentum_long["ticker"] == ticker]
252
+ if rows.empty:
253
+ return [None] * len(all_dates)
254
+ series = pd.Series(
255
+ rows["momentum_20d"].to_numpy(dtype=float),
256
+ index=pd.Index(rows["date"], name="date"),
257
+ ).sort_index()
258
+ filled = series.reindex(all_dates).ffill()
259
+ return [None if pd.isna(value) else float(value) for value in filled]
260
+
261
+ long_value = _asof_values(references["long_duration"])
262
+ short_value = _asof_values(references["short_duration"])
263
+ high_yield_value = _asof_values(references["high_yield"])
264
+ investment_grade_value = _asof_values(references["investment_grade"])
265
+ crypto_value = _asof_values(references["crypto"])
266
+
267
+ result = pd.DataFrame({"date": all_dates})
268
+ result["macro_yield_curve_slope_proxy"] = [
269
+ yield_curve_slope_proxy(long_v, short_v) for long_v, short_v in zip(long_value, short_value)
270
+ ]
271
+ result["macro_credit_spread_proxy"] = [
272
+ credit_spread_proxy(hy, ig) for hy, ig in zip(high_yield_value, investment_grade_value)
273
+ ]
274
+ result["macro_crypto_risk_appetite_proxy"] = [
275
+ crypto_risk_appetite_proxy(c) for c in crypto_value
276
+ ]
277
+ return result
278
+
279
+
280
+ def compute_cs_momentum_rank_replica(momentum_long: pd.DataFrame) -> pd.DataFrame:
281
+ """EXACT replica of build_cross_sectional_momentum_rank_features()
282
+ (train.py:2118-2153): per-date dict over every ticker WITH A ROW that
283
+ date - values may be NaN for young series; a NaN counts in the shared
284
+ function's len() DENOMINATOR while never satisfying its </== comparisons
285
+ - and thin dates (<2 entries) return the 0.5 neutral."""
286
+ records: list[tuple[str, str, float]] = []
287
+ for date, group in momentum_long.groupby("date", sort=True):
288
+ momentum_by_symbol = dict(zip(group["ticker"], group["momentum_20d"]))
289
+ for ticker in momentum_by_symbol:
290
+ rank = cross_sectional_momentum_rank(momentum_by_symbol, ticker)
291
+ records.append((date, ticker, rank))
292
+ return pd.DataFrame(records, columns=["date", "ticker", "cs_momentum_rank_recomputed"])
293
+
294
+
295
+ def summarize_comparison(
296
+ stored_scaled: np.ndarray,
297
+ recomputed_raw: np.ndarray,
298
+ mean: float,
299
+ scale: float,
300
+ clip: float,
301
+ warmup_excluded: int = 0,
302
+ ) -> dict[str, Any]:
303
+ """Forward-direction comparison for ONE feature over ONE ticker slice:
304
+ recomputed raw -> scaler artifact -> stored column. Warmup rows (where a
305
+ replica legitimately cannot see pre-dataset history) are excluded from
306
+ the head of each ticker's series by the caller before this runs."""
307
+ valid = ~(np.isnan(stored_scaled) | np.isnan(recomputed_raw))
308
+ stored_valid = stored_scaled[valid]
309
+ expected = forward_scale(recomputed_raw[valid], mean, scale, clip)
310
+ delta = np.abs(expected - stored_valid)
311
+ saturated = int(np.sum(np.abs(stored_valid) >= clip - 1e-9))
312
+ mismatches = int(np.sum(delta > 1e-9))
313
+ return {
314
+ "n_compared": int(valid.sum()),
315
+ "warmup_excluded_per_ticker": warmup_excluded,
316
+ "max_abs_delta": float(delta.max()) if delta.size else None,
317
+ "mean_abs_delta": float(delta.mean()) if delta.size else None,
318
+ "n_mismatch_gt_1e-9": mismatches,
319
+ "n_saturated_at_clip": saturated,
320
+ "verdict": "PASS" if mismatches == 0 else "FAIL",
321
+ }