aether-quant 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aether_quant-0.0.1.dist-info/METADATA +7 -0
- aether_quant-0.0.1.dist-info/RECORD +99 -0
- aether_quant-0.0.1.dist-info/WHEEL +5 -0
- aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
- aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
- aether_quant-0.0.1.dist-info/top_level.txt +13 -0
- analyzer/README.md +69 -0
- analyzer/__init__.py +17 -0
- analyzer/market_analyzer.py +521 -0
- aq_cli.py +3633 -0
- audit/README.md +47 -0
- audit/__init__.py +38 -0
- audit/hash_chain.py +52 -0
- audit/postgres_audit.py +61 -0
- audit/postgres_worker.py +402 -0
- audit/redis_queue.py +124 -0
- audit/status_export.py +62 -0
- data_pipeline/README.md +257 -0
- data_pipeline/__init__.py +6 -0
- data_pipeline/bar_synthesis.py +97 -0
- data_pipeline/dividend_backfill.py +243 -0
- data_pipeline/factor_file_backfill.py +276 -0
- data_pipeline/fetch.py +206 -0
- data_pipeline/fred_backfill.py +435 -0
- data_pipeline/ib_backfill.py +287 -0
- data_pipeline/v2_manifest.py +75 -0
- data_pipeline/yfinance_backfill.py +501 -0
- evaluation/README.md +152 -0
- evaluation/__init__.py +73 -0
- evaluation/ablation.py +310 -0
- evaluation/benchmark_comparison.py +195 -0
- evaluation/confidence_threshold_calibration.py +157 -0
- evaluation/factor_neutralization_check.py +94 -0
- evaluation/feature_parity.py +321 -0
- evaluation/feature_reconciliation.py +286 -0
- evaluation/impact_model.py +77 -0
- evaluation/kill_switch_replay.py +186 -0
- evaluation/limit_fill_simulator.py +152 -0
- evaluation/model_predictions.py +192 -0
- evaluation/monte_carlo.py +161 -0
- evaluation/public_benchmarks.py +117 -0
- evaluation/rank_book_simulator.py +494 -0
- evaluation/rank_ic_core.py +204 -0
- evaluation/rank_signal_calibration.py +690 -0
- evaluation/rolling_ic_gate_calibration.py +190 -0
- evaluation/rolling_ic_gate_replay.py +138 -0
- execution/README.md +281 -0
- execution/__init__.py +80 -0
- execution/config_cache.py +48 -0
- execution/cost_model.py +213 -0
- execution/lean_config_render.py +131 -0
- execution/live_credentials.py +38 -0
- execution/live_credentials_io.py +52 -0
- execution/order_gate.py +388 -0
- execution/paper_readiness.py +181 -0
- execution/paper_readiness_io.py +54 -0
- execution/paper_readiness_report.py +103 -0
- execution/paper_readiness_scheduler.py +137 -0
- execution/reconciliation.py +160 -0
- execution/runtime_config_io.py +38 -0
- execution/secret_scan.py +73 -0
- features/README.md +77 -0
- features/__init__.py +137 -0
- features/alt_data_features.py +154 -0
- features/bond_features.py +275 -0
- features/cross_asset_sensitivity.py +79 -0
- features/derivatives_macro_features.py +72 -0
- features/macro_features.py +79 -0
- features/options_greeks.py +329 -0
- features/sector_map.py +53 -0
- features/technical_indicators.py +185 -0
- generate_backtest_report.py +311 -0
- generate_evaluation_report.py +712 -0
- inference/README.md +105 -0
- inference/__init__.py +36 -0
- inference/exported_model.py +929 -0
- inference/parallel_inference.py +160 -0
- inference/strategy_selector_inference.py +75 -0
- json_safety.py +130 -0
- portfolio/README.md +447 -0
- portfolio/__init__.py +41 -0
- portfolio/book_construction.py +786 -0
- portfolio/book_neutrality.py +178 -0
- portfolio/hrp_allocation.py +159 -0
- portfolio/options_arbitrage_detector.py +232 -0
- portfolio/options_assignment_risk.py +119 -0
- portfolio/options_margin_sizing.py +333 -0
- portfolio/options_strategy.py +1999 -0
- portfolio/rank_signal.py +219 -0
- portfolio/rolling_ic_gate.py +183 -0
- risk/README.md +275 -0
- risk/__init__.py +38 -0
- risk/asset_class_router.py +584 -0
- risk/forex_risk.py +158 -0
- risk/futures_risk.py +202 -0
- risk/kill_switch.py +261 -0
- risk/manual_override.py +118 -0
- risk/position_sizing.py +303 -0
- risk/rl_sizing.py +189 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Calibrates `phase6.risk.min_confidence_to_trade` (and, when book
|
|
2
|
+
selection data is available, `phase6.risk.min_confidence_to_trade_book_selected`)
|
|
3
|
+
from real data (V5.2.6, development/Problems.md) - the same "derive the
|
|
4
|
+
number, don't guess it" discipline `aq evaluate --calibrate-edge` and
|
|
5
|
+
`aq evaluate --calibrate-book-spread` already established for
|
|
6
|
+
`phase_v2.costs.edge_bps_per_rank_unit` and
|
|
7
|
+
`phase_v2.portfolio_book.min_rank_confidence_spread`.
|
|
8
|
+
|
|
9
|
+
Deliberately reuses, never re-derives: `portfolio.rank_signal.cross_sectional_rank_scores()`
|
|
10
|
+
(the exact normalization main.py applies before book selection) and
|
|
11
|
+
`portfolio.book_construction.build_rank_based_book()` (the exact
|
|
12
|
+
top_n/bottom_n selection main.py's live book uses, for the book-selected
|
|
13
|
+
split) and `analyzer.market_analyzer.compute_trade_metric()` (the exact
|
|
14
|
+
confidence-metric formula the live gate compares against
|
|
15
|
+
min_confidence_to_trade). A calibrated threshold computed here can never
|
|
16
|
+
silently disagree with what the live gate would have done on the same
|
|
17
|
+
historical day, because it is the same code, not a parallel
|
|
18
|
+
approximation.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
import pandas as pd
|
|
25
|
+
|
|
26
|
+
from analyzer.market_analyzer import compute_trade_metric
|
|
27
|
+
from portfolio.book_construction import build_rank_based_book
|
|
28
|
+
from portfolio.rank_signal import cross_sectional_rank_scores
|
|
29
|
+
|
|
30
|
+
_CONFIDENCE_DISTRIBUTION_PERCENTILES = {"p10": 0.10, "p25": 0.25, "median": 0.50, "p75": 0.75}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _empty_distribution() -> dict:
|
|
34
|
+
return {"min": None, "p10": None, "p25": None, "median": None, "p75": None, "max": None}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _percentile_of_paying_confidence(confidences: list[float], percentile: float) -> tuple[float, dict]:
|
|
38
|
+
if not confidences:
|
|
39
|
+
return 0.0, _empty_distribution()
|
|
40
|
+
array = np.array(confidences, dtype=np.float64)
|
|
41
|
+
distribution = {"min": float(np.min(array)), "max": float(np.max(array))}
|
|
42
|
+
for name, quantile in _CONFIDENCE_DISTRIBUTION_PERCENTILES.items():
|
|
43
|
+
distribution[name] = float(np.quantile(array, quantile))
|
|
44
|
+
return float(np.quantile(array, percentile)), distribution
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def calibrate_confidence_threshold(
|
|
48
|
+
dataset: pd.DataFrame,
|
|
49
|
+
*,
|
|
50
|
+
raw_score_column: str,
|
|
51
|
+
forward_return_column: str,
|
|
52
|
+
ticker_column: str = "ticker",
|
|
53
|
+
date_column: str = "date",
|
|
54
|
+
top_n: int,
|
|
55
|
+
bottom_n: int,
|
|
56
|
+
round_trip_cost_fraction: float = 0.001,
|
|
57
|
+
percentile: float = 0.10,
|
|
58
|
+
) -> dict:
|
|
59
|
+
"""Per date (sorted): normalizes that date's raw scores exactly as
|
|
60
|
+
main.py's Pass 1 does (`cross_sectional_rank_scores()`), computes each
|
|
61
|
+
symbol's book-member confidence via the SAME formula main.py uses for
|
|
62
|
+
a book-selected symbol (`min(1.0, abs(normalized_rank - 0.5) * 2.0)` -
|
|
63
|
+
see main.py's Pass 2, `confidence = min(1.0, abs(book_allocation.predicted_rank_20d - 0.5) * 2.0)`),
|
|
64
|
+
and separately determines whether that date's natural top_n/bottom_n
|
|
65
|
+
selection (`build_rank_based_book()`, no gate, no hysteresis - same
|
|
66
|
+
"measure each day's independent natural dispersion" reasoning
|
|
67
|
+
`calibrate_book_confidence_spread()` already uses) would have picked
|
|
68
|
+
each symbol, for the book-selected/non-book-selected split.
|
|
69
|
+
|
|
70
|
+
A row "would have been worth taking" when its DIRECTIONAL realized
|
|
71
|
+
edge clears round_trip_cost_fraction: `direction = 1 if
|
|
72
|
+
normalized_rank >= 0.5 else -1`, `realized_edge = direction *
|
|
73
|
+
forward_return - round_trip_cost_fraction`. The calibrated threshold
|
|
74
|
+
is the given percentile of confidence AMONG ONLY the rows that
|
|
75
|
+
actually paid - the same "percentile of a real, achieved distribution,
|
|
76
|
+
not a guess" discipline `calibrate_book_confidence_spread()` already
|
|
77
|
+
established, applied to confidence instead of spread.
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
{"calibrated_min_confidence_to_trade": float,
|
|
81
|
+
"calibrated_min_confidence_to_trade_book_selected": float,
|
|
82
|
+
"percentile": float,
|
|
83
|
+
"num_rows_used": int,
|
|
84
|
+
"num_rows_used_book_selected": int,
|
|
85
|
+
"num_dates_skipped_thin_universe": int,
|
|
86
|
+
"confidence_distribution": {...},
|
|
87
|
+
"confidence_distribution_book_selected": {...}}
|
|
88
|
+
|
|
89
|
+
An empty/all-thin dataset returns both calibrated values as 0.0 (the
|
|
90
|
+
gate's own "always trade" no-op value) and all-None distributions,
|
|
91
|
+
never raises."""
|
|
92
|
+
paying_confidences: list[float] = []
|
|
93
|
+
paying_confidences_book_selected: list[float] = []
|
|
94
|
+
num_dates_skipped_thin_universe = 0
|
|
95
|
+
|
|
96
|
+
for _date, group in dataset.groupby(date_column, sort=True):
|
|
97
|
+
raw_scores = {
|
|
98
|
+
str(ticker): float(score)
|
|
99
|
+
for ticker, score in zip(group[ticker_column], group[raw_score_column])
|
|
100
|
+
if score is not None and not (isinstance(score, float) and np.isnan(score))
|
|
101
|
+
}
|
|
102
|
+
normalized = cross_sectional_rank_scores(raw_scores)
|
|
103
|
+
if not normalized:
|
|
104
|
+
num_dates_skipped_thin_universe += 1
|
|
105
|
+
continue
|
|
106
|
+
|
|
107
|
+
candidates = {
|
|
108
|
+
symbol: {"predicted_rank_20d": rank, "trading_eligible": True} for symbol, rank in normalized.items()
|
|
109
|
+
}
|
|
110
|
+
book_allocations = build_rank_based_book(
|
|
111
|
+
candidates,
|
|
112
|
+
top_n=top_n,
|
|
113
|
+
bottom_n=bottom_n,
|
|
114
|
+
min_rank_confidence_spread=0.0,
|
|
115
|
+
spread_check_ranks=raw_scores,
|
|
116
|
+
)
|
|
117
|
+
book_selected_symbols = set(book_allocations.keys())
|
|
118
|
+
|
|
119
|
+
forward_return_by_ticker = {
|
|
120
|
+
str(ticker): float(value)
|
|
121
|
+
for ticker, value in zip(group[ticker_column], group[forward_return_column])
|
|
122
|
+
if value is not None and not (isinstance(value, float) and np.isnan(value))
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
for symbol, normalized_rank in normalized.items():
|
|
126
|
+
forward_return = forward_return_by_ticker.get(symbol)
|
|
127
|
+
if forward_return is None:
|
|
128
|
+
continue
|
|
129
|
+
confidence = compute_trade_metric(
|
|
130
|
+
confidence=min(1.0, abs(normalized_rank - 0.5) * 2.0),
|
|
131
|
+
regime_confidence=0.0,
|
|
132
|
+
topology={},
|
|
133
|
+
liquidity={},
|
|
134
|
+
use_composite_signal_score=False,
|
|
135
|
+
)
|
|
136
|
+
direction = 1.0 if normalized_rank >= 0.5 else -1.0
|
|
137
|
+
realized_edge = direction * forward_return - round_trip_cost_fraction
|
|
138
|
+
if realized_edge > 0.0:
|
|
139
|
+
paying_confidences.append(confidence)
|
|
140
|
+
if symbol in book_selected_symbols:
|
|
141
|
+
paying_confidences_book_selected.append(confidence)
|
|
142
|
+
|
|
143
|
+
calibrated_general, distribution_general = _percentile_of_paying_confidence(paying_confidences, percentile)
|
|
144
|
+
calibrated_book_selected, distribution_book_selected = _percentile_of_paying_confidence(
|
|
145
|
+
paying_confidences_book_selected, percentile
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
return {
|
|
149
|
+
"calibrated_min_confidence_to_trade": calibrated_general,
|
|
150
|
+
"calibrated_min_confidence_to_trade_book_selected": calibrated_book_selected,
|
|
151
|
+
"percentile": percentile,
|
|
152
|
+
"num_rows_used": len(paying_confidences),
|
|
153
|
+
"num_rows_used_book_selected": len(paying_confidences_book_selected),
|
|
154
|
+
"num_dates_skipped_thin_universe": num_dates_skipped_thin_universe,
|
|
155
|
+
"confidence_distribution": distribution_general,
|
|
156
|
+
"confidence_distribution_book_selected": distribution_book_selected,
|
|
157
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""V5.4.3 - factor-neutralization check: decomposes the offline rank book's
|
|
2
|
+
realized returns into systematic (market/sector/momentum) and residual
|
|
3
|
+
(pure alpha) components, answering "how much of the claimed Sharpe is
|
|
4
|
+
actually skill vs. factor exposure?"
|
|
5
|
+
|
|
6
|
+
Pure: numpy/pandas only, no I/O, no torch."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def compute_factor_exposure(
|
|
15
|
+
book_returns: pd.Series,
|
|
16
|
+
market_returns: pd.Series,
|
|
17
|
+
*,
|
|
18
|
+
momentum_factor: pd.Series | None = None,
|
|
19
|
+
sector_returns: dict[str, pd.Series] | None = None,
|
|
20
|
+
) -> dict:
|
|
21
|
+
"""Regresses book returns onto factor columns and reports loadings,
|
|
22
|
+
R², residual Sharpe, and % of variance explained by each factor.
|
|
23
|
+
|
|
24
|
+
All input series must be aligned by date (same index or positional).
|
|
25
|
+
Missing values are forward-filled then dropped.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
{"alpha_sharpe": float, "total_sharpe": float, "r_squared": float,
|
|
29
|
+
"loadings": {factor: beta}, "alpha_annualized_return_pct": float,
|
|
30
|
+
"factor_contribution_pct": {factor: pct_of_variance},
|
|
31
|
+
"n_obs": int}
|
|
32
|
+
"""
|
|
33
|
+
book = _align(book_returns)
|
|
34
|
+
market = _align(market_returns)
|
|
35
|
+
n = min(len(book), len(market))
|
|
36
|
+
book, market = book[:n], market[:n]
|
|
37
|
+
|
|
38
|
+
factors = {"market": market}
|
|
39
|
+
if momentum_factor is not None:
|
|
40
|
+
mom = _align(momentum_factor)[:n]
|
|
41
|
+
if len(mom) == n:
|
|
42
|
+
factors["momentum"] = mom
|
|
43
|
+
if sector_returns:
|
|
44
|
+
for name, series in sector_returns.items():
|
|
45
|
+
s = _align(series)
|
|
46
|
+
if len(s) >= n:
|
|
47
|
+
factors[f"sector_{name}"] = s[:n]
|
|
48
|
+
|
|
49
|
+
# Build design matrix [1, factor_1, ..., factor_k]
|
|
50
|
+
names = list(factors.keys())
|
|
51
|
+
X = np.column_stack([np.ones(n)] + [factors[name] for name in names])
|
|
52
|
+
y = book
|
|
53
|
+
|
|
54
|
+
# OLS: beta = (X'X)^-1 X'y
|
|
55
|
+
XtX = X.T @ X
|
|
56
|
+
Xty = X.T @ y
|
|
57
|
+
try:
|
|
58
|
+
betas = np.linalg.solve(XtX, Xty)
|
|
59
|
+
except np.linalg.LinAlgError:
|
|
60
|
+
return {"error": "singular design matrix", "n_obs": n}
|
|
61
|
+
|
|
62
|
+
residuals = y - X @ betas
|
|
63
|
+
ss_total = float(np.sum((y - y.mean()) ** 2))
|
|
64
|
+
ss_resid = float(np.sum(residuals ** 2))
|
|
65
|
+
r_squared = 1.0 - ss_resid / ss_total if ss_total > 0 else 0.0
|
|
66
|
+
|
|
67
|
+
# Residual alpha Sharpe (annualized with sqrt(252))
|
|
68
|
+
resid_std = np.std(residuals, ddof=1) if len(residuals) > 1 else 1e-12
|
|
69
|
+
alpha_sharpe = float(np.mean(residuals) / max(resid_std, 1e-12) * np.sqrt(252))
|
|
70
|
+
total_sharpe = float(np.mean(y) / max(np.std(y, ddof=1), 1e-12) * np.sqrt(252))
|
|
71
|
+
|
|
72
|
+
# Factor contribution: variance explained by each factor alone
|
|
73
|
+
contributions = {}
|
|
74
|
+
for i, name in enumerate(names):
|
|
75
|
+
col = X[:, i + 1]
|
|
76
|
+
ss_factor = float(np.sum((col - col.mean()) ** 2))
|
|
77
|
+
beta_i = betas[i + 1]
|
|
78
|
+
contribution = beta_i ** 2 * ss_factor
|
|
79
|
+
contributions[name] = round(contribution / ss_total * 100, 2) if ss_total > 0 else 0.0
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
"alpha_sharpe": round(alpha_sharpe, 4),
|
|
83
|
+
"total_sharpe": round(total_sharpe, 4),
|
|
84
|
+
"r_squared": round(r_squared, 4),
|
|
85
|
+
"loadings": {name: round(float(betas[i + 1]), 6) for i, name in enumerate(names)},
|
|
86
|
+
"alpha_annualized_return_pct": round(float(np.mean(residuals) * 252 * 100), 4),
|
|
87
|
+
"factor_contribution_pct": contributions,
|
|
88
|
+
"n_obs": n,
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _align(series: pd.Series) -> np.ndarray:
|
|
93
|
+
values = pd.Series(series).ffill().dropna().to_numpy(dtype=float)
|
|
94
|
+
return values
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"""V5.3.6 Workstream A - the train-vs-live feature-parity audit's pure core.
|
|
2
|
+
|
|
3
|
+
Background (development/Problems.md #91/#98/#100 lineage): XOM is the one
|
|
4
|
+
tracked ticker whose live-vs-offline book-selection mismatch never resolved,
|
|
5
|
+
and two prior root causes WERE feature-computation drifts between the
|
|
6
|
+
offline training pipeline and the live Lean path (the bond duration-beta's
|
|
7
|
+
rolling-vs-full-history drift, V5.2.5; missing split/dividend factor files,
|
|
8
|
+
#100). This module is the systematic version of those one-off checks: for
|
|
9
|
+
every model-input feature family that can be recomputed from data already
|
|
10
|
+
on disk, it re-derives the RAW value and pushes it through the shipped
|
|
11
|
+
scaler artifact (`ml/scaler_stats.json`) in the forward direction -
|
|
12
|
+
`clip((raw - mean) / scale, +-clip_sigma)` - comparing against the scaled
|
|
13
|
+
column actually stored in `ml/datasets/full_dataset.csv`.
|
|
14
|
+
|
|
15
|
+
Three layers are covered here:
|
|
16
|
+
1. **Scaler-artifact consistency** - does `scaler_stats.json` still describe
|
|
17
|
+
the dataset it is supposed to scale? (A stale artifact after a dataset
|
|
18
|
+
rebuild would mis-scale live inference systemically - an XOM-class bug.)
|
|
19
|
+
2. **Formula parity** - the per-ticker OHLCV-recomputable families (10 base
|
|
20
|
+
technicals + 6 shared indicators + 2 liquidity) re-derived row-faithfully
|
|
21
|
+
from `train.py::engineer_features()` / `add_liquidity_features()`.
|
|
22
|
+
3. **Cross-sectional + macro families** - cs_momentum_rank_20 and the three
|
|
23
|
+
macro proxies, rebuilt from a close-price pivot using the same shared
|
|
24
|
+
functions both sides import.
|
|
25
|
+
|
|
26
|
+
Families deliberately NOT recomputable offline (regime payload chain, alt-
|
|
27
|
+
data FRED inputs, cross-asset sensitivity regressions' driver series, bond
|
|
28
|
+
treasury inputs, topology/peer correlation graphs) get structured verdicts
|
|
29
|
+
from the driver instead: "SHARED_FUNCTION_CALLSITE" where both sides call
|
|
30
|
+
one `features/*.py` function (verified by inspection), so the residual risk
|
|
31
|
+
is call-site parameters only.
|
|
32
|
+
|
|
33
|
+
Pure: no I/O, no torch, no Lean imports - mirrors the repo's established
|
|
34
|
+
"pure core + thin CLI driver" convention (see feature_reconciliation.py).
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
import numpy as np
|
|
42
|
+
import pandas as pd
|
|
43
|
+
|
|
44
|
+
from features.macro_features import (
|
|
45
|
+
credit_spread_proxy,
|
|
46
|
+
crypto_risk_appetite_proxy,
|
|
47
|
+
yield_curve_slope_proxy,
|
|
48
|
+
)
|
|
49
|
+
from features.technical_indicators import (
|
|
50
|
+
average_true_range_pct,
|
|
51
|
+
bollinger_pctb,
|
|
52
|
+
cross_sectional_momentum_rank,
|
|
53
|
+
distance_from_52w_high,
|
|
54
|
+
macd_histogram_normalized,
|
|
55
|
+
relative_strength_index,
|
|
56
|
+
volume_zscore,
|
|
57
|
+
)
|
|
58
|
+
from liquidity.market_liquidity import estimate_high_low_spread
|
|
59
|
+
|
|
60
|
+
DEFAULT_LONG_LOOKBACK_WINDOW_BARS = 260
|
|
61
|
+
DEFAULT_CROSS_SECTIONAL_WINDOW_BARS = 25
|
|
62
|
+
|
|
63
|
+
BASE_TECHNICAL_FEATURES = [
|
|
64
|
+
"close_to_close_return_1d",
|
|
65
|
+
"close_to_close_return_5d",
|
|
66
|
+
"close_to_close_return_20d",
|
|
67
|
+
"rolling_volatility_5d",
|
|
68
|
+
"rolling_volatility_20d",
|
|
69
|
+
"momentum_5d",
|
|
70
|
+
"momentum_20d",
|
|
71
|
+
"high_low_range_pct",
|
|
72
|
+
"open_close_range_pct",
|
|
73
|
+
"volume_change_1d",
|
|
74
|
+
]
|
|
75
|
+
INDICATOR_FEATURES = [
|
|
76
|
+
"rsi_14",
|
|
77
|
+
"atr_pct_14",
|
|
78
|
+
"bollinger_pctb_20",
|
|
79
|
+
"volume_zscore_20",
|
|
80
|
+
"macd_histogram_norm",
|
|
81
|
+
"dist_52w_high",
|
|
82
|
+
]
|
|
83
|
+
LIQUIDITY_FEATURES = ["liquidity_log_dollar_volume", "liquidity_spread_proxy"]
|
|
84
|
+
MACRO_PROXIES_BY_KEY = {
|
|
85
|
+
"long_duration": ("macro_yield_curve_slope_proxy", yield_curve_slope_proxy),
|
|
86
|
+
"high_yield": ("macro_credit_spread_proxy", credit_spread_proxy),
|
|
87
|
+
"crypto": ("macro_crypto_risk_appetite_proxy", crypto_risk_appetite_proxy),
|
|
88
|
+
}
|
|
89
|
+
DEFAULT_MACRO_REFERENCE_TICKERS = {
|
|
90
|
+
"long_duration": "TLT",
|
|
91
|
+
"short_duration": "SHY",
|
|
92
|
+
"high_yield": "HYG",
|
|
93
|
+
"investment_grade": "LQD",
|
|
94
|
+
"crypto": "BTCUSD",
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def load_scaler_mapping(scaler_stats: dict[str, Any]) -> tuple[dict[str, tuple[float, float, float]], list[str]]:
|
|
99
|
+
"""Maps `ml/scaler_stats.json` into {feature_name: (mean, scale, clip)}
|
|
100
|
+
and reports which schema names it does NOT cover (by design: the binary
|
|
101
|
+
regime/topology/asset-class one-hots skip StandardScaler entirely)."""
|
|
102
|
+
names = list(scaler_stats["feature_names"])
|
|
103
|
+
means = scaler_stats["mean"]
|
|
104
|
+
scales = scaler_stats["scale"]
|
|
105
|
+
clip = float(scaler_stats.get("clip_sigma", 10.0))
|
|
106
|
+
mapping = {name: (float(means[i]), float(scales[i]), clip) for i, name in enumerate(names)}
|
|
107
|
+
return mapping, []
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def forward_scale(raw_values: np.ndarray, mean: float, scale: float, clip: float) -> np.ndarray:
|
|
111
|
+
"""The exact live/offline transform: clip((raw - mean) / scale, +-clip)."""
|
|
112
|
+
return np.clip((np.asarray(raw_values, dtype=float) - mean) / scale, -clip, clip)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def compute_base_technicals_replica(frame: pd.DataFrame, *, long_window_bars: int = DEFAULT_LONG_LOOKBACK_WINDOW_BARS) -> dict[str, np.ndarray]:
|
|
116
|
+
"""Row-faithful replica of train.py::engineer_features()'s inline base
|
|
117
|
+
technicals + Phase-6 indicator block (train.py:656-748), which itself is
|
|
118
|
+
line-mirrored by main.py::_build_model_input() (both files carry explicit
|
|
119
|
+
cross-reference comments asserting parity). First row is NaN exactly like
|
|
120
|
+
train's `[np.nan]` seed convention; volatility degrades to 0.0 under two
|
|
121
|
+
observations exactly like train's guard."""
|
|
122
|
+
closes = frame["close"].tolist()
|
|
123
|
+
opens = frame["open"].tolist()
|
|
124
|
+
highs = frame["high"].tolist()
|
|
125
|
+
lows = frame["low"].tolist()
|
|
126
|
+
volumes = frame["volume"].tolist()
|
|
127
|
+
n = len(frame)
|
|
128
|
+
|
|
129
|
+
columns: dict[str, list[float]] = {
|
|
130
|
+
**{name: [np.nan] for name in BASE_TECHNICAL_FEATURES},
|
|
131
|
+
**{name: [np.nan] for name in INDICATOR_FEATURES},
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
for index in range(1, n):
|
|
135
|
+
previous_close = closes[index - 1]
|
|
136
|
+
previous_volume = volumes[index - 1]
|
|
137
|
+
current_close = closes[index]
|
|
138
|
+
current_open = opens[index]
|
|
139
|
+
|
|
140
|
+
close_5 = closes[max(0, index - 5)]
|
|
141
|
+
close_20 = closes[max(0, index - 20)]
|
|
142
|
+
|
|
143
|
+
all_recent_returns = [
|
|
144
|
+
closes[position] / closes[position - 1] - 1.0
|
|
145
|
+
for position in range(1, index + 1)
|
|
146
|
+
if closes[position - 1] != 0
|
|
147
|
+
]
|
|
148
|
+
recent_returns_5 = all_recent_returns[-5:]
|
|
149
|
+
recent_returns_20 = all_recent_returns[-20:]
|
|
150
|
+
|
|
151
|
+
def _sample_std(values: list[float]) -> float:
|
|
152
|
+
if len(values) < 2:
|
|
153
|
+
return 0.0
|
|
154
|
+
mean_value = sum(values) / len(values)
|
|
155
|
+
variance = sum((value - mean_value) ** 2 for value in values) / (len(values) - 1)
|
|
156
|
+
return float(np.sqrt(max(variance, 0.0)))
|
|
157
|
+
|
|
158
|
+
columns["close_to_close_return_1d"].append(current_close / previous_close - 1.0 if previous_close else 0.0)
|
|
159
|
+
columns["close_to_close_return_5d"].append(current_close / close_5 - 1.0 if close_5 else 0.0)
|
|
160
|
+
columns["close_to_close_return_20d"].append(current_close / close_20 - 1.0 if close_20 else 0.0)
|
|
161
|
+
columns["rolling_volatility_5d"].append(_sample_std(recent_returns_5))
|
|
162
|
+
columns["rolling_volatility_20d"].append(_sample_std(recent_returns_20))
|
|
163
|
+
columns["momentum_5d"].append(current_close / close_5 - 1.0 if close_5 else 0.0)
|
|
164
|
+
columns["momentum_20d"].append(current_close / close_20 - 1.0 if close_20 else 0.0)
|
|
165
|
+
columns["high_low_range_pct"].append((highs[index] - lows[index]) / current_close if current_close else 0.0)
|
|
166
|
+
columns["open_close_range_pct"].append((current_close - current_open) / current_open if current_open else 0.0)
|
|
167
|
+
raw_volume_change = 0.0 if previous_volume == 0 else volumes[index] / previous_volume - 1.0
|
|
168
|
+
columns["volume_change_1d"].append(max(-1.0, min(20.0, raw_volume_change)))
|
|
169
|
+
|
|
170
|
+
trailing_closes = closes[: index + 1]
|
|
171
|
+
columns["rsi_14"].append(relative_strength_index(trailing_closes, period=14))
|
|
172
|
+
columns["atr_pct_14"].append(
|
|
173
|
+
average_true_range_pct(highs[: index + 1], lows[: index + 1], trailing_closes, period=14)
|
|
174
|
+
)
|
|
175
|
+
columns["bollinger_pctb_20"].append(bollinger_pctb(trailing_closes, period=20))
|
|
176
|
+
columns["volume_zscore_20"].append(volume_zscore(volumes[: index + 1], period=20))
|
|
177
|
+
|
|
178
|
+
long_window_start = max(0, index + 1 - long_window_bars)
|
|
179
|
+
long_trailing_closes = closes[long_window_start : index + 1]
|
|
180
|
+
columns["macd_histogram_norm"].append(macd_histogram_normalized(long_trailing_closes))
|
|
181
|
+
columns["dist_52w_high"].append(distance_from_52w_high(long_trailing_closes, window=long_window_bars))
|
|
182
|
+
|
|
183
|
+
return {name: np.asarray(values, dtype=float) for name, values in columns.items()}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def compute_liquidity_replica(
|
|
187
|
+
frame: pd.DataFrame,
|
|
188
|
+
*,
|
|
189
|
+
security_type: str = "equity",
|
|
190
|
+
cross_sectional_window_bars: int = DEFAULT_CROSS_SECTIONAL_WINDOW_BARS,
|
|
191
|
+
typical_spread_by_type: dict[str, float] | None = None,
|
|
192
|
+
) -> dict[str, np.ndarray]:
|
|
193
|
+
"""Row-faithful replica of train.py::add_liquidity_features()
|
|
194
|
+
(train.py:946-988): log1p dollar volume plus the Corwin-Schultz spread
|
|
195
|
+
proxy over a trailing window, static-spread fallback below two bars."""
|
|
196
|
+
from liquidity.market_liquidity import TYPICAL_SPREAD_BY_TYPE
|
|
197
|
+
|
|
198
|
+
spreads = typical_spread_by_type or TYPICAL_SPREAD_BY_TYPE
|
|
199
|
+
fallback = float(spreads.get(str(security_type), 0.001))
|
|
200
|
+
closes = frame["close"].tolist()
|
|
201
|
+
volumes = frame["volume"].tolist()
|
|
202
|
+
highs = frame["high"].tolist()
|
|
203
|
+
lows = frame["low"].tolist()
|
|
204
|
+
|
|
205
|
+
log_dollar_volume: list[float] = []
|
|
206
|
+
spread_proxy: list[float] = []
|
|
207
|
+
for index in range(len(frame)):
|
|
208
|
+
log_dollar_volume.append(float(np.log1p(max(closes[index] * volumes[index], 0.0))))
|
|
209
|
+
window_start = max(0, index + 1 - cross_sectional_window_bars)
|
|
210
|
+
window_highs = highs[window_start : index + 1]
|
|
211
|
+
window_lows = lows[window_start : index + 1]
|
|
212
|
+
estimated = estimate_high_low_spread(window_highs, window_lows) if len(window_highs) >= 2 else None
|
|
213
|
+
spread_proxy.append(float(estimated) if estimated is not None else fallback)
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
"liquidity_log_dollar_volume": np.asarray(log_dollar_volume, dtype=float),
|
|
217
|
+
"liquidity_spread_proxy": np.asarray(spread_proxy, dtype=float),
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def compute_momentum_long(dataset: pd.DataFrame) -> pd.DataFrame:
|
|
222
|
+
"""Per-ticker momentum_20d computed on EACH TICKER'S OWN trading rows -
|
|
223
|
+
position-shift(20) within that ticker's sorted history, exactly how
|
|
224
|
+
engineer_features() consumed each asset's raw frame. Deliberately NOT a
|
|
225
|
+
union-calendar pivot shift: cross-asset calendars differ (crypto trades
|
|
226
|
+
weekends, bonds don't), so a pivot-row shift spans different numbers of
|
|
227
|
+
each ticker's REAL bars - measured drift up to 0.16 absolute on TLT."""
|
|
228
|
+
def _momentum(closes: pd.Series) -> pd.Series:
|
|
229
|
+
return closes / closes.shift(20) - 1.0
|
|
230
|
+
|
|
231
|
+
out = dataset[["date", "ticker"]].copy()
|
|
232
|
+
out["momentum_20d"] = dataset.groupby("ticker", sort=False)["close"].transform(_momentum)
|
|
233
|
+
return out
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def compute_macro_proxies_replica(
|
|
237
|
+
all_dates: list[str],
|
|
238
|
+
momentum_long: pd.DataFrame,
|
|
239
|
+
*,
|
|
240
|
+
reference_tickers: dict[str, str] | None = None,
|
|
241
|
+
) -> pd.DataFrame:
|
|
242
|
+
"""Replica of train.py::build_macro_features_by_date()
|
|
243
|
+
(train.py:1126-1189): per-date AS-OF (last value at-or-before the date)
|
|
244
|
+
momentum_20d of five fixed reference tickers fed through the three
|
|
245
|
+
shared macro proxy functions; missing references degrade to each
|
|
246
|
+
function's neutral default. As-of uses each reference ticker's OWN date
|
|
247
|
+
series - never a union calendar."""
|
|
248
|
+
references = {**DEFAULT_MACRO_REFERENCE_TICKERS, **(reference_tickers or {})}
|
|
249
|
+
|
|
250
|
+
def _asof_values(ticker: str) -> list[float | None]:
|
|
251
|
+
rows = momentum_long[momentum_long["ticker"] == ticker]
|
|
252
|
+
if rows.empty:
|
|
253
|
+
return [None] * len(all_dates)
|
|
254
|
+
series = pd.Series(
|
|
255
|
+
rows["momentum_20d"].to_numpy(dtype=float),
|
|
256
|
+
index=pd.Index(rows["date"], name="date"),
|
|
257
|
+
).sort_index()
|
|
258
|
+
filled = series.reindex(all_dates).ffill()
|
|
259
|
+
return [None if pd.isna(value) else float(value) for value in filled]
|
|
260
|
+
|
|
261
|
+
long_value = _asof_values(references["long_duration"])
|
|
262
|
+
short_value = _asof_values(references["short_duration"])
|
|
263
|
+
high_yield_value = _asof_values(references["high_yield"])
|
|
264
|
+
investment_grade_value = _asof_values(references["investment_grade"])
|
|
265
|
+
crypto_value = _asof_values(references["crypto"])
|
|
266
|
+
|
|
267
|
+
result = pd.DataFrame({"date": all_dates})
|
|
268
|
+
result["macro_yield_curve_slope_proxy"] = [
|
|
269
|
+
yield_curve_slope_proxy(long_v, short_v) for long_v, short_v in zip(long_value, short_value)
|
|
270
|
+
]
|
|
271
|
+
result["macro_credit_spread_proxy"] = [
|
|
272
|
+
credit_spread_proxy(hy, ig) for hy, ig in zip(high_yield_value, investment_grade_value)
|
|
273
|
+
]
|
|
274
|
+
result["macro_crypto_risk_appetite_proxy"] = [
|
|
275
|
+
crypto_risk_appetite_proxy(c) for c in crypto_value
|
|
276
|
+
]
|
|
277
|
+
return result
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def compute_cs_momentum_rank_replica(momentum_long: pd.DataFrame) -> pd.DataFrame:
|
|
281
|
+
"""EXACT replica of build_cross_sectional_momentum_rank_features()
|
|
282
|
+
(train.py:2118-2153): per-date dict over every ticker WITH A ROW that
|
|
283
|
+
date - values may be NaN for young series; a NaN counts in the shared
|
|
284
|
+
function's len() DENOMINATOR while never satisfying its </== comparisons
|
|
285
|
+
- and thin dates (<2 entries) return the 0.5 neutral."""
|
|
286
|
+
records: list[tuple[str, str, float]] = []
|
|
287
|
+
for date, group in momentum_long.groupby("date", sort=True):
|
|
288
|
+
momentum_by_symbol = dict(zip(group["ticker"], group["momentum_20d"]))
|
|
289
|
+
for ticker in momentum_by_symbol:
|
|
290
|
+
rank = cross_sectional_momentum_rank(momentum_by_symbol, ticker)
|
|
291
|
+
records.append((date, ticker, rank))
|
|
292
|
+
return pd.DataFrame(records, columns=["date", "ticker", "cs_momentum_rank_recomputed"])
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def summarize_comparison(
|
|
296
|
+
stored_scaled: np.ndarray,
|
|
297
|
+
recomputed_raw: np.ndarray,
|
|
298
|
+
mean: float,
|
|
299
|
+
scale: float,
|
|
300
|
+
clip: float,
|
|
301
|
+
warmup_excluded: int = 0,
|
|
302
|
+
) -> dict[str, Any]:
|
|
303
|
+
"""Forward-direction comparison for ONE feature over ONE ticker slice:
|
|
304
|
+
recomputed raw -> scaler artifact -> stored column. Warmup rows (where a
|
|
305
|
+
replica legitimately cannot see pre-dataset history) are excluded from
|
|
306
|
+
the head of each ticker's series by the caller before this runs."""
|
|
307
|
+
valid = ~(np.isnan(stored_scaled) | np.isnan(recomputed_raw))
|
|
308
|
+
stored_valid = stored_scaled[valid]
|
|
309
|
+
expected = forward_scale(recomputed_raw[valid], mean, scale, clip)
|
|
310
|
+
delta = np.abs(expected - stored_valid)
|
|
311
|
+
saturated = int(np.sum(np.abs(stored_valid) >= clip - 1e-9))
|
|
312
|
+
mismatches = int(np.sum(delta > 1e-9))
|
|
313
|
+
return {
|
|
314
|
+
"n_compared": int(valid.sum()),
|
|
315
|
+
"warmup_excluded_per_ticker": warmup_excluded,
|
|
316
|
+
"max_abs_delta": float(delta.max()) if delta.size else None,
|
|
317
|
+
"mean_abs_delta": float(delta.mean()) if delta.size else None,
|
|
318
|
+
"n_mismatch_gt_1e-9": mismatches,
|
|
319
|
+
"n_saturated_at_clip": saturated,
|
|
320
|
+
"verdict": "PASS" if mismatches == 0 else "FAIL",
|
|
321
|
+
}
|