aether-quant 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aether_quant-0.0.1.dist-info/METADATA +7 -0
- aether_quant-0.0.1.dist-info/RECORD +99 -0
- aether_quant-0.0.1.dist-info/WHEEL +5 -0
- aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
- aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
- aether_quant-0.0.1.dist-info/top_level.txt +13 -0
- analyzer/README.md +69 -0
- analyzer/__init__.py +17 -0
- analyzer/market_analyzer.py +521 -0
- aq_cli.py +3633 -0
- audit/README.md +47 -0
- audit/__init__.py +38 -0
- audit/hash_chain.py +52 -0
- audit/postgres_audit.py +61 -0
- audit/postgres_worker.py +402 -0
- audit/redis_queue.py +124 -0
- audit/status_export.py +62 -0
- data_pipeline/README.md +257 -0
- data_pipeline/__init__.py +6 -0
- data_pipeline/bar_synthesis.py +97 -0
- data_pipeline/dividend_backfill.py +243 -0
- data_pipeline/factor_file_backfill.py +276 -0
- data_pipeline/fetch.py +206 -0
- data_pipeline/fred_backfill.py +435 -0
- data_pipeline/ib_backfill.py +287 -0
- data_pipeline/v2_manifest.py +75 -0
- data_pipeline/yfinance_backfill.py +501 -0
- evaluation/README.md +152 -0
- evaluation/__init__.py +73 -0
- evaluation/ablation.py +310 -0
- evaluation/benchmark_comparison.py +195 -0
- evaluation/confidence_threshold_calibration.py +157 -0
- evaluation/factor_neutralization_check.py +94 -0
- evaluation/feature_parity.py +321 -0
- evaluation/feature_reconciliation.py +286 -0
- evaluation/impact_model.py +77 -0
- evaluation/kill_switch_replay.py +186 -0
- evaluation/limit_fill_simulator.py +152 -0
- evaluation/model_predictions.py +192 -0
- evaluation/monte_carlo.py +161 -0
- evaluation/public_benchmarks.py +117 -0
- evaluation/rank_book_simulator.py +494 -0
- evaluation/rank_ic_core.py +204 -0
- evaluation/rank_signal_calibration.py +690 -0
- evaluation/rolling_ic_gate_calibration.py +190 -0
- evaluation/rolling_ic_gate_replay.py +138 -0
- execution/README.md +281 -0
- execution/__init__.py +80 -0
- execution/config_cache.py +48 -0
- execution/cost_model.py +213 -0
- execution/lean_config_render.py +131 -0
- execution/live_credentials.py +38 -0
- execution/live_credentials_io.py +52 -0
- execution/order_gate.py +388 -0
- execution/paper_readiness.py +181 -0
- execution/paper_readiness_io.py +54 -0
- execution/paper_readiness_report.py +103 -0
- execution/paper_readiness_scheduler.py +137 -0
- execution/reconciliation.py +160 -0
- execution/runtime_config_io.py +38 -0
- execution/secret_scan.py +73 -0
- features/README.md +77 -0
- features/__init__.py +137 -0
- features/alt_data_features.py +154 -0
- features/bond_features.py +275 -0
- features/cross_asset_sensitivity.py +79 -0
- features/derivatives_macro_features.py +72 -0
- features/macro_features.py +79 -0
- features/options_greeks.py +329 -0
- features/sector_map.py +53 -0
- features/technical_indicators.py +185 -0
- generate_backtest_report.py +311 -0
- generate_evaluation_report.py +712 -0
- inference/README.md +105 -0
- inference/__init__.py +36 -0
- inference/exported_model.py +929 -0
- inference/parallel_inference.py +160 -0
- inference/strategy_selector_inference.py +75 -0
- json_safety.py +130 -0
- portfolio/README.md +447 -0
- portfolio/__init__.py +41 -0
- portfolio/book_construction.py +786 -0
- portfolio/book_neutrality.py +178 -0
- portfolio/hrp_allocation.py +159 -0
- portfolio/options_arbitrage_detector.py +232 -0
- portfolio/options_assignment_risk.py +119 -0
- portfolio/options_margin_sizing.py +333 -0
- portfolio/options_strategy.py +1999 -0
- portfolio/rank_signal.py +219 -0
- portfolio/rolling_ic_gate.py +183 -0
- risk/README.md +275 -0
- risk/__init__.py +38 -0
- risk/asset_class_router.py +584 -0
- risk/forex_risk.py +158 -0
- risk/futures_risk.py +202 -0
- risk/kill_switch.py +261 -0
- risk/manual_override.py +118 -0
- risk/position_sizing.py +303 -0
- risk/rl_sizing.py +189 -0
evaluation/__init__.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Offline, cost-aware evaluation of the cross-sectional rank book (V5.1
|
|
2
|
+
Phase 0) - turns "how good is this rank prediction" into "what would this
|
|
3
|
+
have actually earned net of fees", the question nothing else in this
|
|
4
|
+
codebase answers (train.py::compute_strategy_metrics() simulates a
|
|
5
|
+
long-only 1-day-direction strategy gross of costs, not the rank book, and
|
|
6
|
+
not net-of-cost). Deliberately its own package, not folded into
|
|
7
|
+
retraining/ or performance/: pure numpy/pandas, NO torch import, so both of
|
|
8
|
+
those (and any future Postgres-fed monitoring job) can import it without
|
|
9
|
+
pulling in the training stack - the same torch-free-core convention
|
|
10
|
+
train.py::_rank_ic_from_arrays() already established for its own metric.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .ablation import (
|
|
14
|
+
ABLATION_VARIANTS,
|
|
15
|
+
NOT_OFFLINE_MEASURABLE_VARIANTS,
|
|
16
|
+
compare_static_vs_retrained,
|
|
17
|
+
run_ablation,
|
|
18
|
+
simulate_static_baseline,
|
|
19
|
+
)
|
|
20
|
+
from .model_predictions import (
|
|
21
|
+
build_sequence_windows,
|
|
22
|
+
predict_head,
|
|
23
|
+
predict_multitask_head,
|
|
24
|
+
predict_sequence_head,
|
|
25
|
+
select_context_date_range,
|
|
26
|
+
)
|
|
27
|
+
from .rank_book_simulator import (
|
|
28
|
+
RankBookSimulationResult,
|
|
29
|
+
capacity_curve,
|
|
30
|
+
simulate_rank_book,
|
|
31
|
+
stress_test_costs,
|
|
32
|
+
summarize_metric_stability,
|
|
33
|
+
)
|
|
34
|
+
from .confidence_threshold_calibration import calibrate_confidence_threshold
|
|
35
|
+
from .rank_signal_calibration import (
|
|
36
|
+
calibrate_book_confidence_spread,
|
|
37
|
+
compute_blended_raw_scores,
|
|
38
|
+
reconcile_book_history_date,
|
|
39
|
+
replay_book_history_reconciliation,
|
|
40
|
+
segment_logged_records_by_run,
|
|
41
|
+
summarize_book_history_reconciliation,
|
|
42
|
+
summarize_book_member_diversion,
|
|
43
|
+
summarize_universe_presence_by_symbol,
|
|
44
|
+
summarize_universe_snapshot_by_security_type,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"ABLATION_VARIANTS",
|
|
49
|
+
"NOT_OFFLINE_MEASURABLE_VARIANTS",
|
|
50
|
+
"RankBookSimulationResult",
|
|
51
|
+
"build_sequence_windows",
|
|
52
|
+
"calibrate_book_confidence_spread",
|
|
53
|
+
"calibrate_confidence_threshold",
|
|
54
|
+
"capacity_curve",
|
|
55
|
+
"compare_static_vs_retrained",
|
|
56
|
+
"compute_blended_raw_scores",
|
|
57
|
+
"predict_head",
|
|
58
|
+
"predict_multitask_head",
|
|
59
|
+
"predict_sequence_head",
|
|
60
|
+
"reconcile_book_history_date",
|
|
61
|
+
"replay_book_history_reconciliation",
|
|
62
|
+
"run_ablation",
|
|
63
|
+
"segment_logged_records_by_run",
|
|
64
|
+
"select_context_date_range",
|
|
65
|
+
"simulate_rank_book",
|
|
66
|
+
"simulate_static_baseline",
|
|
67
|
+
"stress_test_costs",
|
|
68
|
+
"summarize_book_history_reconciliation",
|
|
69
|
+
"summarize_book_member_diversion",
|
|
70
|
+
"summarize_metric_stability",
|
|
71
|
+
"summarize_universe_presence_by_symbol",
|
|
72
|
+
"summarize_universe_snapshot_by_security_type",
|
|
73
|
+
]
|
evaluation/ablation.py
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""V5.1 Phase 5 (item 9): declarative ablation harness over the offline
|
|
2
|
+
rank-book simulator - "how much does each piece of the adaptive machinery
|
|
3
|
+
actually contribute", answered honestly. Only pieces genuinely represented
|
|
4
|
+
in the offline dataset/simulator get a real number; everything runtime-only
|
|
5
|
+
(the MoE gating blend, learned-topology sizing, rank-magnitude position
|
|
6
|
+
sizing, the portfolio-book selection process itself, the net-edge cost
|
|
7
|
+
gate) gets an explicit `{"status": "not_offline_measurable", "reason": ...}`
|
|
8
|
+
sentinel, never a fabricated comparison. A fabricated ablation number is
|
|
9
|
+
worse than an absent one - this repo's own convention of recording honest
|
|
10
|
+
negatives (development/Changelog.md) supports being blunt here.
|
|
11
|
+
|
|
12
|
+
Deliberately its own module, not folded into rank_book_simulator.py: this
|
|
13
|
+
module composes multiple simulate_rank_book() runs into one declarative
|
|
14
|
+
table; rank_book_simulator.py stays the single-run primitive both this
|
|
15
|
+
module and `aq evaluate --rank-book` build on.
|
|
16
|
+
|
|
17
|
+
Torch-free (see evaluation/__init__.py's docstring) - only imports
|
|
18
|
+
rank_book_simulator.py, itself pure numpy/pandas plus the pure
|
|
19
|
+
portfolio/book_construction.py and portfolio/book_neutrality.py.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import math
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
import pandas as pd
|
|
28
|
+
|
|
29
|
+
from .rank_book_simulator import simulate_rank_book
|
|
30
|
+
|
|
31
|
+
# Variants whose live-path mechanism has NO representation anywhere in the
|
|
32
|
+
# offline dataset or in simulate_rank_book() - "ablating" them here would
|
|
33
|
+
# either compare the baseline against itself (a mechanism that was never
|
|
34
|
+
# modeled offline in the first place) or would require runtime state this
|
|
35
|
+
# module structurally cannot see. Each reason is specific enough to explain
|
|
36
|
+
# to a reader why the number is genuinely absent, not just "not implemented
|
|
37
|
+
# yet".
|
|
38
|
+
NOT_OFFLINE_MEASURABLE_VARIANTS: dict[str, str] = {
|
|
39
|
+
"no_gating": (
|
|
40
|
+
"The MoE gating blend (moe/gating.py) drives probability_up/magnitude/"
|
|
41
|
+
"volatility - none of those enter the rank book at all. Absent from "
|
|
42
|
+
"the offline dataset; there is nothing to ablate."
|
|
43
|
+
),
|
|
44
|
+
"no_topology_sizing": (
|
|
45
|
+
"Topology-derived sizing (risk/position_sizing.py::topology_sizing_multiplier()) "
|
|
46
|
+
"depends on per-bar learned-topology runtime state (confidence/disagreement) "
|
|
47
|
+
"that is computed live in main.py and never written to the offline dataset."
|
|
48
|
+
),
|
|
49
|
+
"no_rank_sizing": (
|
|
50
|
+
"Rank-magnitude position sizing (risk/position_sizing.py::rank_sizing_multiplier()) "
|
|
51
|
+
"is a LIVE-ONLY behavior driven by main.py's runtime confidence signal. "
|
|
52
|
+
"simulate_rank_book() has never modeled it - every leg is already flat, "
|
|
53
|
+
"equal-weighted offline (see _simulate_rank_book_core()'s own per_name = "
|
|
54
|
+
"gross_exposure/2/len(leg) formula) - 'ablating' it here would compare the "
|
|
55
|
+
"baseline against itself, not a real finding."
|
|
56
|
+
),
|
|
57
|
+
"no_portfolio_book": (
|
|
58
|
+
"The 'portfolio book' IS build_rank_based_book()'s cross-sectional top/"
|
|
59
|
+
"bottom-N selection - the thing simulate_rank_book() exists to simulate. "
|
|
60
|
+
"There is no simpler, non-book trading logic implemented anywhere offline "
|
|
61
|
+
"to fall back to (main.py's non-book code path is a different, per-symbol-"
|
|
62
|
+
"independent decision path with no offline dataset representation)."
|
|
63
|
+
),
|
|
64
|
+
"no_net_edge_gate": (
|
|
65
|
+
"The net-edge cost gate (execution/cost_model.py::build_net_edge_decision(), "
|
|
66
|
+
"wired into analyzer/market_analyzer.py's Priority 6.5) is evaluated per-bar "
|
|
67
|
+
"against live liquidity/order-value context main.py computes at runtime. "
|
|
68
|
+
"simulate_rank_book() has no representation of liquidity or this gate at all."
|
|
69
|
+
),
|
|
70
|
+
# "no_retraining" is handled separately by compare_static_vs_retrained()
|
|
71
|
+
# below - it IS offline-measurable, but only from Phase 4's walk-forward
|
|
72
|
+
# artifacts, not from a single simulate_rank_book() call the way every
|
|
73
|
+
# other variant in ABLATION_VARIANTS is. See run_ablation()'s docstring.
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
# Variants that ARE offline-measurable: kwarg overrides applied on top of
|
|
77
|
+
# the caller's base_kwargs before calling simulate_rank_book(). Each one
|
|
78
|
+
# documents exactly which live mechanism it approximates, and the one
|
|
79
|
+
# respect in which it is only an approximation - never silently pretend
|
|
80
|
+
# the offline number is a perfect stand-in for the live behavior.
|
|
81
|
+
ABLATION_VARIANTS: dict[str, dict] = {
|
|
82
|
+
"no_neutrality": {
|
|
83
|
+
"dollar_neutral": False,
|
|
84
|
+
"sector_neutral": False,
|
|
85
|
+
},
|
|
86
|
+
# Bypasses apply_book_neutrality()'s two constraint steps. The per-name
|
|
87
|
+
# cap and gross-exposure cap (both still config-driven via base_kwargs)
|
|
88
|
+
# are NOT part of "neutrality" in this codebase's own naming (see
|
|
89
|
+
# portfolio/book_neutrality.py's own 4-step docstring) and stay active.
|
|
90
|
+
"no_hysteresis": {
|
|
91
|
+
"hysteresis_rank_margin": 0.0,
|
|
92
|
+
},
|
|
93
|
+
# build_rank_based_book()'s own documented no-op default - every
|
|
94
|
+
# rebalance re-selects top/bottom-N purely by that bar's current rank,
|
|
95
|
+
# no incumbent retention across rebalances.
|
|
96
|
+
"no_cost_model": {
|
|
97
|
+
"cost_bps_per_side": 0.0,
|
|
98
|
+
"commission_bps": 0.0,
|
|
99
|
+
},
|
|
100
|
+
# Zeroes the SIMULATOR's own cost terms - the offline analogue of
|
|
101
|
+
# execution/cost_model.py's round-trip cost estimate, simplified here to
|
|
102
|
+
# a fixed bps rate rather than main.py's per-bar liquidity-derived one
|
|
103
|
+
# (that per-bar estimate is itself absent from the offline dataset, the
|
|
104
|
+
# same reason no_net_edge_gate above is not offline-measurable).
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def simulate_static_baseline(
|
|
109
|
+
frame: pd.DataFrame,
|
|
110
|
+
*,
|
|
111
|
+
forward_return_column: str = "target_return_1d",
|
|
112
|
+
ticker_column: str = "ticker",
|
|
113
|
+
date_column: str = "date",
|
|
114
|
+
min_universe_size: int = 20,
|
|
115
|
+
trading_days_per_year: int = 252,
|
|
116
|
+
) -> dict:
|
|
117
|
+
"""The "did the adaptive machinery add anything over just holding
|
|
118
|
+
everything" reference point every other variant's delta is measured
|
|
119
|
+
against. Equal-weight, long-only, buy-and-hold: the universe is fixed
|
|
120
|
+
ONCE, on the first date with >= min_universe_size eligible names, and
|
|
121
|
+
never rebalanced again (by construction, zero turnover and zero cost
|
|
122
|
+
after the single initial buy - deliberately NOT re-equal-weighting the
|
|
123
|
+
surviving membership each date, which would itself be a form of active
|
|
124
|
+
rebalancing and defeat the point of a static reference).
|
|
125
|
+
|
|
126
|
+
A ticker that later has a missing/NaN forward return on some date is
|
|
127
|
+
simply excluded from that date's equal-weighted average (not
|
|
128
|
+
renormalized against the full original count) - the same "skip, don't
|
|
129
|
+
zero-fill" convention _simulate_rank_book_core() already follows for
|
|
130
|
+
thin dates.
|
|
131
|
+
|
|
132
|
+
Returns the SAME shape as RankBookSimulationResult.to_dict() (a plain
|
|
133
|
+
dict, not the frozen dataclass itself, since this function has no
|
|
134
|
+
turnover/rebalance concept to compute num_rebalances/annualized_turnover
|
|
135
|
+
from - both are reported as 0)."""
|
|
136
|
+
working = frame.dropna(subset=[forward_return_column]).copy()
|
|
137
|
+
unique_dates = sorted(working[date_column].unique())
|
|
138
|
+
|
|
139
|
+
initial_tickers: set | None = None
|
|
140
|
+
net_returns: list[float] = []
|
|
141
|
+
per_date_used: list = []
|
|
142
|
+
names_held_series: list[int] = []
|
|
143
|
+
|
|
144
|
+
for date in unique_dates:
|
|
145
|
+
date_frame = working[working[date_column] == date]
|
|
146
|
+
if initial_tickers is None:
|
|
147
|
+
if len(date_frame) < min_universe_size:
|
|
148
|
+
continue
|
|
149
|
+
initial_tickers = set(date_frame[ticker_column])
|
|
150
|
+
|
|
151
|
+
held_frame = date_frame[date_frame[ticker_column].isin(initial_tickers)]
|
|
152
|
+
if held_frame.empty:
|
|
153
|
+
continue
|
|
154
|
+
bar_return = float(held_frame[forward_return_column].mean())
|
|
155
|
+
net_returns.append(bar_return)
|
|
156
|
+
per_date_used.append(str(date))
|
|
157
|
+
names_held_series.append(len(held_frame))
|
|
158
|
+
|
|
159
|
+
if not per_date_used:
|
|
160
|
+
return {
|
|
161
|
+
"gross_sharpe": 0.0, "net_sharpe": 0.0, "gross_total_return": 0.0,
|
|
162
|
+
"net_total_return": 0.0, "net_max_drawdown": 0.0, "annualized_turnover": 0.0,
|
|
163
|
+
"cost_drag_annual_bps": 0.0, "num_rebalances": 0, "num_dates_used": 0,
|
|
164
|
+
"mean_names_long": 0.0, "mean_names_short": 0.0,
|
|
165
|
+
"per_date_net_return": [], "per_date": [],
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
net_array = np.asarray(net_returns, dtype=float)
|
|
169
|
+
cumprod = np.cumprod(1.0 + net_array)
|
|
170
|
+
running_peak = np.maximum.accumulate(np.concatenate(([1.0], cumprod)))[1:]
|
|
171
|
+
max_drawdown = float(np.min(cumprod / running_peak - 1.0))
|
|
172
|
+
sharpe = 0.0
|
|
173
|
+
if len(net_array) >= 2:
|
|
174
|
+
std = float(net_array.std(ddof=1))
|
|
175
|
+
if std > 0.0:
|
|
176
|
+
sharpe = float(net_array.mean() / std) * math.sqrt(trading_days_per_year)
|
|
177
|
+
|
|
178
|
+
return {
|
|
179
|
+
"gross_sharpe": sharpe,
|
|
180
|
+
"net_sharpe": sharpe,
|
|
181
|
+
"gross_total_return": float(cumprod[-1] - 1.0),
|
|
182
|
+
"net_total_return": float(cumprod[-1] - 1.0),
|
|
183
|
+
"net_max_drawdown": max_drawdown,
|
|
184
|
+
"annualized_turnover": 0.0,
|
|
185
|
+
"cost_drag_annual_bps": 0.0,
|
|
186
|
+
"num_rebalances": 1 if initial_tickers else 0,
|
|
187
|
+
"num_dates_used": len(per_date_used),
|
|
188
|
+
"mean_names_long": float(np.mean(names_held_series)) if names_held_series else 0.0,
|
|
189
|
+
"mean_names_short": 0.0,
|
|
190
|
+
"per_date_net_return": [float(v) for v in net_returns],
|
|
191
|
+
"per_date": per_date_used,
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def run_ablation(frame: pd.DataFrame, base_kwargs: dict, variants: list[str]) -> dict:
|
|
196
|
+
"""Runs simulate_rank_book() once per requested variant (base_kwargs
|
|
197
|
+
overridden per ABLATION_VARIANTS' declarative dict) on the SAME
|
|
198
|
+
predictions and dates, plus simulate_static_baseline() if
|
|
199
|
+
"static_baseline" is requested. Any requested variant in
|
|
200
|
+
NOT_OFFLINE_MEASURABLE_VARIANTS or equal to "no_retraining" returns the
|
|
201
|
+
honesty-contract sentinel instead of a number - "no_retraining" is
|
|
202
|
+
offline-measurable, but only via compare_static_vs_retrained() below,
|
|
203
|
+
which needs Phase 4 walk-forward artifacts this function does not
|
|
204
|
+
receive (a single frame/base_kwargs pair cannot answer a
|
|
205
|
+
trained-once-vs-retrained-per-window question).
|
|
206
|
+
|
|
207
|
+
Returns {variant_name: {...RankBookSimulationResult shape...,
|
|
208
|
+
"delta_vs_static_baseline": float} | {"status": "not_offline_measurable"
|
|
209
|
+
| "unknown_variant", "reason": str}}."""
|
|
210
|
+
results: dict[str, dict] = {}
|
|
211
|
+
|
|
212
|
+
static_result = simulate_static_baseline(
|
|
213
|
+
frame,
|
|
214
|
+
forward_return_column=base_kwargs.get("forward_return_column", "target_return_1d"),
|
|
215
|
+
ticker_column=base_kwargs.get("ticker_column", "ticker"),
|
|
216
|
+
date_column=base_kwargs.get("date_column", "date"),
|
|
217
|
+
min_universe_size=base_kwargs.get("min_universe_size", 20),
|
|
218
|
+
trading_days_per_year=base_kwargs.get("trading_days_per_year", 252),
|
|
219
|
+
)
|
|
220
|
+
static_sharpe = static_result["net_sharpe"]
|
|
221
|
+
if "static_baseline" in variants:
|
|
222
|
+
results["static_baseline"] = {**static_result, "delta_vs_static_baseline": 0.0}
|
|
223
|
+
|
|
224
|
+
for name in variants:
|
|
225
|
+
if name == "static_baseline":
|
|
226
|
+
continue
|
|
227
|
+
if name == "no_retraining":
|
|
228
|
+
results[name] = {
|
|
229
|
+
"status": "not_offline_measurable",
|
|
230
|
+
"reason": (
|
|
231
|
+
"Offline-measurable, but only via compare_static_vs_retrained() "
|
|
232
|
+
"against a Phase 4 walk-forward run's artifacts - run_ablation() "
|
|
233
|
+
"only has a single frame/base_kwargs pair, not per-window models."
|
|
234
|
+
),
|
|
235
|
+
}
|
|
236
|
+
continue
|
|
237
|
+
if name in NOT_OFFLINE_MEASURABLE_VARIANTS:
|
|
238
|
+
results[name] = {"status": "not_offline_measurable", "reason": NOT_OFFLINE_MEASURABLE_VARIANTS[name]}
|
|
239
|
+
continue
|
|
240
|
+
if name not in ABLATION_VARIANTS:
|
|
241
|
+
results[name] = {"status": "unknown_variant", "reason": f"{name!r} is not a recognized ablation variant"}
|
|
242
|
+
continue
|
|
243
|
+
|
|
244
|
+
variant_kwargs = {**base_kwargs, **ABLATION_VARIANTS[name]}
|
|
245
|
+
result = simulate_rank_book(frame, **variant_kwargs)
|
|
246
|
+
results[name] = {**result.to_dict(), "delta_vs_static_baseline": result.net_sharpe - static_sharpe}
|
|
247
|
+
|
|
248
|
+
return results
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def compare_static_vs_retrained(
|
|
252
|
+
frozen_net_performance_by_window: dict[int, dict],
|
|
253
|
+
retrained_net_performance_by_window: dict[int, dict],
|
|
254
|
+
*,
|
|
255
|
+
min_windows: int = 2,
|
|
256
|
+
) -> dict:
|
|
257
|
+
"""The "no_retraining" ablation - pure comparison of two ALREADY-COMPUTED
|
|
258
|
+
result sets, matching train.py::assess_net_performance_quality()'s own
|
|
259
|
+
"assess what's given, never run a simulation itself" convention.
|
|
260
|
+
|
|
261
|
+
frozen_net_performance_by_window: {window_index: <net_performance_by_window
|
|
262
|
+
entry shape, "simulation" key with net_sharpe>} - window 0's model's OWN
|
|
263
|
+
predictions run (via evaluation.predict_head() + run_net_performance_simulation(),
|
|
264
|
+
by the caller) against every LATER window's own backtest frame, i.e.
|
|
265
|
+
"what if we had never retrained past window 0".
|
|
266
|
+
retrained_net_performance_by_window: the corresponding entries already
|
|
267
|
+
in a Phase 4 walk_forward_summary.json's net_performance_by_window
|
|
268
|
+
(window N's OWN freshly-trained model on window N's own data) - "what
|
|
269
|
+
the walk-forward run actually did".
|
|
270
|
+
|
|
271
|
+
Both dicts must share window_index keys for windows 1+ (window 0 is
|
|
272
|
+
excluded from the comparison by construction - it IS the frozen model,
|
|
273
|
+
comparing it against itself is not a retraining ablation). Returns
|
|
274
|
+
{"status": "insufficient_windows", "reason": ...} when fewer than
|
|
275
|
+
min_windows comparable windows are present in both dicts - never a
|
|
276
|
+
fabricated delta from 0 or 1 windows."""
|
|
277
|
+
comparable_windows = sorted(
|
|
278
|
+
(set(frozen_net_performance_by_window) & set(retrained_net_performance_by_window)) - {0}
|
|
279
|
+
)
|
|
280
|
+
if len(comparable_windows) < min_windows:
|
|
281
|
+
return {
|
|
282
|
+
"status": "insufficient_windows",
|
|
283
|
+
"reason": (
|
|
284
|
+
f"need >= {min_windows} windows (excluding window 0, the frozen model itself) "
|
|
285
|
+
f"present in both frozen and retrained result sets, found {len(comparable_windows)}"
|
|
286
|
+
),
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
per_window: list[dict] = []
|
|
290
|
+
deltas: list[float] = []
|
|
291
|
+
for window_index in comparable_windows:
|
|
292
|
+
frozen_sharpe = float(frozen_net_performance_by_window[window_index]["simulation"]["net_sharpe"])
|
|
293
|
+
retrained_sharpe = float(retrained_net_performance_by_window[window_index]["simulation"]["net_sharpe"])
|
|
294
|
+
delta = retrained_sharpe - frozen_sharpe
|
|
295
|
+
deltas.append(delta)
|
|
296
|
+
per_window.append(
|
|
297
|
+
{
|
|
298
|
+
"window_index": window_index,
|
|
299
|
+
"frozen_net_sharpe": frozen_sharpe,
|
|
300
|
+
"retrained_net_sharpe": retrained_sharpe,
|
|
301
|
+
"delta": delta,
|
|
302
|
+
}
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
return {
|
|
306
|
+
"status": "walk_forward_derived",
|
|
307
|
+
"num_windows": len(comparable_windows),
|
|
308
|
+
"per_window": per_window,
|
|
309
|
+
"mean_delta_retrained_minus_frozen": float(np.mean(deltas)),
|
|
310
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""V5.4.3 - benchmark comparison: computes naive baseline strategies
|
|
2
|
+
(momentum, mean-reversion, random-entry) on the same dataset/window as the
|
|
3
|
+
rank-book simulation, so the model's edge can be stated relative to simple
|
|
4
|
+
benchmarks rather than in isolation.
|
|
5
|
+
|
|
6
|
+
V5.4.4 - `run_all_baselines()` aggregates these with
|
|
7
|
+
`public_benchmarks.py`'s SP500/60-40 baselines into the single report dict
|
|
8
|
+
`aq evaluate --benchmarks` persists (and the README section + webui panel
|
|
9
|
+
render). Pure: pandas/numpy only."""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import pandas as pd
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def momentum_baseline(
|
|
18
|
+
close_pivot: pd.DataFrame,
|
|
19
|
+
*,
|
|
20
|
+
lookback: int = 20,
|
|
21
|
+
top_n: int = 6,
|
|
22
|
+
bottom_n: int = 6,
|
|
23
|
+
forward_return_column: str = "target_return_1d",
|
|
24
|
+
rebalance_every: int = 10,
|
|
25
|
+
) -> dict:
|
|
26
|
+
"""Buy the `top_n` assets with the HIGHEST past `lookback`-day return;
|
|
27
|
+
sell short the `bottom_n` with the LOWEST. Rebalances every
|
|
28
|
+
`rebalance_every` unique dates. Equal-weight within each leg."""
|
|
29
|
+
mom = close_pivot / close_pivot.shift(lookback) - 1.0
|
|
30
|
+
dates = close_pivot.index.tolist()
|
|
31
|
+
rebalance_indices = list(range(0, len(dates), rebalance_every))
|
|
32
|
+
|
|
33
|
+
daily_returns = []
|
|
34
|
+
held_weights = {}
|
|
35
|
+
for di, date in enumerate(dates):
|
|
36
|
+
if di in rebalance_indices and di >= lookback:
|
|
37
|
+
row = mom.loc[date].dropna()
|
|
38
|
+
if len(row) < top_n + bottom_n:
|
|
39
|
+
held_weights = {}
|
|
40
|
+
else:
|
|
41
|
+
sorted_vals = row.sort_values(ascending=False)
|
|
42
|
+
longs = sorted_vals.head(top_n).index.tolist()
|
|
43
|
+
shorts = sorted_vals.tail(bottom_n).index.tolist()
|
|
44
|
+
w = 1.0 / max(top_n, 1)
|
|
45
|
+
held_weights = {t: w for t in longs}
|
|
46
|
+
for t in shorts:
|
|
47
|
+
held_weights[t] = -w
|
|
48
|
+
|
|
49
|
+
# Compute portfolio return from next day's close-to-close
|
|
50
|
+
if di + 1 < len(dates):
|
|
51
|
+
port_ret = 0.0
|
|
52
|
+
for ticker, weight in held_weights.items():
|
|
53
|
+
if ticker in close_pivot.columns and date in close_pivot.index:
|
|
54
|
+
cur_idx = close_pivot.index.get_loc(date)
|
|
55
|
+
nxt_idx = cur_idx + 1
|
|
56
|
+
if nxt_idx < len(close_pivot.index):
|
|
57
|
+
nxt = close_pivot.index[nxt_idx]
|
|
58
|
+
c0 = close_pivot.loc[date, ticker] if date in close_pivot.index and not pd.isna(close_pivot.loc[date, ticker]) else np.nan
|
|
59
|
+
c1 = close_pivot.loc[nxt, ticker] if nxt in close_pivot.index and not pd.isna(close_pivot.loc[nxt, ticker]) else np.nan
|
|
60
|
+
if not (pd.isna(c0) or pd.isna(c1) or c0 == 0):
|
|
61
|
+
port_ret += weight * (c1 / c0 - 1.0)
|
|
62
|
+
daily_returns.append(port_ret)
|
|
63
|
+
else:
|
|
64
|
+
daily_returns.append(0.0)
|
|
65
|
+
|
|
66
|
+
arr = np.array(daily_returns)
|
|
67
|
+
sharpe = float(arr.mean() / max(np.std(arr, ddof=1), 1e-12) * np.sqrt(252)) if len(arr) > 1 else 0.0
|
|
68
|
+
total_return = float(np.prod(1 + arr) - 1)
|
|
69
|
+
return {"strategy": "momentum", "net_sharpe": round(sharpe, 4), "total_return_pct": round(total_return * 100, 2)}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def mean_reversion_baseline(
|
|
73
|
+
close_pivot: pd.DataFrame,
|
|
74
|
+
*,
|
|
75
|
+
lookback: int = 5,
|
|
76
|
+
top_n: int = 6,
|
|
77
|
+
bottom_n: int = 6,
|
|
78
|
+
rebalance_every: int = 10,
|
|
79
|
+
) -> dict:
|
|
80
|
+
"""Buy bottom-N by recent `lookback`-day return (mean-reversion);
|
|
81
|
+
sell short the top-N. Rebalances every `rebalance_every` unique dates.
|
|
82
|
+
Equal-weight within each leg. Same next-day close-to-close return
|
|
83
|
+
accrual as momentum_baseline above - V5.4.4 fix: this baseline
|
|
84
|
+
previously accrued a hardcoded placeholder (`weight_sum * 0.001`)
|
|
85
|
+
instead of realized returns, so its Sharpe was structurally 0.0 and
|
|
86
|
+
the "benchmark" row it contributed to the comparison table was
|
|
87
|
+
meaningless."""
|
|
88
|
+
ret = close_pivot.pct_change(periods=lookback, fill_method=None)
|
|
89
|
+
dates = close_pivot.index.tolist()
|
|
90
|
+
rebalance_indices = set(range(lookback, len(dates), rebalance_every))
|
|
91
|
+
|
|
92
|
+
daily_returns = []
|
|
93
|
+
held_weights = {}
|
|
94
|
+
for di, date in enumerate(dates):
|
|
95
|
+
if di in rebalance_indices:
|
|
96
|
+
row = ret.loc[date].dropna() if date in ret.index else pd.Series(dtype=float)
|
|
97
|
+
if len(row) >= top_n + bottom_n:
|
|
98
|
+
sorted_vals = row.sort_values(ascending=True)
|
|
99
|
+
longs = sorted_vals.head(top_n).index.tolist()
|
|
100
|
+
shorts = sorted_vals.tail(bottom_n).index.tolist()
|
|
101
|
+
w = 1.0 / max(top_n, 1)
|
|
102
|
+
held_weights = {t: w for t in longs}
|
|
103
|
+
for t in shorts:
|
|
104
|
+
held_weights[t] = -w
|
|
105
|
+
# Accrue NEXT day's close-to-close return on the held book (same
|
|
106
|
+
# convention as momentum_baseline: decide on today's close, earn
|
|
107
|
+
# today-close -> next-close).
|
|
108
|
+
if di + 1 < len(dates):
|
|
109
|
+
next_date = dates[di + 1]
|
|
110
|
+
port_ret = 0.0
|
|
111
|
+
for ticker, weight in held_weights.items():
|
|
112
|
+
if ticker in close_pivot.columns:
|
|
113
|
+
c0 = close_pivot.loc[date, ticker] if date in close_pivot.index else np.nan
|
|
114
|
+
c1 = close_pivot.loc[next_date, ticker] if next_date in close_pivot.index else np.nan
|
|
115
|
+
if not (pd.isna(c0) or pd.isna(c1) or c0 == 0):
|
|
116
|
+
port_ret += weight * (c1 / c0 - 1.0)
|
|
117
|
+
daily_returns.append(port_ret)
|
|
118
|
+
else:
|
|
119
|
+
daily_returns.append(0.0)
|
|
120
|
+
|
|
121
|
+
arr = np.array(daily_returns)
|
|
122
|
+
sharpe = float(arr.mean() / max(np.std(arr, ddof=1), 1e-12) * np.sqrt(252)) if len(arr) > 1 else 0.0
|
|
123
|
+
total_return = float(np.prod(1 + arr) - 1)
|
|
124
|
+
return {"strategy": "mean_reversion", "net_sharpe": round(sharpe, 4), "total_return_pct": round(total_return * 100, 2)}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def random_entry_baseline(
|
|
128
|
+
tickers: list[str],
|
|
129
|
+
*,
|
|
130
|
+
n_days: int = 500,
|
|
131
|
+
top_n: int = 6,
|
|
132
|
+
bottom_n: int = 6,
|
|
133
|
+
seed: int = 42,
|
|
134
|
+
daily_vol: float = 0.01,
|
|
135
|
+
) -> dict:
|
|
136
|
+
"""Random entry/exit on the same universe with the same book shape.
|
|
137
|
+
Expected Sharpe ≈ 0; serves as a statistical floor."""
|
|
138
|
+
rng = np.random.default_rng(seed)
|
|
139
|
+
n_names = max(top_n + bottom_n, 1)
|
|
140
|
+
daily_vol_per_name = daily_vol / np.sqrt(n_names)
|
|
141
|
+
daily_port_vol = daily_vol_per_name * np.sqrt(n_names)
|
|
142
|
+
returns = rng.normal(0, daily_port_vol, n_days)
|
|
143
|
+
arr = np.asarray(returns)
|
|
144
|
+
sharpe = float(arr.mean() / max(np.std(arr, ddof=1), 1e-12) * np.sqrt(252)) if len(arr) > 1 else 0.0
|
|
145
|
+
total_return = float(np.prod(1 + arr) - 1)
|
|
146
|
+
return {"strategy": "random_entry", "net_sharpe": round(sharpe, 4), "total_return_pct": round(total_return * 100, 2)}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def close_pivot_from_frame(
|
|
150
|
+
frame: pd.DataFrame,
|
|
151
|
+
*,
|
|
152
|
+
date_column: str = "date",
|
|
153
|
+
ticker_column: str = "ticker",
|
|
154
|
+
close_column: str = "close",
|
|
155
|
+
) -> pd.DataFrame:
|
|
156
|
+
"""(dates × tickers) close-price pivot from the long-format
|
|
157
|
+
full_dataset frame - the single input every baseline (strategy and
|
|
158
|
+
public) consumes. Empty pivot when the frame has no `close` column
|
|
159
|
+
(callers then get per-baseline SKIPPED results, never a crash).
|
|
160
|
+
`aggfunc="last"` keeps a (rare) duplicated (date, ticker) row from
|
|
161
|
+
raising - same degrade-don't-raise convention as the rest of this
|
|
162
|
+
package; dates sorted ascending so every baseline's window logic sees
|
|
163
|
+
chronological order regardless of the CSV's row order."""
|
|
164
|
+
if close_column not in frame.columns:
|
|
165
|
+
return pd.DataFrame()
|
|
166
|
+
pivot = frame.pivot_table(index=date_column, columns=ticker_column, values=close_column, aggfunc="last")
|
|
167
|
+
return pivot.sort_index()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def run_all_baselines(
|
|
171
|
+
close_pivot: pd.DataFrame,
|
|
172
|
+
*,
|
|
173
|
+
n_days: int = 500,
|
|
174
|
+
seed: int = 42,
|
|
175
|
+
) -> dict:
|
|
176
|
+
"""Runs ALL five baselines over one close pivot - the three strategy
|
|
177
|
+
baselines above plus public_benchmarks' SP500 buy-and-hold and
|
|
178
|
+
daily-rebalanced 60-40 - into the report dict `aq evaluate
|
|
179
|
+
--benchmarks` persists. Every entry carries the same
|
|
180
|
+
strategy/net_sharpe/total_return_pct keys (public ones add `status`),
|
|
181
|
+
so the README table and webui panel can render them uniformly; a
|
|
182
|
+
baseline that can't compute (missing SPY/TLT, thin universe) degrades
|
|
183
|
+
to SKIPPED/None metrics inside its own entry without affecting the
|
|
184
|
+
others."""
|
|
185
|
+
from evaluation.public_benchmarks import compute_60_40_baseline, compute_sp500_baseline
|
|
186
|
+
|
|
187
|
+
tickers = list(close_pivot.columns)
|
|
188
|
+
baselines = [
|
|
189
|
+
momentum_baseline(close_pivot),
|
|
190
|
+
mean_reversion_baseline(close_pivot),
|
|
191
|
+
random_entry_baseline(tickers, n_days=n_days, seed=seed),
|
|
192
|
+
compute_sp500_baseline(close_pivot),
|
|
193
|
+
compute_60_40_baseline(close_pivot),
|
|
194
|
+
]
|
|
195
|
+
return {"baselines": baselines}
|