aether-quant 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aether_quant-0.0.1.dist-info/METADATA +7 -0
- aether_quant-0.0.1.dist-info/RECORD +99 -0
- aether_quant-0.0.1.dist-info/WHEEL +5 -0
- aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
- aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
- aether_quant-0.0.1.dist-info/top_level.txt +13 -0
- analyzer/README.md +69 -0
- analyzer/__init__.py +17 -0
- analyzer/market_analyzer.py +521 -0
- aq_cli.py +3633 -0
- audit/README.md +47 -0
- audit/__init__.py +38 -0
- audit/hash_chain.py +52 -0
- audit/postgres_audit.py +61 -0
- audit/postgres_worker.py +402 -0
- audit/redis_queue.py +124 -0
- audit/status_export.py +62 -0
- data_pipeline/README.md +257 -0
- data_pipeline/__init__.py +6 -0
- data_pipeline/bar_synthesis.py +97 -0
- data_pipeline/dividend_backfill.py +243 -0
- data_pipeline/factor_file_backfill.py +276 -0
- data_pipeline/fetch.py +206 -0
- data_pipeline/fred_backfill.py +435 -0
- data_pipeline/ib_backfill.py +287 -0
- data_pipeline/v2_manifest.py +75 -0
- data_pipeline/yfinance_backfill.py +501 -0
- evaluation/README.md +152 -0
- evaluation/__init__.py +73 -0
- evaluation/ablation.py +310 -0
- evaluation/benchmark_comparison.py +195 -0
- evaluation/confidence_threshold_calibration.py +157 -0
- evaluation/factor_neutralization_check.py +94 -0
- evaluation/feature_parity.py +321 -0
- evaluation/feature_reconciliation.py +286 -0
- evaluation/impact_model.py +77 -0
- evaluation/kill_switch_replay.py +186 -0
- evaluation/limit_fill_simulator.py +152 -0
- evaluation/model_predictions.py +192 -0
- evaluation/monte_carlo.py +161 -0
- evaluation/public_benchmarks.py +117 -0
- evaluation/rank_book_simulator.py +494 -0
- evaluation/rank_ic_core.py +204 -0
- evaluation/rank_signal_calibration.py +690 -0
- evaluation/rolling_ic_gate_calibration.py +190 -0
- evaluation/rolling_ic_gate_replay.py +138 -0
- execution/README.md +281 -0
- execution/__init__.py +80 -0
- execution/config_cache.py +48 -0
- execution/cost_model.py +213 -0
- execution/lean_config_render.py +131 -0
- execution/live_credentials.py +38 -0
- execution/live_credentials_io.py +52 -0
- execution/order_gate.py +388 -0
- execution/paper_readiness.py +181 -0
- execution/paper_readiness_io.py +54 -0
- execution/paper_readiness_report.py +103 -0
- execution/paper_readiness_scheduler.py +137 -0
- execution/reconciliation.py +160 -0
- execution/runtime_config_io.py +38 -0
- execution/secret_scan.py +73 -0
- features/README.md +77 -0
- features/__init__.py +137 -0
- features/alt_data_features.py +154 -0
- features/bond_features.py +275 -0
- features/cross_asset_sensitivity.py +79 -0
- features/derivatives_macro_features.py +72 -0
- features/macro_features.py +79 -0
- features/options_greeks.py +329 -0
- features/sector_map.py +53 -0
- features/technical_indicators.py +185 -0
- generate_backtest_report.py +311 -0
- generate_evaluation_report.py +712 -0
- inference/README.md +105 -0
- inference/__init__.py +36 -0
- inference/exported_model.py +929 -0
- inference/parallel_inference.py +160 -0
- inference/strategy_selector_inference.py +75 -0
- json_safety.py +130 -0
- portfolio/README.md +447 -0
- portfolio/__init__.py +41 -0
- portfolio/book_construction.py +786 -0
- portfolio/book_neutrality.py +178 -0
- portfolio/hrp_allocation.py +159 -0
- portfolio/options_arbitrage_detector.py +232 -0
- portfolio/options_assignment_risk.py +119 -0
- portfolio/options_margin_sizing.py +333 -0
- portfolio/options_strategy.py +1999 -0
- portfolio/rank_signal.py +219 -0
- portfolio/rolling_ic_gate.py +183 -0
- risk/README.md +275 -0
- risk/__init__.py +38 -0
- risk/asset_class_router.py +584 -0
- risk/forex_risk.py +158 -0
- risk/futures_risk.py +202 -0
- risk/kill_switch.py +261 -0
- risk/manual_override.py +118 -0
- risk/position_sizing.py +303 -0
- risk/rl_sizing.py +189 -0
|
@@ -0,0 +1,712 @@
|
|
|
1
|
+
"""Regenerates README.md's Offline Evaluation, Walk-Forward Training/Testing,
|
|
2
|
+
and Other Metrics subsections (siblings of Lean Backtest, which
|
|
3
|
+
generate_backtest_report.py already owns) from whatever `aq evaluate`/`aq
|
|
4
|
+
train --walk-forward` last wrote to ml/evaluation/ and ml/versions/.
|
|
5
|
+
|
|
6
|
+
Same practical-substitute-for-live-update approach as generate_backtest_report.py:
|
|
7
|
+
`aq evaluate` calls this module after any of --rank-book/--capacity/--stress/
|
|
8
|
+
--all/--reconcile-book-history/--replay-kill-switch/--walk-forward-summary,
|
|
9
|
+
and it rewrites the relevant marker-delimited sections in-place. Every
|
|
10
|
+
section degrades gracefully (an em-dash placeholder table row, not a
|
|
11
|
+
crash/traceback) when its source file doesn't exist yet - a fresh checkout
|
|
12
|
+
that has never run `aq evaluate` still gets a readable README, matching
|
|
13
|
+
monitoring/evaluation_state.py's own "not_evaluated" contract.
|
|
14
|
+
|
|
15
|
+
Data sources (all optional, independently missing-tolerant):
|
|
16
|
+
- ml/evaluation/rank_book_simulation_{sequence,multitask}.json (this
|
|
17
|
+
module's own additive per-model copies - see aq_cli.py::cmd_evaluate())
|
|
18
|
+
- ml/evaluation/capacity_report_{sequence,multitask}.json
|
|
19
|
+
- ml/evaluation/cost_stress_report_{sequence,multitask}.json
|
|
20
|
+
- ml/versions/walk-forward-*/walk_forward_summary.json (newest by mtime)
|
|
21
|
+
- ml/evaluation/book_history_reconciliation.json
|
|
22
|
+
- ml/evaluation/kill_switch_replay.json
|
|
23
|
+
- The latest real Lean backtest result JSON (reuses
|
|
24
|
+
generate_backtest_report.py's own loader - same "latest complete run"
|
|
25
|
+
definition, so the Lean-vs-offline comparison in Other Metrics never
|
|
26
|
+
compares against a stale/different backtest than what Lean Backtest
|
|
27
|
+
itself displays).
|
|
28
|
+
|
|
29
|
+
Usage:
|
|
30
|
+
python generate_evaluation_report.py [--readme README.md]
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import argparse
|
|
36
|
+
import json
|
|
37
|
+
import sys
|
|
38
|
+
from datetime import datetime, timezone
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
|
|
41
|
+
from generate_backtest_report import (
|
|
42
|
+
README_PATH,
|
|
43
|
+
_atomic_write,
|
|
44
|
+
_replace_between_markers,
|
|
45
|
+
find_latest_backtest_result_json,
|
|
46
|
+
load_backtest_summary,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
ROOT_DIR = Path(__file__).resolve().parent
|
|
50
|
+
ML_DIR = ROOT_DIR / "ml"
|
|
51
|
+
EVALUATION_DIR = ML_DIR / "evaluation"
|
|
52
|
+
VERSIONS_DIR = ML_DIR / "versions"
|
|
53
|
+
|
|
54
|
+
EVAL_MARKER_START = "<!-- AQ:EVAL_START -->"
|
|
55
|
+
EVAL_MARKER_END = "<!-- AQ:EVAL_END -->"
|
|
56
|
+
EVAL_FULL_STATS_MARKER_START = "<!-- AQ:EVAL_FULL_STATS_START -->"
|
|
57
|
+
EVAL_FULL_STATS_MARKER_END = "<!-- AQ:EVAL_FULL_STATS_END -->"
|
|
58
|
+
|
|
59
|
+
WALKFORWARD_MARKER_START = "<!-- AQ:WALKFORWARD_START -->"
|
|
60
|
+
WALKFORWARD_MARKER_END = "<!-- AQ:WALKFORWARD_END -->"
|
|
61
|
+
WALKFORWARD_FULL_STATS_MARKER_START = "<!-- AQ:WALKFORWARD_FULL_STATS_START -->"
|
|
62
|
+
WALKFORWARD_FULL_STATS_MARKER_END = "<!-- AQ:WALKFORWARD_FULL_STATS_END -->"
|
|
63
|
+
|
|
64
|
+
OTHER_METRICS_MARKER_START = "<!-- AQ:OTHER_METRICS_START -->"
|
|
65
|
+
OTHER_METRICS_MARKER_END = "<!-- AQ:OTHER_METRICS_END -->"
|
|
66
|
+
|
|
67
|
+
MONTECARLO_MARKER_START = "<!-- AQ:MONTECARLO_START -->"
|
|
68
|
+
MONTECARLO_MARKER_END = "<!-- AQ:MONTECARLO_END -->"
|
|
69
|
+
MONTECARLO_FULL_STATS_MARKER_START = "<!-- AQ:MONTECARLO_FULL_STATS_START -->"
|
|
70
|
+
MONTECARLO_FULL_STATS_MARKER_END = "<!-- AQ:MONTECARLO_FULL_STATS_END -->"
|
|
71
|
+
MONTECARLO_CHART_PATH = ROOT_DIR / "development" / "monte_carlo_equity_curves.png"
|
|
72
|
+
|
|
73
|
+
BENCHMARK_MARKER_START = "<!-- AQ:BENCHMARK_START -->"
|
|
74
|
+
BENCHMARK_MARKER_END = "<!-- AQ:BENCHMARK_END -->"
|
|
75
|
+
|
|
76
|
+
_MODEL_KINDS = ("sequence", "multitask")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _load_json(path: Path) -> dict | list | None:
|
|
80
|
+
if not path.exists():
|
|
81
|
+
return None
|
|
82
|
+
try:
|
|
83
|
+
with path.open("r", encoding="utf-8") as f:
|
|
84
|
+
return json.load(f)
|
|
85
|
+
except (json.JSONDecodeError, OSError):
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _fmt(value: object, spec: str = "") -> str:
|
|
90
|
+
"""Formats a number with `spec`, or returns an em-dash for
|
|
91
|
+
None/missing - every table cell in this module goes through this so a
|
|
92
|
+
partial `aq evaluate` run degrades to a clean placeholder, never a
|
|
93
|
+
KeyError/TypeError."""
|
|
94
|
+
if value is None:
|
|
95
|
+
return "—"
|
|
96
|
+
try:
|
|
97
|
+
return format(value, spec) if spec else str(value)
|
|
98
|
+
except (ValueError, TypeError):
|
|
99
|
+
return str(value)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _updated_at() -> str:
|
|
103
|
+
return datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# --------------------------------------------------------------------------
|
|
107
|
+
# Offline Evaluation
|
|
108
|
+
# --------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def load_offline_evaluation_summary() -> dict[str, dict | None]:
|
|
112
|
+
"""Per-model rank_book/capacity/stress reports, keyed by model_kind -
|
|
113
|
+
each value is None (not the whole function) when that model has never
|
|
114
|
+
been evaluated, so a report with only a sequence run still renders a
|
|
115
|
+
complete multitask="not yet run" row instead of omitting the section."""
|
|
116
|
+
summary: dict[str, dict | None] = {}
|
|
117
|
+
for model_kind in _MODEL_KINDS:
|
|
118
|
+
rank_book = _load_json(EVALUATION_DIR / f"rank_book_simulation_{model_kind}.json")
|
|
119
|
+
capacity = _load_json(EVALUATION_DIR / f"capacity_report_{model_kind}.json")
|
|
120
|
+
stress = _load_json(EVALUATION_DIR / f"cost_stress_report_{model_kind}.json")
|
|
121
|
+
if rank_book is None and capacity is None and stress is None:
|
|
122
|
+
summary[model_kind] = None
|
|
123
|
+
continue
|
|
124
|
+
summary[model_kind] = {"rank_book": rank_book, "capacity": capacity, "stress": stress}
|
|
125
|
+
return summary
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _build_eval_compact_markdown(summary: dict[str, dict | None]) -> str:
|
|
129
|
+
lines = [
|
|
130
|
+
"| Metric | Multitask | Sequence |",
|
|
131
|
+
"|---|---|---|",
|
|
132
|
+
]
|
|
133
|
+
row_specs: list[tuple[str, str, str]] = [
|
|
134
|
+
("Gross Sharpe", "rank_book.gross_sharpe", ".3f"),
|
|
135
|
+
("Net Sharpe", "rank_book.net_sharpe", ".3f"),
|
|
136
|
+
("Net total return", "rank_book.net_total_return", ".2%"),
|
|
137
|
+
("Max drawdown", "rank_book.net_max_drawdown", ".2%"),
|
|
138
|
+
("Annualized turnover", "rank_book.annualized_turnover", ".2f"),
|
|
139
|
+
("Cost drag (bps/yr)", "rank_book.cost_drag_annual_bps", ".1f"),
|
|
140
|
+
("Capacity (USD)", "capacity.capacity_usd", ",.0f"),
|
|
141
|
+
]
|
|
142
|
+
for label, dotted_path, spec in row_specs:
|
|
143
|
+
section, field = dotted_path.split(".")
|
|
144
|
+
cells = []
|
|
145
|
+
for model_kind in ("multitask", "sequence"):
|
|
146
|
+
model_summary = summary.get(model_kind)
|
|
147
|
+
value = (model_summary.get(section) or {}).get(field) if model_summary else None
|
|
148
|
+
cells.append(_fmt(value, spec))
|
|
149
|
+
lines.append(f"| {label} | {cells[0]} | {cells[1]} |")
|
|
150
|
+
body = "\n".join(lines)
|
|
151
|
+
body += f"\n\n_Backtest split, full history. Last updated {_updated_at()} (auto-generated by `aq evaluate --all`)._"
|
|
152
|
+
return body
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _build_eval_full_stats_markdown(summary: dict[str, dict | None]) -> str:
|
|
156
|
+
sections: list[str] = []
|
|
157
|
+
for model_kind in ("multitask", "sequence"):
|
|
158
|
+
model_summary = summary.get(model_kind)
|
|
159
|
+
sections.append(f"**{model_kind.capitalize()} model**")
|
|
160
|
+
if model_summary is None:
|
|
161
|
+
sections.append(f"_Not yet evaluated - run `aq evaluate --all --model {model_kind}`._")
|
|
162
|
+
continue
|
|
163
|
+
|
|
164
|
+
rank_book = model_summary.get("rank_book") or {}
|
|
165
|
+
lines = ["| Metric | Value |", "|---|---|"]
|
|
166
|
+
for key in (
|
|
167
|
+
"gross_sharpe", "net_sharpe", "gross_total_return", "net_total_return", "net_max_drawdown",
|
|
168
|
+
"annualized_turnover", "cost_drag_annual_bps", "num_rebalances", "num_dates_used",
|
|
169
|
+
"mean_names_long", "mean_names_short",
|
|
170
|
+
):
|
|
171
|
+
if key in rank_book:
|
|
172
|
+
lines.append(f"| {key} | {rank_book[key]} |")
|
|
173
|
+
sections.append("\n".join(lines))
|
|
174
|
+
|
|
175
|
+
capacity = model_summary.get("capacity")
|
|
176
|
+
if capacity:
|
|
177
|
+
cap_lines = [
|
|
178
|
+
"",
|
|
179
|
+
f"Capacity: ${capacity.get('capacity_usd', 0):,.0f} (binding: {capacity.get('binding_ticker', '—')})",
|
|
180
|
+
"",
|
|
181
|
+
"| top_n | Net Sharpe |",
|
|
182
|
+
"|---|---|",
|
|
183
|
+
]
|
|
184
|
+
for row in capacity.get("per_top_n", []):
|
|
185
|
+
cap_lines.append(f"| {row.get('top_n')} | {row.get('net_sharpe', 0):.4f} |")
|
|
186
|
+
sections.append("\n".join(cap_lines))
|
|
187
|
+
|
|
188
|
+
stress = model_summary.get("stress")
|
|
189
|
+
if stress and stress.get("stress"):
|
|
190
|
+
stress_lines = ["", "| Cost multiplier | Gross Sharpe | Net Sharpe | Cost drag (bps/yr) |", "|---|---|---|---|"]
|
|
191
|
+
for row in stress["stress"]:
|
|
192
|
+
stress_lines.append(
|
|
193
|
+
f"| {row.get('cost_multiplier')}x | {row.get('gross_sharpe', 0):.4f} | "
|
|
194
|
+
f"{row.get('net_sharpe', 0):.4f} | {row.get('cost_drag_annual_bps', 0):.1f} |"
|
|
195
|
+
)
|
|
196
|
+
sections.append("\n".join(stress_lines))
|
|
197
|
+
|
|
198
|
+
return "\n\n".join(sections)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# --------------------------------------------------------------------------
|
|
202
|
+
# Walk-Forward Training/Testing
|
|
203
|
+
# --------------------------------------------------------------------------
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def load_latest_walk_forward_summary() -> dict | None:
|
|
207
|
+
"""Newest ml/versions/walk-forward-*/walk_forward_summary.json by
|
|
208
|
+
mtime - same "latest run" definition as
|
|
209
|
+
monitoring/evaluation_state.py::_latest_walk_forward_summary()."""
|
|
210
|
+
if not VERSIONS_DIR.exists():
|
|
211
|
+
return None
|
|
212
|
+
candidates = sorted(
|
|
213
|
+
VERSIONS_DIR.glob("walk-forward-*/walk_forward_summary.json"),
|
|
214
|
+
key=lambda path: path.stat().st_mtime,
|
|
215
|
+
reverse=True,
|
|
216
|
+
)
|
|
217
|
+
return _load_json(candidates[0]) if candidates else None
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _build_walk_forward_compact_markdown(summary: dict | None) -> str:
|
|
221
|
+
if summary is None:
|
|
222
|
+
return "_No walk-forward run found yet - run `aq train --walk-forward` first._"
|
|
223
|
+
|
|
224
|
+
run_id = summary.get("run_id", "unknown")
|
|
225
|
+
num_windows = summary.get("num_windows", 0)
|
|
226
|
+
stability = summary.get("stability_by_metric") or {}
|
|
227
|
+
net_perf = summary.get("net_performance_by_window") or []
|
|
228
|
+
|
|
229
|
+
lines = ["| Metric | Mean | 95% CI | Stable? |", "|---|---|---|---|"]
|
|
230
|
+
for metric_name in ("backtest_mcc", "rank_5d_ic", "rank_20d_ic", "residual_rank_20d_ic"):
|
|
231
|
+
stats = stability.get(metric_name)
|
|
232
|
+
if not stats:
|
|
233
|
+
continue
|
|
234
|
+
bootstrap = stats.get("bootstrap", {})
|
|
235
|
+
stable_label = "yes" if stats.get("stable") else "**NO**"
|
|
236
|
+
lines.append(
|
|
237
|
+
f"| {metric_name} | {_fmt(stats.get('mean'), '.4f')} | "
|
|
238
|
+
f"[{_fmt(bootstrap.get('lower_bound'), '.4f')}, {_fmt(bootstrap.get('upper_bound'), '.4f')}] | "
|
|
239
|
+
f"{stable_label} |"
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
if net_perf:
|
|
243
|
+
net_sharpes = [w.get("simulation", {}).get("net_sharpe") for w in net_perf if w.get("simulation")]
|
|
244
|
+
net_sharpes = [s for s in net_sharpes if s is not None]
|
|
245
|
+
if net_sharpes:
|
|
246
|
+
mean_sharpe = sum(net_sharpes) / len(net_sharpes)
|
|
247
|
+
num_positive = sum(1 for s in net_sharpes if s > 0)
|
|
248
|
+
lines.append(
|
|
249
|
+
f"| net_sharpe (per-window) | {mean_sharpe:.3f} | — | "
|
|
250
|
+
f"{num_positive}/{len(net_sharpes)} windows positive |"
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
body = "\n".join(lines)
|
|
254
|
+
body += (
|
|
255
|
+
f"\n\n{num_windows} expanding/rolling windows, run `{run_id}`. "
|
|
256
|
+
f"Last updated {_updated_at()} (auto-generated by `aq train --walk-forward`)."
|
|
257
|
+
)
|
|
258
|
+
return body
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _build_walk_forward_full_stats_markdown(summary: dict | None) -> str:
|
|
262
|
+
if summary is None:
|
|
263
|
+
return "_No walk-forward run found yet._"
|
|
264
|
+
|
|
265
|
+
window_results = summary.get("window_results") or []
|
|
266
|
+
net_perf_by_window = {w.get("window_index"): w for w in (summary.get("net_performance_by_window") or [])}
|
|
267
|
+
|
|
268
|
+
lines = [
|
|
269
|
+
"| Window | Backtest period | Model | Backtest MCC | Gross Sharpe | Net Sharpe | Net return | Max DD | Turnover |",
|
|
270
|
+
"|---|---|---|---|---|---|---|---|---|",
|
|
271
|
+
]
|
|
272
|
+
for index, window in enumerate(window_results):
|
|
273
|
+
backtest_window = window.get("window", {}).get("backtest", {})
|
|
274
|
+
period = f"{backtest_window.get('start', '?')} → {backtest_window.get('end', '?')}"
|
|
275
|
+
net_perf = net_perf_by_window.get(index, {})
|
|
276
|
+
sim = net_perf.get("simulation", {})
|
|
277
|
+
lines.append(
|
|
278
|
+
f"| {index} | {period} | {net_perf.get('model_kind', '—')} | "
|
|
279
|
+
f"{_fmt(window.get('backtest_mcc'), '.4f')} | {_fmt(sim.get('gross_sharpe'), '.3f')} | "
|
|
280
|
+
f"{_fmt(sim.get('net_sharpe'), '.3f')} | {_fmt(sim.get('net_total_return'), '.2%')} | "
|
|
281
|
+
f"{_fmt(sim.get('net_max_drawdown'), '.2%')} | {_fmt(sim.get('annualized_turnover'), '.2f')}x |"
|
|
282
|
+
)
|
|
283
|
+
return "\n".join(lines)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
# --------------------------------------------------------------------------
|
|
287
|
+
# Other Metrics (Lean vs. offline comparison, reconciliation, kill-switch)
|
|
288
|
+
# --------------------------------------------------------------------------
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _load_lean_sharpe() -> tuple[float | None, str | None]:
|
|
292
|
+
"""Returns (sharpe, backtest_window_label) from the same latest
|
|
293
|
+
complete Lean run generate_backtest_report.py's own Lean Backtest
|
|
294
|
+
section displays - never a different/staler run, so the comparison
|
|
295
|
+
below is always apples-to-apples with what's shown just above it."""
|
|
296
|
+
result_json_path = find_latest_backtest_result_json()
|
|
297
|
+
if result_json_path is None:
|
|
298
|
+
return None, None
|
|
299
|
+
lean_summary = load_backtest_summary(result_json_path)
|
|
300
|
+
statistics = lean_summary["statistics"]
|
|
301
|
+
sharpe_str = statistics.get("Sharpe Ratio")
|
|
302
|
+
try:
|
|
303
|
+
sharpe = float(sharpe_str) if sharpe_str is not None else None
|
|
304
|
+
except ValueError:
|
|
305
|
+
sharpe = None
|
|
306
|
+
start_date, end_date = lean_summary["start_date"], lean_summary["end_date"]
|
|
307
|
+
window = f"{start_date.date().isoformat()} to {end_date.date().isoformat()}" if start_date and end_date else None
|
|
308
|
+
return sharpe, window
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _count_real_kill_switch_trips() -> int | None:
|
|
312
|
+
"""V5.3.1 (development/Problems.md #91/#97): always returns None now -
|
|
313
|
+
this used to count `kill_switch_tripped` occurrences in the real
|
|
314
|
+
backtest's Lean text log, but that was never a real measurement.
|
|
315
|
+
main.py's actual trip-audit event (`self._audit_queue.push(...
|
|
316
|
+
"kill_switch_tripped"...)`) is pushed ONLY to a Redis Stream
|
|
317
|
+
(audit/redis_queue.py::AuditQueue), never to the text log, and
|
|
318
|
+
AuditQueue.push() fails silently with no Redis reachable - which a
|
|
319
|
+
standalone `lean backtest .` run never has. So the old count was
|
|
320
|
+
ALWAYS 0 regardless of how many real trips occurred, not a genuine
|
|
321
|
+
"0 trips" finding (V5.2.10's Changelog entry stated this as if it
|
|
322
|
+
were real - that was wrong, corrected here). No cheap, reliable
|
|
323
|
+
offline proxy exists either: `trade_lock_active` transitions are only
|
|
324
|
+
durably logged per-bar into book_history.jsonl's
|
|
325
|
+
`book_member_decisions`, itself gated to rebalance bars and book-
|
|
326
|
+
selected symbols only - too sparse to reconstruct true rising-edge
|
|
327
|
+
trip counts. Kept as a function (not deleted) so
|
|
328
|
+
_build_other_metrics_markdown() has one clear place to render an
|
|
329
|
+
honest "not measurable from a standalone backtest" caveat instead."""
|
|
330
|
+
return None
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _build_other_metrics_markdown(
|
|
334
|
+
eval_summary: dict[str, dict | None],
|
|
335
|
+
walk_forward_summary: dict | None,
|
|
336
|
+
reconciliation: dict | None,
|
|
337
|
+
kill_switch_replay: dict | None,
|
|
338
|
+
) -> str:
|
|
339
|
+
lean_sharpe, lean_window = _load_lean_sharpe()
|
|
340
|
+
sequence_net_sharpe = (eval_summary.get("sequence") or {}).get("rank_book", {}).get("net_sharpe")
|
|
341
|
+
multitask_net_sharpe = (eval_summary.get("multitask") or {}).get("rank_book", {}).get("net_sharpe")
|
|
342
|
+
walk_forward_net_perf = (walk_forward_summary or {}).get("net_performance_by_window") or []
|
|
343
|
+
walk_forward_net_sharpes = [
|
|
344
|
+
w.get("simulation", {}).get("net_sharpe") for w in walk_forward_net_perf if w.get("simulation")
|
|
345
|
+
]
|
|
346
|
+
walk_forward_net_sharpes = [s for s in walk_forward_net_sharpes if s is not None]
|
|
347
|
+
walk_forward_mean_sharpe = (
|
|
348
|
+
sum(walk_forward_net_sharpes) / len(walk_forward_net_sharpes) if walk_forward_net_sharpes else None
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
lines = [
|
|
352
|
+
"**Sharpe: real Lean backtest vs. offline estimates** — offline numbers are consistently more "
|
|
353
|
+
"optimistic than the real backtest; treat the gap itself as the headline number, not either side alone.",
|
|
354
|
+
"",
|
|
355
|
+
"| Source | Sharpe |",
|
|
356
|
+
"|---|---|",
|
|
357
|
+
f"| Real Lean backtest{f' ({lean_window})' if lean_window else ''} | {_fmt(lean_sharpe, '.3f')} |",
|
|
358
|
+
f"| Offline evaluation — sequence model (full backtest split) | {_fmt(sequence_net_sharpe, '.3f')} |",
|
|
359
|
+
f"| Offline evaluation — multitask model (full backtest split) | {_fmt(multitask_net_sharpe, '.3f')} |",
|
|
360
|
+
f"| Walk-forward mean (out-of-sample, per-window) | {_fmt(walk_forward_mean_sharpe, '.3f')} |",
|
|
361
|
+
]
|
|
362
|
+
|
|
363
|
+
if lean_sharpe is not None and sequence_net_sharpe is not None:
|
|
364
|
+
lines.append(f"| Gap: sequence offline − real Lean | {sequence_net_sharpe - lean_sharpe:+.3f} |")
|
|
365
|
+
if lean_sharpe is not None and walk_forward_mean_sharpe is not None:
|
|
366
|
+
lines.append(f"| Gap: walk-forward − real Lean | {walk_forward_mean_sharpe - lean_sharpe:+.3f} |")
|
|
367
|
+
|
|
368
|
+
lines.append("")
|
|
369
|
+
lines.append("**Book-history reconciliation** (real Lean selections vs. a fresh offline re-derivation of the same dates)")
|
|
370
|
+
lines.append("")
|
|
371
|
+
if reconciliation and reconciliation.get("summary"):
|
|
372
|
+
recon_summary = reconciliation["summary"]
|
|
373
|
+
lines.extend(
|
|
374
|
+
[
|
|
375
|
+
"| Metric | Value |",
|
|
376
|
+
"|---|---|",
|
|
377
|
+
f"| Dates reconciled | {recon_summary.get('num_dates', '—')} |",
|
|
378
|
+
f"| Exact matches | {recon_summary.get('num_dates_exact_match', '—')} |",
|
|
379
|
+
f"| Mean overlap fraction | {_fmt(recon_summary.get('mean_overlap_fraction'), '.2%')} |",
|
|
380
|
+
f"| Mean raw-score delta | {_fmt(recon_summary.get('mean_raw_score_delta_abs'), '.4f')} |",
|
|
381
|
+
f"| Replay mode | {reconciliation.get('mode', '—')} |",
|
|
382
|
+
]
|
|
383
|
+
)
|
|
384
|
+
diversion = reconciliation.get("diversion_summary") or {}
|
|
385
|
+
action_counts = diversion.get("action_counts") or {}
|
|
386
|
+
if action_counts:
|
|
387
|
+
total_decisions = sum(action_counts.values())
|
|
388
|
+
lines.append("")
|
|
389
|
+
lines.append(f"Book-member decision outcomes ({total_decisions} total, real Lean run):")
|
|
390
|
+
lines.append("")
|
|
391
|
+
lines.append("| Action | Count | Share |")
|
|
392
|
+
lines.append("|---|---|---|")
|
|
393
|
+
for action, count in sorted(action_counts.items(), key=lambda item: -item[1]):
|
|
394
|
+
lines.append(f"| {action} | {count} | {count / total_decisions:.1%} |")
|
|
395
|
+
else:
|
|
396
|
+
lines.append("_Not yet reconciled - run `aq evaluate --reconcile-book-history --replay-hysteresis` first._")
|
|
397
|
+
|
|
398
|
+
lines.append("")
|
|
399
|
+
lines.append("**Kill-switch: real trips vs. offline replay estimate**")
|
|
400
|
+
lines.append("")
|
|
401
|
+
# V5.3.1 (development/Problems.md #91/#97) - _count_real_kill_switch_trips()
|
|
402
|
+
# always returns None now (see its own docstring for why the old
|
|
403
|
+
# count was never a real measurement) - render an explicit caveat
|
|
404
|
+
# instead of a bare "—" that could be misread as "zero trips".
|
|
405
|
+
if kill_switch_replay and kill_switch_replay.get("summary"):
|
|
406
|
+
ks_summary = kill_switch_replay["summary"]
|
|
407
|
+
lines.append("| Source | Trips | Locked days |")
|
|
408
|
+
lines.append("|---|---|---|")
|
|
409
|
+
lines.append(
|
|
410
|
+
f"| Real Lean backtest{f' ({lean_window})' if lean_window else ''} | "
|
|
411
|
+
"_not measurable from a standalone backtest (see Disclaimer)_ | — |"
|
|
412
|
+
)
|
|
413
|
+
lines.append(
|
|
414
|
+
f"| Offline replay (approximation, see Disclaimer) | {ks_summary.get('trip_count', '—')} | "
|
|
415
|
+
f"{_fmt(ks_summary.get('locked_day_fraction'), '.1%')} |"
|
|
416
|
+
)
|
|
417
|
+
else:
|
|
418
|
+
lines.append("_Not yet available - run `aq evaluate --replay-kill-switch`._")
|
|
419
|
+
|
|
420
|
+
lines.append("")
|
|
421
|
+
lines.append(f"_Last updated {_updated_at()} (auto-generated by `aq evaluate`)._")
|
|
422
|
+
return "\n".join(lines)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
# --------------------------------------------------------------------------
|
|
426
|
+
# Wiring
|
|
427
|
+
# --------------------------------------------------------------------------
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def load_monte_carlo() -> tuple[dict | None, object]:
|
|
431
|
+
"""Loads ml/evaluation/monte_carlo.json (+ curves .npz) - (summary, runs)
|
|
432
|
+
where runs is the downsampled per-run curve matrix for the PNG, or
|
|
433
|
+
(None, None) when either file is missing/unreadable."""
|
|
434
|
+
summary = _load_json(EVALUATION_DIR / "monte_carlo.json")
|
|
435
|
+
if not isinstance(summary, dict):
|
|
436
|
+
return None, None
|
|
437
|
+
runs = None
|
|
438
|
+
npz_path = EVALUATION_DIR / "monte_carlo_curves.npz"
|
|
439
|
+
try:
|
|
440
|
+
import numpy as np
|
|
441
|
+
|
|
442
|
+
with np.load(npz_path) as payload:
|
|
443
|
+
runs = payload["runs_curves"]
|
|
444
|
+
except Exception:
|
|
445
|
+
runs = None
|
|
446
|
+
return summary, runs
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _render_monte_carlo_chart(runs, avg_curve: list[float], config: dict, out_path: Path) -> bool:
|
|
450
|
+
"""Draws every resampled run as a faint line plus the red average -
|
|
451
|
+
regenerated on every evaluate run so the README visual is always in sync."""
|
|
452
|
+
try:
|
|
453
|
+
import matplotlib
|
|
454
|
+
|
|
455
|
+
matplotlib.use("Agg")
|
|
456
|
+
import numpy as np
|
|
457
|
+
import matplotlib.pyplot as plt
|
|
458
|
+
except Exception:
|
|
459
|
+
return False
|
|
460
|
+
if runs is None or getattr(runs, "size", 0) == 0 or not avg_curve:
|
|
461
|
+
return False
|
|
462
|
+
fig, ax = plt.subplots(figsize=(11, 6), dpi=140)
|
|
463
|
+
x = np.arange(runs.shape[1])
|
|
464
|
+
for row in runs:
|
|
465
|
+
ax.plot(x, row, color="#6f9fc4", alpha=0.05, lw=0.5, zorder=1)
|
|
466
|
+
ax.plot(x, avg_curve, color="red", lw=2.4, zorder=3,
|
|
467
|
+
label=f"Average of all {config.get('n_runs', '?')} runs")
|
|
468
|
+
ax.axhline(1.0, color="black", lw=0.8, ls="--", zorder=2, label="Start (1.0x)")
|
|
469
|
+
ax.set_xlabel("Trading days (resampled)")
|
|
470
|
+
ax.set_ylabel("Equity multiple (start = 1.0x)")
|
|
471
|
+
ax.set_title(
|
|
472
|
+
f"Monte Carlo equity curves - {config.get('n_runs')} runs, "
|
|
473
|
+
f"method={config.get('method')}, block={config.get('block_size')}d, seed={config.get('seed')}"
|
|
474
|
+
)
|
|
475
|
+
ax.grid(alpha=0.25)
|
|
476
|
+
ax.legend(loc="best")
|
|
477
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
478
|
+
fig.savefig(out_path)
|
|
479
|
+
plt.close(fig)
|
|
480
|
+
return True
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _build_monte_carlo_compact_markdown(summary: dict | None) -> str:
|
|
484
|
+
if not isinstance(summary, dict):
|
|
485
|
+
return (
|
|
486
|
+
"Monte Carlo simulation has not been run yet - generate it with\n"
|
|
487
|
+
"`aq evaluate --rank-book --monte-carlo` (see CLI Reference)."
|
|
488
|
+
)
|
|
489
|
+
fr = summary["final_return_pct"]
|
|
490
|
+
sh = summary["sharpe"]
|
|
491
|
+
md = summary["max_drawdown"]
|
|
492
|
+
cfg = summary["config"]
|
|
493
|
+
lines = [
|
|
494
|
+
f"{cfg['n_runs']}-run stationary-block bootstrap of the offline rank book's daily net returns",
|
|
495
|
+
f"(method=`{cfg['method']}`, block={cfg['block_size']}d, seed={cfg['seed']}). The red line marks the"
|
|
496
|
+
"\naverage of all generated runs; faint lines are individual resampled histories.",
|
|
497
|
+
"",
|
|
498
|
+
f".as_posix()})",
|
|
499
|
+
"",
|
|
500
|
+
"| Metric | p5 | p50 | p95 |",
|
|
501
|
+
"|---|---|---|---|",
|
|
502
|
+
f"| Final return | {fr['p5']}% | {fr['p50']}% | {fr['p95']}% |",
|
|
503
|
+
f"| Annualized Sharpe | {sh['p5']} | {sh['p50']} | {sh['p95']} |",
|
|
504
|
+
f"| Max drawdown | {md['p5']}% | {md['p50']}% | {md['p95']}% |",
|
|
505
|
+
f"| P(negative total return): **{summary['prob_negative_return']}** |||",
|
|
506
|
+
]
|
|
507
|
+
return "\n".join(lines)
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _build_monte_carlo_full_stats_markdown(summary: dict | None) -> str:
|
|
511
|
+
if not isinstance(summary, dict):
|
|
512
|
+
return "No Monte Carlo data on disk yet."
|
|
513
|
+
cfg = summary["config"]
|
|
514
|
+
|
|
515
|
+
def table(title: str, stats: dict) -> list[str]:
|
|
516
|
+
return [
|
|
517
|
+
f"**{title}**",
|
|
518
|
+
"",
|
|
519
|
+
"| p5 | p25 | p50 | p75 | p95 |",
|
|
520
|
+
"|---|---|---|---|---|",
|
|
521
|
+
f"| {stats['p5']} | {stats['p25']} | {stats['p50']} | {stats['p75']} | {stats['p95']} |",
|
|
522
|
+
"",
|
|
523
|
+
]
|
|
524
|
+
|
|
525
|
+
lines = [
|
|
526
|
+
"Deeper distribution statistics across all Monte Carlo runs:",
|
|
527
|
+
"",
|
|
528
|
+
*table("Final total return (%)", summary["final_return_pct"]),
|
|
529
|
+
*table("Annualized Sharpe", summary["sharpe"]),
|
|
530
|
+
*table("Max drawdown (%)", summary["max_drawdown"]),
|
|
531
|
+
f"- Best run total return: **{summary['best_run_total_return']}%**",
|
|
532
|
+
f"- Worst run total return: **{summary['worst_run_total_return']}%**",
|
|
533
|
+
f"- Probability of a negative total return: **{summary['prob_negative_return']}**",
|
|
534
|
+
f"- Config: n_runs={cfg['n_runs']}, method=`{cfg['method']}`, block_size={cfg['block_size']}d, "
|
|
535
|
+
f"seed={cfg['seed']} (pinned - the section only changes when underlying results change)",
|
|
536
|
+
"",
|
|
537
|
+
"The red line in the chart above is the element-wise AVERAGE of all run "
|
|
538
|
+
"equity curves; faint blue lines are the individual bootstrap histories.",
|
|
539
|
+
]
|
|
540
|
+
return "\n".join(lines)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def load_benchmark_comparison() -> dict | None:
|
|
544
|
+
"""Loads ml/evaluation/benchmark_comparison.json - `aq evaluate
|
|
545
|
+
--benchmarks`' persisted report (all five baselines + run context), or
|
|
546
|
+
None when it has never been written."""
|
|
547
|
+
return _load_json(EVALUATION_DIR / "benchmark_comparison.json")
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _build_benchmark_markdown(summary: dict | None, eval_summary: dict[str, dict | None]) -> str:
|
|
551
|
+
"""Renders the Benchmark Comparison section: the rank book's own net
|
|
552
|
+
Sharpe rows first (the number being benchmarked, per model), then every
|
|
553
|
+
baseline. All from the same dataset window/split, so the comparison is
|
|
554
|
+
like-for-like; a baseline that couldn't compute shows its SKIPPED
|
|
555
|
+
status instead of fabricated numbers."""
|
|
556
|
+
if not isinstance(summary, dict) or not summary.get("baselines"):
|
|
557
|
+
return (
|
|
558
|
+
"Benchmark comparison has not been run yet - generate it with\n"
|
|
559
|
+
"`aq evaluate --benchmarks` (or `aq evaluate --all`)."
|
|
560
|
+
)
|
|
561
|
+
lines = [
|
|
562
|
+
"| Strategy | Net Sharpe | Total return | Note |",
|
|
563
|
+
"|---|---|---|---|",
|
|
564
|
+
]
|
|
565
|
+
for model_kind in _MODEL_KINDS:
|
|
566
|
+
rank_book = (eval_summary or {}).get(model_kind, {}) or {}
|
|
567
|
+
rank_book_report = rank_book.get("rank_book") or {}
|
|
568
|
+
if rank_book_report:
|
|
569
|
+
lines.append(
|
|
570
|
+
f"| **Rank book ({model_kind})** | {_fmt(rank_book_report.get('net_sharpe'), '.3f')} | "
|
|
571
|
+
f"{_fmt(rank_book_report.get('net_total_return'), '.2%')} | the strategy being benchmarked |"
|
|
572
|
+
)
|
|
573
|
+
for baseline in summary["baselines"]:
|
|
574
|
+
note = baseline.get("status", "OK")
|
|
575
|
+
if note == "OK":
|
|
576
|
+
note = "public benchmark" if baseline["strategy"] in ("sp500", "60_40") else "naive strategy baseline"
|
|
577
|
+
lines.append(
|
|
578
|
+
f"| {baseline['strategy']} | {_fmt(baseline.get('net_sharpe'), '.3f')} | "
|
|
579
|
+
f"{_fmt(baseline.get('total_return_pct'), '.2f')}% | {note} |"
|
|
580
|
+
)
|
|
581
|
+
lines.append("")
|
|
582
|
+
lines.append(
|
|
583
|
+
f"_Same dataset window/split (`{summary.get('split', 'backtest')}`) as the rank-book simulation "
|
|
584
|
+
f"above; Sharpe formula identical across every row. Last updated {_updated_at()} "
|
|
585
|
+
"(auto-generated by `aq evaluate --benchmarks`)._"
|
|
586
|
+
)
|
|
587
|
+
return "\n".join(lines)
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def update_readme_evaluation_sections(readme_path: Path = README_PATH) -> bool:
|
|
594
|
+
"""Idempotently rebuilds Offline Evaluation, Walk-Forward Training/
|
|
595
|
+
Testing, and Other Metrics from whatever currently exists on disk -
|
|
596
|
+
never raises (mirrors generate_backtest_report.py::
|
|
597
|
+
update_readme_backtest_section()'s own contract), so a report-
|
|
598
|
+
generation bug never fails the `aq evaluate`/`aq train` command that
|
|
599
|
+
triggered it. Returns True if the README was actually modified."""
|
|
600
|
+
if not readme_path.exists():
|
|
601
|
+
return False
|
|
602
|
+
text = readme_path.read_text(encoding="utf-8")
|
|
603
|
+
changed = False
|
|
604
|
+
|
|
605
|
+
eval_summary = load_offline_evaluation_summary()
|
|
606
|
+
walk_forward_summary = load_latest_walk_forward_summary()
|
|
607
|
+
reconciliation = _load_json(EVALUATION_DIR / "book_history_reconciliation.json")
|
|
608
|
+
kill_switch_replay = _load_json(EVALUATION_DIR / "kill_switch_replay.json")
|
|
609
|
+
|
|
610
|
+
mc_summary, mc_runs = load_monte_carlo()
|
|
611
|
+
# V5.4.4 guard (development/Problems.md #114): the chart is a repo
|
|
612
|
+
# asset OF the repo README - only (re)render it when this call is
|
|
613
|
+
# actually refreshing the real README. A caller passing a scratch/
|
|
614
|
+
# tmp readme (the tests, offline experiments) previously rendered
|
|
615
|
+
# whatever its EVALUATION_DIR contained straight over the REAL
|
|
616
|
+
# development/monte_carlo_equity_curves.png - which is exactly how
|
|
617
|
+
# the committed chart got replaced by a degenerate test-fixture
|
|
618
|
+
# render (10 flat runs, 2-point x-axis): every full test-suite run
|
|
619
|
+
# silently clobbered it, and the README showed only the red average
|
|
620
|
+
# line until the next `aq evaluate --monte-carlo` regenerated it.
|
|
621
|
+
if (
|
|
622
|
+
mc_summary is not None
|
|
623
|
+
and mc_summary.get("status") == "OK"
|
|
624
|
+
and readme_path.resolve() == README_PATH.resolve()
|
|
625
|
+
):
|
|
626
|
+
_render_monte_carlo_chart(mc_runs, mc_summary.get("avg_curve", []), mc_summary.get("config", {}), MONTECARLO_CHART_PATH)
|
|
627
|
+
|
|
628
|
+
updated = _replace_between_markers(text, MONTECARLO_MARKER_START, MONTECARLO_MARKER_END, _build_monte_carlo_compact_markdown(mc_summary))
|
|
629
|
+
if updated is not None:
|
|
630
|
+
text, changed = updated, True
|
|
631
|
+
|
|
632
|
+
# V5.4.4 - benchmark comparison section (rank book vs strategy +
|
|
633
|
+
# public baselines, all on the same window). Rendered from the
|
|
634
|
+
# persisted ml/evaluation/benchmark_comparison.json; like every other
|
|
635
|
+
# section here it degrades to its not-run-yet fallback when that file
|
|
636
|
+
# doesn't exist yet.
|
|
637
|
+
benchmark_summary = load_benchmark_comparison()
|
|
638
|
+
updated = _replace_between_markers(
|
|
639
|
+
text,
|
|
640
|
+
BENCHMARK_MARKER_START,
|
|
641
|
+
BENCHMARK_MARKER_END,
|
|
642
|
+
_build_benchmark_markdown(benchmark_summary, eval_summary),
|
|
643
|
+
)
|
|
644
|
+
if updated is not None:
|
|
645
|
+
text, changed = updated, True
|
|
646
|
+
|
|
647
|
+
updated = _replace_between_markers(
|
|
648
|
+
text,
|
|
649
|
+
MONTECARLO_FULL_STATS_MARKER_START,
|
|
650
|
+
MONTECARLO_FULL_STATS_MARKER_END,
|
|
651
|
+
_build_monte_carlo_full_stats_markdown(mc_summary),
|
|
652
|
+
)
|
|
653
|
+
if updated is not None:
|
|
654
|
+
text, changed = updated, True
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
updated = _replace_between_markers(text, EVAL_MARKER_START, EVAL_MARKER_END, _build_eval_compact_markdown(eval_summary))
|
|
658
|
+
if updated is not None:
|
|
659
|
+
text, changed = updated, True
|
|
660
|
+
|
|
661
|
+
updated = _replace_between_markers(
|
|
662
|
+
text, EVAL_FULL_STATS_MARKER_START, EVAL_FULL_STATS_MARKER_END, _build_eval_full_stats_markdown(eval_summary)
|
|
663
|
+
)
|
|
664
|
+
if updated is not None:
|
|
665
|
+
text, changed = updated, True
|
|
666
|
+
|
|
667
|
+
updated = _replace_between_markers(
|
|
668
|
+
text, WALKFORWARD_MARKER_START, WALKFORWARD_MARKER_END, _build_walk_forward_compact_markdown(walk_forward_summary)
|
|
669
|
+
)
|
|
670
|
+
if updated is not None:
|
|
671
|
+
text, changed = updated, True
|
|
672
|
+
|
|
673
|
+
updated = _replace_between_markers(
|
|
674
|
+
text,
|
|
675
|
+
WALKFORWARD_FULL_STATS_MARKER_START,
|
|
676
|
+
WALKFORWARD_FULL_STATS_MARKER_END,
|
|
677
|
+
_build_walk_forward_full_stats_markdown(walk_forward_summary),
|
|
678
|
+
)
|
|
679
|
+
if updated is not None:
|
|
680
|
+
text, changed = updated, True
|
|
681
|
+
|
|
682
|
+
updated = _replace_between_markers(
|
|
683
|
+
text,
|
|
684
|
+
OTHER_METRICS_MARKER_START,
|
|
685
|
+
OTHER_METRICS_MARKER_END,
|
|
686
|
+
_build_other_metrics_markdown(eval_summary, walk_forward_summary, reconciliation, kill_switch_replay),
|
|
687
|
+
)
|
|
688
|
+
if updated is not None:
|
|
689
|
+
text, changed = updated, True
|
|
690
|
+
|
|
691
|
+
if changed:
|
|
692
|
+
_atomic_write(readme_path, text)
|
|
693
|
+
return changed
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def main() -> int:
|
|
697
|
+
parser = argparse.ArgumentParser(
|
|
698
|
+
description="Regenerate README.md's Offline Evaluation / Walk-Forward / Other Metrics sections."
|
|
699
|
+
)
|
|
700
|
+
parser.add_argument("--readme", type=Path, default=README_PATH)
|
|
701
|
+
args = parser.parse_args()
|
|
702
|
+
|
|
703
|
+
updated = update_readme_evaluation_sections(args.readme)
|
|
704
|
+
if updated:
|
|
705
|
+
print(f"Updated {args.readme} with the latest evaluation/walk-forward results.")
|
|
706
|
+
return 0
|
|
707
|
+
print("Nothing to update (no source files found, or README markers missing).", file=sys.stderr)
|
|
708
|
+
return 1
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
if __name__ == "__main__":
|
|
712
|
+
sys.exit(main())
|