aether-quant 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. aether_quant-0.0.1.dist-info/METADATA +7 -0
  2. aether_quant-0.0.1.dist-info/RECORD +99 -0
  3. aether_quant-0.0.1.dist-info/WHEEL +5 -0
  4. aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
  5. aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
  6. aether_quant-0.0.1.dist-info/top_level.txt +13 -0
  7. analyzer/README.md +69 -0
  8. analyzer/__init__.py +17 -0
  9. analyzer/market_analyzer.py +521 -0
  10. aq_cli.py +3633 -0
  11. audit/README.md +47 -0
  12. audit/__init__.py +38 -0
  13. audit/hash_chain.py +52 -0
  14. audit/postgres_audit.py +61 -0
  15. audit/postgres_worker.py +402 -0
  16. audit/redis_queue.py +124 -0
  17. audit/status_export.py +62 -0
  18. data_pipeline/README.md +257 -0
  19. data_pipeline/__init__.py +6 -0
  20. data_pipeline/bar_synthesis.py +97 -0
  21. data_pipeline/dividend_backfill.py +243 -0
  22. data_pipeline/factor_file_backfill.py +276 -0
  23. data_pipeline/fetch.py +206 -0
  24. data_pipeline/fred_backfill.py +435 -0
  25. data_pipeline/ib_backfill.py +287 -0
  26. data_pipeline/v2_manifest.py +75 -0
  27. data_pipeline/yfinance_backfill.py +501 -0
  28. evaluation/README.md +152 -0
  29. evaluation/__init__.py +73 -0
  30. evaluation/ablation.py +310 -0
  31. evaluation/benchmark_comparison.py +195 -0
  32. evaluation/confidence_threshold_calibration.py +157 -0
  33. evaluation/factor_neutralization_check.py +94 -0
  34. evaluation/feature_parity.py +321 -0
  35. evaluation/feature_reconciliation.py +286 -0
  36. evaluation/impact_model.py +77 -0
  37. evaluation/kill_switch_replay.py +186 -0
  38. evaluation/limit_fill_simulator.py +152 -0
  39. evaluation/model_predictions.py +192 -0
  40. evaluation/monte_carlo.py +161 -0
  41. evaluation/public_benchmarks.py +117 -0
  42. evaluation/rank_book_simulator.py +494 -0
  43. evaluation/rank_ic_core.py +204 -0
  44. evaluation/rank_signal_calibration.py +690 -0
  45. evaluation/rolling_ic_gate_calibration.py +190 -0
  46. evaluation/rolling_ic_gate_replay.py +138 -0
  47. execution/README.md +281 -0
  48. execution/__init__.py +80 -0
  49. execution/config_cache.py +48 -0
  50. execution/cost_model.py +213 -0
  51. execution/lean_config_render.py +131 -0
  52. execution/live_credentials.py +38 -0
  53. execution/live_credentials_io.py +52 -0
  54. execution/order_gate.py +388 -0
  55. execution/paper_readiness.py +181 -0
  56. execution/paper_readiness_io.py +54 -0
  57. execution/paper_readiness_report.py +103 -0
  58. execution/paper_readiness_scheduler.py +137 -0
  59. execution/reconciliation.py +160 -0
  60. execution/runtime_config_io.py +38 -0
  61. execution/secret_scan.py +73 -0
  62. features/README.md +77 -0
  63. features/__init__.py +137 -0
  64. features/alt_data_features.py +154 -0
  65. features/bond_features.py +275 -0
  66. features/cross_asset_sensitivity.py +79 -0
  67. features/derivatives_macro_features.py +72 -0
  68. features/macro_features.py +79 -0
  69. features/options_greeks.py +329 -0
  70. features/sector_map.py +53 -0
  71. features/technical_indicators.py +185 -0
  72. generate_backtest_report.py +311 -0
  73. generate_evaluation_report.py +712 -0
  74. inference/README.md +105 -0
  75. inference/__init__.py +36 -0
  76. inference/exported_model.py +929 -0
  77. inference/parallel_inference.py +160 -0
  78. inference/strategy_selector_inference.py +75 -0
  79. json_safety.py +130 -0
  80. portfolio/README.md +447 -0
  81. portfolio/__init__.py +41 -0
  82. portfolio/book_construction.py +786 -0
  83. portfolio/book_neutrality.py +178 -0
  84. portfolio/hrp_allocation.py +159 -0
  85. portfolio/options_arbitrage_detector.py +232 -0
  86. portfolio/options_assignment_risk.py +119 -0
  87. portfolio/options_margin_sizing.py +333 -0
  88. portfolio/options_strategy.py +1999 -0
  89. portfolio/rank_signal.py +219 -0
  90. portfolio/rolling_ic_gate.py +183 -0
  91. risk/README.md +275 -0
  92. risk/__init__.py +38 -0
  93. risk/asset_class_router.py +584 -0
  94. risk/forex_risk.py +158 -0
  95. risk/futures_risk.py +202 -0
  96. risk/kill_switch.py +261 -0
  97. risk/manual_override.py +118 -0
  98. risk/position_sizing.py +303 -0
  99. risk/rl_sizing.py +189 -0
@@ -0,0 +1,712 @@
1
+ """Regenerates README.md's Offline Evaluation, Walk-Forward Training/Testing,
2
+ and Other Metrics subsections (siblings of Lean Backtest, which
3
+ generate_backtest_report.py already owns) from whatever `aq evaluate`/`aq
4
+ train --walk-forward` last wrote to ml/evaluation/ and ml/versions/.
5
+
6
+ Same practical-substitute-for-live-update approach as generate_backtest_report.py:
7
+ `aq evaluate` calls this module after any of --rank-book/--capacity/--stress/
8
+ --all/--reconcile-book-history/--replay-kill-switch/--walk-forward-summary,
9
+ and it rewrites the relevant marker-delimited sections in-place. Every
10
+ section degrades gracefully (an em-dash placeholder table row, not a
11
+ crash/traceback) when its source file doesn't exist yet - a fresh checkout
12
+ that has never run `aq evaluate` still gets a readable README, matching
13
+ monitoring/evaluation_state.py's own "not_evaluated" contract.
14
+
15
+ Data sources (all optional, independently missing-tolerant):
16
+ - ml/evaluation/rank_book_simulation_{sequence,multitask}.json (this
17
+ module's own additive per-model copies - see aq_cli.py::cmd_evaluate())
18
+ - ml/evaluation/capacity_report_{sequence,multitask}.json
19
+ - ml/evaluation/cost_stress_report_{sequence,multitask}.json
20
+ - ml/versions/walk-forward-*/walk_forward_summary.json (newest by mtime)
21
+ - ml/evaluation/book_history_reconciliation.json
22
+ - ml/evaluation/kill_switch_replay.json
23
+ - The latest real Lean backtest result JSON (reuses
24
+ generate_backtest_report.py's own loader - same "latest complete run"
25
+ definition, so the Lean-vs-offline comparison in Other Metrics never
26
+ compares against a stale/different backtest than what Lean Backtest
27
+ itself displays).
28
+
29
+ Usage:
30
+ python generate_evaluation_report.py [--readme README.md]
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import argparse
36
+ import json
37
+ import sys
38
+ from datetime import datetime, timezone
39
+ from pathlib import Path
40
+
41
+ from generate_backtest_report import (
42
+ README_PATH,
43
+ _atomic_write,
44
+ _replace_between_markers,
45
+ find_latest_backtest_result_json,
46
+ load_backtest_summary,
47
+ )
48
+
49
+ ROOT_DIR = Path(__file__).resolve().parent
50
+ ML_DIR = ROOT_DIR / "ml"
51
+ EVALUATION_DIR = ML_DIR / "evaluation"
52
+ VERSIONS_DIR = ML_DIR / "versions"
53
+
54
+ EVAL_MARKER_START = "<!-- AQ:EVAL_START -->"
55
+ EVAL_MARKER_END = "<!-- AQ:EVAL_END -->"
56
+ EVAL_FULL_STATS_MARKER_START = "<!-- AQ:EVAL_FULL_STATS_START -->"
57
+ EVAL_FULL_STATS_MARKER_END = "<!-- AQ:EVAL_FULL_STATS_END -->"
58
+
59
+ WALKFORWARD_MARKER_START = "<!-- AQ:WALKFORWARD_START -->"
60
+ WALKFORWARD_MARKER_END = "<!-- AQ:WALKFORWARD_END -->"
61
+ WALKFORWARD_FULL_STATS_MARKER_START = "<!-- AQ:WALKFORWARD_FULL_STATS_START -->"
62
+ WALKFORWARD_FULL_STATS_MARKER_END = "<!-- AQ:WALKFORWARD_FULL_STATS_END -->"
63
+
64
+ OTHER_METRICS_MARKER_START = "<!-- AQ:OTHER_METRICS_START -->"
65
+ OTHER_METRICS_MARKER_END = "<!-- AQ:OTHER_METRICS_END -->"
66
+
67
+ MONTECARLO_MARKER_START = "<!-- AQ:MONTECARLO_START -->"
68
+ MONTECARLO_MARKER_END = "<!-- AQ:MONTECARLO_END -->"
69
+ MONTECARLO_FULL_STATS_MARKER_START = "<!-- AQ:MONTECARLO_FULL_STATS_START -->"
70
+ MONTECARLO_FULL_STATS_MARKER_END = "<!-- AQ:MONTECARLO_FULL_STATS_END -->"
71
+ MONTECARLO_CHART_PATH = ROOT_DIR / "development" / "monte_carlo_equity_curves.png"
72
+
73
+ BENCHMARK_MARKER_START = "<!-- AQ:BENCHMARK_START -->"
74
+ BENCHMARK_MARKER_END = "<!-- AQ:BENCHMARK_END -->"
75
+
76
+ _MODEL_KINDS = ("sequence", "multitask")
77
+
78
+
79
+ def _load_json(path: Path) -> dict | list | None:
80
+ if not path.exists():
81
+ return None
82
+ try:
83
+ with path.open("r", encoding="utf-8") as f:
84
+ return json.load(f)
85
+ except (json.JSONDecodeError, OSError):
86
+ return None
87
+
88
+
89
+ def _fmt(value: object, spec: str = "") -> str:
90
+ """Formats a number with `spec`, or returns an em-dash for
91
+ None/missing - every table cell in this module goes through this so a
92
+ partial `aq evaluate` run degrades to a clean placeholder, never a
93
+ KeyError/TypeError."""
94
+ if value is None:
95
+ return "—"
96
+ try:
97
+ return format(value, spec) if spec else str(value)
98
+ except (ValueError, TypeError):
99
+ return str(value)
100
+
101
+
102
+ def _updated_at() -> str:
103
+ return datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
104
+
105
+
106
+ # --------------------------------------------------------------------------
107
+ # Offline Evaluation
108
+ # --------------------------------------------------------------------------
109
+
110
+
111
+ def load_offline_evaluation_summary() -> dict[str, dict | None]:
112
+ """Per-model rank_book/capacity/stress reports, keyed by model_kind -
113
+ each value is None (not the whole function) when that model has never
114
+ been evaluated, so a report with only a sequence run still renders a
115
+ complete multitask="not yet run" row instead of omitting the section."""
116
+ summary: dict[str, dict | None] = {}
117
+ for model_kind in _MODEL_KINDS:
118
+ rank_book = _load_json(EVALUATION_DIR / f"rank_book_simulation_{model_kind}.json")
119
+ capacity = _load_json(EVALUATION_DIR / f"capacity_report_{model_kind}.json")
120
+ stress = _load_json(EVALUATION_DIR / f"cost_stress_report_{model_kind}.json")
121
+ if rank_book is None and capacity is None and stress is None:
122
+ summary[model_kind] = None
123
+ continue
124
+ summary[model_kind] = {"rank_book": rank_book, "capacity": capacity, "stress": stress}
125
+ return summary
126
+
127
+
128
+ def _build_eval_compact_markdown(summary: dict[str, dict | None]) -> str:
129
+ lines = [
130
+ "| Metric | Multitask | Sequence |",
131
+ "|---|---|---|",
132
+ ]
133
+ row_specs: list[tuple[str, str, str]] = [
134
+ ("Gross Sharpe", "rank_book.gross_sharpe", ".3f"),
135
+ ("Net Sharpe", "rank_book.net_sharpe", ".3f"),
136
+ ("Net total return", "rank_book.net_total_return", ".2%"),
137
+ ("Max drawdown", "rank_book.net_max_drawdown", ".2%"),
138
+ ("Annualized turnover", "rank_book.annualized_turnover", ".2f"),
139
+ ("Cost drag (bps/yr)", "rank_book.cost_drag_annual_bps", ".1f"),
140
+ ("Capacity (USD)", "capacity.capacity_usd", ",.0f"),
141
+ ]
142
+ for label, dotted_path, spec in row_specs:
143
+ section, field = dotted_path.split(".")
144
+ cells = []
145
+ for model_kind in ("multitask", "sequence"):
146
+ model_summary = summary.get(model_kind)
147
+ value = (model_summary.get(section) or {}).get(field) if model_summary else None
148
+ cells.append(_fmt(value, spec))
149
+ lines.append(f"| {label} | {cells[0]} | {cells[1]} |")
150
+ body = "\n".join(lines)
151
+ body += f"\n\n_Backtest split, full history. Last updated {_updated_at()} (auto-generated by `aq evaluate --all`)._"
152
+ return body
153
+
154
+
155
+ def _build_eval_full_stats_markdown(summary: dict[str, dict | None]) -> str:
156
+ sections: list[str] = []
157
+ for model_kind in ("multitask", "sequence"):
158
+ model_summary = summary.get(model_kind)
159
+ sections.append(f"**{model_kind.capitalize()} model**")
160
+ if model_summary is None:
161
+ sections.append(f"_Not yet evaluated - run `aq evaluate --all --model {model_kind}`._")
162
+ continue
163
+
164
+ rank_book = model_summary.get("rank_book") or {}
165
+ lines = ["| Metric | Value |", "|---|---|"]
166
+ for key in (
167
+ "gross_sharpe", "net_sharpe", "gross_total_return", "net_total_return", "net_max_drawdown",
168
+ "annualized_turnover", "cost_drag_annual_bps", "num_rebalances", "num_dates_used",
169
+ "mean_names_long", "mean_names_short",
170
+ ):
171
+ if key in rank_book:
172
+ lines.append(f"| {key} | {rank_book[key]} |")
173
+ sections.append("\n".join(lines))
174
+
175
+ capacity = model_summary.get("capacity")
176
+ if capacity:
177
+ cap_lines = [
178
+ "",
179
+ f"Capacity: ${capacity.get('capacity_usd', 0):,.0f} (binding: {capacity.get('binding_ticker', '—')})",
180
+ "",
181
+ "| top_n | Net Sharpe |",
182
+ "|---|---|",
183
+ ]
184
+ for row in capacity.get("per_top_n", []):
185
+ cap_lines.append(f"| {row.get('top_n')} | {row.get('net_sharpe', 0):.4f} |")
186
+ sections.append("\n".join(cap_lines))
187
+
188
+ stress = model_summary.get("stress")
189
+ if stress and stress.get("stress"):
190
+ stress_lines = ["", "| Cost multiplier | Gross Sharpe | Net Sharpe | Cost drag (bps/yr) |", "|---|---|---|---|"]
191
+ for row in stress["stress"]:
192
+ stress_lines.append(
193
+ f"| {row.get('cost_multiplier')}x | {row.get('gross_sharpe', 0):.4f} | "
194
+ f"{row.get('net_sharpe', 0):.4f} | {row.get('cost_drag_annual_bps', 0):.1f} |"
195
+ )
196
+ sections.append("\n".join(stress_lines))
197
+
198
+ return "\n\n".join(sections)
199
+
200
+
201
+ # --------------------------------------------------------------------------
202
+ # Walk-Forward Training/Testing
203
+ # --------------------------------------------------------------------------
204
+
205
+
206
+ def load_latest_walk_forward_summary() -> dict | None:
207
+ """Newest ml/versions/walk-forward-*/walk_forward_summary.json by
208
+ mtime - same "latest run" definition as
209
+ monitoring/evaluation_state.py::_latest_walk_forward_summary()."""
210
+ if not VERSIONS_DIR.exists():
211
+ return None
212
+ candidates = sorted(
213
+ VERSIONS_DIR.glob("walk-forward-*/walk_forward_summary.json"),
214
+ key=lambda path: path.stat().st_mtime,
215
+ reverse=True,
216
+ )
217
+ return _load_json(candidates[0]) if candidates else None
218
+
219
+
220
+ def _build_walk_forward_compact_markdown(summary: dict | None) -> str:
221
+ if summary is None:
222
+ return "_No walk-forward run found yet - run `aq train --walk-forward` first._"
223
+
224
+ run_id = summary.get("run_id", "unknown")
225
+ num_windows = summary.get("num_windows", 0)
226
+ stability = summary.get("stability_by_metric") or {}
227
+ net_perf = summary.get("net_performance_by_window") or []
228
+
229
+ lines = ["| Metric | Mean | 95% CI | Stable? |", "|---|---|---|---|"]
230
+ for metric_name in ("backtest_mcc", "rank_5d_ic", "rank_20d_ic", "residual_rank_20d_ic"):
231
+ stats = stability.get(metric_name)
232
+ if not stats:
233
+ continue
234
+ bootstrap = stats.get("bootstrap", {})
235
+ stable_label = "yes" if stats.get("stable") else "**NO**"
236
+ lines.append(
237
+ f"| {metric_name} | {_fmt(stats.get('mean'), '.4f')} | "
238
+ f"[{_fmt(bootstrap.get('lower_bound'), '.4f')}, {_fmt(bootstrap.get('upper_bound'), '.4f')}] | "
239
+ f"{stable_label} |"
240
+ )
241
+
242
+ if net_perf:
243
+ net_sharpes = [w.get("simulation", {}).get("net_sharpe") for w in net_perf if w.get("simulation")]
244
+ net_sharpes = [s for s in net_sharpes if s is not None]
245
+ if net_sharpes:
246
+ mean_sharpe = sum(net_sharpes) / len(net_sharpes)
247
+ num_positive = sum(1 for s in net_sharpes if s > 0)
248
+ lines.append(
249
+ f"| net_sharpe (per-window) | {mean_sharpe:.3f} | — | "
250
+ f"{num_positive}/{len(net_sharpes)} windows positive |"
251
+ )
252
+
253
+ body = "\n".join(lines)
254
+ body += (
255
+ f"\n\n{num_windows} expanding/rolling windows, run `{run_id}`. "
256
+ f"Last updated {_updated_at()} (auto-generated by `aq train --walk-forward`)."
257
+ )
258
+ return body
259
+
260
+
261
+ def _build_walk_forward_full_stats_markdown(summary: dict | None) -> str:
262
+ if summary is None:
263
+ return "_No walk-forward run found yet._"
264
+
265
+ window_results = summary.get("window_results") or []
266
+ net_perf_by_window = {w.get("window_index"): w for w in (summary.get("net_performance_by_window") or [])}
267
+
268
+ lines = [
269
+ "| Window | Backtest period | Model | Backtest MCC | Gross Sharpe | Net Sharpe | Net return | Max DD | Turnover |",
270
+ "|---|---|---|---|---|---|---|---|---|",
271
+ ]
272
+ for index, window in enumerate(window_results):
273
+ backtest_window = window.get("window", {}).get("backtest", {})
274
+ period = f"{backtest_window.get('start', '?')} → {backtest_window.get('end', '?')}"
275
+ net_perf = net_perf_by_window.get(index, {})
276
+ sim = net_perf.get("simulation", {})
277
+ lines.append(
278
+ f"| {index} | {period} | {net_perf.get('model_kind', '—')} | "
279
+ f"{_fmt(window.get('backtest_mcc'), '.4f')} | {_fmt(sim.get('gross_sharpe'), '.3f')} | "
280
+ f"{_fmt(sim.get('net_sharpe'), '.3f')} | {_fmt(sim.get('net_total_return'), '.2%')} | "
281
+ f"{_fmt(sim.get('net_max_drawdown'), '.2%')} | {_fmt(sim.get('annualized_turnover'), '.2f')}x |"
282
+ )
283
+ return "\n".join(lines)
284
+
285
+
286
+ # --------------------------------------------------------------------------
287
+ # Other Metrics (Lean vs. offline comparison, reconciliation, kill-switch)
288
+ # --------------------------------------------------------------------------
289
+
290
+
291
+ def _load_lean_sharpe() -> tuple[float | None, str | None]:
292
+ """Returns (sharpe, backtest_window_label) from the same latest
293
+ complete Lean run generate_backtest_report.py's own Lean Backtest
294
+ section displays - never a different/staler run, so the comparison
295
+ below is always apples-to-apples with what's shown just above it."""
296
+ result_json_path = find_latest_backtest_result_json()
297
+ if result_json_path is None:
298
+ return None, None
299
+ lean_summary = load_backtest_summary(result_json_path)
300
+ statistics = lean_summary["statistics"]
301
+ sharpe_str = statistics.get("Sharpe Ratio")
302
+ try:
303
+ sharpe = float(sharpe_str) if sharpe_str is not None else None
304
+ except ValueError:
305
+ sharpe = None
306
+ start_date, end_date = lean_summary["start_date"], lean_summary["end_date"]
307
+ window = f"{start_date.date().isoformat()} to {end_date.date().isoformat()}" if start_date and end_date else None
308
+ return sharpe, window
309
+
310
+
311
+ def _count_real_kill_switch_trips() -> int | None:
312
+ """V5.3.1 (development/Problems.md #91/#97): always returns None now -
313
+ this used to count `kill_switch_tripped` occurrences in the real
314
+ backtest's Lean text log, but that was never a real measurement.
315
+ main.py's actual trip-audit event (`self._audit_queue.push(...
316
+ "kill_switch_tripped"...)`) is pushed ONLY to a Redis Stream
317
+ (audit/redis_queue.py::AuditQueue), never to the text log, and
318
+ AuditQueue.push() fails silently with no Redis reachable - which a
319
+ standalone `lean backtest .` run never has. So the old count was
320
+ ALWAYS 0 regardless of how many real trips occurred, not a genuine
321
+ "0 trips" finding (V5.2.10's Changelog entry stated this as if it
322
+ were real - that was wrong, corrected here). No cheap, reliable
323
+ offline proxy exists either: `trade_lock_active` transitions are only
324
+ durably logged per-bar into book_history.jsonl's
325
+ `book_member_decisions`, itself gated to rebalance bars and book-
326
+ selected symbols only - too sparse to reconstruct true rising-edge
327
+ trip counts. Kept as a function (not deleted) so
328
+ _build_other_metrics_markdown() has one clear place to render an
329
+ honest "not measurable from a standalone backtest" caveat instead."""
330
+ return None
331
+
332
+
333
+ def _build_other_metrics_markdown(
334
+ eval_summary: dict[str, dict | None],
335
+ walk_forward_summary: dict | None,
336
+ reconciliation: dict | None,
337
+ kill_switch_replay: dict | None,
338
+ ) -> str:
339
+ lean_sharpe, lean_window = _load_lean_sharpe()
340
+ sequence_net_sharpe = (eval_summary.get("sequence") or {}).get("rank_book", {}).get("net_sharpe")
341
+ multitask_net_sharpe = (eval_summary.get("multitask") or {}).get("rank_book", {}).get("net_sharpe")
342
+ walk_forward_net_perf = (walk_forward_summary or {}).get("net_performance_by_window") or []
343
+ walk_forward_net_sharpes = [
344
+ w.get("simulation", {}).get("net_sharpe") for w in walk_forward_net_perf if w.get("simulation")
345
+ ]
346
+ walk_forward_net_sharpes = [s for s in walk_forward_net_sharpes if s is not None]
347
+ walk_forward_mean_sharpe = (
348
+ sum(walk_forward_net_sharpes) / len(walk_forward_net_sharpes) if walk_forward_net_sharpes else None
349
+ )
350
+
351
+ lines = [
352
+ "**Sharpe: real Lean backtest vs. offline estimates** — offline numbers are consistently more "
353
+ "optimistic than the real backtest; treat the gap itself as the headline number, not either side alone.",
354
+ "",
355
+ "| Source | Sharpe |",
356
+ "|---|---|",
357
+ f"| Real Lean backtest{f' ({lean_window})' if lean_window else ''} | {_fmt(lean_sharpe, '.3f')} |",
358
+ f"| Offline evaluation — sequence model (full backtest split) | {_fmt(sequence_net_sharpe, '.3f')} |",
359
+ f"| Offline evaluation — multitask model (full backtest split) | {_fmt(multitask_net_sharpe, '.3f')} |",
360
+ f"| Walk-forward mean (out-of-sample, per-window) | {_fmt(walk_forward_mean_sharpe, '.3f')} |",
361
+ ]
362
+
363
+ if lean_sharpe is not None and sequence_net_sharpe is not None:
364
+ lines.append(f"| Gap: sequence offline − real Lean | {sequence_net_sharpe - lean_sharpe:+.3f} |")
365
+ if lean_sharpe is not None and walk_forward_mean_sharpe is not None:
366
+ lines.append(f"| Gap: walk-forward − real Lean | {walk_forward_mean_sharpe - lean_sharpe:+.3f} |")
367
+
368
+ lines.append("")
369
+ lines.append("**Book-history reconciliation** (real Lean selections vs. a fresh offline re-derivation of the same dates)")
370
+ lines.append("")
371
+ if reconciliation and reconciliation.get("summary"):
372
+ recon_summary = reconciliation["summary"]
373
+ lines.extend(
374
+ [
375
+ "| Metric | Value |",
376
+ "|---|---|",
377
+ f"| Dates reconciled | {recon_summary.get('num_dates', '—')} |",
378
+ f"| Exact matches | {recon_summary.get('num_dates_exact_match', '—')} |",
379
+ f"| Mean overlap fraction | {_fmt(recon_summary.get('mean_overlap_fraction'), '.2%')} |",
380
+ f"| Mean raw-score delta | {_fmt(recon_summary.get('mean_raw_score_delta_abs'), '.4f')} |",
381
+ f"| Replay mode | {reconciliation.get('mode', '—')} |",
382
+ ]
383
+ )
384
+ diversion = reconciliation.get("diversion_summary") or {}
385
+ action_counts = diversion.get("action_counts") or {}
386
+ if action_counts:
387
+ total_decisions = sum(action_counts.values())
388
+ lines.append("")
389
+ lines.append(f"Book-member decision outcomes ({total_decisions} total, real Lean run):")
390
+ lines.append("")
391
+ lines.append("| Action | Count | Share |")
392
+ lines.append("|---|---|---|")
393
+ for action, count in sorted(action_counts.items(), key=lambda item: -item[1]):
394
+ lines.append(f"| {action} | {count} | {count / total_decisions:.1%} |")
395
+ else:
396
+ lines.append("_Not yet reconciled - run `aq evaluate --reconcile-book-history --replay-hysteresis` first._")
397
+
398
+ lines.append("")
399
+ lines.append("**Kill-switch: real trips vs. offline replay estimate**")
400
+ lines.append("")
401
+ # V5.3.1 (development/Problems.md #91/#97) - _count_real_kill_switch_trips()
402
+ # always returns None now (see its own docstring for why the old
403
+ # count was never a real measurement) - render an explicit caveat
404
+ # instead of a bare "—" that could be misread as "zero trips".
405
+ if kill_switch_replay and kill_switch_replay.get("summary"):
406
+ ks_summary = kill_switch_replay["summary"]
407
+ lines.append("| Source | Trips | Locked days |")
408
+ lines.append("|---|---|---|")
409
+ lines.append(
410
+ f"| Real Lean backtest{f' ({lean_window})' if lean_window else ''} | "
411
+ "_not measurable from a standalone backtest (see Disclaimer)_ | — |"
412
+ )
413
+ lines.append(
414
+ f"| Offline replay (approximation, see Disclaimer) | {ks_summary.get('trip_count', '—')} | "
415
+ f"{_fmt(ks_summary.get('locked_day_fraction'), '.1%')} |"
416
+ )
417
+ else:
418
+ lines.append("_Not yet available - run `aq evaluate --replay-kill-switch`._")
419
+
420
+ lines.append("")
421
+ lines.append(f"_Last updated {_updated_at()} (auto-generated by `aq evaluate`)._")
422
+ return "\n".join(lines)
423
+
424
+
425
+ # --------------------------------------------------------------------------
426
+ # Wiring
427
+ # --------------------------------------------------------------------------
428
+
429
+
430
+ def load_monte_carlo() -> tuple[dict | None, object]:
431
+ """Loads ml/evaluation/monte_carlo.json (+ curves .npz) - (summary, runs)
432
+ where runs is the downsampled per-run curve matrix for the PNG, or
433
+ (None, None) when either file is missing/unreadable."""
434
+ summary = _load_json(EVALUATION_DIR / "monte_carlo.json")
435
+ if not isinstance(summary, dict):
436
+ return None, None
437
+ runs = None
438
+ npz_path = EVALUATION_DIR / "monte_carlo_curves.npz"
439
+ try:
440
+ import numpy as np
441
+
442
+ with np.load(npz_path) as payload:
443
+ runs = payload["runs_curves"]
444
+ except Exception:
445
+ runs = None
446
+ return summary, runs
447
+
448
+
449
+ def _render_monte_carlo_chart(runs, avg_curve: list[float], config: dict, out_path: Path) -> bool:
450
+ """Draws every resampled run as a faint line plus the red average -
451
+ regenerated on every evaluate run so the README visual is always in sync."""
452
+ try:
453
+ import matplotlib
454
+
455
+ matplotlib.use("Agg")
456
+ import numpy as np
457
+ import matplotlib.pyplot as plt
458
+ except Exception:
459
+ return False
460
+ if runs is None or getattr(runs, "size", 0) == 0 or not avg_curve:
461
+ return False
462
+ fig, ax = plt.subplots(figsize=(11, 6), dpi=140)
463
+ x = np.arange(runs.shape[1])
464
+ for row in runs:
465
+ ax.plot(x, row, color="#6f9fc4", alpha=0.05, lw=0.5, zorder=1)
466
+ ax.plot(x, avg_curve, color="red", lw=2.4, zorder=3,
467
+ label=f"Average of all {config.get('n_runs', '?')} runs")
468
+ ax.axhline(1.0, color="black", lw=0.8, ls="--", zorder=2, label="Start (1.0x)")
469
+ ax.set_xlabel("Trading days (resampled)")
470
+ ax.set_ylabel("Equity multiple (start = 1.0x)")
471
+ ax.set_title(
472
+ f"Monte Carlo equity curves - {config.get('n_runs')} runs, "
473
+ f"method={config.get('method')}, block={config.get('block_size')}d, seed={config.get('seed')}"
474
+ )
475
+ ax.grid(alpha=0.25)
476
+ ax.legend(loc="best")
477
+ out_path.parent.mkdir(parents=True, exist_ok=True)
478
+ fig.savefig(out_path)
479
+ plt.close(fig)
480
+ return True
481
+
482
+
483
+ def _build_monte_carlo_compact_markdown(summary: dict | None) -> str:
484
+ if not isinstance(summary, dict):
485
+ return (
486
+ "Monte Carlo simulation has not been run yet - generate it with\n"
487
+ "`aq evaluate --rank-book --monte-carlo` (see CLI Reference)."
488
+ )
489
+ fr = summary["final_return_pct"]
490
+ sh = summary["sharpe"]
491
+ md = summary["max_drawdown"]
492
+ cfg = summary["config"]
493
+ lines = [
494
+ f"{cfg['n_runs']}-run stationary-block bootstrap of the offline rank book's daily net returns",
495
+ f"(method=`{cfg['method']}`, block={cfg['block_size']}d, seed={cfg['seed']}). The red line marks the"
496
+ "\naverage of all generated runs; faint lines are individual resampled histories.",
497
+ "",
498
+ f"![Monte Carlo equity curves]({MONTECARLO_CHART_PATH.relative_to(ROOT_DIR).as_posix()})",
499
+ "",
500
+ "| Metric | p5 | p50 | p95 |",
501
+ "|---|---|---|---|",
502
+ f"| Final return | {fr['p5']}% | {fr['p50']}% | {fr['p95']}% |",
503
+ f"| Annualized Sharpe | {sh['p5']} | {sh['p50']} | {sh['p95']} |",
504
+ f"| Max drawdown | {md['p5']}% | {md['p50']}% | {md['p95']}% |",
505
+ f"| P(negative total return): **{summary['prob_negative_return']}** |||",
506
+ ]
507
+ return "\n".join(lines)
508
+
509
+
510
+ def _build_monte_carlo_full_stats_markdown(summary: dict | None) -> str:
511
+ if not isinstance(summary, dict):
512
+ return "No Monte Carlo data on disk yet."
513
+ cfg = summary["config"]
514
+
515
+ def table(title: str, stats: dict) -> list[str]:
516
+ return [
517
+ f"**{title}**",
518
+ "",
519
+ "| p5 | p25 | p50 | p75 | p95 |",
520
+ "|---|---|---|---|---|",
521
+ f"| {stats['p5']} | {stats['p25']} | {stats['p50']} | {stats['p75']} | {stats['p95']} |",
522
+ "",
523
+ ]
524
+
525
+ lines = [
526
+ "Deeper distribution statistics across all Monte Carlo runs:",
527
+ "",
528
+ *table("Final total return (%)", summary["final_return_pct"]),
529
+ *table("Annualized Sharpe", summary["sharpe"]),
530
+ *table("Max drawdown (%)", summary["max_drawdown"]),
531
+ f"- Best run total return: **{summary['best_run_total_return']}%**",
532
+ f"- Worst run total return: **{summary['worst_run_total_return']}%**",
533
+ f"- Probability of a negative total return: **{summary['prob_negative_return']}**",
534
+ f"- Config: n_runs={cfg['n_runs']}, method=`{cfg['method']}`, block_size={cfg['block_size']}d, "
535
+ f"seed={cfg['seed']} (pinned - the section only changes when underlying results change)",
536
+ "",
537
+ "The red line in the chart above is the element-wise AVERAGE of all run "
538
+ "equity curves; faint blue lines are the individual bootstrap histories.",
539
+ ]
540
+ return "\n".join(lines)
541
+
542
+
543
+ def load_benchmark_comparison() -> dict | None:
544
+ """Loads ml/evaluation/benchmark_comparison.json - `aq evaluate
545
+ --benchmarks`' persisted report (all five baselines + run context), or
546
+ None when it has never been written."""
547
+ return _load_json(EVALUATION_DIR / "benchmark_comparison.json")
548
+
549
+
550
+ def _build_benchmark_markdown(summary: dict | None, eval_summary: dict[str, dict | None]) -> str:
551
+ """Renders the Benchmark Comparison section: the rank book's own net
552
+ Sharpe rows first (the number being benchmarked, per model), then every
553
+ baseline. All from the same dataset window/split, so the comparison is
554
+ like-for-like; a baseline that couldn't compute shows its SKIPPED
555
+ status instead of fabricated numbers."""
556
+ if not isinstance(summary, dict) or not summary.get("baselines"):
557
+ return (
558
+ "Benchmark comparison has not been run yet - generate it with\n"
559
+ "`aq evaluate --benchmarks` (or `aq evaluate --all`)."
560
+ )
561
+ lines = [
562
+ "| Strategy | Net Sharpe | Total return | Note |",
563
+ "|---|---|---|---|",
564
+ ]
565
+ for model_kind in _MODEL_KINDS:
566
+ rank_book = (eval_summary or {}).get(model_kind, {}) or {}
567
+ rank_book_report = rank_book.get("rank_book") or {}
568
+ if rank_book_report:
569
+ lines.append(
570
+ f"| **Rank book ({model_kind})** | {_fmt(rank_book_report.get('net_sharpe'), '.3f')} | "
571
+ f"{_fmt(rank_book_report.get('net_total_return'), '.2%')} | the strategy being benchmarked |"
572
+ )
573
+ for baseline in summary["baselines"]:
574
+ note = baseline.get("status", "OK")
575
+ if note == "OK":
576
+ note = "public benchmark" if baseline["strategy"] in ("sp500", "60_40") else "naive strategy baseline"
577
+ lines.append(
578
+ f"| {baseline['strategy']} | {_fmt(baseline.get('net_sharpe'), '.3f')} | "
579
+ f"{_fmt(baseline.get('total_return_pct'), '.2f')}% | {note} |"
580
+ )
581
+ lines.append("")
582
+ lines.append(
583
+ f"_Same dataset window/split (`{summary.get('split', 'backtest')}`) as the rank-book simulation "
584
+ f"above; Sharpe formula identical across every row. Last updated {_updated_at()} "
585
+ "(auto-generated by `aq evaluate --benchmarks`)._"
586
+ )
587
+ return "\n".join(lines)
588
+
589
+
590
+
591
+
592
+
593
+ def update_readme_evaluation_sections(readme_path: Path = README_PATH) -> bool:
594
+ """Idempotently rebuilds Offline Evaluation, Walk-Forward Training/
595
+ Testing, and Other Metrics from whatever currently exists on disk -
596
+ never raises (mirrors generate_backtest_report.py::
597
+ update_readme_backtest_section()'s own contract), so a report-
598
+ generation bug never fails the `aq evaluate`/`aq train` command that
599
+ triggered it. Returns True if the README was actually modified."""
600
+ if not readme_path.exists():
601
+ return False
602
+ text = readme_path.read_text(encoding="utf-8")
603
+ changed = False
604
+
605
+ eval_summary = load_offline_evaluation_summary()
606
+ walk_forward_summary = load_latest_walk_forward_summary()
607
+ reconciliation = _load_json(EVALUATION_DIR / "book_history_reconciliation.json")
608
+ kill_switch_replay = _load_json(EVALUATION_DIR / "kill_switch_replay.json")
609
+
610
+ mc_summary, mc_runs = load_monte_carlo()
611
+ # V5.4.4 guard (development/Problems.md #114): the chart is a repo
612
+ # asset OF the repo README - only (re)render it when this call is
613
+ # actually refreshing the real README. A caller passing a scratch/
614
+ # tmp readme (the tests, offline experiments) previously rendered
615
+ # whatever its EVALUATION_DIR contained straight over the REAL
616
+ # development/monte_carlo_equity_curves.png - which is exactly how
617
+ # the committed chart got replaced by a degenerate test-fixture
618
+ # render (10 flat runs, 2-point x-axis): every full test-suite run
619
+ # silently clobbered it, and the README showed only the red average
620
+ # line until the next `aq evaluate --monte-carlo` regenerated it.
621
+ if (
622
+ mc_summary is not None
623
+ and mc_summary.get("status") == "OK"
624
+ and readme_path.resolve() == README_PATH.resolve()
625
+ ):
626
+ _render_monte_carlo_chart(mc_runs, mc_summary.get("avg_curve", []), mc_summary.get("config", {}), MONTECARLO_CHART_PATH)
627
+
628
+ updated = _replace_between_markers(text, MONTECARLO_MARKER_START, MONTECARLO_MARKER_END, _build_monte_carlo_compact_markdown(mc_summary))
629
+ if updated is not None:
630
+ text, changed = updated, True
631
+
632
+ # V5.4.4 - benchmark comparison section (rank book vs strategy +
633
+ # public baselines, all on the same window). Rendered from the
634
+ # persisted ml/evaluation/benchmark_comparison.json; like every other
635
+ # section here it degrades to its not-run-yet fallback when that file
636
+ # doesn't exist yet.
637
+ benchmark_summary = load_benchmark_comparison()
638
+ updated = _replace_between_markers(
639
+ text,
640
+ BENCHMARK_MARKER_START,
641
+ BENCHMARK_MARKER_END,
642
+ _build_benchmark_markdown(benchmark_summary, eval_summary),
643
+ )
644
+ if updated is not None:
645
+ text, changed = updated, True
646
+
647
+ updated = _replace_between_markers(
648
+ text,
649
+ MONTECARLO_FULL_STATS_MARKER_START,
650
+ MONTECARLO_FULL_STATS_MARKER_END,
651
+ _build_monte_carlo_full_stats_markdown(mc_summary),
652
+ )
653
+ if updated is not None:
654
+ text, changed = updated, True
655
+
656
+
657
+ updated = _replace_between_markers(text, EVAL_MARKER_START, EVAL_MARKER_END, _build_eval_compact_markdown(eval_summary))
658
+ if updated is not None:
659
+ text, changed = updated, True
660
+
661
+ updated = _replace_between_markers(
662
+ text, EVAL_FULL_STATS_MARKER_START, EVAL_FULL_STATS_MARKER_END, _build_eval_full_stats_markdown(eval_summary)
663
+ )
664
+ if updated is not None:
665
+ text, changed = updated, True
666
+
667
+ updated = _replace_between_markers(
668
+ text, WALKFORWARD_MARKER_START, WALKFORWARD_MARKER_END, _build_walk_forward_compact_markdown(walk_forward_summary)
669
+ )
670
+ if updated is not None:
671
+ text, changed = updated, True
672
+
673
+ updated = _replace_between_markers(
674
+ text,
675
+ WALKFORWARD_FULL_STATS_MARKER_START,
676
+ WALKFORWARD_FULL_STATS_MARKER_END,
677
+ _build_walk_forward_full_stats_markdown(walk_forward_summary),
678
+ )
679
+ if updated is not None:
680
+ text, changed = updated, True
681
+
682
+ updated = _replace_between_markers(
683
+ text,
684
+ OTHER_METRICS_MARKER_START,
685
+ OTHER_METRICS_MARKER_END,
686
+ _build_other_metrics_markdown(eval_summary, walk_forward_summary, reconciliation, kill_switch_replay),
687
+ )
688
+ if updated is not None:
689
+ text, changed = updated, True
690
+
691
+ if changed:
692
+ _atomic_write(readme_path, text)
693
+ return changed
694
+
695
+
696
+ def main() -> int:
697
+ parser = argparse.ArgumentParser(
698
+ description="Regenerate README.md's Offline Evaluation / Walk-Forward / Other Metrics sections."
699
+ )
700
+ parser.add_argument("--readme", type=Path, default=README_PATH)
701
+ args = parser.parse_args()
702
+
703
+ updated = update_readme_evaluation_sections(args.readme)
704
+ if updated:
705
+ print(f"Updated {args.readme} with the latest evaluation/walk-forward results.")
706
+ return 0
707
+ print("Nothing to update (no source files found, or README markers missing).", file=sys.stderr)
708
+ return 1
709
+
710
+
711
+ if __name__ == "__main__":
712
+ sys.exit(main())