aether-quant 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. aether_quant-0.0.1.dist-info/METADATA +7 -0
  2. aether_quant-0.0.1.dist-info/RECORD +99 -0
  3. aether_quant-0.0.1.dist-info/WHEEL +5 -0
  4. aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
  5. aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
  6. aether_quant-0.0.1.dist-info/top_level.txt +13 -0
  7. analyzer/README.md +69 -0
  8. analyzer/__init__.py +17 -0
  9. analyzer/market_analyzer.py +521 -0
  10. aq_cli.py +3633 -0
  11. audit/README.md +47 -0
  12. audit/__init__.py +38 -0
  13. audit/hash_chain.py +52 -0
  14. audit/postgres_audit.py +61 -0
  15. audit/postgres_worker.py +402 -0
  16. audit/redis_queue.py +124 -0
  17. audit/status_export.py +62 -0
  18. data_pipeline/README.md +257 -0
  19. data_pipeline/__init__.py +6 -0
  20. data_pipeline/bar_synthesis.py +97 -0
  21. data_pipeline/dividend_backfill.py +243 -0
  22. data_pipeline/factor_file_backfill.py +276 -0
  23. data_pipeline/fetch.py +206 -0
  24. data_pipeline/fred_backfill.py +435 -0
  25. data_pipeline/ib_backfill.py +287 -0
  26. data_pipeline/v2_manifest.py +75 -0
  27. data_pipeline/yfinance_backfill.py +501 -0
  28. evaluation/README.md +152 -0
  29. evaluation/__init__.py +73 -0
  30. evaluation/ablation.py +310 -0
  31. evaluation/benchmark_comparison.py +195 -0
  32. evaluation/confidence_threshold_calibration.py +157 -0
  33. evaluation/factor_neutralization_check.py +94 -0
  34. evaluation/feature_parity.py +321 -0
  35. evaluation/feature_reconciliation.py +286 -0
  36. evaluation/impact_model.py +77 -0
  37. evaluation/kill_switch_replay.py +186 -0
  38. evaluation/limit_fill_simulator.py +152 -0
  39. evaluation/model_predictions.py +192 -0
  40. evaluation/monte_carlo.py +161 -0
  41. evaluation/public_benchmarks.py +117 -0
  42. evaluation/rank_book_simulator.py +494 -0
  43. evaluation/rank_ic_core.py +204 -0
  44. evaluation/rank_signal_calibration.py +690 -0
  45. evaluation/rolling_ic_gate_calibration.py +190 -0
  46. evaluation/rolling_ic_gate_replay.py +138 -0
  47. execution/README.md +281 -0
  48. execution/__init__.py +80 -0
  49. execution/config_cache.py +48 -0
  50. execution/cost_model.py +213 -0
  51. execution/lean_config_render.py +131 -0
  52. execution/live_credentials.py +38 -0
  53. execution/live_credentials_io.py +52 -0
  54. execution/order_gate.py +388 -0
  55. execution/paper_readiness.py +181 -0
  56. execution/paper_readiness_io.py +54 -0
  57. execution/paper_readiness_report.py +103 -0
  58. execution/paper_readiness_scheduler.py +137 -0
  59. execution/reconciliation.py +160 -0
  60. execution/runtime_config_io.py +38 -0
  61. execution/secret_scan.py +73 -0
  62. features/README.md +77 -0
  63. features/__init__.py +137 -0
  64. features/alt_data_features.py +154 -0
  65. features/bond_features.py +275 -0
  66. features/cross_asset_sensitivity.py +79 -0
  67. features/derivatives_macro_features.py +72 -0
  68. features/macro_features.py +79 -0
  69. features/options_greeks.py +329 -0
  70. features/sector_map.py +53 -0
  71. features/technical_indicators.py +185 -0
  72. generate_backtest_report.py +311 -0
  73. generate_evaluation_report.py +712 -0
  74. inference/README.md +105 -0
  75. inference/__init__.py +36 -0
  76. inference/exported_model.py +929 -0
  77. inference/parallel_inference.py +160 -0
  78. inference/strategy_selector_inference.py +75 -0
  79. json_safety.py +130 -0
  80. portfolio/README.md +447 -0
  81. portfolio/__init__.py +41 -0
  82. portfolio/book_construction.py +786 -0
  83. portfolio/book_neutrality.py +178 -0
  84. portfolio/hrp_allocation.py +159 -0
  85. portfolio/options_arbitrage_detector.py +232 -0
  86. portfolio/options_assignment_risk.py +119 -0
  87. portfolio/options_margin_sizing.py +333 -0
  88. portfolio/options_strategy.py +1999 -0
  89. portfolio/rank_signal.py +219 -0
  90. portfolio/rolling_ic_gate.py +183 -0
  91. risk/README.md +275 -0
  92. risk/__init__.py +38 -0
  93. risk/asset_class_router.py +584 -0
  94. risk/forex_risk.py +158 -0
  95. risk/futures_risk.py +202 -0
  96. risk/kill_switch.py +261 -0
  97. risk/manual_override.py +118 -0
  98. risk/position_sizing.py +303 -0
  99. risk/rl_sizing.py +189 -0
aq_cli.py ADDED
@@ -0,0 +1,3633 @@
1
+ """`aq` — a thin convenience CLI wrapping Aether Quant's day-to-day commands.
2
+
3
+ Matches this codebase's existing CLI convention exactly (see
4
+ `retraining/orchestrator.py`'s `argparse` + `subparsers.add_parser(...)`
5
+ shape) - a single-file dispatcher, not a framework. Every subcommand other
6
+ than `trade-lock` and `fetch` is a thin `subprocess.run(...)` wrapper around
7
+ a command that already exists and is already documented elsewhere
8
+ (README.md, development/infrastructure.md) - no logic is reimplemented
9
+ here, this file only saves typing. `trade-lock` and `fetch` are the two
10
+ exceptions: they call `risk/manual_override.py` and
11
+ `data_pipeline/fetch.py` directly, in-process, no subprocess.
12
+
13
+ Deliberately scoped for v1 - wraps the commands already in daily use, not
14
+ every command mentioned anywhere in the project. Designed to be extended
15
+ incrementally: add a new `subparsers.add_parser(...)` block plus one `elif`
16
+ branch in `main()` for each new command, following the existing pattern.
17
+
18
+ Install once (registers the `aq` command on PATH inside the active venv):
19
+ pip install -e .
20
+ Then:
21
+ aq --help
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import argparse
27
+ import copy
28
+ import json
29
+ import os
30
+ import re
31
+ import shutil
32
+ import subprocess
33
+ import sys
34
+ import time
35
+ import urllib.request
36
+ import uuid
37
+ from collections.abc import Iterator, Mapping
38
+ from datetime import date
39
+ from importlib.metadata import version as installed_version
40
+ from pathlib import Path
41
+
42
+ from data_pipeline.fetch import ASSET_CLASSES, fetch_adhoc_asset
43
+ from data_pipeline.ib_backfill import (
44
+ IBNotConfiguredError,
45
+ attempt_connection,
46
+ connect_ib,
47
+ disconnect_ib,
48
+ fetch_future_historical_bars,
49
+ fetch_option_historical_bars,
50
+ ib_readiness_status,
51
+ load_futures_contract_specs,
52
+ )
53
+ from risk.manual_override import (
54
+ read_kill_switch_manual_override,
55
+ read_manual_trade_lock_override,
56
+ write_kill_switch_manual_override,
57
+ write_manual_trade_lock_override,
58
+ )
59
+
60
+ IB_ASSET_CLASSES = ("futures", "options")
61
+
62
+ ROOT_DIR = Path(__file__).resolve().parent
63
+ CONFIG_PATH = ROOT_DIR / "config.json"
64
+ LEAN_JSON_PATH = ROOT_DIR / "lean.json"
65
+ WEBUI_DIR = ROOT_DIR / "webui"
66
+ README_PATH = ROOT_DIR / "README.md"
67
+ ML_DIR = ROOT_DIR / "ml"
68
+
69
+ PACKAGE_NAME = "aether-quant"
70
+ UPDATE_CACHE_PATH = Path.home() / ".aq" / "update_check.json"
71
+ UPDATE_CHECK_INTERVAL_SECONDS = 24 * 60 * 60
72
+ UPDATE_CHECK_TIMEOUT_SECONDS = 2
73
+
74
+ # `lean backtest .` resolves quantconnect/lean:latest by default when no
75
+ # --image is given - a MUTABLE tag, so Docker re-checks and re-pulls
76
+ # whatever layers changed on every single run, even against an already-
77
+ # fully-cached local image (confirmed directly: a ~42.5GB re-pull kicked
78
+ # off on this exact machine because Docker Hub's `latest` had moved to a
79
+ # newer build than what was cached). Pinned here to a specific numbered
80
+ # QuantConnect build tag instead (confirmed to exist on Docker Hub) so
81
+ # every clone of this repo - not just this machine - gets a byte-
82
+ # identical, one-time-download engine image that never silently re-pulls.
83
+ # To intentionally move to a newer Lean engine: pull the new tag by hand
84
+ # first (`docker pull quantconnect/lean:<new-tag>`), confirm it works,
85
+ # then bump this constant - never let it drift back to `latest`.
86
+ PINNED_LEAN_ENGINE_IMAGE = "quantconnect/lean:17900"
87
+ # Local derivative used by `aq backtest` by default. It is built once from
88
+ # PINNED_LEAN_ENGINE_IMAGE and installs the packages this project needs but
89
+ # the official image does not provide (redis, httpx - see requirements/
90
+ # lean-runtime.txt). Keeping them in the image avoids Lean CLI's Windows-host
91
+ # bind mount of a generated requirements.txt file, which fails on some
92
+ # Docker Desktop setups. Tag is DERIVED from PINNED_LEAN_ENGINE_IMAGE's own
93
+ # tag, never a second hardcoded copy - _ensure_local_lean_engine_image()'s
94
+ # cache check is a `docker image inspect` on this exact tag, so if the two
95
+ # constants could drift apart, bumping the pinned base without remembering
96
+ # to bump this one too would silently keep reusing a stale local image
97
+ # built from the old base forever.
98
+ LOCAL_LEAN_ENGINE_IMAGE = f"aether-quant-lean:{PINNED_LEAN_ENGINE_IMAGE.rsplit(':', 1)[-1]}"
99
+
100
+ _ANSI_ESCAPE_PATTERN = re.compile(r"\x1b\[[0-9;]*m")
101
+ _TEST_BADGE_MARKER_START = "<!-- AQ:TEST_BADGE_START -->"
102
+ _TEST_BADGE_MARKER_END = "<!-- AQ:TEST_BADGE_END -->"
103
+ _TEST_COUNT_MARKER_START = "<!-- AQ:TEST_COUNT_START -->"
104
+ _TEST_COUNT_MARKER_END = "<!-- AQ:TEST_COUNT_END -->"
105
+
106
+
107
+ def _run(cmd: list[str], cwd: Path = ROOT_DIR) -> int:
108
+ """Runs a command with inherited stdout/stderr (live output), returns its exit code."""
109
+ result = subprocess.run(cmd, cwd=str(cwd))
110
+ return result.returncode
111
+
112
+
113
+ def _find_quantconnect_lean_binary() -> str | None:
114
+ """Plain `lean` on PATH is ambiguous on machines with `elan` (Lean 4, the
115
+ theorem prover) installed - it ships its own `lean` binary under the same
116
+ name as QuantConnect's Lean CLI (`pip install lean`). Disambiguate by
117
+ checking `--version` output (Lean 4 prints "Lean (version 4...."; the
118
+ QuantConnect CLI does not), preferring this repo's own venv first."""
119
+ bin_dir_name = "Scripts" if sys.platform == "win32" else "bin"
120
+ binary_name = "lean.exe" if sys.platform == "win32" else "lean"
121
+ candidates = [str(ROOT_DIR / ".venv" / bin_dir_name / binary_name)]
122
+ on_path = shutil.which("lean")
123
+ if on_path:
124
+ candidates.append(on_path)
125
+
126
+ for candidate in candidates:
127
+ if candidate != on_path and not Path(candidate).exists():
128
+ continue
129
+ try:
130
+ result = subprocess.run([candidate, "--version"], capture_output=True, text=True, timeout=30)
131
+ except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
132
+ continue
133
+ if "Lean (version" not in (result.stdout or "") + (result.stderr or ""):
134
+ return candidate
135
+ return None
136
+
137
+
138
+ def _ensure_local_lean_engine_image() -> bool:
139
+ """Build the small project-specific LEAN image when it is not cached."""
140
+ image_check = subprocess.run(
141
+ ["docker", "image", "inspect", LOCAL_LEAN_ENGINE_IMAGE],
142
+ cwd=str(ROOT_DIR),
143
+ stdout=subprocess.DEVNULL,
144
+ stderr=subprocess.DEVNULL,
145
+ )
146
+ if image_check.returncode == 0:
147
+ return True
148
+
149
+ dockerfile = ROOT_DIR / "Dockerfile.lean"
150
+ if not dockerfile.is_file():
151
+ print(f"error: missing local LEAN image definition: {dockerfile}", file=sys.stderr)
152
+ return False
153
+
154
+ print(
155
+ f"Building {LOCAL_LEAN_ENGINE_IMAGE} from {PINNED_LEAN_ENGINE_IMAGE} "
156
+ "(one-time setup; the large base image is reused)..."
157
+ )
158
+ return _run(
159
+ [
160
+ "docker",
161
+ "build",
162
+ "--file",
163
+ str(dockerfile),
164
+ "--tag",
165
+ LOCAL_LEAN_ENGINE_IMAGE,
166
+ "--build-arg",
167
+ f"LEAN_BASE_IMAGE={PINNED_LEAN_ENGINE_IMAGE}",
168
+ str(ROOT_DIR),
169
+ ]
170
+ ) == 0
171
+
172
+
173
+ def _parse_simple_version(value: str) -> tuple[int, ...] | None:
174
+ """Best-effort "X.Y.Z" -> (X, Y, Z) parse. Returns None for anything that
175
+ isn't a clean dotted-integer release version - dev/local builds (e.g.
176
+ setuptools-scm's "0.1.dev35+gc744f9ca4.d20260704" fallback for untagged
177
+ installs) simply never get flagged as outdated, which is the correct
178
+ behavior here."""
179
+ try:
180
+ return tuple(int(part) for part in value.split("."))
181
+ except ValueError:
182
+ return None
183
+
184
+
185
+ def _iso_date(value: str) -> str:
186
+ """Validates --start/--end as strict ISO 8601 YYYY-MM-DD, matching the
187
+ convention used everywhere else in this repo (config.json,
188
+ yfinance_backfill.py) - rejects other formats (e.g. DD.MM.YYYY) with a
189
+ clear error instead of a confusing downstream yfinance failure."""
190
+ try:
191
+ date.fromisoformat(value)
192
+ except ValueError as exc:
193
+ raise argparse.ArgumentTypeError(f"invalid date {value!r} - expected ISO 8601 YYYY-MM-DD") from exc
194
+ return value
195
+
196
+
197
+ def _read_update_cache() -> dict:
198
+ if not UPDATE_CACHE_PATH.exists():
199
+ return {}
200
+ try:
201
+ return json.loads(UPDATE_CACHE_PATH.read_text(encoding="utf-8"))
202
+ except (OSError, ValueError):
203
+ return {}
204
+
205
+
206
+ def _write_update_cache(latest_version: str) -> None:
207
+ UPDATE_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
208
+ payload = {"last_checked": time.time(), "latest_version": latest_version}
209
+ UPDATE_CACHE_PATH.write_text(json.dumps(payload), encoding="utf-8")
210
+
211
+
212
+ def _fetch_latest_version_from_pypi() -> str | None:
213
+ try:
214
+ url = f"https://pypi.org/pypi/{PACKAGE_NAME}/json"
215
+ with urllib.request.urlopen(url, timeout=UPDATE_CHECK_TIMEOUT_SECONDS) as response: # noqa: S310
216
+ payload = json.loads(response.read().decode("utf-8"))
217
+ return payload["info"]["version"]
218
+ except Exception:
219
+ return None
220
+
221
+
222
+ def _latest_known_version() -> str | None:
223
+ cache = _read_update_cache()
224
+ last_checked = cache.get("last_checked", 0)
225
+ if time.time() - last_checked < UPDATE_CHECK_INTERVAL_SECONDS:
226
+ return cache.get("latest_version")
227
+
228
+ # Update the cache timestamp even on a failed fetch, so an offline user
229
+ # doesn't pay the network timeout again on every single command - only
230
+ # once per interval.
231
+ latest = _fetch_latest_version_from_pypi()
232
+ _write_update_cache(latest or cache.get("latest_version", ""))
233
+ return latest or cache.get("latest_version")
234
+
235
+
236
+ def check_for_update() -> None:
237
+ """Prints a one-line notice to stderr if a newer aether-quant release is
238
+ available on PyPI. Never raises, never blocks a real command by more
239
+ than the short network timeout, and only actually checks PyPI once per
240
+ 24h (cached in ~/.aq/update_check.json). Opt out with
241
+ AQ_SKIP_UPDATE_CHECK=1 (e.g. for CI/scripted usage)."""
242
+ if os.environ.get("AQ_SKIP_UPDATE_CHECK"):
243
+ return
244
+ try:
245
+ installed = installed_version(PACKAGE_NAME)
246
+ latest = _latest_known_version()
247
+ if not latest:
248
+ return
249
+ installed_tuple = _parse_simple_version(installed)
250
+ latest_tuple = _parse_simple_version(latest)
251
+ if installed_tuple is None or latest_tuple is None:
252
+ return
253
+ if installed_tuple < latest_tuple:
254
+ print(
255
+ f"aq: a newer version is available ({latest}, you have {installed}) - "
256
+ f"upgrade with: pip install --upgrade {PACKAGE_NAME}",
257
+ file=sys.stderr,
258
+ )
259
+ except Exception:
260
+ pass
261
+
262
+
263
+ def cmd_train(args: argparse.Namespace) -> int:
264
+ seed = getattr(args, "seed", None)
265
+ ranking_objective = getattr(args, "ranking_objective", None)
266
+ if (seed is not None or ranking_objective is not None) and not (args.multitask_only or args.sequence_only):
267
+ print(
268
+ "aq train: --seed/--ranking-objective only apply to --multitask-only/--sequence-only - ignored here.",
269
+ file=sys.stderr,
270
+ )
271
+ # V5.1 Phase 4 (item 4) - same not-silently-ignored precedent as
272
+ # --seed/--ranking-objective above.
273
+ if (
274
+ getattr(args, "include_multitask", False)
275
+ or getattr(args, "include_sequence", False)
276
+ or getattr(args, "metrics", None) is not None
277
+ ) and not args.walk_forward:
278
+ print(
279
+ "aq train: --include-multitask/--include-sequence/--metrics only apply to --walk-forward - ignored here.",
280
+ file=sys.stderr,
281
+ )
282
+ if args.gating_only:
283
+ return _train_gating_only()
284
+ if args.multitask_only:
285
+ return _train_multitask_only(seed=seed, ranking_objective=ranking_objective)
286
+ if args.sequence_only:
287
+ return _train_sequence_only(seed=seed, ranking_objective=ranking_objective)
288
+ if args.topology_only:
289
+ return _train_topology_only()
290
+ if args.strategy_selector_only:
291
+ return _train_strategy_selector_only()
292
+ if args.rl_sizing_only:
293
+ return _train_rl_sizing_only()
294
+ cmd = [sys.executable, "train.py"]
295
+ # A fully bare `aq train` (no scope flag at all) is the only case that
296
+ # chains gating/multitask/sequence training on top of train.py's own
297
+ # baseline+experts run below - --dataset-only/--init-only/--experts-only/
298
+ # --walk-forward all still mean exactly what they say and stop there.
299
+ bare_run = not (args.dataset_only or args.init_only or args.experts_only or args.walk_forward)
300
+ if args.dataset_only:
301
+ cmd.append("--dataset-only")
302
+ elif args.init_only:
303
+ cmd.append("--init-only")
304
+ elif args.experts_only:
305
+ cmd.append("--experts-only")
306
+ elif args.walk_forward:
307
+ cmd.append("--walk-forward")
308
+ if args.step_days is not None:
309
+ cmd += ["--step-days", str(args.step_days)]
310
+ if args.mode is not None:
311
+ cmd += ["--mode", args.mode]
312
+ if getattr(args, "include_multitask", False):
313
+ cmd.append("--include-multitask")
314
+ if getattr(args, "include_sequence", False):
315
+ cmd.append("--include-sequence")
316
+ if getattr(args, "metrics", None) is not None:
317
+ cmd += ["--metrics", args.metrics]
318
+
319
+ returncode = _run(cmd)
320
+ if returncode != 0 or not bare_run:
321
+ return returncode
322
+
323
+ # Each of these installs straight into active ml/ the same way its own
324
+ # --gating-only/--multitask-only/--sequence-only flag does - a bare
325
+ # `aq train` genuinely trains baseline + experts + gating + multitask +
326
+ # sequence, not just the first two. Stops at the first failure rather
327
+ # than silently continuing with a partially-updated active model.
328
+ for stage_name, stage_fn in (
329
+ ("gating", _train_gating_only),
330
+ ("multitask", _train_multitask_only),
331
+ ("sequence", _train_sequence_only),
332
+ ):
333
+ returncode = stage_fn()
334
+ if returncode != 0:
335
+ print(f"aq train: stopped after {stage_name} training failed (exit {returncode}).", file=sys.stderr)
336
+ return returncode
337
+ return 0
338
+
339
+
340
+ def _train_gating_only() -> int:
341
+ """`aq train --gating-only`: trains the learned gating blend
342
+ (train_gating.py) and installs it straight into active ml/, mirroring
343
+ what `train.py --experts-only` already does for the expert models.
344
+
345
+ train_gating.py always writes to ml/versions/<version_id>/ (same
346
+ versioned-candidate convention every other trainer in this project
347
+ uses), so this generates a throwaway version-id, runs the trainer,
348
+ then copies the 3 resulting artifacts into active ml/ - the same
349
+ manual promotion-simulation step already documented for verifying this
350
+ trainer, skipping the full retraining/validate/backtest/commit/promote
351
+ pipeline since this is an ad-hoc, user-triggered run, not a scheduled
352
+ candidate."""
353
+ version_id = f"gating-only-{uuid.uuid4()}"
354
+ returncode = _run([sys.executable, "train_gating.py", "--version-id", version_id])
355
+ if returncode != 0:
356
+ return returncode
357
+
358
+ version_dir = ROOT_DIR / "ml" / "versions" / version_id
359
+ artifact_names = ("gating_model.json", "gating_feature_schema.json", "gating_training_metrics.json")
360
+ if any(not (version_dir / name).exists() for name in artifact_names):
361
+ print(
362
+ "aq train --gating-only: train_gating.py exited 0 but skipped writing artifacts "
363
+ "(likely insufficient validation/backtest rows) - active ml/ left unchanged.",
364
+ file=sys.stderr,
365
+ )
366
+ return 0
367
+
368
+ ml_dir = ROOT_DIR / "ml"
369
+ for name in artifact_names:
370
+ shutil.copy2(version_dir / name, ml_dir / name)
371
+ print(f"aq train --gating-only: copied {', '.join(artifact_names)} into active ml/.")
372
+ return 0
373
+
374
+
375
+ def _train_multitask_only(seed: int | None = None, ranking_objective: str | None = None) -> int:
376
+ """`aq train --multitask-only`: trains the joint direction+magnitude+
377
+ volatility model (train_multitask.py) and installs it straight into
378
+ active ml/ - identical shape to _train_gating_only() above, including
379
+ the throwaway version-id / manual promotion-simulation / "skipped must
380
+ never look like failed" handling.
381
+
382
+ seed/ranking_objective (V5.1 Phase 3, item 1) - passed straight
383
+ through to train_multitask.py's own --seed/--ranking-objective flags
384
+ when provided, the `aq`-level entry point for the seed-ensembling/
385
+ ranking-objective-A/B workflow that previously required calling the
386
+ script directly."""
387
+ version_id = f"multitask-only-{uuid.uuid4()}"
388
+ cmd = [sys.executable, "train_multitask.py", "--version-id", version_id]
389
+ if seed is not None:
390
+ cmd += ["--seed", str(seed)]
391
+ if ranking_objective is not None:
392
+ cmd += ["--ranking-objective", ranking_objective]
393
+ returncode = _run(cmd)
394
+ if returncode != 0:
395
+ return returncode
396
+
397
+ version_dir = ROOT_DIR / "ml" / "versions" / version_id
398
+ artifact_names = ("multitask_model.json", "multitask_feature_schema.json", "multitask_training_metrics.json")
399
+ if any(not (version_dir / name).exists() for name in artifact_names):
400
+ print(
401
+ "aq train --multitask-only: train_multitask.py exited 0 but skipped writing artifacts "
402
+ "(likely insufficient train/validation/backtest rows) - active ml/ left unchanged.",
403
+ file=sys.stderr,
404
+ )
405
+ return 0
406
+
407
+ ml_dir = ROOT_DIR / "ml"
408
+ for name in artifact_names:
409
+ shutil.copy2(version_dir / name, ml_dir / name)
410
+ print(f"aq train --multitask-only: copied {', '.join(artifact_names)} into active ml/.")
411
+ return 0
412
+
413
+
414
+ def _train_sequence_only(seed: int | None = None, ranking_objective: str | None = None) -> int:
415
+ """`aq train --sequence-only`: trains the Phase 2 causal-TCN sequence
416
+ encoder (train_sequence.py) and installs it straight into active ml/ -
417
+ identical shape to _train_multitask_only()/_train_gating_only() above.
418
+
419
+ seed/ranking_objective - see _train_multitask_only()'s identical
420
+ docstring."""
421
+ version_id = f"sequence-only-{uuid.uuid4()}"
422
+ cmd = [sys.executable, "train_sequence.py", "--version-id", version_id]
423
+ if seed is not None:
424
+ cmd += ["--seed", str(seed)]
425
+ if ranking_objective is not None:
426
+ cmd += ["--ranking-objective", ranking_objective]
427
+ returncode = _run(cmd)
428
+ if returncode != 0:
429
+ return returncode
430
+
431
+ version_dir = ROOT_DIR / "ml" / "versions" / version_id
432
+ artifact_names = ("sequence_model.json", "sequence_feature_schema.json", "sequence_training_metrics.json")
433
+ if any(not (version_dir / name).exists() for name in artifact_names):
434
+ print(
435
+ "aq train --sequence-only: train_sequence.py exited 0 but skipped writing artifacts "
436
+ "(likely insufficient train/validation/backtest rows) - active ml/ left unchanged.",
437
+ file=sys.stderr,
438
+ )
439
+ return 0
440
+
441
+ ml_dir = ROOT_DIR / "ml"
442
+ for name in artifact_names:
443
+ shutil.copy2(version_dir / name, ml_dir / name)
444
+ print(f"aq train --sequence-only: copied {', '.join(artifact_names)} into active ml/.")
445
+ return 0
446
+
447
+
448
+ def _train_topology_only() -> int:
449
+ """`aq train --topology-only`: trains the learned topology overlay
450
+ (train_topology.py) and installs it straight into active ml/ -
451
+ identical shape to _train_multitask_only()/_train_gating_only()/
452
+ _train_sequence_only() above.
453
+
454
+ Different data source and a real prerequisite the other three
455
+ trainers don't have: train_topology.py fits over realized trading
456
+ outcomes pulled from Postgres (AETHER_POSTGRES_DSN env var, read by
457
+ train_topology.py itself - no DSN plumbing needed here), and needs
458
+ at least phase_v2.topology_learning.training.min_training_events
459
+ (default 500) usable events from a lookback_days (default 90) window.
460
+ That data only accumulates from runs with the full stack (Postgres +
461
+ the audit worker) up - on a fresh checkout, or before Postgres has
462
+ been running long enough, this is expected to hit the "skipped"
463
+ branch below, not the "artifacts written" one. development/
464
+ Problems.md #56 has the full offset-scale story this trainer feeds."""
465
+ version_id = f"topology-only-{uuid.uuid4()}"
466
+ returncode = _run([sys.executable, "train_topology.py", "--version-id", version_id])
467
+ if returncode != 0:
468
+ return returncode
469
+
470
+ version_dir = ROOT_DIR / "ml" / "versions" / version_id
471
+ artifact_names = ("topology_model.json", "topology_feature_schema.json", "topology_training_metrics.json")
472
+ if any(not (version_dir / name).exists() for name in artifact_names):
473
+ print(
474
+ "aq train --topology-only: train_topology.py exited 0 but skipped writing artifacts "
475
+ "(fewer than min_training_events realized-outcome events in the lookback window - needs "
476
+ "Postgres up and the audit worker running long enough to accumulate them) - active ml/ left "
477
+ "unchanged.",
478
+ file=sys.stderr,
479
+ )
480
+ return 0
481
+
482
+ ml_dir = ROOT_DIR / "ml"
483
+ for name in artifact_names:
484
+ shutil.copy2(version_dir / name, ml_dir / name)
485
+ print(f"aq train --topology-only: copied {', '.join(artifact_names)} into active ml/.")
486
+ return 0
487
+
488
+
489
+ def _train_strategy_selector_only() -> int:
490
+ """`aq train --strategy-selector-only`: trains the learned multi-leg
491
+ strategy-selector model (train_strategy_selector.py, V4.7,
492
+ development/Problems.md #29's own framing) and installs it straight
493
+ into active ml/ - identical shape to _train_topology_only()/
494
+ _train_multitask_only()/_train_gating_only()/_train_sequence_only()
495
+ above.
496
+
497
+ Different, and much harder to satisfy, prerequisite than any of the
498
+ other four: train_strategy_selector.py fits over
499
+ "option_strategy_outcome" experience events (Postgres,
500
+ AETHER_POSTGRES_DSN env var), which only main.py::
501
+ _emit_option_strategy_outcome_if_pending() can ever emit - and that
502
+ only fires for a multi-leg OPTION position that was actually opened
503
+ AND closed. Unlike train_topology.py (dormant only until enough
504
+ Postgres VOLUME accumulates from an already-running system), this
505
+ trainer has NO data source at all until real option positions
506
+ actually trade - confirmed during scoping research that every options
507
+ code path in this repo is still IB-unverified with zero option assets
508
+ in the live universe. Expect the "skipped" branch below on every run
509
+ in this environment, indefinitely - see train_strategy_selector.py's
510
+ own module docstring for the full story."""
511
+ version_id = f"strategy-selector-only-{uuid.uuid4()}"
512
+ returncode = _run([sys.executable, "train_strategy_selector.py", "--version-id", version_id])
513
+ if returncode != 0:
514
+ return returncode
515
+
516
+ version_dir = ROOT_DIR / "ml" / "versions" / version_id
517
+ artifact_names = (
518
+ "strategy_selector_model.json",
519
+ "strategy_selector_feature_schema.json",
520
+ "strategy_selector_training_metrics.json",
521
+ )
522
+ if any(not (version_dir / name).exists() for name in artifact_names):
523
+ print(
524
+ "aq train --strategy-selector-only: train_strategy_selector.py exited 0 but skipped writing "
525
+ "artifacts (no option_strategy_outcome events yet - needs real option positions to have actually "
526
+ "traded and closed) - active ml/ left unchanged.",
527
+ file=sys.stderr,
528
+ )
529
+ return 0
530
+
531
+ ml_dir = ROOT_DIR / "ml"
532
+ for name in artifact_names:
533
+ shutil.copy2(version_dir / name, ml_dir / name)
534
+ print(f"aq train --strategy-selector-only: copied {', '.join(artifact_names)} into active ml/.")
535
+ return 0
536
+
537
+
538
+ def _train_rl_sizing_only() -> int:
539
+ """`aq train --rl-sizing-only`: trains the offline contextual-bandit
540
+ sizing overlay (train_rl_sizing.py, development/Problems.md #71) and
541
+ installs it straight into active ml/ - identical shape to
542
+ _train_topology_only()/_train_strategy_selector_only() above.
543
+
544
+ Different, but much more tractable, prerequisite than either of those
545
+ two: train_rl_sizing.py reads ml/datasets/{validation,backtest}_dataset.csv
546
+ - NOT Postgres, NOT real option trades - so this can run any time a
547
+ normal `python train.py` has already produced a multitask model and
548
+ the standard dataset files. Requires phase1.features.input_set to
549
+ already include Component D's 3 alt-data feature names (a retrain
550
+ with those in the schema) - see train_rl_sizing.py's own module
551
+ docstring for the full honest framing (full-information contextual
552
+ bandit, NOT off-policy RL) and risk/rl_sizing.py for the abandon
553
+ criteria this feature should be judged against before ever being
554
+ enabled by default."""
555
+ version_id = f"rl-sizing-only-{uuid.uuid4()}"
556
+ returncode = _run([sys.executable, "train_rl_sizing.py", "--version-id", version_id])
557
+ if returncode != 0:
558
+ return returncode
559
+
560
+ version_dir = ROOT_DIR / "ml" / "versions" / version_id
561
+ artifact_names = ("rl_sizing_model.json", "rl_sizing_feature_schema.json", "rl_sizing_training_metrics.json")
562
+ if any(not (version_dir / name).exists() for name in artifact_names):
563
+ print(
564
+ "aq train --rl-sizing-only: train_rl_sizing.py exited 0 but skipped writing artifacts "
565
+ "(fewer than min_training_rows usable rows, or the multitask model/dataset files are missing) - "
566
+ "active ml/ left unchanged.",
567
+ file=sys.stderr,
568
+ )
569
+ return 0
570
+
571
+ ml_dir = ROOT_DIR / "ml"
572
+ for name in artifact_names:
573
+ shutil.copy2(version_dir / name, ml_dir / name)
574
+ print(f"aq train --rl-sizing-only: copied {', '.join(artifact_names)} into active ml/.")
575
+ return 0
576
+
577
+
578
+ def _update_readme_test_badge(passed: int, failed: int) -> None:
579
+ """Atomically rewrites the shields.io test-count badge AND every
580
+ AQ:TEST_COUNT-marked "N tests" prose mention (Test Suite section,
581
+ Module Documentation table's tests/ row) in README.md, so neither ever
582
+ drifts from the real collected-test total. Mirrors the equivalent
583
+ mechanism in the sibling Aether-Vault project's `av test`. Never
584
+ raises - a badge-update bug must never fail `aq test` itself."""
585
+ total = passed + failed
586
+ if total == 0:
587
+ return # nothing collected - leave the badge/count alone rather than zero them out
588
+ if not README_PATH.is_file():
589
+ return
590
+ text = README_PATH.read_text(encoding="utf-8")
591
+
592
+ if _TEST_BADGE_MARKER_START in text and _TEST_BADGE_MARKER_END in text:
593
+ color = "brightgreen" if failed == 0 else "red"
594
+ badge = (
595
+ f'<img src="https://img.shields.io/badge/tests-{passed}%2F{total}%20passing-{color}'
596
+ f'?style=flat-square&labelColor=1A1A1A" alt="{passed} of {total} tests passing">'
597
+ )
598
+ badge_pattern = re.compile(
599
+ re.escape(_TEST_BADGE_MARKER_START) + r".*?" + re.escape(_TEST_BADGE_MARKER_END), re.DOTALL
600
+ )
601
+ text = badge_pattern.sub(f"{_TEST_BADGE_MARKER_START}{badge}{_TEST_BADGE_MARKER_END}", text, count=1)
602
+
603
+ if _TEST_COUNT_MARKER_START in text and _TEST_COUNT_MARKER_END in text:
604
+ count_pattern = re.compile(
605
+ re.escape(_TEST_COUNT_MARKER_START) + r".*?" + re.escape(_TEST_COUNT_MARKER_END), re.DOTALL
606
+ )
607
+ text = count_pattern.sub(f"{_TEST_COUNT_MARKER_START}{total}{_TEST_COUNT_MARKER_END}", text)
608
+
609
+ original_text = README_PATH.read_text(encoding="utf-8")
610
+ if text == original_text:
611
+ return
612
+
613
+ tmp_path = README_PATH.with_suffix(README_PATH.suffix + ".tmp")
614
+ tmp_path.write_text(text, encoding="utf-8")
615
+ tmp_path.replace(README_PATH)
616
+ print(f"Updated README.md test badge/count: {passed}/{total} passing")
617
+
618
+
619
+ def _run_captured(cmd: list[str], cwd: Path = ROOT_DIR) -> tuple[int, str]:
620
+ """Like _run(), but also captures combined stdout+stderr while still
621
+ streaming it live to the terminal - used only by cmd_test, which needs
622
+ the captured text afterward to parse the real pass/fail count for the
623
+ README badge. Kept as its own function, separate from _run() (every
624
+ other subprocess-wrapping command's single choke point - see this
625
+ module's test file docstring), specifically so tests can mock this one
626
+ choke point without silently falling through to a real subprocess call
627
+ the way mocking only `_run` would (that exact gap previously let
628
+ `aq test`'s own test recursively spawn a real, full pytest run on every
629
+ invocation of the suite)."""
630
+ process = subprocess.Popen(
631
+ cmd,
632
+ cwd=str(cwd),
633
+ stdout=subprocess.PIPE,
634
+ stderr=subprocess.STDOUT,
635
+ text=True,
636
+ bufsize=1,
637
+ )
638
+ output_lines = []
639
+ for line in process.stdout:
640
+ print(line, end="")
641
+ output_lines.append(line)
642
+ process.wait()
643
+ return process.returncode, "".join(output_lines)
644
+
645
+
646
+ # Subsystem -> tests/*.py filenames, used only when the user passes one or
647
+ # more --<subsystem> flags to filter the run. Not required to be exhaustive
648
+ # (the default, flag-less `aq test` just runs tests/ directly - marker-based
649
+ # exclusion, not this mapping, is what makes that complete) but kept
650
+ # reasonably complete so the flags are actually useful for "test every
651
+ # subsystem" one at a time. test_lean_backtest_ml_coverage.py is
652
+ # deliberately absent from every bucket - it's gated by the lean_backtest
653
+ # marker (see --lean/--full below), never by a subsystem flag.
654
+ #
655
+ # Hand-maintained, not derived from the filesystem - real drift found and
656
+ # fixed once already (~14 files, including a whole missing "audit" bucket,
657
+ # had silently fallen off every group). tests/test_aq_cli.py::
658
+ # test_subsystem_test_files_maps_every_real_test_file_to_exactly_one_bucket()
659
+ # now locks in "every real test file (except the one deliberate exclusion
660
+ # above) is in exactly one bucket" so a new test file missing from every
661
+ # bucket fails CI instead of quietly working only via the flag-less
662
+ # default run.
663
+ _SUBSYSTEM_TEST_FILES: dict[str, list[str]] = {
664
+ "cli": [
665
+ "test_aq_cli.py", "test_generate_backtest_report.py", "test_generate_evaluation_report.py",
666
+ "test_lean_config_render.py", "test_dockerignore_secrets.py", "test_secret_scan.py",
667
+ "test_profile_inference.py", "test_profile_subsystems.py", "test_lean_runtime_imports.py",
668
+ "test_order_events_audit.py",
669
+ # V5.4.7 - json_safety.py + the CLI help-surface/README-sync guards.
670
+ "test_json_safety.py", "test_cli_help_surface.py",
671
+ ],
672
+ "audit": [
673
+ "test_hash_chain.py", "test_audit_queue.py", "test_postgres_audit.py",
674
+ "test_audit_postgres_worker.py", "test_audit_status_export.py",
675
+ ],
676
+ "risk": [
677
+ "test_risk_controls.py", "test_asset_class_router.py", "test_futures_risk.py",
678
+ "test_forex_risk.py",
679
+ "test_order_gate.py", "test_position_sizing.py", "test_backtest_gate.py",
680
+ "test_validation_gate.py", "test_manual_override.py", "test_rl_sizing.py",
681
+ # V5.1 Phase 6 (production safety) - risk/kill_switch.py,
682
+ # execution/reconciliation.py (grouped here, not a dedicated
683
+ # "execution" bucket, alongside every other production-safety
684
+ # risk mechanism in this list).
685
+ "test_kill_switch.py", "test_reconciliation.py",
686
+ ],
687
+ "portfolio": [
688
+ "test_portfolio_book_construction.py", "test_options_strategy.py",
689
+ "test_options_strategy_multileg.py", "test_options_arbitrage_detector.py",
690
+ "test_options_margin_sizing.py",
691
+ "test_options_greeks.py", "test_simulated_portfolio.py", "test_options_assignment_risk.py",
692
+ # V5.1 Phase 1 (development/Problems.md #73) - portfolio/rank_signal.py.
693
+ "test_rank_signal.py",
694
+ # V5.3.5 (development/Problems.md #102) - portfolio/rolling_ic_gate.py.
695
+ "test_rolling_ic_gate.py",
696
+ ],
697
+ "features": [
698
+ "test_bond_features.py", "test_derivatives_macro_features.py", "test_macro_features.py",
699
+ "test_technical_indicators.py", "test_train_bond_features.py",
700
+ "test_train_derivatives_macro_features.py", "test_train_macro_features.py",
701
+ "test_train_asset_class_context_features.py", "test_train_cross_sectional_features.py",
702
+ "test_train_indicators.py", "test_alt_data_features.py", "test_train_alt_data_features.py",
703
+ # V5.1 Phase 2 (item 8 / F2) - features/cross_asset_sensitivity.py.
704
+ "test_cross_asset_sensitivity.py", "test_train_cross_asset_sensitivity.py",
705
+ ],
706
+ "data-pipeline": [
707
+ "test_fetch.py", "test_ib_backfill.py", "test_fred_backfill.py", "test_yfinance_backfill.py",
708
+ "test_dividend_backfill.py", "test_factor_file_backfill.py",
709
+ ],
710
+ "webui": [
711
+ "test_neural_network_state.py", "test_assets_status.py", "test_status_export.py",
712
+ "test_rank_ic_monitor.py", "test_observation_metrics.py", "test_strategy_catalog.py",
713
+ "test_api_server.py",
714
+ ],
715
+ "ml": [
716
+ "test_expert_models.py", "test_expert_datasets.py", "test_gating_network.py", "test_train_gating.py",
717
+ "test_train_multitask.py", "test_train_multitask_architecture.py", "test_train_sequence.py",
718
+ "test_train_sequence_architecture.py", "test_train_pipeline.py", "test_train_ranking_validation.py",
719
+ "test_train_walk_forward_windows.py", "test_train_topology.py", "test_learned_topology.py",
720
+ "test_exported_model.py", "test_market_topology.py", "test_market_regime.py",
721
+ "test_market_analyzer.py", "test_market_liquidity.py",
722
+ "test_train_strategy_selector.py", "test_strategy_selector_inference.py",
723
+ "test_parallel_inference.py", "test_train_threshold_and_early_stop.py",
724
+ "test_train_select_model_context_columns.py", "test_train_rl_sizing.py",
725
+ # V5.1 Phase 2 (item 5 of the roadmap) - train.py::build_residual_rank_targets().
726
+ "test_train_residual_rank_targets.py",
727
+ # V5.1 Phase 3 (items 1, 10, 11) - cross-sectional batching, ranking
728
+ # losses, optimizer/schedule/SWA/smoothing helpers.
729
+ "test_train_cross_sectional_batching.py", "test_train_ranking_loss.py", "test_train_optimizer_and_swa.py",
730
+ # V5.1 Phase 4 (item 4) - multi-model walk-forward subprocess/net-
731
+ # performance-simulation helpers.
732
+ "test_walk_forward_multimodel.py",
733
+ # V5.2.8 (Problems.md #94) - train.py::compute_gate_friendliness_weight_by_date().
734
+ "test_train_gate_friendliness.py",
735
+ ],
736
+ "retraining": [
737
+ "test_retraining_artifacts.py", "test_retraining_orchestrator.py", "test_retraining_planning.py",
738
+ "test_retraining_postgres_registry.py", "test_retraining_worker.py", "test_trigger_worker.py",
739
+ "test_triggers.py", "test_vault_client.py", "test_vault_commands.py", "test_lean_backtest.py",
740
+ "test_v2_pipeline_manifest.py",
741
+ # V5.1 Phase 6 (production safety) - retraining/auto_rollback.py.
742
+ "test_auto_rollback.py",
743
+ ],
744
+ "notifications": ["test_telegram_alerts.py", "test_telegram_client.py", "test_telegram_worker.py", "test_postgres_telegram.py"],
745
+ "storage": ["test_postgres_triggers.py", "test_postgres_worker.py", "test_config_cache.py", "test_runtime_config_io.py", "test_experience_queue.py"],
746
+ "live": [
747
+ "test_live_credentials.py", "test_live_credentials_io.py", "test_paper_readiness.py",
748
+ "test_paper_readiness_io.py", "test_paper_readiness_report.py", "test_paper_readiness_scheduler.py",
749
+ ],
750
+ "evaluation": [
751
+ "test_rank_book_simulator.py", "test_book_neutrality.py", "test_cost_model.py",
752
+ "test_model_predictions.py", "test_evaluation_state.py", "test_sector_map.py",
753
+ # V5.1 Phase 5 (item 9) - evaluation/ablation.py.
754
+ "test_ablation.py",
755
+ # V5.1 (Problems.md) - evaluation/rank_signal_calibration.py.
756
+ "test_rank_signal_calibration.py",
757
+ # V5.2.6 (Problems.md) - evaluation/confidence_threshold_calibration.py.
758
+ "test_confidence_threshold_calibration.py",
759
+ # V5.2.8 (Problems.md #94) - evaluation/kill_switch_replay.py.
760
+ "test_kill_switch_replay.py",
761
+ # V5.3.1 (Problems.md #34/#96) - evaluation/limit_fill_simulator.py.
762
+ "test_limit_fill_simulator.py",
763
+ # V5.3.5 (Problems.md #102) - evaluation/rank_ic_core.py,
764
+ # evaluation/rolling_ic_gate_calibration.py, evaluation/
765
+ # rolling_ic_gate_replay.py.
766
+ "test_rank_ic_core.py", "test_rolling_ic_gate_calibration.py", "test_rolling_ic_gate_replay.py",
767
+ # V5.3.5.3 (Problems.md #91/#100) - evaluation/feature_reconciliation.py.
768
+ "test_feature_reconciliation.py",
769
+ "test_feature_parity.py",
770
+ "test_promotion_gate_era_rule.py",
771
+ "test_aq_test_ruff_flag.py",
772
+ "test_ci_workflows.py",
773
+ "test_impact_model.py",
774
+ "test_v543_new_features.py",
775
+ "test_bar_synthesis_parity.py",
776
+ "test_rollback_hardening.py",
777
+ "test_rl_sizing_v541.py",
778
+ "test_monte_carlo.py",
779
+ # V5.4.4 - evaluation/public_benchmarks.py (SP500 + 60-40 public
780
+ # baselines) and the HRP live-book-sizing integration tests.
781
+ "test_public_benchmarks.py",
782
+ "test_hrp_book_sizing.py",
783
+ # V5.4.7 - full-system bug-hunt round file (#120-#125).
784
+ "test_v547_bug_hunt.py",
785
+ ],
786
+ }
787
+
788
+
789
+ def cmd_test(args: argparse.Namespace) -> int:
790
+ """Runs pytest with live-streamed output (same UX as a plain
791
+ subprocess.run), while also capturing it so the real pass/fail count can
792
+ be parsed afterward and used to refresh README.md's test badge - mirrors
793
+ the sibling Aether-Vault project's `av test` exactly.
794
+
795
+ Default (no flags): excludes tests/test_lean_backtest_ml_coverage.py's
796
+ lean_backtest-marked tests - a real `lean backtest .` run there takes
797
+ over an hour wall-clock, and this repo's own .venv happens to have a
798
+ real Lean CLI installed, so that file's skipif alone never actually
799
+ skipped it. --lean/--full opts back in. --parallel adds pytest-xdist's
800
+ -n auto (off by default - multiple workers each importing torch is a
801
+ real OOM risk on memory-constrained dev machines). One or more
802
+ --<subsystem> flags restrict the run to _SUBSYSTEM_TEST_FILES'
803
+ filenames for those subsystems instead of the whole tree.
804
+
805
+ V5.3.10 - --ruff: runs `ruff check .` BEFORE pytest so a lint failure
806
+ fails fast without paying the full suite's wall-clock first (mirrors
807
+ ci.yml's python-lint job; single source of truth for the invocation)."""
808
+ if getattr(args, "ruff", False):
809
+ import subprocess as _sp
810
+
811
+ ruff_exit = _sp.run(["ruff", "check", "."]).returncode
812
+ if ruff_exit != 0:
813
+ print(f"ruff check failed (exit {ruff_exit}) - fix lint findings before running the suite.")
814
+ return ruff_exit
815
+
816
+ cmd = [sys.executable, "-m", "pytest", "--color=yes", "--durations=15"]
817
+
818
+ subsystem_files: list[str] = []
819
+ for name in _SUBSYSTEM_TEST_FILES:
820
+ if getattr(args, name.replace("-", "_"), False):
821
+ subsystem_files.extend(_SUBSYSTEM_TEST_FILES[name])
822
+ is_filtered_run = bool(subsystem_files)
823
+
824
+ if is_filtered_run:
825
+ cmd.extend(f"tests/{name}" for name in dict.fromkeys(subsystem_files))
826
+ else:
827
+ cmd.append("tests/")
828
+
829
+ if getattr(args, "lean", False) or getattr(args, "full", False):
830
+ if is_filtered_run:
831
+ cmd.append("tests/test_lean_backtest_ml_coverage.py")
832
+ else:
833
+ cmd.extend(["-m", "not lean_backtest"])
834
+
835
+ if getattr(args, "parallel", False):
836
+ cmd.extend(["-n", "auto"])
837
+
838
+ exit_code, output = _run_captured(cmd)
839
+
840
+ captured = _ANSI_ESCAPE_PATTERN.sub("", output)
841
+ passed_match = re.search(r"(\d+) passed", captured)
842
+ failed_match = re.search(r"(\d+) failed", captured)
843
+ error_match = re.search(r"(\d+) error", captured)
844
+ # Only the full, unfiltered default run's pass/fail count reflects the
845
+ # whole suite - updating the badge from a --cli-only or --lean-only
846
+ # partial run would make it silently report a subset as if it were
847
+ # everything.
848
+ if passed_match and not is_filtered_run:
849
+ passed = int(passed_match.group(1))
850
+ failed = (int(failed_match.group(1)) if failed_match else 0) + (int(error_match.group(1)) if error_match else 0)
851
+ _update_readme_test_badge(passed, failed)
852
+
853
+ return exit_code
854
+
855
+
856
+ def cmd_backtest(args: argparse.Namespace) -> int:
857
+ lean_binary = _find_quantconnect_lean_binary()
858
+ if lean_binary is None:
859
+ print("error: QuantConnect Lean CLI not found (checked .venv and PATH).", file=sys.stderr)
860
+ return 1
861
+ # The default is a small local derivative of the pinned official image.
862
+ # It contains redis in the image itself, so Lean CLI does not need to
863
+ # create and Windows-bind-mount a generated requirements.txt file.
864
+ # --image remains an escape hatch for deliberately selecting another
865
+ # engine image without editing this file.
866
+ if args.image:
867
+ engine_image = args.image
868
+ else:
869
+ if not _ensure_local_lean_engine_image():
870
+ return 1
871
+ engine_image = LOCAL_LEAN_ENGINE_IMAGE
872
+ lean_command = [lean_binary, "backtest", ".", "--image", engine_image]
873
+ # Lean CLI creates config/startup files with tempfile.mkdtemp(). On some
874
+ # Windows Docker Desktop installations those Python-created directories
875
+ # are not readable by Docker, while ordinary host files are. Run through
876
+ # the project wrapper so every generated temp directory receives
877
+ # Docker-readable inherited permissions before the container is created.
878
+ windows_wrapper = ROOT_DIR / "scripts" / "run_lean_cli_windows.py"
879
+ if sys.platform == "win32" and windows_wrapper.is_file():
880
+ lean_command = [sys.executable, str(windows_wrapper), *lean_command[1:]]
881
+ exit_code = _run(lean_command)
882
+ # Attempt the README update regardless of exit_code, NOT only on == 0.
883
+ # On a resource-constrained machine Lean's Python-interpreter teardown
884
+ # can exceed its own hardcoded 10-second shutdown-isolator budget and
885
+ # make `lean backtest` return a NON-zero exit code even though the
886
+ # backtest itself fully completed and wrote valid results - a cosmetic
887
+ # "PythonInitializer.Shutdown() ... Operation timed out" error that
888
+ # fires AFTER statistics are already saved (see development/Problems.md).
889
+ # update_readme_from_latest_backtest() self-guards: it only ever picks a
890
+ # backtest folder whose result JSON has BOTH non-empty statistics and a
891
+ # non-empty equity curve, so calling it after a genuinely-failed run
892
+ # (e.g. an init-timeout that produced no statistics) is a safe no-op.
893
+ # Decoupling the README update from Lean's flaky shutdown-phase exit
894
+ # code is the robust fix - the exit code is still returned to the caller.
895
+ try:
896
+ from generate_backtest_report import update_readme_from_latest_backtest
897
+
898
+ if update_readme_from_latest_backtest():
899
+ print("Updated README.md's Backtest Results section.")
900
+ except Exception as error: # noqa: BLE001 - must never fail the backtest command itself
901
+ print(f"warning: failed to update README.md's backtest results ({error})", file=sys.stderr)
902
+ # Also refreshes Other Metrics (Lean-vs-offline comparison) with this
903
+ # backtest's fresh Sharpe, rather than leaving it stale until the next
904
+ # `aq evaluate` run - same self-guarding, never-fail contract as above.
905
+ _refresh_readme_evaluation_sections()
906
+ if exit_code != 0:
907
+ print(
908
+ "note: `lean backtest` returned a non-zero exit code. If the run reached the end date and "
909
+ "statistics were written, this is most likely the benign Python-shutdown timeout on a "
910
+ "resource-constrained machine (development/Problems.md) - the README was still updated from "
911
+ "the completed results above. Check the backtest log if unsure the run actually finished.",
912
+ file=sys.stderr,
913
+ )
914
+ return exit_code
915
+
916
+
917
+ # One flag per scripts/profile_subsystems.py subsystem - same
918
+ # established loop-generated-flags convention as _SUBSYSTEM_TEST_FILES
919
+ # above (`aq test --cli --risk` etc.), applied to `aq profile` instead of
920
+ # `aq test`. Values aren't used (dispatch is by attribute presence, same
921
+ # as cmd_test()) - a plain tuple would do, kept as a dict of Nones so the
922
+ # iteration pattern below reads identically to _SUBSYSTEM_TEST_FILES's.
923
+ _PROFILE_SUBSYSTEM_FLAGS: dict[str, None] = {
924
+ "regime": None, "topology": None, "topology-cached": None, "learned-topology": None, "liquidity": None,
925
+ "gating": None, "analyzer": None, "indicators": None, "options": None,
926
+ }
927
+
928
+
929
+ def cmd_profile(args: argparse.Namespace) -> int:
930
+ """Wraps scripts/profile_inference.py (default) - the cProfile+wall-
931
+ clock harness for main.py's per-bar inference hot path (see
932
+ development/Problems.md for what it found: weight-array/batched-stack
933
+ caching, expert-loop batching, and _conv1d_causal vectorization,
934
+ -89.2% total profiled cost) - or, when any --<subsystem> flag is set,
935
+ scripts/profile_subsystems.py instead (regime/topology/topology-cached/
936
+ liquidity/gating/analyzer/indicators/options - everything else main.py
937
+ calls per bar that inference profiling never covered; --topology-cached
938
+ exercises development/Problems.md#36's correlation-stability cache
939
+ against slowly-drifting synthetic data, since --topology's fully
940
+ independent per-iteration returns can never show that cache's
941
+ benefit; --options exercises risk/asset_class_router.py::
942
+ route_multi_leg_option_sizing() end-to-end, V4.9 Priority 7). Same
943
+ subprocess-wrapper convention every other non-`trade-lock`/`fetch`
944
+ command follows (_run(), not an in-process import).
945
+
946
+ --batched/--no-gc/--bucket-report/--parallel/--pool-workers/
947
+ --symbols-per-bar only have meaning for the inference path (no batched
948
+ variant, GC-isolation/bucketing diagnostic, or process-pool benchmark
949
+ exists for these pure subsystem functions) - combining any of them
950
+ with a subsystem flag is a user error, rejected loudly rather than
951
+ silently ignored. --parallel (V4.9 Priority 6) runs a real
952
+ ProcessPoolExecutor benchmark of inference/parallel_inference.py's
953
+ run_symbol_inference() against a sequential baseline, answering that
954
+ module's own never-measured IPC/pickling break-even warning."""
955
+ subsystem_flags = [name for name in _PROFILE_SUBSYSTEM_FLAGS if getattr(args, name.replace("-", "_"), False)]
956
+ inference_only_flags = args.batched or args.no_gc or args.bucket_report or args.parallel
957
+ if subsystem_flags and inference_only_flags:
958
+ print(
959
+ "error: --batched/--no-gc/--bucket-report/--parallel only apply to inference profiling, not --<subsystem> flags",
960
+ file=sys.stderr,
961
+ )
962
+ return 1
963
+
964
+ if subsystem_flags:
965
+ cmd = [sys.executable, "scripts/profile_subsystems.py"]
966
+ if args.iterations is not None:
967
+ cmd.extend(["--iterations", str(args.iterations)])
968
+ cmd.extend(["--sort", args.sort])
969
+ cmd.extend(f"--{name}" for name in subsystem_flags)
970
+ return _run(cmd)
971
+
972
+ cmd = [sys.executable, "scripts/profile_inference.py"]
973
+ if args.iterations is not None:
974
+ cmd.extend(["--iterations", str(args.iterations)])
975
+ cmd.extend(["--sort", args.sort])
976
+ if args.batched:
977
+ cmd.append("--batched")
978
+ if args.no_gc:
979
+ cmd.append("--no-gc")
980
+ if args.bucket_report:
981
+ cmd.append("--bucket-report")
982
+ if args.parallel:
983
+ cmd.append("--parallel")
984
+ if args.pool_workers is not None:
985
+ cmd.extend(["--pool-workers", str(args.pool_workers)])
986
+ if args.symbols_per_bar is not None:
987
+ cmd.extend(["--symbols-per-bar", str(args.symbols_per_bar)])
988
+ return _run(cmd)
989
+
990
+
991
+ def cmd_report(args: argparse.Namespace) -> int:
992
+ lean_binary = _find_quantconnect_lean_binary()
993
+ if lean_binary is None:
994
+ print("error: QuantConnect Lean CLI not found (checked .venv and PATH).", file=sys.stderr)
995
+ return 1
996
+ backtest_dir = ROOT_DIR / "backtests" / args.backtest_dir
997
+ return _run(
998
+ [
999
+ lean_binary,
1000
+ "report",
1001
+ "--backtest-results",
1002
+ str(backtest_dir / f"{args.result_id}.json"),
1003
+ "--report-destination",
1004
+ str(backtest_dir / "report.html"),
1005
+ "--overwrite",
1006
+ ]
1007
+ )
1008
+
1009
+
1010
+ def cmd_api(_args: argparse.Namespace) -> int:
1011
+ return _run([sys.executable, "-m", "uvicorn", "monitoring.api_server:app", "--port", "8001", "--reload"])
1012
+
1013
+
1014
+ def cmd_webui(_args: argparse.Namespace) -> int:
1015
+ npm = "npm.cmd" if sys.platform == "win32" else "npm"
1016
+ return _run([npm, "run", "dev"], cwd=WEBUI_DIR)
1017
+
1018
+
1019
+ def cmd_docker_up(args: argparse.Namespace) -> int:
1020
+ if args.all:
1021
+ # "engine" is the one consolidated build (aether-quant-engine) that
1022
+ # backs the app AND every worker below - see docker-compose.yml's
1023
+ # own comment on the `engine` service and requirements/README.md.
1024
+ services = [
1025
+ "redis",
1026
+ "postgres",
1027
+ "engine",
1028
+ "experience-worker",
1029
+ "audit-worker",
1030
+ "performance-trigger-worker",
1031
+ "retraining-worker",
1032
+ "telegram-worker",
1033
+ ]
1034
+ return _run(["docker", "compose", "up", "-d", *services])
1035
+ if args.lean:
1036
+ return _run(["docker", "compose", "--profile", "lean", "up", "-d"])
1037
+ return _run(["docker", "compose", "up", "-d", "redis", "postgres"])
1038
+
1039
+
1040
+ def cmd_docker_build(_args: argparse.Namespace) -> int:
1041
+ return _run(["docker", "compose", "build", "engine"])
1042
+
1043
+
1044
+ def cmd_render_lean_config(args: argparse.Namespace) -> int:
1045
+ """Render the gitignored lean.live.json from .env.live / AETHER_* env vars.
1046
+ The tracked lean.json stays all-empty; live/paper deploy uses the rendered
1047
+ file via `--lean-config`. See execution/lean_config_render.py."""
1048
+ from execution.lean_config_render import build_render_environment, write_rendered_config
1049
+
1050
+ base = Path(args.base) if args.base else LEAN_JSON_PATH
1051
+ out = Path(args.out) if args.out else ROOT_DIR / "lean.live.json"
1052
+ env_file = Path(args.env_file) if args.env_file else ROOT_DIR / ".env.live"
1053
+
1054
+ try:
1055
+ filled = write_rendered_config(base, out, build_render_environment(env_file=env_file))
1056
+ except FileNotFoundError as exc:
1057
+ print(f"Error: {exc}", file=sys.stderr)
1058
+ return 1
1059
+
1060
+ # Credential-load audit event (development/Problems.md #42) - field
1061
+ # NAMES only, matching the print statement below. A short-lived local
1062
+ # AuditQueue (this is a one-shot CLI invocation, not a long-running
1063
+ # process like main.py's) - never blocks/raises on a Redis hiccup.
1064
+ try:
1065
+ from audit import AuditQueue, CREDENTIAL_LOAD, build_audit_event
1066
+
1067
+ config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) if CONFIG_PATH.exists() else {}
1068
+ audit_config = config.get("phase_v2", {}).get("audit_log", {})
1069
+ AuditQueue(
1070
+ enabled=bool(audit_config.get("enabled", True)),
1071
+ stream_name=str(audit_config.get("redis_stream", "aether:audit")),
1072
+ ).push(build_audit_event(CREDENTIAL_LOAD, {"loaded_fields": filled}, actor="cli"))
1073
+ except Exception: # noqa: BLE001 - audit logging must never fail this command
1074
+ pass
1075
+
1076
+ if filled:
1077
+ # Field NAMES only - never the secret values.
1078
+ print(f"Rendered {out} with {len(filled)} field(s): {', '.join(filled)}")
1079
+ else:
1080
+ print(
1081
+ f"Rendered {out}, but NO secret fields were populated - check that "
1082
+ f"{env_file} exists and its AETHER_* values are set."
1083
+ )
1084
+ print(f"Deploy against it with Lean's --lean-config {out}")
1085
+ return 0
1086
+
1087
+
1088
+ def cmd_secrets_check(_args: argparse.Namespace) -> int:
1089
+ """Fail (non-zero) if secrets are about to be committed: a populated
1090
+ secret field in the tracked lean.json, or a tracked real `.env` file.
1091
+ Backs .githooks/pre-commit. Pure detection lives in
1092
+ execution/secret_scan.py."""
1093
+ from execution.secret_scan import find_populated_secret_fields, is_tracked_env_secret
1094
+
1095
+ problems: list[str] = []
1096
+
1097
+ if LEAN_JSON_PATH.exists():
1098
+ try:
1099
+ lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
1100
+ except (json.JSONDecodeError, OSError) as exc:
1101
+ print(f"Error reading lean.json: {exc}", file=sys.stderr)
1102
+ return 1
1103
+ populated = find_populated_secret_fields(lean_config)
1104
+ if populated:
1105
+ problems.append(
1106
+ "lean.json has populated secret field(s) that must not be committed "
1107
+ f"(render them into gitignored lean.live.json instead): {', '.join(populated)}"
1108
+ )
1109
+
1110
+ # Ask git which files are tracked; flag any real .env among them. Degrades
1111
+ # gracefully (skips this check) outside a git repo or without git on PATH.
1112
+ try:
1113
+ tracked = subprocess.run(
1114
+ ["git", "ls-files"], cwd=str(ROOT_DIR), capture_output=True, text=True
1115
+ )
1116
+ if tracked.returncode == 0:
1117
+ for path in tracked.stdout.splitlines():
1118
+ if is_tracked_env_secret(path):
1119
+ problems.append(f"tracked secret file must not be committed: {path}")
1120
+ except (OSError, FileNotFoundError):
1121
+ pass
1122
+
1123
+ if problems:
1124
+ print("aq secrets-check FAILED:", file=sys.stderr)
1125
+ for problem in problems:
1126
+ print(f" - {problem}", file=sys.stderr)
1127
+ return 1
1128
+
1129
+ print("aq secrets-check passed: no committed secrets detected.")
1130
+ return 0
1131
+
1132
+
1133
+ def cmd_audit_log(args: argparse.Namespace) -> int:
1134
+ """Query the tamper-evident audit log (development/Problems.md #42) -
1135
+ order placement, credential loads, live-mode transitions. Requires
1136
+ AETHER_POSTGRES_DSN (same var every other Postgres-backed `aq` command
1137
+ uses) - the audit-worker service must have drained at least one batch
1138
+ from Redis into Postgres for anything to show up here (see
1139
+ docker-compose.yml's audit-worker service / `python -m audit.postgres_worker`)."""
1140
+ import psycopg
1141
+
1142
+ from audit import fetch_all_events_ordered, fetch_recent_events, verify_chain
1143
+
1144
+ dsn = os.environ.get("AETHER_POSTGRES_DSN", "")
1145
+ if not dsn:
1146
+ print("error: AETHER_POSTGRES_DSN is not set.", file=sys.stderr)
1147
+ return 1
1148
+
1149
+ try:
1150
+ conn = psycopg.connect(dsn, autocommit=False)
1151
+ except Exception as exc:
1152
+ print(f"error: could not connect to Postgres: {exc}", file=sys.stderr)
1153
+ return 1
1154
+
1155
+ try:
1156
+ if args.verify:
1157
+ rows = fetch_all_events_ordered(conn)
1158
+ valid, broken_index = verify_chain(rows)
1159
+ if not rows:
1160
+ print("aq audit-log --verify: table is empty — trivially valid.")
1161
+ return 0
1162
+ if valid:
1163
+ print(f"aq audit-log --verify: chain intact across {len(rows)} entries.")
1164
+ return 0
1165
+ broken_row = rows[broken_index]
1166
+ print(
1167
+ f"aq audit-log --verify: CHAIN BROKEN at entry {broken_index} "
1168
+ f"(event_id={broken_row['event_id']}, event_type={broken_row['event_type']}, "
1169
+ f"created_at={broken_row['created_at']}) — tampering or data loss detected.",
1170
+ file=sys.stderr,
1171
+ )
1172
+ return 1
1173
+
1174
+ since = None
1175
+ if args.since:
1176
+ from datetime import datetime as _datetime, timezone as _timezone
1177
+
1178
+ since = _datetime.fromisoformat(args.since).replace(tzinfo=_timezone.utc)
1179
+ rows = fetch_recent_events(conn, event_type=args.event_type, since=since, limit=args.limit)
1180
+ if not rows:
1181
+ print("aq audit-log: no matching entries.")
1182
+ return 0
1183
+ for row in rows:
1184
+ print(f"{row['created_at']} {row['event_type']:<20} {row['actor']:<8} {json.dumps(row['payload'])}")
1185
+ return 0
1186
+ finally:
1187
+ conn.close()
1188
+
1189
+
1190
+ def cmd_retrain(args: argparse.Namespace) -> int:
1191
+ return _run([sys.executable, "-m", "retraining.orchestrator", args.stage, *args.retrain_args])
1192
+
1193
+
1194
+ # Phase 4.8 - whole-universe bulk-backfill scripts, each its own standalone
1195
+ # argparse/__main__ module with a PLURAL filter (--tickers/--series nargs="*",
1196
+ # --apply) - genuinely different from cmd_fetch()'s single-ticker/ad-hoc IB
1197
+ # tool below, which does NOT wire to or overlap with any of these three.
1198
+ # None of them was reachable via `aq` before this - purely a discoverability
1199
+ # wrapper, zero new backfill logic (each script's own main()/argparse is
1200
+ # unchanged and still works standalone via `python -m data_pipeline.<module>`).
1201
+ _BACKFILL_MODULE_BY_TARGET: dict[str, str] = {
1202
+ "dividends": "data_pipeline.dividend_backfill",
1203
+ "fred": "data_pipeline.fred_backfill",
1204
+ "yfinance": "data_pipeline.yfinance_backfill",
1205
+ }
1206
+
1207
+
1208
+ def cmd_backfill(args: argparse.Namespace) -> int:
1209
+ return _run([sys.executable, "-m", _BACKFILL_MODULE_BY_TARGET[args.target], *args.backfill_args])
1210
+
1211
+
1212
+ def cmd_paper_readiness(_args: argparse.Namespace) -> int:
1213
+ return _run([sys.executable, "-m", "execution.paper_readiness_report"])
1214
+
1215
+
1216
+ def cmd_trade_lock(args: argparse.Namespace) -> int:
1217
+ if args.on:
1218
+ write_manual_trade_lock_override(True, CONFIG_PATH)
1219
+ print("Trade lock override: ON (trading forced paused).")
1220
+ elif args.off:
1221
+ write_manual_trade_lock_override(False, CONFIG_PATH)
1222
+ print("Trade lock override: OFF (trading forced resumed, even past a sticky total-drawdown lock).")
1223
+ elif args.auto:
1224
+ write_manual_trade_lock_override(None, CONFIG_PATH)
1225
+ print("Trade lock override: AUTO (back to today's default automatic behavior).")
1226
+ else: # status
1227
+ override = read_manual_trade_lock_override(CONFIG_PATH)
1228
+ label = {True: "ON (forced paused)", False: "OFF (forced resumed)", None: "AUTO (automatic behavior)"}[override]
1229
+ print(f"Trade lock override: {label}")
1230
+ return 0
1231
+
1232
+
1233
+ def _cmd_kill_switch_history(args: argparse.Namespace) -> int:
1234
+ """Queries the SAME tamper-evident audit log as `aq audit-log`
1235
+ (development/Problems.md #42), filtering for kill-switch trips. main.py's
1236
+ Phase 6 wiring pushes a trip as a LIVE_MODE_TRANSITION event with
1237
+ payload {"event": "kill_switch_tripped", ...} - LIVE_MODE_TRANSITION is
1238
+ reused rather than registering a new EVENT_TYPES entry (audit/redis_queue.py
1239
+ fixes that tuple; build_audit_event() raises ValueError on anything else)."""
1240
+ import psycopg
1241
+
1242
+ from audit import LIVE_MODE_TRANSITION, fetch_recent_events
1243
+
1244
+ dsn = os.environ.get("AETHER_POSTGRES_DSN", "")
1245
+ if not dsn:
1246
+ print("error: AETHER_POSTGRES_DSN is not set.", file=sys.stderr)
1247
+ return 1
1248
+
1249
+ try:
1250
+ conn = psycopg.connect(dsn, autocommit=False)
1251
+ except Exception as exc:
1252
+ print(f"error: could not connect to Postgres: {exc}", file=sys.stderr)
1253
+ return 1
1254
+
1255
+ try:
1256
+ rows = fetch_recent_events(conn, event_type=LIVE_MODE_TRANSITION, limit=args.limit)
1257
+ trip_rows = [row for row in rows if row["payload"].get("event") == "kill_switch_tripped"]
1258
+ if not trip_rows:
1259
+ print("aq kill-switch --history: no kill-switch trips recorded.")
1260
+ return 0
1261
+ for row in trip_rows:
1262
+ print(f"{row['created_at']} {json.dumps(row['payload'])}")
1263
+ return 0
1264
+ finally:
1265
+ conn.close()
1266
+
1267
+
1268
+ def cmd_kill_switch(args: argparse.Namespace) -> int:
1269
+ if args.arm:
1270
+ write_kill_switch_manual_override(True, CONFIG_PATH)
1271
+ print("Kill switch override: ARMED (forced evaluation active, even if phase_v2.risk.kill_switch.enabled is false).")
1272
+ elif args.disarm:
1273
+ write_kill_switch_manual_override(False, CONFIG_PATH)
1274
+ print("Kill switch override: DISARMED (forced off, even if phase_v2.risk.kill_switch.enabled is true).")
1275
+ elif args.auto:
1276
+ write_kill_switch_manual_override(None, CONFIG_PATH)
1277
+ print("Kill switch override: AUTO (defers to phase_v2.risk.kill_switch.enabled).")
1278
+ elif args.history:
1279
+ return _cmd_kill_switch_history(args)
1280
+ else: # status
1281
+ override = read_kill_switch_manual_override(CONFIG_PATH)
1282
+ label = {True: "ARMED (forced on)", False: "DISARMED (forced off)", None: "AUTO (defers to config)"}[override]
1283
+ print(f"Kill switch override: {label}")
1284
+ return 0
1285
+
1286
+
1287
+ class ConfigPathError(Exception):
1288
+ """Raised by _get_config_value/_set_config_value for a bad dotted path."""
1289
+
1290
+
1291
+ def _get_config_value(config: dict, dotted_path: str) -> object:
1292
+ node = config
1293
+ walked: list[str] = []
1294
+ for segment in dotted_path.split("."):
1295
+ walked.append(segment)
1296
+ if not isinstance(node, dict) or segment not in node:
1297
+ raise ConfigPathError(f"no such config key: {'.'.join(walked)!r}")
1298
+ node = node[segment]
1299
+ return node
1300
+
1301
+
1302
+ def _coerce_config_value(raw: str) -> object:
1303
+ """JSON-first parsing so true/false/123/0.5/[...]/{...} all become their
1304
+ real type automatically; anything that isn't valid JSON on its own
1305
+ (e.g. a bare word) is kept as a plain string."""
1306
+ try:
1307
+ return json.loads(raw)
1308
+ except json.JSONDecodeError:
1309
+ return raw
1310
+
1311
+
1312
+ def _set_config_value(config: dict, dotted_path: str, raw_value: str) -> tuple[object, object, bool]:
1313
+ """Mutates `config` in place. Returns (old_value, new_value, type_changed) -
1314
+ deliberately does not refuse to overwrite a list/dict: the caller wants
1315
+ full read/write access to every key, not just scalars. Safety instead
1316
+ comes from always reporting old -> new (and a type-change warning) to
1317
+ the caller, plus the automatic config.json.bak snapshot cmd_config()
1318
+ writes before every set."""
1319
+ *parents, leaf = dotted_path.split(".")
1320
+ node = config
1321
+ for segment in parents:
1322
+ if not isinstance(node, dict) or segment not in node:
1323
+ raise ConfigPathError(f"no such config key: {dotted_path!r}")
1324
+ node = node[segment]
1325
+ if not isinstance(node, dict) or leaf not in node:
1326
+ raise ConfigPathError(f"no such config key: {dotted_path!r}")
1327
+ old_value = node[leaf]
1328
+ new_value = _coerce_config_value(raw_value)
1329
+ node[leaf] = new_value
1330
+ return old_value, new_value, type(old_value) is not type(new_value)
1331
+
1332
+
1333
+ def _iter_leaf_paths(node: object, prefix: str = "") -> Iterator[str]:
1334
+ """Recursively yields every dot-joined leaf path under `node`. A "leaf"
1335
+ is any non-dict value (or an empty dict) - list-valued keys show up as
1336
+ one leaf, never expanded per-element."""
1337
+ if isinstance(node, dict) and node:
1338
+ for key, value in node.items():
1339
+ yield from _iter_leaf_paths(value, f"{prefix}.{key}" if prefix else key)
1340
+ else:
1341
+ yield prefix
1342
+
1343
+
1344
+ def _dispatch_json_config_command(args: argparse.Namespace, json_path: Path, command_attr: str) -> int:
1345
+ """Shared dispatch for `aq config`/`aq lean` - both are the same
1346
+ dump/get/set/keys tool over a single flat JSON file, just pointed at a
1347
+ different path. See cmd_config()/cmd_lean()."""
1348
+ data = json.loads(json_path.read_text(encoding="utf-8"))
1349
+ command = getattr(args, command_attr, None)
1350
+
1351
+ try:
1352
+ if command is None:
1353
+ print(json.dumps(data, indent=2))
1354
+ return 0
1355
+
1356
+ if command == "get":
1357
+ value = _get_config_value(data, args.dotted_path)
1358
+ print(value if isinstance(value, str) else json.dumps(value, indent=2))
1359
+ return 0
1360
+
1361
+ if command == "keys":
1362
+ root = _get_config_value(data, args.dotted_prefix) if args.dotted_prefix else data
1363
+ for path in _iter_leaf_paths(root, args.dotted_prefix or ""):
1364
+ print(path)
1365
+ return 0
1366
+
1367
+ if command == "set":
1368
+ shutil.copy2(json_path, json_path.with_suffix(".json.bak"))
1369
+ old_value, new_value, type_changed = _set_config_value(data, args.dotted_path, args.value)
1370
+ json_path.write_text(json.dumps(data, indent=4) + "\n", encoding="utf-8")
1371
+ print(f"{args.dotted_path}: {old_value!r} -> {new_value!r}")
1372
+ if type_changed:
1373
+ print(
1374
+ f"WARNING: type changed from {type(old_value).__name__} to {type(new_value).__name__} "
1375
+ f"for {args.dotted_path}",
1376
+ file=sys.stderr,
1377
+ )
1378
+ if json_path == CONFIG_PATH and args.dotted_path == "phase_v2.inference_parallelism.enabled" and new_value:
1379
+ print(
1380
+ "WARNING: phase_v2.inference_parallelism.enabled=true - development/Problems.md #65 "
1381
+ "measured this ProcessPoolExecutor pool as dramatically slower than sequential on "
1382
+ "Windows' spawn start method (never measured on Linux/fork). main.py will also log "
1383
+ "this at startup on Windows if the pool actually starts.",
1384
+ file=sys.stderr,
1385
+ )
1386
+ return 0
1387
+
1388
+ if command == "preset":
1389
+ return _dispatch_config_preset_command(args, data, json_path)
1390
+ except ConfigPathError as error:
1391
+ print(f"error: {error}", file=sys.stderr)
1392
+ return 1
1393
+
1394
+ return 1
1395
+
1396
+
1397
+ def _dispatch_config_preset_command(args: argparse.Namespace, data: dict, json_path: Path) -> int:
1398
+ """`aq config preset` (V5.1 Phase 1) - list/show/apply a named
1399
+ dotted-key-override block from phase_v2.presets. Reuses
1400
+ _get_config_value()/_set_config_value() so backup/type-coercion/old->new
1401
+ printing are IDENTICAL to a manual `aq config set` per key - a preset
1402
+ is just several `set` calls applied together, never a separate write
1403
+ path. Only registered under `config` (not `lean` - presets are a
1404
+ config.json-only concept)."""
1405
+ presets_root = data.get("phase_v2", {}).get("presets", {})
1406
+ preset_names = sorted(name for name in presets_root if name != "active")
1407
+
1408
+ if args.preset_list:
1409
+ active = presets_root.get("active")
1410
+ for name in preset_names:
1411
+ print(f"{name}{' (active)' if name == active else ''}")
1412
+ return 0
1413
+
1414
+ if args.preset_show:
1415
+ preset_values = presets_root.get(args.preset_show)
1416
+ if preset_values is None:
1417
+ print(f"error: no such preset {args.preset_show!r} - available: {preset_names}", file=sys.stderr)
1418
+ return 1
1419
+ print(json.dumps(preset_values, indent=2))
1420
+ return 0
1421
+
1422
+ if args.preset_apply:
1423
+ preset_values = presets_root.get(args.preset_apply)
1424
+ if preset_values is None:
1425
+ print(f"error: no such preset {args.preset_apply!r} - available: {preset_names}", file=sys.stderr)
1426
+ return 1
1427
+
1428
+ # Validate every dotted path resolves BEFORE writing anything -
1429
+ # apply the whole preset or none. A partial apply that failed
1430
+ # halfway through would leave config.json in a mixed state no
1431
+ # preset actually describes.
1432
+ for dotted_path in preset_values:
1433
+ _get_config_value(data, dotted_path)
1434
+
1435
+ if args.preset_dry_run:
1436
+ for dotted_path, new_value in preset_values.items():
1437
+ old_value = _get_config_value(data, dotted_path)
1438
+ print(f"{dotted_path}: {old_value!r} -> {new_value!r} (dry run)")
1439
+ return 0
1440
+
1441
+ shutil.copy2(json_path, json_path.with_suffix(".json.bak"))
1442
+ for dotted_path, new_value in preset_values.items():
1443
+ old_value, applied_value, type_changed = _set_config_value(data, dotted_path, json.dumps(new_value))
1444
+ print(f"{dotted_path}: {old_value!r} -> {applied_value!r}")
1445
+ if type_changed:
1446
+ print(f"WARNING: type changed for {dotted_path}", file=sys.stderr)
1447
+ if "active" in presets_root:
1448
+ _set_config_value(data, "phase_v2.presets.active", json.dumps(args.preset_apply))
1449
+ json_path.write_text(json.dumps(data, indent=4) + "\n", encoding="utf-8")
1450
+ return 0
1451
+
1452
+ # No flag given - print the whole presets block, same "bare command
1453
+ # dumps everything" convention as `aq config` itself.
1454
+ print(json.dumps(presets_root, indent=2))
1455
+ return 0
1456
+
1457
+
1458
+ def cmd_config(args: argparse.Namespace) -> int:
1459
+ return _dispatch_json_config_command(args, CONFIG_PATH, "config_command")
1460
+
1461
+
1462
+ def cmd_lean(args: argparse.Namespace) -> int:
1463
+ return _dispatch_json_config_command(args, LEAN_JSON_PATH, "lean_command")
1464
+
1465
+ return 1
1466
+
1467
+
1468
+ def cmd_fetch(args: argparse.Namespace) -> int:
1469
+ ib = None
1470
+ fetch_fn = None
1471
+ extra_metadata = None
1472
+ if args.asset_class in IB_ASSET_CLASSES:
1473
+ if args.expiry is None:
1474
+ print(f"error: --expiry is required for asset_class={args.asset_class!r}", file=sys.stderr)
1475
+ return 1
1476
+ if args.asset_class == "options" and (args.strike is None or args.right is None):
1477
+ print("error: --strike and --right are required for asset_class='options'", file=sys.stderr)
1478
+ return 1
1479
+
1480
+ config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
1481
+ lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
1482
+ try:
1483
+ ib = connect_ib(config, lean_config)
1484
+ except IBNotConfiguredError as error:
1485
+ print(f"error: {error}", file=sys.stderr)
1486
+ return 1
1487
+
1488
+ family_ticker = args.family_ticker or args.ticker.upper()
1489
+ if args.asset_class == "futures":
1490
+ contract_spec = load_futures_contract_specs().get(args.ticker.upper(), {})
1491
+ fetch_fn = lambda symbol, start, end: fetch_future_historical_bars( # noqa: E731
1492
+ ib, symbol, contract_spec, start, end, contract_month=args.contract_month
1493
+ )
1494
+ extra_metadata = {"family_ticker": family_ticker}
1495
+ if args.contract_month:
1496
+ extra_metadata["contract_month"] = args.contract_month
1497
+ else:
1498
+ fetch_fn = lambda symbol, start, end: fetch_option_historical_bars( # noqa: E731
1499
+ ib, symbol, args.expiry, args.strike, args.right, start, end
1500
+ )
1501
+ extra_metadata = {
1502
+ "family_ticker": family_ticker,
1503
+ "strike": args.strike,
1504
+ "expiry": args.expiry,
1505
+ "right": args.right,
1506
+ }
1507
+
1508
+ try:
1509
+ fetch_kwargs = {"fetch_fn": fetch_fn} if fetch_fn is not None else {}
1510
+ report = fetch_adhoc_asset(
1511
+ args.asset_class, args.ticker, args.start, args.end, apply=args.apply,
1512
+ extra_metadata=extra_metadata, **fetch_kwargs,
1513
+ )
1514
+ finally:
1515
+ if ib is not None:
1516
+ disconnect_ib(ib)
1517
+
1518
+ label = "APPLY" if args.apply else "DRY RUN"
1519
+ print(f"{label} — {report['ticker']} ({report['yahoo_symbol']}): {report['action']}, rows_fetched={report['rows_fetched']}")
1520
+ if report["suggested_available_from"]:
1521
+ print(f" date range fetched: {report['suggested_available_from']} .. {report['suggested_available_to']}")
1522
+ print(f" data_path: {report['data_path']}")
1523
+
1524
+ if report["config_status"] == "added":
1525
+ print(f" config.json: added a new {report['ticker']} asset block to phase1.universe.assets[]")
1526
+ elif report["config_status"] == "already_exists":
1527
+ print(
1528
+ f" config.json: {report['ticker']} is already configured - left untouched. "
1529
+ "Use data_pipeline/yfinance_backfill.py to extend an existing asset's date range instead."
1530
+ )
1531
+
1532
+ if not args.apply:
1533
+ print("\nDry run only — nothing was written. Re-run with --apply to write the zip file and update config.json.")
1534
+ elif report["action"] == "written":
1535
+ print("\nReady to prepare training: run `python train.py --dataset-only` to confirm this ticker's asset quality, then `python train.py` when ready.")
1536
+
1537
+ return 1 if report["action"] == "no_data_returned" else 0
1538
+
1539
+
1540
+ def cmd_ib(_args: argparse.Namespace) -> int:
1541
+ config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
1542
+ lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
1543
+ status = ib_readiness_status(config, lean_config)
1544
+
1545
+ if status == "disabled":
1546
+ print("IB: disabled (phase_v2.ib.enabled is false)")
1547
+ print(" Enable with: aq config set phase_v2.ib.enabled true")
1548
+ return 0
1549
+
1550
+ if status == "enabled_but_lean_credentials_missing":
1551
+ print("IB: enabled in config.json, but lean.json's IB credentials are not filled in")
1552
+ print(" Set them with: aq lean set ib-account <ACCOUNT> and aq lean set ib-user-name <USERNAME>")
1553
+ return 1
1554
+
1555
+ reachable, detail = attempt_connection(config, lean_config)
1556
+ if reachable:
1557
+ print(f"IB: {detail} (connected to {config['phase_v2']['ib'].get('host')}:{config['phase_v2']['ib'].get('port')})")
1558
+ return 0
1559
+
1560
+ print(f"IB: enabled and credentialed, but not reachable — {detail}")
1561
+ print(" Check that TWS or IB Gateway is running and logged in on the configured host/port.")
1562
+ return 1
1563
+
1564
+
1565
+ def cmd_assets(_args: argparse.Namespace) -> int:
1566
+ """`aq assets status`: one command reporting full multi-asset-class
1567
+ readiness at a glance - IB, the futures_risk/options_risk feature
1568
+ flags, how many futures contract specs are loaded, how much of the
1569
+ FRED yield-curve cache is populated, and how many futures/options
1570
+ assets are actually configured in config.json's universe. Read-only
1571
+ reporting only - basic enable/disable of any of these already works
1572
+ today via the generic
1573
+ `aq config set phase_v2.{ib,futures_risk,options_risk}.enabled true|false`
1574
+ (_dispatch_json_config_command), same as cmd_ib's own "Enable with"
1575
+ hint.
1576
+
1577
+ The actual report is built by monitoring/assets_status.py::
1578
+ build_assets_status() - shared with the webui's `/api/assets-status`
1579
+ endpoint so the readiness logic is defined exactly once."""
1580
+ from monitoring.assets_status import build_assets_status
1581
+
1582
+ config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
1583
+ lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
1584
+ report = build_assets_status(config, lean_config)
1585
+
1586
+ print(f"IB: {report['ib_status']}")
1587
+ print(f"futures_risk.enabled: {report['futures_risk_enabled']}")
1588
+ print(f"options_risk.enabled: {report['options_risk_enabled']}")
1589
+ tickers = ", ".join(report["futures_contract_specs_tickers"]) or "none"
1590
+ print(f"Futures contract specs loaded: {report['futures_contract_specs_loaded']} ({tickers})")
1591
+ most_recent = report["fred_cache_most_recent_date"] or "never populated"
1592
+ print(f"FRED cache: {report['fred_cache_series_count']} series populated, most recent date: {most_recent}")
1593
+ print(f"Configured futures assets: {report['configured_futures_assets']}")
1594
+ print(f"Configured options assets: {report['configured_options_assets']}")
1595
+ # V5.1 Phase 2, Step 2.5 - development/Problems.md: derivatives macro
1596
+ # features are computed every dataset build, but deliberately left out
1597
+ # of phase1.features.input_set (constant 0.0 with no futures/options
1598
+ # contract subscribed - see build_dataset_manifest()'s
1599
+ # computed_but_unused_features list for the code-enforced version of
1600
+ # this same fact).
1601
+ derivatives_in_input_set = config.get("phase1", {}).get("features", {}).get("derivatives_in_input_set", False)
1602
+ status = "in input_set" if derivatives_in_input_set else "computed, NOT in input_set"
1603
+ print(
1604
+ "Derivatives macro features (futures_term_structure_slope, options_put_call_ratio, "
1605
+ f"options_implied_vol_skew): {status} - needs a futures/options contract subscribed "
1606
+ "(IB-gated) to carry real signal; flip phase1.features.derivatives_in_input_set "
1607
+ "once one is."
1608
+ )
1609
+
1610
+ return 0
1611
+
1612
+
1613
+ def _apply_preset_in_memory(config: dict, preset: dict) -> dict:
1614
+ """Applies a phase_v2.presets.<name> dotted-key overlay to an IN-MEMORY
1615
+ deep copy of `config` only - `aq evaluate --preset` must never write
1616
+ config.json (that is `aq config preset --apply`'s job, a separate
1617
+ command). Reuses _set_config_value()'s own dotted-path resolution -
1618
+ json.dumps() round-trips each already-typed preset value through the
1619
+ same string-coercion path `aq config set` uses, so both entry points
1620
+ share one implementation and can never silently disagree on how a
1621
+ dotted path resolves."""
1622
+ overlaid = copy.deepcopy(config)
1623
+ for dotted_path, value in preset.items():
1624
+ _set_config_value(overlaid, dotted_path, json.dumps(value))
1625
+ return overlaid
1626
+
1627
+
1628
+ def _write_evaluation_json(path: Path, payload: object) -> None:
1629
+ path.parent.mkdir(parents=True, exist_ok=True)
1630
+ path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
1631
+
1632
+
1633
+ def _refresh_readme_evaluation_sections() -> None:
1634
+ """Best-effort call to generate_evaluation_report.py after any `aq
1635
+ evaluate` flag that could have changed what's on ml/evaluation/ or
1636
+ ml/versions/walk-forward-*/ - mirrors cmd_backtest()'s own "a report-
1637
+ generation bug must never fail the real command" contract. Lazily
1638
+ imported for the same reason as this function's pandas/torch imports
1639
+ above: every other `aq` subcommand's startup must stay unaffected."""
1640
+ try:
1641
+ from generate_evaluation_report import update_readme_evaluation_sections
1642
+
1643
+ update_readme_evaluation_sections()
1644
+ except Exception as exc: # noqa: BLE001 - report generation must never break `aq evaluate`
1645
+ print(f"warning: README evaluation-section refresh failed: {exc}", file=sys.stderr)
1646
+
1647
+
1648
+ _EVALUATE_MODEL_ARTIFACTS = {
1649
+ "sequence": ("sequence_model.json", "sequence_feature_schema.json"),
1650
+ "multitask": ("multitask_model.json", "multitask_feature_schema.json"),
1651
+ }
1652
+
1653
+
1654
+ def cmd_evaluate(args: argparse.Namespace) -> int:
1655
+ """`aq evaluate`: offline, cost-aware evaluation of the cross-sectional
1656
+ rank book (V5.1 Phase 0, development/Problems.md) - turns "how good is
1657
+ this rank prediction" into "what would this have earned net of fees",
1658
+ the question nothing else in this codebase answers
1659
+ (train.py::compute_strategy_metrics() simulates a long-only
1660
+ 1-day-direction strategy gross of costs, not the rank book, not net of
1661
+ cost). Runs every dataset row through inference/exported_model.py (the
1662
+ SAME torch-free interpreter main.py's live decision path uses) and
1663
+ evaluation/rank_book_simulator.py's simulate_rank_book() (which itself
1664
+ reuses portfolio/book_construction.py's build_rank_based_book() and
1665
+ portfolio/book_neutrality.py's apply_book_neutrality()) - so every
1666
+ number this prints is the offline mirror of live behavior, not a
1667
+ separately re-derived approximation.
1668
+
1669
+ Heavy imports (pandas/numpy/evaluation/inference) are deliberately
1670
+ LAZY, inside this function body, not at module top - the same
1671
+ convention cmd_assets() already follows for monitoring.assets_status -
1672
+ so every other `aq` subcommand's startup stays exactly as fast as
1673
+ before this command existed.
1674
+ """
1675
+ import pandas as pd
1676
+
1677
+ from evaluation import capacity_curve, predict_head, simulate_rank_book, stress_test_costs
1678
+ from features import load_sector_mapping
1679
+
1680
+ config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
1681
+
1682
+ if getattr(args, "preset", None):
1683
+ presets = config.get("phase_v2", {}).get("presets", {})
1684
+ preset = presets.get(args.preset)
1685
+ if preset is None:
1686
+ available = sorted(name for name in presets if name != "active")
1687
+ print(f"error: unknown preset {args.preset!r} - available: {available or 'none configured yet'}", file=sys.stderr)
1688
+ return 1
1689
+ config = _apply_preset_in_memory(config, preset)
1690
+
1691
+ # V5.1 Phase 4 (item 4) - `aq evaluate --walk-forward-summary` reads an
1692
+ # already-written ml/versions/walk-forward-*/walk_forward_summary.json
1693
+ # (train.py::_run_walk_forward()'s output) rather than running anything
1694
+ # itself - a completely separate, lightweight code path from every
1695
+ # other --rank-book/--capacity/--stress/--calibrate-edge flag below,
1696
+ # so it returns early before any of that dataset/model loading.
1697
+ if getattr(args, "walk_forward_summary", False):
1698
+ versions_dir = ML_DIR / "versions"
1699
+ run_id = getattr(args, "run_id", None)
1700
+ if run_id:
1701
+ summary_path = versions_dir / run_id / "walk_forward_summary.json"
1702
+ else:
1703
+ candidates = sorted(versions_dir.glob("walk-forward-*/walk_forward_summary.json"), key=lambda p: p.stat().st_mtime)
1704
+ summary_path = candidates[-1] if candidates else None
1705
+ if summary_path is None or not summary_path.exists():
1706
+ print("error: no walk-forward summary found - run `aq train --walk-forward` first.", file=sys.stderr)
1707
+ return 1
1708
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
1709
+ if args.json:
1710
+ print(json.dumps(summary, indent=2))
1711
+ else:
1712
+ print(f"Walk-forward summary ({summary_path.parent.name}): {summary.get('num_windows', 0)} windows")
1713
+ for name, stats in (summary.get("summary_by_metric") or {}).items():
1714
+ bootstrap = stats.get("cross_window_bootstrap", {})
1715
+ print(
1716
+ f" {name}: mean={bootstrap.get('mean_ic', 0.0):.4f} "
1717
+ f"95% CI=[{bootstrap.get('lower_bound', 0.0):.4f}, {bootstrap.get('upper_bound', 0.0):.4f}]"
1718
+ )
1719
+ for name, stability in (summary.get("stability_by_metric") or {}).items():
1720
+ status = "stable" if stability.get("stable") else "UNSTABLE"
1721
+ print(
1722
+ f" {name} stability: {status} "
1723
+ f"(sign_flip_fraction={stability.get('sign_flip_fraction', 0.0):.2f}, "
1724
+ f"num_windows={stability.get('num_windows', 0)})"
1725
+ )
1726
+ net_performance_windows = summary.get("net_performance_by_window") or []
1727
+ if net_performance_windows:
1728
+ print(f" net_performance: {len(net_performance_windows)} window(s) with a simulated book")
1729
+ _refresh_readme_evaluation_sections()
1730
+ return 0
1731
+
1732
+ # V5.3.5.3 Workstream B (development/Problems.md #91/#100) -
1733
+ # `aq evaluate --reconcile-features --symbol XOM` diffs the RAW feature
1734
+ # values the live path logged for one symbol (the optional
1735
+ # "feature_snapshot" key on book_history.jsonl records, written when
1736
+ # phase_v2.diagnostics.book_history.include_feature_snapshot is on and
1737
+ # the symbol is in feature_snapshot_symbols' allowlist - see main.py's
1738
+ # write site) against the SAME (ticker, date) row of full_dataset.csv -
1739
+ # per-feature, not a single mismatch number: XOM is the one tracked
1740
+ # ticker whose live-vs-offline divergence never resolved across
1741
+ # #91/#97/#99/#100, and this is the tool that either names the exact
1742
+ # diverging feature(s) or rules the lead out cleanly.
1743
+ # Early-return, same precedent as --reconcile-book-history below, and
1744
+ # deliberately MUCH lighter: no model artifacts, no re-inference at all
1745
+ # - a pure log-vs-dataset diff. Not included in --all,
1746
+ # investigation-only like --replay-kill-switch.
1747
+ if getattr(args, "reconcile_features", False):
1748
+ from evaluation import segment_logged_records_by_run
1749
+ from evaluation.feature_reconciliation import (
1750
+ reconcile_feature_snapshot,
1751
+ summarize_feature_reconciliation,
1752
+ )
1753
+
1754
+ target_symbol = str(getattr(args, "symbol", "") or "").strip().upper()
1755
+ if not target_symbol:
1756
+ print(
1757
+ "error: --reconcile-features requires --symbol <TICKER> "
1758
+ "(e.g. --symbol XOM).",
1759
+ file=sys.stderr,
1760
+ )
1761
+ return 1
1762
+
1763
+ book_history_path_arg = getattr(args, "book_history_path", None)
1764
+ diagnostics_config = config.get("phase_v2", {}).get("diagnostics", {}).get("book_history", {})
1765
+ book_history_path = (
1766
+ Path(book_history_path_arg)
1767
+ if book_history_path_arg
1768
+ else ROOT_DIR / diagnostics_config.get("output_path", "visualization/book_history.jsonl")
1769
+ )
1770
+ if not book_history_path.exists():
1771
+ print(
1772
+ f"error: {book_history_path} not found - run a backtest with "
1773
+ "phase_v2.diagnostics.book_history.enabled=true first.",
1774
+ file=sys.stderr,
1775
+ )
1776
+ return 1
1777
+
1778
+ logged_records: list[dict] = []
1779
+ with open(book_history_path, "r", encoding="utf-8") as book_history_file:
1780
+ for line_number, raw_line in enumerate(book_history_file, start=1):
1781
+ stripped_line = raw_line.strip()
1782
+ if not stripped_line:
1783
+ continue
1784
+ try:
1785
+ logged_records.append(json.loads(stripped_line))
1786
+ except json.JSONDecodeError as error:
1787
+ print(
1788
+ f"warning: skipping malformed line {line_number} in {book_history_path}: {error}",
1789
+ file=sys.stderr,
1790
+ )
1791
+ if not logged_records:
1792
+ print(f"error: {book_history_path} contains no usable records.", file=sys.stderr)
1793
+ return 1
1794
+
1795
+ # Same cumulative-log discipline as --reconcile-book-history
1796
+ # (#91/#97/#99): segment FIRST, default to the most recent run only.
1797
+ run_segments = segment_logged_records_by_run(logged_records)
1798
+ if not run_segments:
1799
+ print(f"error: {book_history_path} contains no dated records.", file=sys.stderr)
1800
+ return 1
1801
+ reconcile_all_runs = bool(getattr(args, "reconcile_all_runs", False))
1802
+ reconcile_run_index_arg = getattr(args, "reconcile_run_index", None)
1803
+ requested_indices = (
1804
+ list(range(len(run_segments)))
1805
+ if reconcile_all_runs
1806
+ else [reconcile_run_index_arg if reconcile_run_index_arg is not None else -1]
1807
+ )
1808
+ selected_run_indices: list[int] = []
1809
+ for raw_index in requested_indices:
1810
+ normalized_index = raw_index if raw_index >= 0 else raw_index + len(run_segments)
1811
+ if not (0 <= normalized_index < len(run_segments)):
1812
+ print(
1813
+ f"error: --reconcile-run-index {raw_index} out of range - {len(run_segments)} run(s) "
1814
+ f"detected in {book_history_path}.", file=sys.stderr,
1815
+ )
1816
+ return 1
1817
+ selected_run_indices.append(normalized_index)
1818
+
1819
+ recon_dataset_path = ML_DIR / "datasets" / "full_dataset.csv"
1820
+ if not recon_dataset_path.exists():
1821
+ print(f"error: {recon_dataset_path} not found - run `aq train --dataset-only` first.", file=sys.stderr)
1822
+ return 1
1823
+ full_dataset = pd.read_csv(recon_dataset_path)
1824
+ dataset_has_ticker_column = "ticker" in full_dataset.columns
1825
+ rows_by_date_ticker: dict[tuple[str, str], dict] = {}
1826
+ if dataset_has_ticker_column:
1827
+ # (date, ticker) -> row dict, built vectorized - O(rows) once,
1828
+ # then O(1) lookups per logged snapshot below.
1829
+ rows_by_date_ticker = {
1830
+ (str(date), str(ticker)): row
1831
+ for (date, ticker), row in full_dataset.set_index(["date", "ticker"]).to_dict("index").items()
1832
+ }
1833
+
1834
+ per_run_payloads = []
1835
+ total_compared_dates = 0
1836
+ for run_index in selected_run_indices:
1837
+ run_records = run_segments[run_index]
1838
+ records_with_snapshot = [
1839
+ record for record in run_records if isinstance(record.get("feature_snapshot"), dict)
1840
+ ]
1841
+ matching_snapshots = []
1842
+ snapshot_keys_seen: set[str] = set()
1843
+ num_missing_offline_rows = 0
1844
+ per_date_results = []
1845
+ for record in records_with_snapshot:
1846
+ snapshot_entry = record["feature_snapshot"].get(target_symbol)
1847
+ if snapshot_entry is None:
1848
+ # Allowlist may cover several symbols; only this one's
1849
+ # entries reconcile. Track near-miss keys for the error
1850
+ # message so a ticker/symbol_key spelling mismatch is
1851
+ # immediately visible instead of silently zero.
1852
+ snapshot_keys_seen.update(str(key).upper() for key in record["feature_snapshot"])
1853
+ continue
1854
+ if not isinstance(snapshot_entry, Mapping):
1855
+ continue
1856
+ date_str = str(record.get("date", ""))
1857
+ offline_row = rows_by_date_ticker.get((date_str, target_symbol)) if dataset_has_ticker_column else None
1858
+ if offline_row is None:
1859
+ num_missing_offline_rows += 1
1860
+ continue
1861
+ per_date_results.append(reconcile_feature_snapshot(snapshot_entry, offline_row))
1862
+ matching_snapshots.append(date_str)
1863
+
1864
+ summary = summarize_feature_reconciliation(per_date_results)
1865
+ total_compared_dates += len(per_date_results)
1866
+ run_dates = [record["date"] for record in run_records if record.get("date")]
1867
+ per_run_payloads.append({
1868
+ "run_metadata": {
1869
+ "num_runs_detected": len(run_segments),
1870
+ "selected_run_index": run_index,
1871
+ "start_date": run_dates[0] if run_dates else None,
1872
+ "end_date": run_dates[-1] if run_dates else None,
1873
+ "num_records": len(run_records),
1874
+ "num_records_scanned": len(records_with_snapshot),
1875
+ "num_dates_reconciled": len(per_date_results),
1876
+ "num_dates_missing_offline_row": num_missing_offline_rows,
1877
+ "snapshot_symbol_keys_seen": sorted(snapshot_keys_seen),
1878
+ },
1879
+ "summary": summary,
1880
+ "per_date": [
1881
+ {"date": date_str, **result}
1882
+ for date_str, result in zip(matching_snapshots, per_date_results)
1883
+ ],
1884
+ })
1885
+
1886
+ payload = dict(per_run_payloads[-1]) if per_run_payloads else {}
1887
+ if len(per_run_payloads) > 1:
1888
+ payload["all_runs"] = per_run_payloads
1889
+
1890
+ _write_evaluation_json(ML_DIR / "evaluation" / "feature_reconciliation.json", payload)
1891
+
1892
+ if args.json:
1893
+ print(json.dumps(payload, indent=2, default=str))
1894
+ else:
1895
+ last_meta = payload.get("run_metadata", {})
1896
+ print(
1897
+ f"Feature reconciliation for {target_symbol} ({book_history_path}): "
1898
+ f"{last_meta.get('num_records_scanned', 0)} record(s) carried a feature_snapshot, "
1899
+ f"{last_meta.get('num_dates_reconciled', 0)} reconciled against full_dataset.csv."
1900
+ )
1901
+ if last_meta.get("num_dates_missing_offline_row"):
1902
+ print(
1903
+ f" {last_meta['num_dates_missing_offline_row']} snapshot date(s) had no matching "
1904
+ "(date, ticker) row in the dataset - skipped, not counted as divergences."
1905
+ )
1906
+ summary = payload.get("summary", {})
1907
+ if summary.get("num_dates", 0) == 0 or summary.get("num_dates_with_comparison", 0) == 0:
1908
+ keys_hint = ", ".join(last_meta.get("snapshot_symbol_keys_seen", [])[:10])
1909
+ print(
1910
+ f" NO comparable snapshots found for {target_symbol}. Either the diagnostic flags were "
1911
+ "off during that run (set phase_v2.diagnostics.book_history.include_feature_snapshot=true "
1912
+ f"+ add \"{target_symbol}\" to feature_snapshot_symbols, then re-run a backtest)"
1913
+ + (f"; snapshot keys seen in the log: [{keys_hint}]" if keys_hint else "")
1914
+ + ".",
1915
+ file=sys.stderr,
1916
+ )
1917
+ else:
1918
+ print(
1919
+ f" compared={summary['num_features_compared_per_date']['mean']:.1f} features/date avg "
1920
+ f"divergent_dates={summary['dates_with_any_divergence']}/{summary['num_dates']} "
1921
+ f"total_divergences={summary['total_divergences']}"
1922
+ )
1923
+ top_offenders = summary.get("features_diverged_most_often", [])
1924
+ if top_offenders:
1925
+ print(" Worst offenders by divergence count:")
1926
+ for entry in top_offenders:
1927
+ print(
1928
+ f" {entry['feature']}: diverged on {entry['diverged_count']} date(s) "
1929
+ f"({entry['diverged_fraction']:.1%}), max_abs_delta={entry['max_abs_delta']:.6g}, "
1930
+ f"mean_abs_delta={entry['mean_abs_delta']:.6g}"
1931
+ )
1932
+ never_compared = summary.get("features_never_compared", [])
1933
+ if never_compared:
1934
+ print(f" Features present live but NEVER in the offline row: {never_compared}")
1935
+ return 0
1936
+
1937
+ # V5.2.2 (development/Problems.md) - `aq evaluate --reconcile-book-history`
1938
+ # compares a real Lean backtest's actual per-date book selections
1939
+ # (logged via phase_v2.diagnostics.book_history, see main.py's write
1940
+ # site) against a fresh offline re-derivation of the same raw scores -
1941
+ # true ground truth instead of more indirect offline-vs-live hypothesis
1942
+ # testing. Early-return, same precedent as --walk-forward-summary above:
1943
+ # needs its own UNFILTERED-by-split dataset load, because a recorded
1944
+ # date's required lookback context can reach into rows from a different
1945
+ # split boundary (see select_context_date_range()'s own docstring for
1946
+ # why naively reusing the split-filtered dataset below would be wrong).
1947
+ if getattr(args, "reconcile_book_history", False):
1948
+ from evaluation import (
1949
+ compute_blended_raw_scores,
1950
+ reconcile_book_history_date,
1951
+ replay_book_history_reconciliation,
1952
+ segment_logged_records_by_run,
1953
+ select_context_date_range,
1954
+ summarize_book_history_reconciliation,
1955
+ summarize_book_member_diversion,
1956
+ summarize_universe_presence_by_symbol,
1957
+ summarize_universe_snapshot_by_security_type,
1958
+ )
1959
+ from portfolio.rank_signal import resolve_rank_signal_policy
1960
+
1961
+ book_history_path_arg = getattr(args, "book_history_path", None)
1962
+ diagnostics_config = config.get("phase_v2", {}).get("diagnostics", {}).get("book_history", {})
1963
+ book_history_path = (
1964
+ Path(book_history_path_arg)
1965
+ if book_history_path_arg
1966
+ else ROOT_DIR / diagnostics_config.get("output_path", "visualization/book_history.jsonl")
1967
+ )
1968
+ if not book_history_path.exists():
1969
+ print(
1970
+ f"error: {book_history_path} not found - run a backtest with "
1971
+ "phase_v2.diagnostics.book_history.enabled=true first.",
1972
+ file=sys.stderr,
1973
+ )
1974
+ return 1
1975
+
1976
+ logged_records: list[dict] = []
1977
+ with open(book_history_path, "r", encoding="utf-8") as book_history_file:
1978
+ for line_number, raw_line in enumerate(book_history_file, start=1):
1979
+ stripped_line = raw_line.strip()
1980
+ if not stripped_line:
1981
+ continue
1982
+ try:
1983
+ logged_records.append(json.loads(stripped_line))
1984
+ except json.JSONDecodeError as error:
1985
+ print(
1986
+ f"warning: skipping malformed line {line_number} in {book_history_path}: {error}",
1987
+ file=sys.stderr,
1988
+ )
1989
+
1990
+ if not logged_records:
1991
+ print(f"error: {book_history_path} contains no usable records.", file=sys.stderr)
1992
+ return 1
1993
+
1994
+ # V5.3.2 (development/Problems.md #91/#97/#99) - book_history.jsonl
1995
+ # is a cumulative, NEVER-rotated log: every real backtest run's
1996
+ # records are appended to the same file forever. Reconciling
1997
+ # against it unsegmented (the pre-V5.3.2 behavior) silently merges
1998
+ # unrelated runs together - confirmed against the real file: 160 of
1999
+ # 174 unique dates recur across more than one run, one date in as
2000
+ # many as 8 - and a --replay-hysteresis walk carries held
2001
+ # allocations across a real run boundary as if it were one
2002
+ # continuous backtest. Segment FIRST; default to the MOST RECENT
2003
+ # run only, never a silent cross-run merge.
2004
+ run_segments = segment_logged_records_by_run(logged_records)
2005
+ if not run_segments:
2006
+ print(f"error: {book_history_path} contains no dated records.", file=sys.stderr)
2007
+ return 1
2008
+
2009
+ reconcile_all_runs = bool(getattr(args, "reconcile_all_runs", False))
2010
+ reconcile_run_index_arg = getattr(args, "reconcile_run_index", None)
2011
+ requested_indices = (
2012
+ list(range(len(run_segments)))
2013
+ if reconcile_all_runs
2014
+ else [reconcile_run_index_arg if reconcile_run_index_arg is not None else -1]
2015
+ )
2016
+ selected_run_indices: list[int] = []
2017
+ for raw_index in requested_indices:
2018
+ normalized_index = raw_index if raw_index >= 0 else raw_index + len(run_segments)
2019
+ if not (0 <= normalized_index < len(run_segments)):
2020
+ print(
2021
+ f"error: --reconcile-run-index {raw_index} out of range - {len(run_segments)} run(s) "
2022
+ f"detected in {book_history_path}.", file=sys.stderr,
2023
+ )
2024
+ return 1
2025
+ selected_run_indices.append(normalized_index)
2026
+
2027
+ run_metadata_by_index: dict[int, dict] = {}
2028
+ for i, run_records in enumerate(run_segments):
2029
+ run_dates = [record["date"] for record in run_records if record.get("date")]
2030
+ run_metadata_by_index[i] = {
2031
+ "num_runs_detected": len(run_segments),
2032
+ "selected_run_index": i,
2033
+ "start_date": run_dates[0] if run_dates else None,
2034
+ "end_date": run_dates[-1] if run_dates else None,
2035
+ "num_records": len(run_records),
2036
+ }
2037
+
2038
+ recon_dataset_path = ML_DIR / "datasets" / "full_dataset.csv"
2039
+ if not recon_dataset_path.exists():
2040
+ print(f"error: {recon_dataset_path} not found - run `aq train --dataset-only` first.", file=sys.stderr)
2041
+ return 1
2042
+ full_dataset = pd.read_csv(recon_dataset_path)
2043
+ if "training_eligible" in full_dataset.columns:
2044
+ full_dataset = full_dataset[full_dataset["training_eligible"]].reset_index(drop=True)
2045
+
2046
+ sequence_window_default = config.get("phase_v2", {}).get("sequence_model", {}).get("window_size", 30)
2047
+ training_metrics_by_model: dict[str, dict | None] = {}
2048
+ for model_name, metrics_filename in (
2049
+ ("sequence", "sequence_training_metrics.json"),
2050
+ ("multitask", "multitask_training_metrics.json"),
2051
+ ):
2052
+ metrics_path = ML_DIR / metrics_filename
2053
+ training_metrics_by_model[model_name] = (
2054
+ json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
2055
+ )
2056
+ policy = resolve_rank_signal_policy(training_metrics_by_model, config)
2057
+ active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
2058
+
2059
+ # V5.3.2 (development/Problems.md #91/#97/#99) - the SAME symbol
2060
+ # order main.py's self.symbols uses (config.json's
2061
+ # phase1.universe.assets, in file order) - cross_sectional_rank_scores()/
2062
+ # _select_book_group() are both Python-stable sorts keyed only on
2063
+ # score, so an exact tie between two symbols resolves by dict
2064
+ # insertion order. Live builds its raw-scores dict in self.symbols
2065
+ # order; this reconciliation tool previously built it from a pandas
2066
+ # groupby (dataset row order) - a different, uncorrelated order -
2067
+ # so a tied pair near a top/bottom-N boundary could pick a
2068
+ # different winner offline than live with byte-identical scores.
2069
+ # Re-inserting in this canonical order before it ever reaches
2070
+ # cross_sectional_rank_scores() fixes that, with zero changes to
2071
+ # the live decision path itself.
2072
+ canonical_symbol_order = [
2073
+ str(asset.get("ticker")) for asset in config.get("phase1", {}).get("universe", {}).get("assets", [])
2074
+ ]
2075
+
2076
+ book_config = config.get("phase_v2", {}).get("portfolio_book", {})
2077
+ strategy_mode = config.get("phase5", {}).get("backtest", {}).get("strategy_mode", "long_flat")
2078
+ book_top_n = int(book_config.get("top_n", 6))
2079
+ book_bottom_n = int(book_config.get("bottom_n", 6)) if strategy_mode == "long_short" else 0
2080
+ # V5.2.3 (development/Problems.md #91) - --replay-hysteresis
2081
+ # switches from independently reconciling each date (the
2082
+ # V5.2.2 default - can't tell a real divergence apart from
2083
+ # hysteresis correctly holding an incumbent) to a walk-forward
2084
+ # replay of offline's OWN selection history, carrying hysteresis
2085
+ # forward the same way main.py's live book does.
2086
+ replay_hysteresis = bool(getattr(args, "replay_hysteresis", False))
2087
+ hysteresis_rank_margin = float(book_config.get("hysteresis_rank_margin", 0.0))
2088
+
2089
+ def _reconcile_one_run(run_records: list[dict]) -> dict:
2090
+ # V5.3.2: each run gets its OWN context date range/predictions -
2091
+ # previously computed once over the entire cumulative file's
2092
+ # date span even when only one small run was being reconciled.
2093
+ recorded_dates = sorted({record["date"] for record in run_records if record.get("date")})
2094
+ logged_records_by_date = {record["date"]: record for record in run_records if record.get("date")}
2095
+
2096
+ recon_min_date, recon_max_date = select_context_date_range(
2097
+ full_dataset, recorded_dates, window_size=int(sequence_window_default)
2098
+ )
2099
+ context_dataset = full_dataset[
2100
+ (full_dataset["date"] >= recon_min_date) & (full_dataset["date"] <= recon_max_date)
2101
+ ].reset_index(drop=True)
2102
+
2103
+ predictions_by_model_head: dict[str, dict[str, object]] = {}
2104
+ for model_kind_for_head in policy["model_priority"]:
2105
+ recon_model_filename, recon_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
2106
+ recon_model_path = ML_DIR / recon_model_filename
2107
+ recon_schema_path = ML_DIR / recon_schema_filename
2108
+ if not recon_model_path.exists() or not recon_schema_path.exists():
2109
+ continue # best-effort - same convention as --calibrate-book-spread above
2110
+ recon_model_export = json.loads(recon_model_path.read_text(encoding="utf-8"))
2111
+ recon_feature_schema = json.loads(recon_schema_path.read_text(encoding="utf-8"))
2112
+ recon_feature_names = recon_feature_schema["model_input_names"]
2113
+ head_predictions: dict[str, object] = {}
2114
+ for head_name in active_heads:
2115
+ head_predictions[head_name] = predict_head(
2116
+ context_dataset, recon_model_export, recon_feature_names, head_name,
2117
+ model_kind=model_kind_for_head,
2118
+ sequence_feature_schema=recon_feature_schema if model_kind_for_head == "sequence" else None,
2119
+ configured_window_size=int(sequence_window_default),
2120
+ )
2121
+ predictions_by_model_head[model_kind_for_head] = head_predictions
2122
+
2123
+ context_dataset["raw_blended_score"] = compute_blended_raw_scores(
2124
+ context_dataset, predictions_by_model_head, policy
2125
+ )
2126
+
2127
+ recorded_subset = context_dataset[context_dataset["date"].isin(recorded_dates)]
2128
+ raw_scores_by_date: dict[str, dict[str, float]] = {}
2129
+ for recon_date, group in recorded_subset.groupby("date", sort=True):
2130
+ if recon_date not in logged_records_by_date:
2131
+ continue
2132
+ scores_lookup = {
2133
+ str(ticker): float(score)
2134
+ for ticker, score in zip(group["ticker"], group["raw_blended_score"])
2135
+ if score is not None and not (isinstance(score, float) and pd.isna(score))
2136
+ }
2137
+ ordered_scores = {
2138
+ ticker: scores_lookup[ticker] for ticker in canonical_symbol_order if ticker in scores_lookup
2139
+ }
2140
+ ordered_scores.update({t: s for t, s in scores_lookup.items() if t not in ordered_scores})
2141
+ raw_scores_by_date[recon_date] = ordered_scores
2142
+
2143
+ if replay_hysteresis:
2144
+ ordered_logged_records = [
2145
+ logged_records_by_date[d] for d in recorded_dates if d in raw_scores_by_date
2146
+ ]
2147
+ per_date_results = replay_book_history_reconciliation(
2148
+ ordered_logged_records, raw_scores_by_date,
2149
+ top_n=book_top_n, bottom_n=book_bottom_n, hysteresis_rank_margin=hysteresis_rank_margin,
2150
+ )
2151
+ else:
2152
+ per_date_results = [
2153
+ reconcile_book_history_date(
2154
+ logged_records_by_date[recon_date], raw_scores_by_symbol,
2155
+ top_n=book_top_n, bottom_n=book_bottom_n,
2156
+ )
2157
+ for recon_date, raw_scores_by_symbol in raw_scores_by_date.items()
2158
+ ]
2159
+
2160
+ summary = summarize_book_history_reconciliation(per_date_results)
2161
+ # Cheap and always attempted (pure, reads straight off the log's
2162
+ # own "universe" keys - no re-inference) - degrades to
2163
+ # num_dates_with_universe_data=0 for a log written without
2164
+ # include_full_universe, never raises. V5.3.2: computed over
2165
+ # this RUN's own records only, not the whole cumulative file.
2166
+ universe_summary = summarize_universe_snapshot_by_security_type(run_records)
2167
+ # V5.2.6 (development/Problems.md) - same "cheap, always attempted,
2168
+ # degrades to zeros" contract as universe_summary above - reads
2169
+ # straight off the log's own optional "book_member_decisions" key.
2170
+ diversion_summary = summarize_book_member_diversion(run_records)
2171
+ # V5.3.1 (development/Problems.md #91/#97) - summarize_universe_presence_by_symbol()
2172
+ # does its own internal run-segmentation too, but since run_records
2173
+ # is already a single run here, it always reports exactly one.
2174
+ universe_presence_summary = summarize_universe_presence_by_symbol(run_records)
2175
+ return {
2176
+ "mode": "replay_hysteresis" if replay_hysteresis else "independent",
2177
+ "per_date": per_date_results,
2178
+ "summary": summary,
2179
+ "universe_summary": universe_summary,
2180
+ "diversion_summary": diversion_summary,
2181
+ "universe_presence_summary": universe_presence_summary,
2182
+ }
2183
+
2184
+ if reconcile_all_runs:
2185
+ all_runs_payloads = []
2186
+ for i in selected_run_indices:
2187
+ run_payload = _reconcile_one_run(run_segments[i])
2188
+ run_payload["run_metadata"] = run_metadata_by_index[i]
2189
+ all_runs_payloads.append(run_payload)
2190
+ # Top-level default keys mirror the most-recent run (last in
2191
+ # selected_run_indices, since run_segments is chronological) -
2192
+ # --reconcile-all-runs is additive on top of the normal default
2193
+ # output, never a replacement for it.
2194
+ payload = dict(all_runs_payloads[-1])
2195
+ payload["all_runs"] = all_runs_payloads
2196
+ else:
2197
+ selected_index = selected_run_indices[0]
2198
+ payload = _reconcile_one_run(run_segments[selected_index])
2199
+ payload["run_metadata"] = run_metadata_by_index[selected_index]
2200
+
2201
+ _write_evaluation_json(ML_DIR / "evaluation" / "book_history_reconciliation.json", payload)
2202
+
2203
+ if args.json:
2204
+ print(json.dumps(payload, indent=2, default=str))
2205
+ else:
2206
+ def _print_run_result(run_payload: dict, label: str) -> None:
2207
+ run_summary = run_payload["summary"]
2208
+ run_universe_summary = run_payload["universe_summary"]
2209
+ run_diversion_summary = run_payload["diversion_summary"]
2210
+ run_universe_presence_summary = run_payload["universe_presence_summary"]
2211
+ run_meta = run_payload["run_metadata"]
2212
+ print(f"{label}{run_meta['start_date']}..{run_meta['end_date']}, {run_summary['num_dates']} dates:")
2213
+ print(
2214
+ f" exact_match={run_summary['num_dates_exact_match']}/{run_summary['num_dates']} "
2215
+ f"mean_overlap_fraction={run_summary['mean_overlap_fraction']}"
2216
+ )
2217
+ print(
2218
+ f" mean_raw_score_delta_abs={run_summary['mean_raw_score_delta_abs']} "
2219
+ f"mean_weight_delta_abs={run_summary['mean_weight_delta_abs']} "
2220
+ f"({run_summary['num_dates_with_weight_logged']}/{run_summary['num_dates']} dates had a "
2221
+ "logged weight)"
2222
+ )
2223
+ print(
2224
+ f" symbols_only_logged_total={run_summary['num_symbols_only_logged_total']} "
2225
+ f"symbols_only_offline_total={run_summary['num_symbols_only_offline_total']}"
2226
+ )
2227
+ if run_universe_summary["num_dates_with_universe_data"] > 0:
2228
+ print(
2229
+ f" Universe snapshot ({run_universe_summary['num_dates_with_universe_data']}/"
2230
+ f"{run_summary['num_dates']} dates carry full-universe data):"
2231
+ )
2232
+ for security_type, stats in sorted(run_universe_summary["by_security_type"].items()):
2233
+ print(
2234
+ f" {security_type}: mean_raw_rank_score={stats['mean_raw_rank_score']} "
2235
+ f"feature_ready_rate={stats['feature_ready_rate']} "
2236
+ f"trading_eligible_rate={stats['trading_eligible_rate']} "
2237
+ f"(n={stats['num_symbol_dates']})"
2238
+ )
2239
+ if run_diversion_summary["num_records_with_decisions"] > 0:
2240
+ print(
2241
+ f" Book-member diversion ({run_diversion_summary['num_records_with_decisions']}/"
2242
+ f"{run_summary['num_dates']} dates carry decision data, "
2243
+ f"{run_diversion_summary['total_book_member_dates']} book-member-dates total):"
2244
+ )
2245
+ for action, count in sorted(
2246
+ run_diversion_summary["action_counts"].items(), key=lambda item: -item[1]
2247
+ ):
2248
+ print(f" action={action}: {count}")
2249
+ for reason, count in sorted(
2250
+ run_diversion_summary["reason_counts"].items(), key=lambda item: -item[1]
2251
+ ):
2252
+ print(f" reason={reason}: {count}")
2253
+ if run_universe_presence_summary["num_runs_detected"] > 0:
2254
+ top_absent = sorted(
2255
+ run_universe_presence_summary["runs"][0]["absence_rate_by_symbol"].items(),
2256
+ key=lambda item: -item[1],
2257
+ )[:10]
2258
+ for symbol, absence_rate in top_absent:
2259
+ if absence_rate > 0:
2260
+ print(f" {symbol}: absent from universe on {absence_rate:.1%} of this run's dates")
2261
+
2262
+ print(
2263
+ f"Book-history reconciliation ({book_history_path}): "
2264
+ f"{payload['run_metadata']['num_runs_detected']} run(s) detected total "
2265
+ "(cumulative, never-rotated log)."
2266
+ )
2267
+ if replay_hysteresis:
2268
+ print(
2269
+ " NOTE: this reconciliation REPLAYS hysteresis (--replay-hysteresis) - offline's own "
2270
+ "held allocations are carried forward date-by-date, the same way main.py's live book "
2271
+ "does, WITHIN each run only (V5.3.2: never across a run boundary). The first reconciled "
2272
+ "date of each run still starts from a COLD (empty) held-allocations state regardless of "
2273
+ "the book's true earlier history, so early dates may show a colder-start mismatch than "
2274
+ "mid-series dates."
2275
+ )
2276
+ else:
2277
+ print(
2278
+ " NOTE: this reconciliation replays NO hysteresis (previous_allocations=None) - a "
2279
+ "mismatch can mean either a real divergence OR the live book correctly holding an "
2280
+ "incumbent that day's natural ranking alone wouldn't pick. Read per-date role_mismatches "
2281
+ "before concluding anything is actually wrong. Pass --replay-hysteresis for the "
2282
+ "hysteresis-aware alternative."
2283
+ )
2284
+ if reconcile_all_runs:
2285
+ num_runs = len(payload["all_runs"])
2286
+ for position, run_payload in enumerate(payload["all_runs"]):
2287
+ _print_run_result(run_payload, f"Run {position + 1}/{num_runs} (")
2288
+ else:
2289
+ _print_run_result(payload, "Reconciled run (")
2290
+
2291
+ _refresh_readme_evaluation_sections()
2292
+ return 0
2293
+
2294
+ ranking_config = config.get("phase1", {}).get("target", {}).get("ranking", {})
2295
+ net_perf_config = ranking_config.get("net_performance", {})
2296
+ if not net_perf_config:
2297
+ print("error: phase1.target.ranking.net_performance is not configured.", file=sys.stderr)
2298
+ return 1
2299
+
2300
+ dataset_path = ML_DIR / "datasets" / "full_dataset.csv"
2301
+ if not dataset_path.exists():
2302
+ print(f"error: {dataset_path} not found - run `aq train --dataset-only` first.", file=sys.stderr)
2303
+ return 1
2304
+
2305
+ model_kind = args.model or "sequence"
2306
+ head = args.head or "rank_20d"
2307
+ split = args.split or "backtest"
2308
+
2309
+ model_filename, schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind]
2310
+ model_path = ML_DIR / model_filename
2311
+ schema_path = ML_DIR / schema_filename
2312
+ if not model_path.exists() or not schema_path.exists():
2313
+ print(f"error: {model_path} or {schema_path} not found - run `aq train` first.", file=sys.stderr)
2314
+ return 1
2315
+
2316
+ model_export = json.loads(model_path.read_text(encoding="utf-8"))
2317
+ feature_schema = json.loads(schema_path.read_text(encoding="utf-8"))
2318
+ feature_names = feature_schema["model_input_names"]
2319
+
2320
+ dataset = pd.read_csv(dataset_path)
2321
+ if "training_eligible" in dataset.columns:
2322
+ dataset = dataset[dataset["training_eligible"]].reset_index(drop=True)
2323
+ if split != "all":
2324
+ dataset = dataset[dataset["split"] == split].reset_index(drop=True)
2325
+ if dataset.empty:
2326
+ print(f"error: no rows for split={split!r} after filtering - check `aq train` has run for this window.", file=sys.stderr)
2327
+ return 1
2328
+
2329
+ sequence_window_default = config.get("phase_v2", {}).get("sequence_model", {}).get("window_size", 30)
2330
+ predictions = predict_head(
2331
+ dataset, model_export, feature_names, head,
2332
+ model_kind=model_kind,
2333
+ sequence_feature_schema=feature_schema if model_kind == "sequence" else None,
2334
+ configured_window_size=int(sequence_window_default),
2335
+ )
2336
+ dataset = dataset.copy()
2337
+ dataset["predicted_head"] = predictions
2338
+
2339
+ sector_by_ticker = load_sector_mapping(config)
2340
+ base_kwargs = dict(
2341
+ prediction_column="predicted_head",
2342
+ forward_return_column="target_return_1d",
2343
+ ticker_column="ticker",
2344
+ date_column="date",
2345
+ top_n=int(net_perf_config.get("top_n", 6)),
2346
+ bottom_n=int(net_perf_config.get("bottom_n", 6)),
2347
+ rebalance_every_bars=int(net_perf_config.get("rebalance_every_bars", 10)),
2348
+ cost_bps_per_side=float(net_perf_config.get("cost_bps_per_side", 5.0)),
2349
+ commission_bps=float(net_perf_config.get("commission_bps", 1.0)),
2350
+ gross_exposure=float(net_perf_config.get("gross_exposure", 1.0)),
2351
+ dollar_neutral=bool(net_perf_config.get("dollar_neutral", True)),
2352
+ sector_neutral=bool(net_perf_config.get("sector_neutral", True)),
2353
+ sector_by_ticker=sector_by_ticker,
2354
+ hysteresis_rank_margin=float(net_perf_config.get("hysteresis_rank_margin", 0.0)),
2355
+ max_weight_per_name=float(net_perf_config.get("max_weight_per_name", 0.12)),
2356
+ min_universe_size=int(ranking_config.get("min_universe_size", 20)),
2357
+ # V5.2.1 (development/Problems.md) - both no-op by default (0/0.0/0.0),
2358
+ # matching simulate_rank_book()'s own defaults exactly.
2359
+ entry_lag_bars=int(net_perf_config.get("entry_lag_bars", 0)),
2360
+ min_commission_usd=float(net_perf_config.get("min_commission_usd", 0.0)),
2361
+ assumed_portfolio_value_usd=float(net_perf_config.get("assumed_portfolio_value_usd", 0.0)),
2362
+ )
2363
+
2364
+ run_rank_book = bool(args.rank_book or args.all)
2365
+ # V5.3.8 - Monte Carlo layer rides on top of the base rank-book run's
2366
+ # daily net returns, so requesting it implicitly requests the sim.
2367
+ run_monte_carlo = bool(getattr(args, "monte_carlo", False))
2368
+ if run_monte_carlo:
2369
+ run_rank_book = True
2370
+ # V5.4.4 - benchmark comparison: cheap (pure pandas over the dataset's
2371
+ # own close pivot, no model inference), so it rides in --all and the
2372
+ # README's benchmark section auto-refreshes on every full evaluate run.
2373
+ run_benchmarks = bool(getattr(args, "benchmarks", False) or args.all)
2374
+ run_capacity = bool(args.capacity or args.all)
2375
+ run_stress = bool(args.stress or args.all)
2376
+ run_calibrate = bool(args.calibrate_edge or args.all)
2377
+ # V5.1 Phase 5 (item 9) - deliberately NOT bundled into --all (whose
2378
+ # bundle is rank-book/capacity/stress/calibrate-edge + the V5.4.4
2379
+ # benchmarks) - ablation is a heavier, opt-in-only report.
2380
+ run_ablation_flag = bool(getattr(args, "ablation", False))
2381
+ # V5.1 (Problems.md) - also deliberately NOT bundled into --all, same
2382
+ # reasoning as --ablation: a heavier, two-model, opt-in-only report.
2383
+ run_calibrate_book_spread = bool(getattr(args, "calibrate_book_spread", False))
2384
+ # V5.2.6 (development/Problems.md) - same reasoning as
2385
+ # run_calibrate_book_spread immediately above: a heavier, opt-in-only
2386
+ # report, not bundled into --all.
2387
+ run_calibrate_confidence_threshold = bool(getattr(args, "calibrate_confidence_threshold", False))
2388
+ # V5.3.5 (development/Problems.md #102) - same reasoning as
2389
+ # run_calibrate_book_spread/run_calibrate_confidence_threshold above: a
2390
+ # heavier, opt-in-only report, not bundled into --all.
2391
+ run_calibrate_rolling_ic_floor = bool(getattr(args, "calibrate_rolling_ic_floor", False))
2392
+ # V5.2.8 (development/Problems.md #94) - same reasoning as --ablation/
2393
+ # --calibrate-book-spread above: a heavier, investigation-only report,
2394
+ # not bundled into --all.
2395
+ run_replay_kill_switch = bool(getattr(args, "replay_kill_switch", False))
2396
+ # V5.3.5 (development/Problems.md #102) - same reasoning as
2397
+ # --replay-kill-switch above: a heavier, investigation-only report,
2398
+ # not bundled into --all.
2399
+ run_replay_rolling_ic_gate = bool(getattr(args, "replay_rolling_ic_gate", False))
2400
+ # V5.3.1 (development/Problems.md #34/#96) - same reasoning as
2401
+ # --replay-kill-switch above: a heavier, investigation-only report,
2402
+ # not bundled into --all.
2403
+ run_simulate_limit_fills = bool(getattr(args, "simulate_limit_fills", False))
2404
+ # Bare `aq evaluate` with no flags at all defaults to --rank-book - the
2405
+ # single most useful number ("is the fee drag fixed"), matching every
2406
+ # other `aq` command's "sane default when no scope flag is given"
2407
+ # convention (e.g. `aq test` with no subsystem flags runs everything).
2408
+ # Every opt-in-only flag (not just the --all-bundled ones) must be
2409
+ # included here - Problems.md bug: --calibrate-book-spread alone used
2410
+ # to also silently run a whole extra --rank-book simulation because it
2411
+ # wasn't counted as "a flag was given".
2412
+ if not (
2413
+ run_rank_book or run_capacity or run_stress or run_calibrate or run_ablation_flag
2414
+ or run_calibrate_book_spread or run_calibrate_confidence_threshold or run_calibrate_rolling_ic_floor
2415
+ or run_replay_kill_switch or run_replay_rolling_ic_gate or run_simulate_limit_fills
2416
+ or run_monte_carlo or run_benchmarks
2417
+ ):
2418
+ run_rank_book = True
2419
+
2420
+ report: dict = {}
2421
+ evaluation_dir = ML_DIR / "evaluation"
2422
+
2423
+ if run_rank_book:
2424
+ result = simulate_rank_book(dataset, **base_kwargs)
2425
+ report["rank_book"] = result.to_dict()
2426
+ _write_evaluation_json(evaluation_dir / "rank_book_simulation.json", report["rank_book"])
2427
+ # Additive, per-model copy alongside the unsuffixed "last run" file
2428
+ # above (untouched - monitoring/evaluation_state.py's webui feed
2429
+ # and existing tests depend on that exact fixed name) - lets a
2430
+ # multitask run and a sequence run coexist on disk instead of the
2431
+ # second silently clobbering the first, for callers (e.g.
2432
+ # generate_evaluation_report.py) that want to compare both models
2433
+ # at once rather than only "whichever ran last".
2434
+ _write_evaluation_json(evaluation_dir / f"rank_book_simulation_{model_kind}.json", report["rank_book"])
2435
+ if not args.json:
2436
+ print(f"Rank book ({model_kind}/{head}, split={split}, entry_lag_bars={base_kwargs['entry_lag_bars']}):")
2437
+ print(f" gross_sharpe={result.gross_sharpe:.4f} net_sharpe={result.net_sharpe:.4f}")
2438
+ print(f" gross_total_return={result.gross_total_return:.4%} net_total_return={result.net_total_return:.4%}")
2439
+ print(f" net_max_drawdown={result.net_max_drawdown:.4%}")
2440
+ print(f" annualized_turnover={result.annualized_turnover:.2f}x cost_drag_annual_bps={result.cost_drag_annual_bps:.2f}")
2441
+ print(f" num_rebalances={result.num_rebalances} num_dates_used={result.num_dates_used}")
2442
+ print(f" mean_names_long={result.mean_names_long:.2f} mean_names_short={result.mean_names_short:.2f}")
2443
+
2444
+ # V5.2.1 (development/Problems.md) - a Daily-resolution market
2445
+ # order decided off bar N's close fills at bar N+1's open, not
2446
+ # bar N's close; this simulator's own return accrual previously
2447
+ # had no way to model that. ALWAYS compute and show both
2448
+ # entry_lag_bars=0 (same-bar fill, what this simulator has always
2449
+ # assumed) and =1 (the honest, next-bar-fill "lag tax") here,
2450
+ # regardless of what net_perf_config.entry_lag_bars is configured
2451
+ # to - so the gap this investigation found is never silently
2452
+ # invisible again, whichever value base_kwargs happens to carry.
2453
+ lag_kwargs = dict(base_kwargs)
2454
+ lag_kwargs["entry_lag_bars"] = 1
2455
+ lagged_result = simulate_rank_book(dataset, **lag_kwargs)
2456
+ report["rank_book_entry_lag_1"] = lagged_result.to_dict()
2457
+ _write_evaluation_json(evaluation_dir / "rank_book_simulation_entry_lag_1.json", report["rank_book_entry_lag_1"])
2458
+ _write_evaluation_json(
2459
+ evaluation_dir / f"rank_book_simulation_entry_lag_1_{model_kind}.json", report["rank_book_entry_lag_1"]
2460
+ )
2461
+ if run_monte_carlo:
2462
+ # V5.3.8 - Monte Carlo layer: stationary-block bootstrap of this
2463
+ # base run's daily NET returns into n_runs alternative equity
2464
+ # histories. Pure core in evaluation/monte_carlo.py; heavy runs
2465
+ # matrix persisted as .npz next to the JSON (README renderer owns
2466
+ # the PNG), summary + avg/band curves go in the JSON.
2467
+ from evaluation.monte_carlo import run_monte_carlo
2468
+
2469
+ import numpy as _np_mc
2470
+
2471
+ mc = run_monte_carlo(
2472
+ result.per_date_net_return,
2473
+ n_runs=int(getattr(args, "mc_runs", 1000)),
2474
+ block_size=int(getattr(args, "mc_block_size", 20)),
2475
+ seed=int(getattr(args, "mc_seed", 42)),
2476
+ method=str(getattr(args, "mc_method", "block")),
2477
+ )
2478
+ mc_payload = {k: v for k, v in mc.items() if k != "runs_curves"}
2479
+ mc_payload["model_kind"] = model_kind
2480
+ mc_payload["head"] = head
2481
+ mc_payload["split"] = split
2482
+ _write_evaluation_json(evaluation_dir / "monte_carlo.json", mc_payload)
2483
+ _np_mc.savez_compressed(evaluation_dir / "monte_carlo_curves.npz", runs_curves=_np_mc.asarray(mc["runs_curves"]))
2484
+ if not args.json:
2485
+ fr = mc["final_return_pct"]
2486
+ md = mc["max_drawdown"]
2487
+ sh = mc["sharpe"]
2488
+ print(f"Monte Carlo ({mc['status']}): {mc['config']['n_runs']} runs "
2489
+ f"(method={mc['config']['method']}, block={mc['config']['block_size']}, seed={mc['config']['seed']})")
2490
+ print(f" final_return p5/p50/p95 = {fr['p5']}% / {fr['p50']}% / {fr['p95']}% | P(neg)={mc['prob_negative_return']}")
2491
+ print(f" sharpe p5/p95 = {sh['p5']} / {sh['p95']} | maxDD p95 = {md['p95']}%")
2492
+ if not args.json:
2493
+ print("Rank book (entry_lag_bars=1, the 'lag tax' - see development/Problems.md):")
2494
+ print(f" gross_sharpe={lagged_result.gross_sharpe:.4f} net_sharpe={lagged_result.net_sharpe:.4f}")
2495
+ print(f" delta_net_sharpe vs entry_lag_bars=0: {lagged_result.net_sharpe - result.net_sharpe:+.4f}")
2496
+
2497
+ # V5.4.4 - benchmark comparison over the SAME dataset window/split the
2498
+ # rank book just ran on (the `dataset` frame is already split-filtered
2499
+ # above, so the pivot - and every baseline number - is automatically
2500
+ # like-for-like with the rank book's own window). Pure pandas over the
2501
+ # dataset's own close columns; zero new data dependencies, zero model
2502
+ # inference. Missing close column or missing SPY/TLT degrades to
2503
+ # SKIPPED entries inside the report, never a crash.
2504
+ if run_benchmarks:
2505
+ from evaluation.benchmark_comparison import close_pivot_from_frame, run_all_baselines
2506
+
2507
+ benchmark_close_pivot = close_pivot_from_frame(dataset)
2508
+ benchmark_report = run_all_baselines(benchmark_close_pivot)
2509
+ benchmark_report["model_kind"] = model_kind
2510
+ benchmark_report["head"] = head
2511
+ benchmark_report["split"] = split
2512
+ report["benchmarks"] = benchmark_report
2513
+ _write_evaluation_json(evaluation_dir / "benchmark_comparison.json", benchmark_report)
2514
+ if not args.json:
2515
+ print(f"Benchmarks (split={split}, {len(benchmark_close_pivot)} dates x {benchmark_close_pivot.shape[1]} tickers):")
2516
+ for baseline in benchmark_report["baselines"]:
2517
+ sharpe = baseline["net_sharpe"]
2518
+ sharpe_str = f"{sharpe:+.4f}" if sharpe is not None else " n/a "
2519
+ print(f" {baseline['strategy']:<14} net_sharpe={sharpe_str} total_return={baseline['total_return_pct']}% [{baseline.get('status', 'OK')}]")
2520
+
2521
+ # V5.2.8 (development/Problems.md #94) - reuses --rank-book's own
2522
+ # `result` (per_date/per_date_net_return) when that flag was also
2523
+ # given, rather than recomputing simulate_rank_book() a second time;
2524
+ # computes it fresh (entry_lag_bars=0, the same default --rank-book
2525
+ # uses) when --replay-kill-switch is passed on its own.
2526
+ if run_replay_kill_switch:
2527
+ from evaluation.kill_switch_replay import (
2528
+ replay_kill_switch_over_dataset,
2529
+ summarize_kill_switch_replay,
2530
+ )
2531
+
2532
+ kill_switch_replay_result = result if run_rank_book else simulate_rank_book(dataset, **base_kwargs)
2533
+ kill_switch_config = config.get("phase_v2", {}).get("risk", {}).get("kill_switch", {})
2534
+ phase6_risk = config.get("phase6", {}).get("risk", {})
2535
+ daily_portfolio_returns = dict(
2536
+ zip(kill_switch_replay_result.per_date, kill_switch_replay_result.per_date_net_return)
2537
+ )
2538
+ replay_records = replay_kill_switch_over_dataset(
2539
+ kill_switch_replay_result.per_date,
2540
+ daily_portfolio_returns,
2541
+ kill_switch_config,
2542
+ max_daily_drawdown_pct=float(phase6_risk.get("max_daily_drawdown_pct", 0.03)),
2543
+ max_total_drawdown_pct=float(phase6_risk.get("max_total_drawdown_pct", 0.12)),
2544
+ )
2545
+ replay_summary = summarize_kill_switch_replay(replay_records)
2546
+ report["kill_switch_replay"] = {"summary": replay_summary, "per_date": replay_records}
2547
+ _write_evaluation_json(evaluation_dir / "kill_switch_replay.json", report["kill_switch_replay"])
2548
+ if not args.json:
2549
+ print(
2550
+ f"Kill-switch replay (APPROXIMATION - see development/Problems.md #94): "
2551
+ f"{replay_summary['trip_count']} trips across {replay_summary['total_dates']} dates, "
2552
+ f"{replay_summary['locked_days']} locked days ({replay_summary['locked_day_fraction']:.2%})"
2553
+ )
2554
+
2555
+ # V5.3.1 (development/Problems.md #34/#96) - offline counterfactual:
2556
+ # "how often would a real limit order have filled" without needing a
2557
+ # live Lean run. Deliberately independent of --rank-book above - it
2558
+ # simulates a signal firing on every row, not just book-selected ones
2559
+ # (see evaluation/limit_fill_simulator.py's own docstring for why
2560
+ # that's a different, complementary question, not a replay of the
2561
+ # real order-events.json evidence).
2562
+ if run_simulate_limit_fills:
2563
+ from evaluation.limit_fill_simulator import simulate_limit_fills, sweep_limit_fill_offsets
2564
+
2565
+ limit_orders_config = config.get("phase_v2", {}).get("limit_orders", {})
2566
+ timeout_bars = int(limit_orders_config.get("unfilled_timeout_bars", 3))
2567
+ offset_sweep_arg = getattr(args, "limit_fill_offset_sweep", None)
2568
+ if offset_sweep_arg:
2569
+ offsets = [float(value) for value in offset_sweep_arg.split(",")]
2570
+ limit_fill_result = sweep_limit_fill_offsets(dataset, unfilled_timeout_bars=timeout_bars, offset_multipliers=offsets)
2571
+ else:
2572
+ configured_offset = float(limit_orders_config.get("offset_multiplier", 1.0))
2573
+ limit_fill_result = simulate_limit_fills(dataset, unfilled_timeout_bars=timeout_bars, offset_multiplier=configured_offset)
2574
+ report["limit_fill_simulation"] = limit_fill_result
2575
+ _write_evaluation_json(evaluation_dir / "limit_fill_simulation.json", limit_fill_result)
2576
+ if not args.json:
2577
+ print(f"Limit-fill simulation (APPROXIMATION, unfilled_timeout_bars={timeout_bars}):")
2578
+ if offset_sweep_arg:
2579
+ for multiplier, result_for_multiplier in limit_fill_result.items():
2580
+ overall = result_for_multiplier["overall"]
2581
+ print(
2582
+ f" offset_multiplier={multiplier}: fill_rate={overall['fill_rate']:.2%} "
2583
+ f"timeout_rate={overall['timeout_rate']:.2%} num_signals={overall['num_signals']}"
2584
+ )
2585
+ else:
2586
+ overall = limit_fill_result["overall"]
2587
+ print(
2588
+ f" overall: fill_rate={overall['fill_rate']:.2%} timeout_rate={overall['timeout_rate']:.2%} "
2589
+ f"num_signals={overall['num_signals']}"
2590
+ )
2591
+ for security_type, stats in sorted(limit_fill_result["by_asset_class"].items()):
2592
+ print(f" {security_type}: fill_rate={stats['fill_rate']:.2%} num_signals={stats['num_signals']}")
2593
+
2594
+ if run_capacity:
2595
+ cap = capacity_curve(
2596
+ dataset,
2597
+ participation_cap=float(net_perf_config.get("capacity_participation_cap", 0.01)),
2598
+ base_kwargs=base_kwargs,
2599
+ top_n_sweep=[int(n) for n in net_perf_config.get("capacity_top_n_sweep", [3, 6, 10, 15])],
2600
+ )
2601
+ report["capacity"] = cap
2602
+ _write_evaluation_json(evaluation_dir / "capacity_report.json", cap)
2603
+ _write_evaluation_json(evaluation_dir / f"capacity_report_{model_kind}.json", cap)
2604
+ if not args.json:
2605
+ print(f"Capacity: ${cap['capacity_usd']:,.0f} (binding: {cap['binding_ticker']})")
2606
+ for row in cap["per_top_n"]:
2607
+ print(f" top_n={row['top_n']}: net_sharpe={row['net_sharpe']:.4f}")
2608
+
2609
+ if run_stress:
2610
+ stress = stress_test_costs(
2611
+ dataset, base_kwargs=base_kwargs,
2612
+ cost_multipliers=tuple(float(m) for m in net_perf_config.get("stress_cost_multipliers", [1.0, 2.0, 3.0])),
2613
+ )
2614
+ report["stress"] = stress
2615
+ _write_evaluation_json(evaluation_dir / "cost_stress_report.json", {"stress": stress})
2616
+ _write_evaluation_json(evaluation_dir / f"cost_stress_report_{model_kind}.json", {"stress": stress})
2617
+ if not args.json:
2618
+ print("Cost stress test:")
2619
+ for row in stress:
2620
+ print(f" {row['cost_multiplier']}x: net_sharpe={row['net_sharpe']:.4f} net_total_return={row['net_total_return']:.4%}")
2621
+
2622
+ if run_calibrate:
2623
+ # A simple OLS slope-through-the-origin of forward return on
2624
+ # (rank - 0.5)*2 - matches execution/cost_model.py::expected_edge_bps()'s
2625
+ # own linear-in-rank-deviation model. That function documents its
2626
+ # edge_bps_per_rank_unit as the FULL predicted move over
2627
+ # phase_v2.costs.horizon_days (it then scales down by
2628
+ # holding_bars/horizon_days itself) - so the regression must use
2629
+ # the SAME horizon's forward return, not a fixed 1-day one, or the
2630
+ # calibrated number is off by roughly horizon_days/1 (a ~20x
2631
+ # understatement of edge at the default horizon_days=20, since a
2632
+ # 1-day slope was previously being treated as a 20-day move and
2633
+ # then halved again downstream).
2634
+ costs_config = config.get("phase_v2", {}).get("costs", {})
2635
+ horizon_days = int(costs_config.get("horizon_days", 20))
2636
+ forward_return_column = f"target_return_{horizon_days}d"
2637
+ if forward_return_column not in dataset.columns:
2638
+ print(
2639
+ f"warning: {forward_return_column!r} not in dataset (horizon_days={horizon_days}); "
2640
+ "falling back to target_return_1d - the calibrated value will then represent a "
2641
+ "1-day move, not the horizon_days move expected_edge_bps() assumes.",
2642
+ file=sys.stderr,
2643
+ )
2644
+ forward_return_column = "target_return_1d"
2645
+ rank_deviation = (dataset["predicted_head"] - 0.5) * 2.0
2646
+ forward_return = dataset[forward_return_column]
2647
+ valid = rank_deviation.notna() & forward_return.notna()
2648
+ x = rank_deviation[valid].to_numpy()
2649
+ y = forward_return[valid].to_numpy()
2650
+ denominator = float((x * x).sum()) if len(x) >= 2 else 0.0
2651
+ slope = float((x * y).sum() / denominator) if denominator > 0 else 0.0
2652
+ calibrated_edge_bps = slope * 10_000.0
2653
+ report["calibrated_edge_bps_per_rank_unit"] = calibrated_edge_bps
2654
+ report["calibrated_edge_forward_return_column"] = forward_return_column
2655
+ if not args.json:
2656
+ print(f"Calibrated edge_bps_per_rank_unit ({forward_return_column}): {calibrated_edge_bps:.4f}")
2657
+ print(f" Apply with: aq config set phase_v2.costs.edge_bps_per_rank_unit {calibrated_edge_bps:.4f}")
2658
+ print(" Then enable the gate: aq config set phase_v2.costs.enabled true")
2659
+
2660
+ if run_calibrate_book_spread:
2661
+ # V5.1 (development/Problems.md) - min_rank_confidence_spread was a
2662
+ # guessed constant (0.2) never checked against this codebase's
2663
+ # actual raw-score scale; evaluation/rank_book_simulator.py's own
2664
+ # book simulation hardcodes min_rank_confidence_spread=0.0 (see
2665
+ # that module's comment), so nothing ever exercised the real gate
2666
+ # before it shipped live. This mirrors --calibrate-edge's own
2667
+ # discipline: derive the number from data, don't guess it.
2668
+ from evaluation import calibrate_book_confidence_spread, compute_blended_raw_scores
2669
+ from portfolio.rank_signal import resolve_rank_signal_policy
2670
+
2671
+ training_metrics_by_model: dict[str, dict | None] = {}
2672
+ for model_name, metrics_filename in (
2673
+ ("sequence", "sequence_training_metrics.json"),
2674
+ ("multitask", "multitask_training_metrics.json"),
2675
+ ):
2676
+ metrics_path = ML_DIR / metrics_filename
2677
+ training_metrics_by_model[model_name] = (
2678
+ json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
2679
+ )
2680
+ policy = resolve_rank_signal_policy(training_metrics_by_model, config)
2681
+ active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
2682
+
2683
+ predictions_by_model_head: dict[str, dict[str, object]] = {}
2684
+ for model_kind_for_head in policy["model_priority"]:
2685
+ calib_model_filename, calib_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
2686
+ calib_model_path = ML_DIR / calib_model_filename
2687
+ calib_schema_path = ML_DIR / calib_schema_filename
2688
+ if not calib_model_path.exists() or not calib_schema_path.exists():
2689
+ continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
2690
+ calib_model_export = json.loads(calib_model_path.read_text(encoding="utf-8"))
2691
+ calib_feature_schema = json.loads(calib_schema_path.read_text(encoding="utf-8"))
2692
+ calib_feature_names = calib_feature_schema["model_input_names"]
2693
+ head_predictions: dict[str, object] = {}
2694
+ for head_name in active_heads:
2695
+ head_predictions[head_name] = predict_head(
2696
+ dataset, calib_model_export, calib_feature_names, head_name,
2697
+ model_kind=model_kind_for_head,
2698
+ sequence_feature_schema=calib_feature_schema if model_kind_for_head == "sequence" else None,
2699
+ configured_window_size=int(sequence_window_default),
2700
+ )
2701
+ predictions_by_model_head[model_kind_for_head] = head_predictions
2702
+
2703
+ dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
2704
+
2705
+ book_config = config.get("phase_v2", {}).get("portfolio_book", {})
2706
+ strategy_mode = config.get("phase5", {}).get("backtest", {}).get("strategy_mode", "long_flat")
2707
+ book_top_n = int(book_config.get("top_n", 6))
2708
+ book_bottom_n = int(book_config.get("bottom_n", 6)) if strategy_mode == "long_short" else 0
2709
+
2710
+ book_spread_result = calibrate_book_confidence_spread(
2711
+ dataset,
2712
+ raw_score_column="raw_blended_score",
2713
+ top_n=book_top_n,
2714
+ bottom_n=book_bottom_n,
2715
+ percentile=float(getattr(args, "book_spread_percentile", 0.10)),
2716
+ )
2717
+ report["book_spread_calibration"] = book_spread_result
2718
+ _write_evaluation_json(evaluation_dir / "book_spread_calibration.json", book_spread_result)
2719
+ if not args.json:
2720
+ calibrated_spread = book_spread_result["calibrated_min_rank_confidence_spread"]
2721
+ distribution = book_spread_result["spread_distribution"]
2722
+ print(
2723
+ f"Calibrated min_rank_confidence_spread (p{book_spread_result['percentile']*100:.0f}, "
2724
+ f"{book_spread_result['num_dates_used']} dates used, "
2725
+ f"{book_spread_result['num_dates_skipped_thin_universe']} skipped thin-universe): "
2726
+ f"{calibrated_spread:.4f}"
2727
+ )
2728
+ print(
2729
+ f" Distribution: min={distribution['min']}, p10={distribution['p10']}, "
2730
+ f"median={distribution['median']}, p75={distribution['p75']}, max={distribution['max']}"
2731
+ )
2732
+ print(f" Apply with: aq config set phase_v2.portfolio_book.min_rank_confidence_spread {calibrated_spread:.4f}")
2733
+ print(
2734
+ " NOTE: this recalibrates the gate's threshold to this model's real achievable dispersion - "
2735
+ "it does not certify the underlying rank heads are skillful. Check ml/*_training_metrics.json's "
2736
+ "quality_status before trusting the resulting book."
2737
+ )
2738
+
2739
+ if run_calibrate_rolling_ic_floor:
2740
+ # V5.3.5 (development/Problems.md #102) - min_rolling_mean_ic was
2741
+ # about to be shipped as a guessed constant, the same
2742
+ # "guessed-then-calibrated" gap min_rank_confidence_spread had
2743
+ # (Problems.md #89) before --calibrate-book-spread fixed it.
2744
+ # Mirrors that tool's exact structure, including recomputing the
2745
+ # blended raw score fresh rather than reusing a value from a
2746
+ # different block - see run_calibrate_book_spread above for why
2747
+ # that duplication is this file's own established convention.
2748
+ from evaluation import compute_blended_raw_scores
2749
+ from evaluation.rolling_ic_gate_calibration import calibrate_rolling_ic_floor
2750
+ from portfolio.rank_signal import resolve_rank_signal_policy
2751
+
2752
+ training_metrics_by_model: dict[str, dict | None] = {}
2753
+ for model_name, metrics_filename in (
2754
+ ("sequence", "sequence_training_metrics.json"),
2755
+ ("multitask", "multitask_training_metrics.json"),
2756
+ ):
2757
+ metrics_path = ML_DIR / metrics_filename
2758
+ training_metrics_by_model[model_name] = (
2759
+ json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
2760
+ )
2761
+ policy = resolve_rank_signal_policy(training_metrics_by_model, config)
2762
+ active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
2763
+
2764
+ predictions_by_model_head: dict[str, dict[str, object]] = {}
2765
+ for model_kind_for_head in policy["model_priority"]:
2766
+ calib_model_filename, calib_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
2767
+ calib_model_path = ML_DIR / calib_model_filename
2768
+ calib_schema_path = ML_DIR / calib_schema_filename
2769
+ if not calib_model_path.exists() or not calib_schema_path.exists():
2770
+ continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
2771
+ calib_model_export = json.loads(calib_model_path.read_text(encoding="utf-8"))
2772
+ calib_feature_schema = json.loads(calib_schema_path.read_text(encoding="utf-8"))
2773
+ calib_feature_names = calib_feature_schema["model_input_names"]
2774
+ head_predictions: dict[str, object] = {}
2775
+ for head_name in active_heads:
2776
+ head_predictions[head_name] = predict_head(
2777
+ dataset, calib_model_export, calib_feature_names, head_name,
2778
+ model_kind=model_kind_for_head,
2779
+ sequence_feature_schema=calib_feature_schema if model_kind_for_head == "sequence" else None,
2780
+ configured_window_size=int(sequence_window_default),
2781
+ )
2782
+ predictions_by_model_head[model_kind_for_head] = head_predictions
2783
+
2784
+ dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
2785
+
2786
+ rolling_ic_gate_config = config.get("phase_v2", {}).get("rolling_ic_gate", {})
2787
+ rolling_ic_result = calibrate_rolling_ic_floor(
2788
+ dataset,
2789
+ raw_score_column="raw_blended_score",
2790
+ horizon_days=int(rolling_ic_gate_config.get("horizon_days", 20)),
2791
+ rolling_window_days=int(rolling_ic_gate_config.get("rolling_window_days", 40)),
2792
+ percentile=float(getattr(args, "rolling_ic_floor_percentile", 0.10)),
2793
+ min_names_per_date=int(rolling_ic_gate_config.get("min_names_per_date", 10)),
2794
+ )
2795
+ report["rolling_ic_floor_calibration"] = rolling_ic_result
2796
+ _write_evaluation_json(evaluation_dir / "rolling_ic_floor_calibration.json", rolling_ic_result)
2797
+ if not args.json:
2798
+ calibrated_floor = rolling_ic_result["calibrated_min_rolling_mean_ic"]
2799
+ distribution = rolling_ic_result["rolling_ic_distribution"]
2800
+ print(
2801
+ f"Calibrated min_rolling_mean_ic (p{rolling_ic_result['percentile']*100:.0f}, "
2802
+ f"{rolling_ic_result['num_samples_used']} samples used, "
2803
+ f"{rolling_ic_result['num_samples_skipped_insufficient_history']} skipped insufficient-history): "
2804
+ f"{calibrated_floor:.4f}"
2805
+ )
2806
+ print(
2807
+ f" Distribution: min={distribution['min']}, p10={distribution['p10']}, "
2808
+ f"median={distribution['median']}, p75={distribution['p75']}, max={distribution['max']}"
2809
+ )
2810
+ print(f" Apply with: aq config set phase_v2.rolling_ic_gate.min_rolling_mean_ic {calibrated_floor:.4f}")
2811
+ print(
2812
+ " NOTE: only sets the threshold - phase_v2.rolling_ic_gate.enabled still defaults to false. "
2813
+ "Check --replay-rolling-ic-gate's per-era breakdown before flipping it on live."
2814
+ )
2815
+
2816
+ if run_replay_rolling_ic_gate:
2817
+ # V5.3.5 (development/Problems.md #102) - the critical pre-main.py
2818
+ # checkpoint: recomputes the blended raw score fresh (same
2819
+ # duplication convention as run_calibrate_rolling_ic_floor above),
2820
+ # then replays the gate day-by-day at the same rebalance cadence
2821
+ # phase_v2.portfolio_book.rebalance_every_bars uses live, and
2822
+ # reports engagement broken down by the three known bad eras.
2823
+ #
2824
+ # Always evaluated as ENABLED regardless of
2825
+ # phase_v2.rolling_ic_gate.enabled's live default (False) - this
2826
+ # is a "what would happen IF this were live" diagnostic, the same
2827
+ # relationship evaluation/kill_switch_replay.py's own replay has
2828
+ # to the live kill switch (which has no master on/off flag to
2829
+ # begin with). Everything else - crucially min_rolling_mean_ic -
2830
+ # is read from config as-is, so this reports honestly against
2831
+ # whatever floor is currently configured (the shipped -1e12
2832
+ # sentinel gives a trivial always-engaged result until
2833
+ # --calibrate-rolling-ic-floor's value has actually been applied
2834
+ # with `aq config set`).
2835
+ from evaluation import compute_blended_raw_scores
2836
+ from evaluation.rolling_ic_gate_replay import (
2837
+ replay_rolling_ic_gate_over_dataset,
2838
+ summarize_rolling_ic_gate_replay,
2839
+ )
2840
+ from portfolio.rank_signal import resolve_rank_signal_policy
2841
+
2842
+ training_metrics_by_model: dict[str, dict | None] = {}
2843
+ for model_name, metrics_filename in (
2844
+ ("sequence", "sequence_training_metrics.json"),
2845
+ ("multitask", "multitask_training_metrics.json"),
2846
+ ):
2847
+ metrics_path = ML_DIR / metrics_filename
2848
+ training_metrics_by_model[model_name] = (
2849
+ json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
2850
+ )
2851
+ policy = resolve_rank_signal_policy(training_metrics_by_model, config)
2852
+ active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
2853
+
2854
+ predictions_by_model_head: dict[str, dict[str, object]] = {}
2855
+ for model_kind_for_head in policy["model_priority"]:
2856
+ replay_model_filename, replay_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
2857
+ replay_model_path = ML_DIR / replay_model_filename
2858
+ replay_schema_path = ML_DIR / replay_schema_filename
2859
+ if not replay_model_path.exists() or not replay_schema_path.exists():
2860
+ continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
2861
+ replay_model_export = json.loads(replay_model_path.read_text(encoding="utf-8"))
2862
+ replay_feature_schema = json.loads(replay_schema_path.read_text(encoding="utf-8"))
2863
+ replay_feature_names = replay_feature_schema["model_input_names"]
2864
+ head_predictions: dict[str, object] = {}
2865
+ for head_name in active_heads:
2866
+ head_predictions[head_name] = predict_head(
2867
+ dataset, replay_model_export, replay_feature_names, head_name,
2868
+ model_kind=model_kind_for_head,
2869
+ sequence_feature_schema=replay_feature_schema if model_kind_for_head == "sequence" else None,
2870
+ configured_window_size=int(sequence_window_default),
2871
+ )
2872
+ predictions_by_model_head[model_kind_for_head] = head_predictions
2873
+
2874
+ dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
2875
+
2876
+ rolling_ic_gate_config = dict(config.get("phase_v2", {}).get("rolling_ic_gate", {}))
2877
+ rolling_ic_gate_config["enabled"] = True
2878
+ rebalance_every_bars = int(config.get("phase_v2", {}).get("portfolio_book", {}).get("rebalance_every_bars", 10))
2879
+ rebalance_dates = sorted(dataset["date"].unique())[::max(1, rebalance_every_bars)]
2880
+
2881
+ replay_records = replay_rolling_ic_gate_over_dataset(
2882
+ dataset,
2883
+ raw_score_column="raw_blended_score",
2884
+ rebalance_dates=rebalance_dates,
2885
+ horizon_days=int(rolling_ic_gate_config.get("horizon_days", 20)),
2886
+ rolling_window_days=int(rolling_ic_gate_config.get("rolling_window_days", 40)),
2887
+ gate_config=rolling_ic_gate_config,
2888
+ min_names_per_date=int(rolling_ic_gate_config.get("min_names_per_date", 10)),
2889
+ )
2890
+ replay_summary = summarize_rolling_ic_gate_replay(replay_records)
2891
+ report["rolling_ic_gate_replay"] = {"summary": replay_summary, "per_date": replay_records}
2892
+ _write_evaluation_json(evaluation_dir / "rolling_ic_gate_replay.json", report["rolling_ic_gate_replay"])
2893
+ if not args.json:
2894
+ overall = replay_summary["overall"]
2895
+ print(
2896
+ f"Rolling-IC gate replay (floor={rolling_ic_gate_config.get('min_rolling_mean_ic')}): "
2897
+ f"{overall['disengaged_days']}/{overall['total_dates']} rebalance dates disengaged "
2898
+ f"({overall['disengaged_day_fraction']:.2%})"
2899
+ )
2900
+ for era_name, era_stats in replay_summary["by_era"].items():
2901
+ fraction = era_stats["disengaged_day_fraction"]
2902
+ fraction_display = f"{fraction:.2%}" if fraction is not None else "n/a (no dates in era)"
2903
+ print(f" {era_name}: {era_stats['disengaged_days']}/{era_stats['total_dates']} disengaged ({fraction_display})")
2904
+
2905
+ if run_calibrate_confidence_threshold:
2906
+ # V5.2.6 (development/Problems.md) - min_confidence_to_trade was a
2907
+ # guessed constant (0.12), never checked against this codebase's
2908
+ # actual confidence-vs-forward-return relationship - the same
2909
+ # "guessed constant, never calibrated" problem
2910
+ # min_rank_confidence_spread had (Problems.md #89) before
2911
+ # --calibrate-book-spread fixed it. Mirrors that tool's exact
2912
+ # structure.
2913
+ from evaluation import calibrate_confidence_threshold, compute_blended_raw_scores
2914
+ from portfolio.rank_signal import resolve_rank_signal_policy
2915
+
2916
+ training_metrics_by_model: dict[str, dict | None] = {}
2917
+ for model_name, metrics_filename in (
2918
+ ("sequence", "sequence_training_metrics.json"),
2919
+ ("multitask", "multitask_training_metrics.json"),
2920
+ ):
2921
+ metrics_path = ML_DIR / metrics_filename
2922
+ training_metrics_by_model[model_name] = (
2923
+ json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
2924
+ )
2925
+ policy = resolve_rank_signal_policy(training_metrics_by_model, config)
2926
+ active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
2927
+
2928
+ predictions_by_model_head: dict[str, dict[str, object]] = {}
2929
+ for model_kind_for_head in policy["model_priority"]:
2930
+ calib_model_filename, calib_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
2931
+ calib_model_path = ML_DIR / calib_model_filename
2932
+ calib_schema_path = ML_DIR / calib_schema_filename
2933
+ if not calib_model_path.exists() or not calib_schema_path.exists():
2934
+ continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
2935
+ calib_model_export = json.loads(calib_model_path.read_text(encoding="utf-8"))
2936
+ calib_feature_schema = json.loads(calib_schema_path.read_text(encoding="utf-8"))
2937
+ calib_feature_names = calib_feature_schema["model_input_names"]
2938
+ head_predictions: dict[str, object] = {}
2939
+ for head_name in active_heads:
2940
+ head_predictions[head_name] = predict_head(
2941
+ dataset, calib_model_export, calib_feature_names, head_name,
2942
+ model_kind=model_kind_for_head,
2943
+ sequence_feature_schema=calib_feature_schema if model_kind_for_head == "sequence" else None,
2944
+ configured_window_size=int(sequence_window_default),
2945
+ )
2946
+ predictions_by_model_head[model_kind_for_head] = head_predictions
2947
+
2948
+ dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
2949
+
2950
+ book_config = config.get("phase_v2", {}).get("portfolio_book", {})
2951
+ strategy_mode = config.get("phase5", {}).get("backtest", {}).get("strategy_mode", "long_flat")
2952
+ book_top_n = int(book_config.get("top_n", 6))
2953
+ book_bottom_n = int(book_config.get("bottom_n", 6)) if strategy_mode == "long_short" else 0
2954
+
2955
+ confidence_forward_return_column = "target_return_20d"
2956
+ if confidence_forward_return_column not in dataset.columns:
2957
+ print(
2958
+ f"warning: {confidence_forward_return_column!r} not in dataset; falling back to "
2959
+ "target_return_1d - the calibrated value will then represent a 1-day move, not the "
2960
+ "20-day move predicted_rank_20d assumes.",
2961
+ file=sys.stderr,
2962
+ )
2963
+ confidence_forward_return_column = "target_return_1d"
2964
+
2965
+ confidence_result = calibrate_confidence_threshold(
2966
+ dataset,
2967
+ raw_score_column="raw_blended_score",
2968
+ forward_return_column=confidence_forward_return_column,
2969
+ top_n=book_top_n,
2970
+ bottom_n=book_bottom_n,
2971
+ round_trip_cost_fraction=float(
2972
+ config.get("phase_v2", {}).get("liquidity", {}).get("max_round_trip_cost_fraction") or 0.001
2973
+ ),
2974
+ percentile=float(getattr(args, "confidence_threshold_percentile", 0.10)),
2975
+ )
2976
+ report["confidence_threshold_calibration"] = confidence_result
2977
+ _write_evaluation_json(evaluation_dir / "confidence_threshold_calibration.json", confidence_result)
2978
+ if not args.json:
2979
+ calibrated_general = confidence_result["calibrated_min_confidence_to_trade"]
2980
+ calibrated_book_selected = confidence_result["calibrated_min_confidence_to_trade_book_selected"]
2981
+ print(
2982
+ f"Calibrated min_confidence_to_trade (p{confidence_result['percentile']*100:.0f}, "
2983
+ f"{confidence_result['num_rows_used']} paying rows used, "
2984
+ f"{confidence_result['num_dates_skipped_thin_universe']} dates skipped thin-universe): "
2985
+ f"{calibrated_general:.4f}"
2986
+ )
2987
+ print(
2988
+ f"Calibrated min_confidence_to_trade_book_selected ({confidence_result['num_rows_used_book_selected']} "
2989
+ f"paying book-selected rows used): {calibrated_book_selected:.4f}"
2990
+ )
2991
+ print(f" Apply with: aq config set phase6.risk.min_confidence_to_trade {calibrated_general:.4f}")
2992
+ print(
2993
+ " Apply with: aq config set phase6.risk.min_confidence_to_trade_book_selected "
2994
+ f"{calibrated_book_selected:.4f}"
2995
+ )
2996
+ print(
2997
+ " NOTE: this recalibrates the gate's threshold to this model's real achievable confidence "
2998
+ "among historically-paying trades - it does not certify the underlying rank heads are "
2999
+ "skillful. Check ml/*_training_metrics.json's quality_status before trusting the result."
3000
+ )
3001
+
3002
+ if run_ablation_flag:
3003
+ from evaluation import run_ablation
3004
+
3005
+ ablation_config = config.get("phase_v2", {}).get("evaluation", {}).get("ablation", {})
3006
+ requested_variants = (
3007
+ [name.strip() for name in args.variants.split(",") if name.strip()]
3008
+ if getattr(args, "variants", None)
3009
+ else list(ablation_config.get("variants", ["static_baseline", "no_neutrality", "no_hysteresis", "no_cost_model"]))
3010
+ )
3011
+ ablation_results = run_ablation(dataset, base_kwargs, requested_variants)
3012
+ report["ablation"] = ablation_results
3013
+ _write_evaluation_json(
3014
+ ROOT_DIR / ablation_config.get("report_path", "ml/evaluation/ablation_report.json"), ablation_results
3015
+ )
3016
+ if not args.json:
3017
+ print(f"Ablation ({model_kind}/{head}, split={split}):")
3018
+ for name, entry in ablation_results.items():
3019
+ if entry.get("status") in {"not_offline_measurable", "unknown_variant", "insufficient_windows"}:
3020
+ print(f" {name}: [{entry['status']}] {entry.get('reason', '')}")
3021
+ else:
3022
+ print(f" {name}: net_sharpe={entry['net_sharpe']:.4f} delta_vs_static_baseline={entry['delta_vs_static_baseline']:+.4f}")
3023
+
3024
+ if args.json:
3025
+ print(json.dumps(report, indent=2, default=str))
3026
+
3027
+ _refresh_readme_evaluation_sections()
3028
+ return 0
3029
+
3030
+
3031
+ def cmd_status(_args: argparse.Namespace) -> int:
3032
+ return _run(["git", "status"])
3033
+
3034
+
3035
+ def build_parser() -> argparse.ArgumentParser:
3036
+ parser = argparse.ArgumentParser(prog="aq", description="Aether Quant convenience CLI")
3037
+ subparsers = parser.add_subparsers(dest="command", required=True)
3038
+
3039
+ train_parser = subparsers.add_parser("train", help="Run the training pipeline (wraps python train.py)")
3040
+ train_group = train_parser.add_mutually_exclusive_group()
3041
+ train_group.add_argument("--dataset-only", action="store_true", help="Build dataset/scaler/manifest only")
3042
+ train_group.add_argument("--init-only", action="store_true", help="Refresh the data inventory only")
3043
+ train_group.add_argument("--experts-only", action="store_true", help="Train the 4 expert models only")
3044
+ train_group.add_argument(
3045
+ "--gating-only", action="store_true", help="Train the learned gating blend only (wraps python train_gating.py)"
3046
+ )
3047
+ train_group.add_argument(
3048
+ "--multitask-only",
3049
+ action="store_true",
3050
+ help="Train the joint direction+magnitude+volatility model only (wraps python train_multitask.py)",
3051
+ )
3052
+ train_group.add_argument(
3053
+ "--sequence-only",
3054
+ action="store_true",
3055
+ help="Train the Phase 2 causal-TCN sequence encoder only (wraps python train_sequence.py)",
3056
+ )
3057
+ train_group.add_argument(
3058
+ "--topology-only",
3059
+ action="store_true",
3060
+ help=(
3061
+ "Train the learned topology overlay only (wraps python train_topology.py). Needs Postgres up "
3062
+ "and enough accumulated realized-outcome events - see the command's own output if it skips."
3063
+ ),
3064
+ )
3065
+ train_group.add_argument(
3066
+ "--strategy-selector-only",
3067
+ action="store_true",
3068
+ help=(
3069
+ "Train the learned multi-leg strategy-selector model only (wraps python "
3070
+ "train_strategy_selector.py, V4.7, development/Problems.md #29's own framing). Needs real "
3071
+ "option_strategy_outcome events (real option positions traded and closed) - expect this to "
3072
+ "skip indefinitely in this environment, see the command's own output."
3073
+ ),
3074
+ )
3075
+ train_group.add_argument(
3076
+ "--rl-sizing-only",
3077
+ action="store_true",
3078
+ help=(
3079
+ "Train the offline contextual-bandit sizing overlay only (wraps python train_rl_sizing.py, "
3080
+ "development/Problems.md #71). Reads ml/datasets/*.csv, NOT Postgres - needs a completed "
3081
+ "`aq train` and Component D's alt-data features in phase1.features.input_set."
3082
+ ),
3083
+ )
3084
+ train_group.add_argument(
3085
+ "--walk-forward",
3086
+ action="store_true",
3087
+ help=(
3088
+ "Phase 4 of the 5/10 -> 9/10 roadmap: run the dataset-build + training pipeline once per "
3089
+ "walk-forward window (wraps python train.py --walk-forward). Never touches active ml/ - "
3090
+ "diagnostic only, writes to ml/versions/<run-id>/window_<i>/."
3091
+ ),
3092
+ )
3093
+ train_parser.add_argument(
3094
+ "--step-days", type=int, default=None, help="Walk-forward step size in days (only with --walk-forward)."
3095
+ )
3096
+ train_parser.add_argument(
3097
+ "--mode",
3098
+ type=str,
3099
+ choices=("rolling", "expanding"),
3100
+ default=None,
3101
+ help="Walk-forward mode: rolling or expanding (only with --walk-forward).",
3102
+ )
3103
+ # V5.1 Phase 4 (item 4) - one-run overrides of config.json's
3104
+ # phase_v2.retraining.walk_forward.{train_multitask,train_sequence,
3105
+ # tracked_metrics}, only meaningful with --walk-forward - passthrough
3106
+ # to train.py --walk-forward's own identical flags.
3107
+ train_parser.add_argument(
3108
+ "--include-multitask",
3109
+ action="store_true",
3110
+ help="Force multitask training per walk-forward window on for this run (only with --walk-forward).",
3111
+ )
3112
+ train_parser.add_argument(
3113
+ "--include-sequence",
3114
+ action="store_true",
3115
+ help="Force sequence training per walk-forward window on for this run (only with --walk-forward).",
3116
+ )
3117
+ train_parser.add_argument(
3118
+ "--metrics",
3119
+ type=str,
3120
+ default=None,
3121
+ help=(
3122
+ "Comma-separated tracked-metric names to restrict walk-forward tracking to for this run "
3123
+ "(only with --walk-forward)."
3124
+ ),
3125
+ )
3126
+ # V5.1 Phase 3 (item 1) - passthrough to --multitask-only/--sequence-only's
3127
+ # own train_multitask.py/train_sequence.py --seed/--ranking-objective
3128
+ # flags (development/Problems.md - previously the seed-ensembling
3129
+ # workflow required calling those scripts directly, bypassing `aq`
3130
+ # entirely). Ignored (with a warning) by every other --*-only mode and
3131
+ # by the default full-pipeline train.py invocation, neither of which
3132
+ # accepts these flags.
3133
+ train_parser.add_argument(
3134
+ "--seed", type=int, default=None, help="Override config.json's seed (only with --multitask-only/--sequence-only)"
3135
+ )
3136
+ train_parser.add_argument(
3137
+ "--ranking-objective",
3138
+ type=str,
3139
+ default=None,
3140
+ choices=["mse", "soft_spearman", "listnet"],
3141
+ help="Override config.json's ranking_loss.objective for this run only (only with --multitask-only/--sequence-only)",
3142
+ )
3143
+ train_parser.set_defaults(func=cmd_train)
3144
+
3145
+ test_parser = subparsers.add_parser("test", help="Run the test suite (wraps pytest tests/)")
3146
+ test_parser.add_argument(
3147
+ "--lean", "--full", dest="lean", action="store_true",
3148
+ help="Include the real `lean backtest .` integration test (tests/test_lean_backtest_ml_coverage.py, over an hour wall-clock) - excluded by default",
3149
+ )
3150
+ test_parser.add_argument(
3151
+ "--parallel", action="store_true",
3152
+ help="Run via pytest-xdist (-n auto) - off by default, multiple workers importing torch risk OOM on memory-constrained machines",
3153
+ )
3154
+ test_parser.add_argument(
3155
+ "--ruff", action="store_true",
3156
+ help="V5.3.10: run `ruff check .` BEFORE pytest - a lint failure fails fast without paying the full suite's wall-clock first (mirrors ci.yml's python-lint job).",
3157
+ )
3158
+ for _subsystem_name in _SUBSYSTEM_TEST_FILES:
3159
+ test_parser.add_argument(
3160
+ f"--{_subsystem_name}",
3161
+ action="store_true",
3162
+ help=f"Run only the {_subsystem_name} subsystem's tests ({', '.join(_SUBSYSTEM_TEST_FILES[_subsystem_name][:2])}, ...)",
3163
+ )
3164
+ test_parser.set_defaults(func=cmd_test)
3165
+
3166
+ backtest_parser = subparsers.add_parser("backtest", help="Run a Lean backtest (wraps lean backtest .)")
3167
+ backtest_parser.add_argument(
3168
+ "--image",
3169
+ default=None,
3170
+ help=f"Override the Lean engine Docker image (default: pinned {PINNED_LEAN_ENGINE_IMAGE}, "
3171
+ "never the mutable :latest tag - see PINNED_LEAN_ENGINE_IMAGE's comment in aq_cli.py)",
3172
+ )
3173
+ backtest_parser.set_defaults(func=cmd_backtest)
3174
+
3175
+ profile_parser = subparsers.add_parser(
3176
+ "profile",
3177
+ help="Profile the per-bar hot path (wraps scripts/profile_inference.py / scripts/profile_subsystems.py)",
3178
+ )
3179
+ # default=None (not 10_000/200) so cmd_profile() can tell "user didn't
3180
+ # pass --iterations" apart from "user explicitly passed the same
3181
+ # number" and let whichever script actually runs use ITS OWN default -
3182
+ # profile_inference.py's 10,000 (cheap ~5ms/call) and
3183
+ # profile_subsystems.py's 200 (build_market_topology() alone costs
3184
+ # ~500-600ms/call at this project's real universe size - 10,000 there
3185
+ # would take over an hour) are deliberately very different, and
3186
+ # hardcoding either one here would silently override the other.
3187
+ profile_parser.add_argument(
3188
+ "--iterations", type=int, default=None,
3189
+ help="Iterations to profile (default: 10000 for inference, 200 for --<subsystem> flags)",
3190
+ )
3191
+ profile_parser.add_argument("--sort", default="cumulative", help="pstats sort key (default: cumulative)")
3192
+ profile_parser.add_argument(
3193
+ "--batched", action="store_true",
3194
+ help="Use the batched expert-inference path (with its precomputed stack caches) instead of a per-expert loop",
3195
+ )
3196
+ profile_parser.add_argument(
3197
+ "--no-gc", action="store_true",
3198
+ help="Disable the GC around the profiled region, to isolate whether GC pauses drive tail latency (inference only)",
3199
+ )
3200
+ profile_parser.add_argument(
3201
+ "--bucket-report", action="store_true",
3202
+ help="Print a 10-bucket-by-iteration-index duration breakdown, to check for a warmup effect (inference only)",
3203
+ )
3204
+ profile_parser.add_argument(
3205
+ "--parallel", action="store_true",
3206
+ help="V4.9 Priority 6: run the ProcessPoolExecutor IPC-overhead benchmark against run_symbol_inference() (inference only)",
3207
+ )
3208
+ profile_parser.add_argument(
3209
+ "--pool-workers", type=int, default=None,
3210
+ help="Worker process count for --parallel (default: profile_inference.py's own default, 4)",
3211
+ )
3212
+ profile_parser.add_argument(
3213
+ "--symbols-per-bar", type=int, default=None,
3214
+ help="Symbols-per-bar grouping for --batched's sequence-batching comparison and --parallel's benchmark (default: profile_inference.py's own default, 74)",
3215
+ )
3216
+ for _profile_subsystem_name in _PROFILE_SUBSYSTEM_FLAGS:
3217
+ profile_parser.add_argument(
3218
+ f"--{_profile_subsystem_name}",
3219
+ action="store_true",
3220
+ help=f"Profile only the {_profile_subsystem_name} subsystem (scripts/profile_subsystems.py)",
3221
+ )
3222
+ profile_parser.set_defaults(func=cmd_profile)
3223
+
3224
+ report_parser = subparsers.add_parser("report", help="Generate a Lean HTML report for a finished backtest")
3225
+ report_parser.add_argument("backtest_dir", help="Folder name under backtests/, e.g. 2026-07-04_13-06-51")
3226
+ report_parser.add_argument("result_id", help="Result JSON id, e.g. 1366365999")
3227
+ report_parser.set_defaults(func=cmd_report)
3228
+
3229
+ api_parser = subparsers.add_parser("api", help="Start the FastAPI monitoring server on :8001")
3230
+ api_parser.set_defaults(func=cmd_api)
3231
+
3232
+ webui_parser = subparsers.add_parser("webui", help="Start the webui dev server (npm run dev)")
3233
+ webui_parser.set_defaults(func=cmd_webui)
3234
+
3235
+ docker_parser = subparsers.add_parser("docker", help="Docker Compose shortcuts")
3236
+ docker_subparsers = docker_parser.add_subparsers(dest="docker_command", required=True)
3237
+
3238
+ docker_up_parser = docker_subparsers.add_parser("up", help="Start infra services")
3239
+ docker_up_group = docker_up_parser.add_mutually_exclusive_group()
3240
+ docker_up_group.add_argument("--lean", action="store_true", help="Start via the lean Compose profile")
3241
+ docker_up_group.add_argument("--all", action="store_true", help="Start the full stack, including all workers")
3242
+ docker_up_parser.set_defaults(func=cmd_docker_up)
3243
+
3244
+ docker_build_parser = docker_subparsers.add_parser(
3245
+ "build", help="Rebuild the consolidated aether-quant-engine image (app + every worker)"
3246
+ )
3247
+ docker_build_parser.set_defaults(func=cmd_docker_build)
3248
+
3249
+ config_parser = subparsers.add_parser("config", help="Show or edit config.json")
3250
+ config_parser.set_defaults(func=cmd_config)
3251
+ config_subparsers = config_parser.add_subparsers(dest="config_command")
3252
+
3253
+ config_get_parser = config_subparsers.add_parser("get", help="Print a config.json value")
3254
+ config_get_parser.add_argument("dotted_path")
3255
+
3256
+ config_keys_parser = config_subparsers.add_parser("keys", help="List leaf key paths (optionally scoped to a prefix)")
3257
+ config_keys_parser.add_argument("dotted_prefix", nargs="?", default=None)
3258
+
3259
+ config_set_parser = config_subparsers.add_parser("set", help="Set a config.json value (JSON-parsed, string fallback)")
3260
+ config_set_parser.add_argument("dotted_path")
3261
+ config_set_parser.add_argument("value")
3262
+
3263
+ # V5.1 Phase 1 - phase_v2.presets, named dotted-key-override blocks
3264
+ # (e.g. "aggressive"/"moderate" turnover profiles). config-only, no
3265
+ # `aq lean preset` equivalent.
3266
+ config_preset_parser = config_subparsers.add_parser(
3267
+ "preset", help="List/show/apply a named config preset (phase_v2.presets)"
3268
+ )
3269
+ config_preset_group = config_preset_parser.add_mutually_exclusive_group()
3270
+ config_preset_group.add_argument("--list", dest="preset_list", action="store_true", help="List available presets")
3271
+ config_preset_group.add_argument("--show", dest="preset_show", metavar="NAME", help="Print one preset's overrides")
3272
+ config_preset_group.add_argument(
3273
+ "--apply", dest="preset_apply", metavar="NAME", help="Apply one preset's overrides to config.json"
3274
+ )
3275
+ config_preset_parser.add_argument(
3276
+ "--dry-run", dest="preset_dry_run", action="store_true", help="With --apply: print old -> new, write nothing"
3277
+ )
3278
+
3279
+ lean_parser = subparsers.add_parser("lean", help="Show or edit lean.json (same shape as `aq config`)")
3280
+ lean_parser.set_defaults(func=cmd_lean)
3281
+ lean_subparsers = lean_parser.add_subparsers(dest="lean_command")
3282
+
3283
+ lean_get_parser = lean_subparsers.add_parser("get", help="Print a lean.json value")
3284
+ lean_get_parser.add_argument("dotted_path")
3285
+
3286
+ lean_keys_parser = lean_subparsers.add_parser("keys", help="List leaf key paths (optionally scoped to a prefix)")
3287
+ lean_keys_parser.add_argument("dotted_prefix", nargs="?", default=None)
3288
+
3289
+ lean_set_parser = lean_subparsers.add_parser("set", help="Set a lean.json value (JSON-parsed, string fallback)")
3290
+ lean_set_parser.add_argument("dotted_path")
3291
+ lean_set_parser.add_argument("value")
3292
+
3293
+ retrain_parser = subparsers.add_parser(
3294
+ "retrain", help="Thin dispatcher to python -m retraining.orchestrator <stage> ..."
3295
+ )
3296
+ retrain_parser.add_argument(
3297
+ "stage",
3298
+ choices=[
3299
+ "plan",
3300
+ "train",
3301
+ "train_topology",
3302
+ "train_gating",
3303
+ "train_multitask",
3304
+ "train_sequence",
3305
+ "train_strategy_selector",
3306
+ "validate",
3307
+ "backtest",
3308
+ "commit",
3309
+ "promote",
3310
+ "rollback",
3311
+ "status",
3312
+ # V5.1 Phase 6 (production safety) - read-only diagnostic,
3313
+ # never itself calls rollback() - see retraining/orchestrator.py::
3314
+ # auto_rollback_status()'s own docstring.
3315
+ "auto-rollback",
3316
+ ],
3317
+ )
3318
+ retrain_parser.add_argument("retrain_args", nargs=argparse.REMAINDER, help="Passed through verbatim, e.g. --version-id <uuid>")
3319
+ retrain_parser.set_defaults(func=cmd_retrain)
3320
+
3321
+ backfill_parser = subparsers.add_parser(
3322
+ "backfill",
3323
+ help=(
3324
+ "Thin dispatcher to python -m data_pipeline.<target>_backfill ... (whole-universe bulk "
3325
+ "backfills). Different from `aq fetch`, which is for one ad-hoc ticker."
3326
+ ),
3327
+ )
3328
+ backfill_parser.add_argument("target", choices=["dividends", "fred", "yfinance"])
3329
+ backfill_parser.add_argument(
3330
+ "backfill_args",
3331
+ nargs=argparse.REMAINDER,
3332
+ help="Passed through verbatim, e.g. --apply --tickers AAPL MSFT (dividends/yfinance), or --series DGS10 (fred)",
3333
+ )
3334
+ backfill_parser.set_defaults(func=cmd_backfill)
3335
+
3336
+ paper_readiness_parser = subparsers.add_parser(
3337
+ "paper-readiness", help="Check whether the system is ready for phase_v2.runtime.mode='paper'"
3338
+ )
3339
+ paper_readiness_parser.set_defaults(func=cmd_paper_readiness)
3340
+
3341
+ trade_lock_parser = subparsers.add_parser(
3342
+ "trade-lock", help="Manually override the sticky total-drawdown trade lock"
3343
+ )
3344
+ trade_lock_group = trade_lock_parser.add_mutually_exclusive_group(required=True)
3345
+ trade_lock_group.add_argument("--on", action="store_true", help="Force trading paused")
3346
+ trade_lock_group.add_argument("--off", action="store_true", help="Force trading resumed")
3347
+ trade_lock_group.add_argument("--auto", action="store_true", help="Return to fully automatic behavior")
3348
+ trade_lock_group.add_argument("--status", dest="status", action="store_true", help="Print the current override state")
3349
+ trade_lock_parser.set_defaults(func=cmd_trade_lock)
3350
+
3351
+ # V5.1 Phase 6 (production safety) - deliberately the same
3352
+ # override-file/mutually-exclusive-group shape as `trade-lock` above (see
3353
+ # risk/manual_override.py's kill_switch_manual_override, the exact same
3354
+ # read/write/cache convention as manual_trade_lock_override) so the two
3355
+ # switches can never disagree about their CLI surface. --history is the
3356
+ # one addition, querying the tamper-evident audit log instead.
3357
+ kill_switch_parser = subparsers.add_parser(
3358
+ "kill-switch", help="Manually override / inspect the automated production kill switch"
3359
+ )
3360
+ kill_switch_group = kill_switch_parser.add_mutually_exclusive_group(required=True)
3361
+ kill_switch_group.add_argument("--arm", action="store_true", help="Force kill-switch evaluation on")
3362
+ kill_switch_group.add_argument("--disarm", action="store_true", help="Force kill-switch evaluation off")
3363
+ kill_switch_group.add_argument("--auto", action="store_true", help="Defer to phase_v2.risk.kill_switch.enabled")
3364
+ kill_switch_group.add_argument("--status", dest="status", action="store_true", help="Print the current override state")
3365
+ kill_switch_group.add_argument("--history", action="store_true", help="List recorded kill-switch trips from the audit log")
3366
+ kill_switch_parser.add_argument("--limit", type=int, default=50, help="Max --history rows (default 50)")
3367
+ kill_switch_parser.set_defaults(func=cmd_kill_switch)
3368
+
3369
+ fetch_parser = subparsers.add_parser(
3370
+ "fetch",
3371
+ help=(
3372
+ "Ad-hoc fetch of historical OHLCV from Yahoo Finance for a ticker not yet in config.json. "
3373
+ "For a whole-universe bulk refresh instead (dividends/FRED series/Yahoo gaps), see `aq backfill`."
3374
+ ),
3375
+ )
3376
+ fetch_parser.add_argument(
3377
+ "asset_class", choices=list(ASSET_CLASSES), help="Asset class (picks the Lean data_path/market convention)"
3378
+ )
3379
+ fetch_parser.add_argument("--ticker", required=True, help="Internal ticker, e.g. AAPL or BTCUSD")
3380
+ fetch_parser.add_argument("--start", required=True, type=_iso_date, help="Start date, ISO 8601 YYYY-MM-DD")
3381
+ fetch_parser.add_argument("--end", required=True, type=_iso_date, help="End date, ISO 8601 YYYY-MM-DD")
3382
+ fetch_parser.add_argument(
3383
+ "--apply", action="store_true", help="Actually write the zip file and update config.json (default: dry run, report only)"
3384
+ )
3385
+ fetch_parser.add_argument(
3386
+ "--expiry", default=None, help="Contract expiry, YYYY-MM-DD (required for asset_class futures/options; requires IB, see 'aq ib status')"
3387
+ )
3388
+ fetch_parser.add_argument("--strike", type=float, default=None, help="Strike price (required for asset_class options)")
3389
+ fetch_parser.add_argument("--right", choices=["call", "put"], default=None, help="Option right (required for asset_class options)")
3390
+ fetch_parser.add_argument(
3391
+ "--contract-month", default=None,
3392
+ help="Futures only: YYYYMM - fetch a specific dated contract instead of the continuous front-month "
3393
+ "(e.g. for building real historical term structure with a second, later --contract-month fetch under "
3394
+ "the same --family-ticker)",
3395
+ )
3396
+ fetch_parser.add_argument(
3397
+ "--family-ticker", default=None,
3398
+ help="Groups multiple fetched contracts under one root for offline training's derivatives-macro features "
3399
+ "(train.py::build_derivatives_macro_features_by_date()) - e.g. two futures/options fetches sharing "
3400
+ "--family-ticker ES. Defaults to --ticker itself (a single, ungrouped contract).",
3401
+ )
3402
+ fetch_parser.set_defaults(func=cmd_fetch)
3403
+
3404
+ ib_parser = subparsers.add_parser("ib", help="Check Interactive Brokers configuration/connectivity")
3405
+ ib_subparsers = ib_parser.add_subparsers(dest="ib_command", required=True)
3406
+ ib_subparsers.add_parser("status", help="Report disabled / credentials-missing / reachable")
3407
+ ib_parser.set_defaults(func=cmd_ib)
3408
+
3409
+ assets_parser = subparsers.add_parser("assets", help="Report multi-asset-class (futures/options/FRED) readiness")
3410
+ assets_subparsers = assets_parser.add_subparsers(dest="assets_command", required=True)
3411
+ assets_subparsers.add_parser("status", help="Report IB/futures/options/FRED readiness at a glance")
3412
+ assets_parser.set_defaults(func=cmd_assets)
3413
+
3414
+ evaluate_parser = subparsers.add_parser(
3415
+ "evaluate",
3416
+ help="Offline, cost-aware evaluation of the cross-sectional rank book (V5.1) - net Sharpe/turnover/"
3417
+ "capacity/cost-stress from the active model + dataset, without a Lean backtest",
3418
+ )
3419
+ evaluate_parser.add_argument("--rank-book", action="store_true", help="Simulate the rank book net of costs")
3420
+ evaluate_parser.add_argument("--capacity", action="store_true", help="Breadth (top_n sweep) + capacity estimate")
3421
+ evaluate_parser.add_argument("--stress", action="store_true", help="Re-run at 1x/2x/3x the configured cost")
3422
+ evaluate_parser.add_argument(
3423
+ "--calibrate-edge", action="store_true",
3424
+ help="Print an edge_bps_per_rank_unit calibrated from this split's realized rank-vs-return relationship",
3425
+ )
3426
+ evaluate_parser.add_argument("--all", action="store_true", help="Run --rank-book, --capacity, --stress, --calibrate-edge and --benchmarks together")
3427
+ evaluate_parser.add_argument(
3428
+ "--walk-forward-summary", action="store_true",
3429
+ help="Print an already-written ml/versions/walk-forward-*/walk_forward_summary.json (V5.1 Phase 4) - "
3430
+ "never runs training itself; run `aq train --walk-forward` first.",
3431
+ )
3432
+ evaluate_parser.add_argument(
3433
+ "--run-id", default=None,
3434
+ help="Specific walk-forward run-id to read with --walk-forward-summary (default: the most recent run).",
3435
+ )
3436
+ evaluate_parser.add_argument(
3437
+ "--ablation", action="store_true",
3438
+ help="V5.1 Phase 5: run the adaptive-machinery ablation harness - runtime-only mechanisms "
3439
+ "(gating, topology sizing, the net-edge gate, ...) honestly report not_offline_measurable "
3440
+ "rather than a fabricated number. Not included in --all.",
3441
+ )
3442
+ evaluate_parser.add_argument(
3443
+ "--variants", default=None,
3444
+ help="Comma-separated ablation variant names to run with --ablation (default: "
3445
+ "phase_v2.evaluation.ablation.variants from config.json).",
3446
+ )
3447
+ evaluate_parser.add_argument(
3448
+ "--calibrate-book-spread", action="store_true",
3449
+ help="Print a min_rank_confidence_spread calibrated from this split's actual raw-score dispersion "
3450
+ "(the same resolved rank-signal blend and book top_n/bottom_n main.py uses live). Not included in "
3451
+ "--all - loads both models' predictions, a heavier run like --ablation.",
3452
+ )
3453
+ evaluate_parser.add_argument(
3454
+ "--book-spread-percentile", type=float, default=0.10,
3455
+ help="Percentile (0-1) of the per-date confidence-spread distribution to use as the calibrated "
3456
+ "floor with --calibrate-book-spread (default: 0.10).",
3457
+ )
3458
+ evaluate_parser.add_argument(
3459
+ "--calibrate-confidence-threshold", action="store_true",
3460
+ help="Print a min_confidence_to_trade (and, when book-selection data is available, a separate "
3461
+ "min_confidence_to_trade_book_selected) calibrated from this split's real confidence-vs-forward-"
3462
+ "return relationship, mirroring --calibrate-book-spread's discipline. Not included in --all.",
3463
+ )
3464
+ evaluate_parser.add_argument(
3465
+ "--confidence-threshold-percentile", type=float, default=0.10,
3466
+ help="Percentile (0-1) of the paying-trade confidence distribution to use as the calibrated floor "
3467
+ "with --calibrate-confidence-threshold (default: 0.10).",
3468
+ )
3469
+ evaluate_parser.add_argument(
3470
+ "--calibrate-rolling-ic-floor", action="store_true",
3471
+ help="V5.3.5 (development/Problems.md #102): print a phase_v2.rolling_ic_gate.min_rolling_mean_ic "
3472
+ "calibrated from this split's real walk-forward rolling trailing-IC distribution (portfolio/"
3473
+ "rolling_ic_gate.py's own aggregation, no-lookahead - see evaluation/rolling_ic_gate_calibration.py's "
3474
+ "docstring), mirroring --calibrate-book-spread's discipline. Not included in --all - loads both "
3475
+ "models' predictions, a heavier run like --ablation.",
3476
+ )
3477
+ evaluate_parser.add_argument(
3478
+ "--rolling-ic-floor-percentile", type=float, default=0.10,
3479
+ help="Percentile (0-1) of the rolling-IC distribution to use as the calibrated floor with "
3480
+ "--calibrate-rolling-ic-floor (default: 0.10).",
3481
+ )
3482
+ evaluate_parser.add_argument(
3483
+ "--reconcile-features", action="store_true",
3484
+ help="V5.3.5.3 Workstream B (development/Problems.md #91/#100): diff ONE symbol's logged live "
3485
+ "feature values (book_history.jsonl's optional 'feature_snapshot' key, written when "
3486
+ "phase_v2.diagnostics.book_history.include_feature_snapshot is on and the symbol is in "
3487
+ "feature_snapshot_symbols) against full_dataset.csv's same (ticker, date) row - per-feature "
3488
+ "deltas, worst offenders first. Requires --symbol. Not included in --all - investigation-only.",
3489
+ )
3490
+ evaluate_parser.add_argument(
3491
+ "--symbol", default=None,
3492
+ help="Ticker to reconcile with --reconcile-features (e.g. --symbol XOM). Matched "
3493
+ "case-insensitively against each record's feature_snapshot keys.",
3494
+ )
3495
+ evaluate_parser.add_argument(
3496
+ "--reconcile-book-history", action="store_true",
3497
+ help="V5.2.2: compare a real Lean backtest's logged book selections "
3498
+ "(phase_v2.diagnostics.book_history) against a fresh offline re-derivation of the same raw "
3499
+ "scores on those same dates - true ground truth for diagnosing offline-vs-live divergence. "
3500
+ "Not included in --all - loads both models' predictions like --calibrate-book-spread.",
3501
+ )
3502
+ evaluate_parser.add_argument(
3503
+ "--book-history-path", default=None,
3504
+ help="Path to the book_history.jsonl log to reconcile with --reconcile-book-history "
3505
+ "(default: phase_v2.diagnostics.book_history.output_path from config.json).",
3506
+ )
3507
+ evaluate_parser.add_argument(
3508
+ "--replay-hysteresis", action="store_true",
3509
+ help="V5.2.3: with --reconcile-book-history, replay offline's own hysteresis-aware selection "
3510
+ "walk-forward across the log's dates (carrying held allocations forward, same as main.py's live "
3511
+ "book) instead of reconciling each date independently - tells a real divergence apart from the "
3512
+ "live book correctly holding an incumbent a from-scratch reselection wouldn't naturally pick.",
3513
+ )
3514
+ reconcile_run_group = evaluate_parser.add_mutually_exclusive_group()
3515
+ reconcile_run_group.add_argument(
3516
+ "--reconcile-run-index", type=int, default=None,
3517
+ help="V5.3.2: with --reconcile-book-history, reconcile only the run at this 0-indexed position "
3518
+ "within the log's own run-segmented history (negative indices count from the end, -1 == most "
3519
+ "recent - the default). book_history.jsonl is a cumulative, never-rotated log - every historical "
3520
+ "real backtest's records are appended forever - so reconciling without this flag defaults to the "
3521
+ "LATEST run only, never a silent cross-run merge (development/Problems.md #91/#97/#99).",
3522
+ )
3523
+ reconcile_run_group.add_argument(
3524
+ "--reconcile-all-runs", action="store_true",
3525
+ help="V5.3.2: with --reconcile-book-history, reconcile EVERY run segment independently (each its "
3526
+ "own held-allocations/summary, never merged across a run boundary) instead of just the most "
3527
+ "recent - for investigating an older run's own numbers, not for combining them into one figure.",
3528
+ )
3529
+ evaluate_parser.add_argument(
3530
+ "--replay-kill-switch", action="store_true",
3531
+ help="V5.2.8: day-by-day OFFLINE replay of the kill-switch + sticky trade-lock state machine "
3532
+ "(evaluation/kill_switch_replay.py) against the rank book's own simulated return series - an "
3533
+ "explicitly approximate estimate of how much of the run would have been locked out, without "
3534
+ "needing a real Lean backtest. Not included in --all - investigation-only, see development/"
3535
+ "Problems.md #94 for the caveats.",
3536
+ )
3537
+ evaluate_parser.add_argument(
3538
+ "--replay-rolling-ic-gate", action="store_true",
3539
+ help="V5.3.5 (development/Problems.md #102): day-by-day, no-lookahead OFFLINE replay of "
3540
+ "portfolio/rolling_ic_gate.py's engagement decision (evaluation/rolling_ic_gate_replay.py) at "
3541
+ "phase_v2.net_performance's rebalance cadence, with a per-era breakdown against the three known "
3542
+ "bad eras this round's research found - the critical pre-main.py checkpoint before flipping "
3543
+ "phase_v2.rolling_ic_gate.enabled live. Not included in --all - investigation-only.",
3544
+ )
3545
+ evaluate_parser.add_argument(
3546
+ "--simulate-limit-fills", action="store_true",
3547
+ help="V5.3.1: offline counterfactual (evaluation/limit_fill_simulator.py) estimating how often a "
3548
+ "real limit order would fill vs. time out, using the existing dataset's own high/low bars and "
3549
+ "phase_v2.limit_orders' pricing/timeout config - without needing a real Lean backtest. Not "
3550
+ "included in --all - investigation-only, see development/Problems.md #34/#96 for the caveats.",
3551
+ )
3552
+ evaluate_parser.add_argument(
3553
+ "--limit-fill-offset-sweep", default=None,
3554
+ help="Comma-separated offset_multiplier values to sweep with --simulate-limit-fills (default: a "
3555
+ "single run at the configured phase_v2.limit_orders.offset_multiplier).",
3556
+ )
3557
+ evaluate_parser.add_argument(
3558
+ "--monte-carlo", action="store_true",
3559
+ help="V5.3.8: Monte Carlo simulation testing layer (evaluation/monte_carlo.py) - stationary-block "
3560
+ "bootstraps the base rank-book run's daily net returns into --mc-runs alternative equity histories, "
3561
+ "writing ml/evaluation/monte_carlo.json + monte_carlo_curves.npz and refreshing the README's Monte "
3562
+ "Carlo section (curves chart + red average line + foldable deep stats). Requires/implies the base "
3563
+ "--rank-book simulation. Not included in --all.",
3564
+ )
3565
+ evaluate_parser.add_argument("--mc-runs", type=int, default=1000,
3566
+ help="Number of Monte Carlo resampled equity runs (default: 1000).")
3567
+ evaluate_parser.add_argument("--mc-block-size", type=int, default=20,
3568
+ help="Stationary block bootstrap block length in days (default: 20).")
3569
+ evaluate_parser.add_argument("--mc-seed", type=int, default=42,
3570
+ help="RNG seed - pinned by default so the README stays stable unless results change.")
3571
+ evaluate_parser.add_argument("--mc-method", choices=["block", "iid"], default="block",
3572
+ help="Resampling method (default: block = stationary block bootstrap).")
3573
+ evaluate_parser.add_argument(
3574
+ "--benchmarks", action="store_true",
3575
+ help="V5.4.4: benchmark comparison (evaluation/benchmark_comparison.py + "
3576
+ "evaluation/public_benchmarks.py) - runs the momentum/mean-reversion/random-entry strategy "
3577
+ "baselines AND the SP500 buy-and-hold + daily-rebalanced 60/40 SPY-TLT public baselines over "
3578
+ "the SAME dataset window/split as the rank book, writing ml/evaluation/benchmark_comparison.json "
3579
+ "and refreshing the README's Benchmark Comparison section. Included in --all.",
3580
+ )
3581
+ evaluate_parser.add_argument("--model", choices=["sequence", "multitask"], default=None, help="Default: sequence")
3582
+ evaluate_parser.add_argument("--head", default=None, help="Model head to evaluate, e.g. rank_20d/rank_5d (default: rank_20d)")
3583
+ evaluate_parser.add_argument("--split", default=None, help="Dataset split to evaluate: train/validation/backtest/all (default: backtest)")
3584
+ evaluate_parser.add_argument(
3585
+ "--preset", default=None,
3586
+ help="Overlay phase_v2.presets.<name> in memory only (never writes config.json) - see `aq config preset --list`",
3587
+ )
3588
+ evaluate_parser.add_argument("--json", action="store_true", help="Print the full report as JSON instead of a summary")
3589
+ evaluate_parser.set_defaults(func=cmd_evaluate)
3590
+
3591
+ render_lean_parser = subparsers.add_parser(
3592
+ "render-lean-config",
3593
+ help="Render gitignored lean.live.json from .env.live / AETHER_* env vars (secrets for live/paper)",
3594
+ )
3595
+ render_lean_parser.add_argument("--base", default=None, help="Empty tracked template (default: lean.json)")
3596
+ render_lean_parser.add_argument("--out", default=None, help="Rendered secret-bearing output (default: lean.live.json)")
3597
+ render_lean_parser.add_argument("--env-file", dest="env_file", default=None, help="Secrets file (default: .env.live)")
3598
+ render_lean_parser.set_defaults(func=cmd_render_lean_config)
3599
+
3600
+ secrets_check_parser = subparsers.add_parser(
3601
+ "secrets-check",
3602
+ help="Fail if a populated secret field in lean.json or a tracked .env is about to be committed (backs the pre-commit hook)",
3603
+ )
3604
+ secrets_check_parser.set_defaults(func=cmd_secrets_check)
3605
+
3606
+ audit_log_parser = subparsers.add_parser(
3607
+ "audit-log",
3608
+ help="Query the tamper-evident audit log (order placement, credential loads, live-mode transitions)",
3609
+ )
3610
+ audit_log_parser.add_argument("--event-type", dest="event_type", default=None,
3611
+ choices=["order_placement", "credential_load", "live_mode_transition"],
3612
+ help="Filter to one event type (default: all)")
3613
+ audit_log_parser.add_argument("--since", default=None, type=_iso_date, help="Only entries after this date, ISO 8601 YYYY-MM-DD")
3614
+ audit_log_parser.add_argument("--limit", type=int, default=100, help="Max entries to show (default: 100)")
3615
+ audit_log_parser.add_argument("--verify", action="store_true", help="Walk the whole hash chain and report the first break, if any")
3616
+ audit_log_parser.set_defaults(func=cmd_audit_log)
3617
+
3618
+ status_parser = subparsers.add_parser("status", help="Show git status")
3619
+ status_parser.set_defaults(func=cmd_status)
3620
+
3621
+ return parser
3622
+
3623
+
3624
+ def main() -> None:
3625
+ parser = build_parser()
3626
+ args = parser.parse_args()
3627
+ exit_code = args.func(args)
3628
+ check_for_update()
3629
+ sys.exit(exit_code)
3630
+
3631
+
3632
+ if __name__ == "__main__":
3633
+ main()