aether-quant 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aether_quant-0.0.1.dist-info/METADATA +7 -0
- aether_quant-0.0.1.dist-info/RECORD +99 -0
- aether_quant-0.0.1.dist-info/WHEEL +5 -0
- aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
- aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
- aether_quant-0.0.1.dist-info/top_level.txt +13 -0
- analyzer/README.md +69 -0
- analyzer/__init__.py +17 -0
- analyzer/market_analyzer.py +521 -0
- aq_cli.py +3633 -0
- audit/README.md +47 -0
- audit/__init__.py +38 -0
- audit/hash_chain.py +52 -0
- audit/postgres_audit.py +61 -0
- audit/postgres_worker.py +402 -0
- audit/redis_queue.py +124 -0
- audit/status_export.py +62 -0
- data_pipeline/README.md +257 -0
- data_pipeline/__init__.py +6 -0
- data_pipeline/bar_synthesis.py +97 -0
- data_pipeline/dividend_backfill.py +243 -0
- data_pipeline/factor_file_backfill.py +276 -0
- data_pipeline/fetch.py +206 -0
- data_pipeline/fred_backfill.py +435 -0
- data_pipeline/ib_backfill.py +287 -0
- data_pipeline/v2_manifest.py +75 -0
- data_pipeline/yfinance_backfill.py +501 -0
- evaluation/README.md +152 -0
- evaluation/__init__.py +73 -0
- evaluation/ablation.py +310 -0
- evaluation/benchmark_comparison.py +195 -0
- evaluation/confidence_threshold_calibration.py +157 -0
- evaluation/factor_neutralization_check.py +94 -0
- evaluation/feature_parity.py +321 -0
- evaluation/feature_reconciliation.py +286 -0
- evaluation/impact_model.py +77 -0
- evaluation/kill_switch_replay.py +186 -0
- evaluation/limit_fill_simulator.py +152 -0
- evaluation/model_predictions.py +192 -0
- evaluation/monte_carlo.py +161 -0
- evaluation/public_benchmarks.py +117 -0
- evaluation/rank_book_simulator.py +494 -0
- evaluation/rank_ic_core.py +204 -0
- evaluation/rank_signal_calibration.py +690 -0
- evaluation/rolling_ic_gate_calibration.py +190 -0
- evaluation/rolling_ic_gate_replay.py +138 -0
- execution/README.md +281 -0
- execution/__init__.py +80 -0
- execution/config_cache.py +48 -0
- execution/cost_model.py +213 -0
- execution/lean_config_render.py +131 -0
- execution/live_credentials.py +38 -0
- execution/live_credentials_io.py +52 -0
- execution/order_gate.py +388 -0
- execution/paper_readiness.py +181 -0
- execution/paper_readiness_io.py +54 -0
- execution/paper_readiness_report.py +103 -0
- execution/paper_readiness_scheduler.py +137 -0
- execution/reconciliation.py +160 -0
- execution/runtime_config_io.py +38 -0
- execution/secret_scan.py +73 -0
- features/README.md +77 -0
- features/__init__.py +137 -0
- features/alt_data_features.py +154 -0
- features/bond_features.py +275 -0
- features/cross_asset_sensitivity.py +79 -0
- features/derivatives_macro_features.py +72 -0
- features/macro_features.py +79 -0
- features/options_greeks.py +329 -0
- features/sector_map.py +53 -0
- features/technical_indicators.py +185 -0
- generate_backtest_report.py +311 -0
- generate_evaluation_report.py +712 -0
- inference/README.md +105 -0
- inference/__init__.py +36 -0
- inference/exported_model.py +929 -0
- inference/parallel_inference.py +160 -0
- inference/strategy_selector_inference.py +75 -0
- json_safety.py +130 -0
- portfolio/README.md +447 -0
- portfolio/__init__.py +41 -0
- portfolio/book_construction.py +786 -0
- portfolio/book_neutrality.py +178 -0
- portfolio/hrp_allocation.py +159 -0
- portfolio/options_arbitrage_detector.py +232 -0
- portfolio/options_assignment_risk.py +119 -0
- portfolio/options_margin_sizing.py +333 -0
- portfolio/options_strategy.py +1999 -0
- portfolio/rank_signal.py +219 -0
- portfolio/rolling_ic_gate.py +183 -0
- risk/README.md +275 -0
- risk/__init__.py +38 -0
- risk/asset_class_router.py +584 -0
- risk/forex_risk.py +158 -0
- risk/futures_risk.py +202 -0
- risk/kill_switch.py +261 -0
- risk/manual_override.py +118 -0
- risk/position_sizing.py +303 -0
- risk/rl_sizing.py +189 -0
aq_cli.py
ADDED
|
@@ -0,0 +1,3633 @@
|
|
|
1
|
+
"""`aq` — a thin convenience CLI wrapping Aether Quant's day-to-day commands.
|
|
2
|
+
|
|
3
|
+
Matches this codebase's existing CLI convention exactly (see
|
|
4
|
+
`retraining/orchestrator.py`'s `argparse` + `subparsers.add_parser(...)`
|
|
5
|
+
shape) - a single-file dispatcher, not a framework. Every subcommand other
|
|
6
|
+
than `trade-lock` and `fetch` is a thin `subprocess.run(...)` wrapper around
|
|
7
|
+
a command that already exists and is already documented elsewhere
|
|
8
|
+
(README.md, development/infrastructure.md) - no logic is reimplemented
|
|
9
|
+
here, this file only saves typing. `trade-lock` and `fetch` are the two
|
|
10
|
+
exceptions: they call `risk/manual_override.py` and
|
|
11
|
+
`data_pipeline/fetch.py` directly, in-process, no subprocess.
|
|
12
|
+
|
|
13
|
+
Deliberately scoped for v1 - wraps the commands already in daily use, not
|
|
14
|
+
every command mentioned anywhere in the project. Designed to be extended
|
|
15
|
+
incrementally: add a new `subparsers.add_parser(...)` block plus one `elif`
|
|
16
|
+
branch in `main()` for each new command, following the existing pattern.
|
|
17
|
+
|
|
18
|
+
Install once (registers the `aq` command on PATH inside the active venv):
|
|
19
|
+
pip install -e .
|
|
20
|
+
Then:
|
|
21
|
+
aq --help
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import argparse
|
|
27
|
+
import copy
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import re
|
|
31
|
+
import shutil
|
|
32
|
+
import subprocess
|
|
33
|
+
import sys
|
|
34
|
+
import time
|
|
35
|
+
import urllib.request
|
|
36
|
+
import uuid
|
|
37
|
+
from collections.abc import Iterator, Mapping
|
|
38
|
+
from datetime import date
|
|
39
|
+
from importlib.metadata import version as installed_version
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
|
|
42
|
+
from data_pipeline.fetch import ASSET_CLASSES, fetch_adhoc_asset
|
|
43
|
+
from data_pipeline.ib_backfill import (
|
|
44
|
+
IBNotConfiguredError,
|
|
45
|
+
attempt_connection,
|
|
46
|
+
connect_ib,
|
|
47
|
+
disconnect_ib,
|
|
48
|
+
fetch_future_historical_bars,
|
|
49
|
+
fetch_option_historical_bars,
|
|
50
|
+
ib_readiness_status,
|
|
51
|
+
load_futures_contract_specs,
|
|
52
|
+
)
|
|
53
|
+
from risk.manual_override import (
|
|
54
|
+
read_kill_switch_manual_override,
|
|
55
|
+
read_manual_trade_lock_override,
|
|
56
|
+
write_kill_switch_manual_override,
|
|
57
|
+
write_manual_trade_lock_override,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
IB_ASSET_CLASSES = ("futures", "options")
|
|
61
|
+
|
|
62
|
+
ROOT_DIR = Path(__file__).resolve().parent
|
|
63
|
+
CONFIG_PATH = ROOT_DIR / "config.json"
|
|
64
|
+
LEAN_JSON_PATH = ROOT_DIR / "lean.json"
|
|
65
|
+
WEBUI_DIR = ROOT_DIR / "webui"
|
|
66
|
+
README_PATH = ROOT_DIR / "README.md"
|
|
67
|
+
ML_DIR = ROOT_DIR / "ml"
|
|
68
|
+
|
|
69
|
+
PACKAGE_NAME = "aether-quant"
|
|
70
|
+
UPDATE_CACHE_PATH = Path.home() / ".aq" / "update_check.json"
|
|
71
|
+
UPDATE_CHECK_INTERVAL_SECONDS = 24 * 60 * 60
|
|
72
|
+
UPDATE_CHECK_TIMEOUT_SECONDS = 2
|
|
73
|
+
|
|
74
|
+
# `lean backtest .` resolves quantconnect/lean:latest by default when no
|
|
75
|
+
# --image is given - a MUTABLE tag, so Docker re-checks and re-pulls
|
|
76
|
+
# whatever layers changed on every single run, even against an already-
|
|
77
|
+
# fully-cached local image (confirmed directly: a ~42.5GB re-pull kicked
|
|
78
|
+
# off on this exact machine because Docker Hub's `latest` had moved to a
|
|
79
|
+
# newer build than what was cached). Pinned here to a specific numbered
|
|
80
|
+
# QuantConnect build tag instead (confirmed to exist on Docker Hub) so
|
|
81
|
+
# every clone of this repo - not just this machine - gets a byte-
|
|
82
|
+
# identical, one-time-download engine image that never silently re-pulls.
|
|
83
|
+
# To intentionally move to a newer Lean engine: pull the new tag by hand
|
|
84
|
+
# first (`docker pull quantconnect/lean:<new-tag>`), confirm it works,
|
|
85
|
+
# then bump this constant - never let it drift back to `latest`.
|
|
86
|
+
PINNED_LEAN_ENGINE_IMAGE = "quantconnect/lean:17900"
|
|
87
|
+
# Local derivative used by `aq backtest` by default. It is built once from
|
|
88
|
+
# PINNED_LEAN_ENGINE_IMAGE and installs the packages this project needs but
|
|
89
|
+
# the official image does not provide (redis, httpx - see requirements/
|
|
90
|
+
# lean-runtime.txt). Keeping them in the image avoids Lean CLI's Windows-host
|
|
91
|
+
# bind mount of a generated requirements.txt file, which fails on some
|
|
92
|
+
# Docker Desktop setups. Tag is DERIVED from PINNED_LEAN_ENGINE_IMAGE's own
|
|
93
|
+
# tag, never a second hardcoded copy - _ensure_local_lean_engine_image()'s
|
|
94
|
+
# cache check is a `docker image inspect` on this exact tag, so if the two
|
|
95
|
+
# constants could drift apart, bumping the pinned base without remembering
|
|
96
|
+
# to bump this one too would silently keep reusing a stale local image
|
|
97
|
+
# built from the old base forever.
|
|
98
|
+
LOCAL_LEAN_ENGINE_IMAGE = f"aether-quant-lean:{PINNED_LEAN_ENGINE_IMAGE.rsplit(':', 1)[-1]}"
|
|
99
|
+
|
|
100
|
+
_ANSI_ESCAPE_PATTERN = re.compile(r"\x1b\[[0-9;]*m")
|
|
101
|
+
_TEST_BADGE_MARKER_START = "<!-- AQ:TEST_BADGE_START -->"
|
|
102
|
+
_TEST_BADGE_MARKER_END = "<!-- AQ:TEST_BADGE_END -->"
|
|
103
|
+
_TEST_COUNT_MARKER_START = "<!-- AQ:TEST_COUNT_START -->"
|
|
104
|
+
_TEST_COUNT_MARKER_END = "<!-- AQ:TEST_COUNT_END -->"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _run(cmd: list[str], cwd: Path = ROOT_DIR) -> int:
|
|
108
|
+
"""Runs a command with inherited stdout/stderr (live output), returns its exit code."""
|
|
109
|
+
result = subprocess.run(cmd, cwd=str(cwd))
|
|
110
|
+
return result.returncode
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _find_quantconnect_lean_binary() -> str | None:
|
|
114
|
+
"""Plain `lean` on PATH is ambiguous on machines with `elan` (Lean 4, the
|
|
115
|
+
theorem prover) installed - it ships its own `lean` binary under the same
|
|
116
|
+
name as QuantConnect's Lean CLI (`pip install lean`). Disambiguate by
|
|
117
|
+
checking `--version` output (Lean 4 prints "Lean (version 4...."; the
|
|
118
|
+
QuantConnect CLI does not), preferring this repo's own venv first."""
|
|
119
|
+
bin_dir_name = "Scripts" if sys.platform == "win32" else "bin"
|
|
120
|
+
binary_name = "lean.exe" if sys.platform == "win32" else "lean"
|
|
121
|
+
candidates = [str(ROOT_DIR / ".venv" / bin_dir_name / binary_name)]
|
|
122
|
+
on_path = shutil.which("lean")
|
|
123
|
+
if on_path:
|
|
124
|
+
candidates.append(on_path)
|
|
125
|
+
|
|
126
|
+
for candidate in candidates:
|
|
127
|
+
if candidate != on_path and not Path(candidate).exists():
|
|
128
|
+
continue
|
|
129
|
+
try:
|
|
130
|
+
result = subprocess.run([candidate, "--version"], capture_output=True, text=True, timeout=30)
|
|
131
|
+
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
|
|
132
|
+
continue
|
|
133
|
+
if "Lean (version" not in (result.stdout or "") + (result.stderr or ""):
|
|
134
|
+
return candidate
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _ensure_local_lean_engine_image() -> bool:
|
|
139
|
+
"""Build the small project-specific LEAN image when it is not cached."""
|
|
140
|
+
image_check = subprocess.run(
|
|
141
|
+
["docker", "image", "inspect", LOCAL_LEAN_ENGINE_IMAGE],
|
|
142
|
+
cwd=str(ROOT_DIR),
|
|
143
|
+
stdout=subprocess.DEVNULL,
|
|
144
|
+
stderr=subprocess.DEVNULL,
|
|
145
|
+
)
|
|
146
|
+
if image_check.returncode == 0:
|
|
147
|
+
return True
|
|
148
|
+
|
|
149
|
+
dockerfile = ROOT_DIR / "Dockerfile.lean"
|
|
150
|
+
if not dockerfile.is_file():
|
|
151
|
+
print(f"error: missing local LEAN image definition: {dockerfile}", file=sys.stderr)
|
|
152
|
+
return False
|
|
153
|
+
|
|
154
|
+
print(
|
|
155
|
+
f"Building {LOCAL_LEAN_ENGINE_IMAGE} from {PINNED_LEAN_ENGINE_IMAGE} "
|
|
156
|
+
"(one-time setup; the large base image is reused)..."
|
|
157
|
+
)
|
|
158
|
+
return _run(
|
|
159
|
+
[
|
|
160
|
+
"docker",
|
|
161
|
+
"build",
|
|
162
|
+
"--file",
|
|
163
|
+
str(dockerfile),
|
|
164
|
+
"--tag",
|
|
165
|
+
LOCAL_LEAN_ENGINE_IMAGE,
|
|
166
|
+
"--build-arg",
|
|
167
|
+
f"LEAN_BASE_IMAGE={PINNED_LEAN_ENGINE_IMAGE}",
|
|
168
|
+
str(ROOT_DIR),
|
|
169
|
+
]
|
|
170
|
+
) == 0
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _parse_simple_version(value: str) -> tuple[int, ...] | None:
|
|
174
|
+
"""Best-effort "X.Y.Z" -> (X, Y, Z) parse. Returns None for anything that
|
|
175
|
+
isn't a clean dotted-integer release version - dev/local builds (e.g.
|
|
176
|
+
setuptools-scm's "0.1.dev35+gc744f9ca4.d20260704" fallback for untagged
|
|
177
|
+
installs) simply never get flagged as outdated, which is the correct
|
|
178
|
+
behavior here."""
|
|
179
|
+
try:
|
|
180
|
+
return tuple(int(part) for part in value.split("."))
|
|
181
|
+
except ValueError:
|
|
182
|
+
return None
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _iso_date(value: str) -> str:
|
|
186
|
+
"""Validates --start/--end as strict ISO 8601 YYYY-MM-DD, matching the
|
|
187
|
+
convention used everywhere else in this repo (config.json,
|
|
188
|
+
yfinance_backfill.py) - rejects other formats (e.g. DD.MM.YYYY) with a
|
|
189
|
+
clear error instead of a confusing downstream yfinance failure."""
|
|
190
|
+
try:
|
|
191
|
+
date.fromisoformat(value)
|
|
192
|
+
except ValueError as exc:
|
|
193
|
+
raise argparse.ArgumentTypeError(f"invalid date {value!r} - expected ISO 8601 YYYY-MM-DD") from exc
|
|
194
|
+
return value
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _read_update_cache() -> dict:
|
|
198
|
+
if not UPDATE_CACHE_PATH.exists():
|
|
199
|
+
return {}
|
|
200
|
+
try:
|
|
201
|
+
return json.loads(UPDATE_CACHE_PATH.read_text(encoding="utf-8"))
|
|
202
|
+
except (OSError, ValueError):
|
|
203
|
+
return {}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _write_update_cache(latest_version: str) -> None:
|
|
207
|
+
UPDATE_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
208
|
+
payload = {"last_checked": time.time(), "latest_version": latest_version}
|
|
209
|
+
UPDATE_CACHE_PATH.write_text(json.dumps(payload), encoding="utf-8")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _fetch_latest_version_from_pypi() -> str | None:
|
|
213
|
+
try:
|
|
214
|
+
url = f"https://pypi.org/pypi/{PACKAGE_NAME}/json"
|
|
215
|
+
with urllib.request.urlopen(url, timeout=UPDATE_CHECK_TIMEOUT_SECONDS) as response: # noqa: S310
|
|
216
|
+
payload = json.loads(response.read().decode("utf-8"))
|
|
217
|
+
return payload["info"]["version"]
|
|
218
|
+
except Exception:
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _latest_known_version() -> str | None:
|
|
223
|
+
cache = _read_update_cache()
|
|
224
|
+
last_checked = cache.get("last_checked", 0)
|
|
225
|
+
if time.time() - last_checked < UPDATE_CHECK_INTERVAL_SECONDS:
|
|
226
|
+
return cache.get("latest_version")
|
|
227
|
+
|
|
228
|
+
# Update the cache timestamp even on a failed fetch, so an offline user
|
|
229
|
+
# doesn't pay the network timeout again on every single command - only
|
|
230
|
+
# once per interval.
|
|
231
|
+
latest = _fetch_latest_version_from_pypi()
|
|
232
|
+
_write_update_cache(latest or cache.get("latest_version", ""))
|
|
233
|
+
return latest or cache.get("latest_version")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def check_for_update() -> None:
|
|
237
|
+
"""Prints a one-line notice to stderr if a newer aether-quant release is
|
|
238
|
+
available on PyPI. Never raises, never blocks a real command by more
|
|
239
|
+
than the short network timeout, and only actually checks PyPI once per
|
|
240
|
+
24h (cached in ~/.aq/update_check.json). Opt out with
|
|
241
|
+
AQ_SKIP_UPDATE_CHECK=1 (e.g. for CI/scripted usage)."""
|
|
242
|
+
if os.environ.get("AQ_SKIP_UPDATE_CHECK"):
|
|
243
|
+
return
|
|
244
|
+
try:
|
|
245
|
+
installed = installed_version(PACKAGE_NAME)
|
|
246
|
+
latest = _latest_known_version()
|
|
247
|
+
if not latest:
|
|
248
|
+
return
|
|
249
|
+
installed_tuple = _parse_simple_version(installed)
|
|
250
|
+
latest_tuple = _parse_simple_version(latest)
|
|
251
|
+
if installed_tuple is None or latest_tuple is None:
|
|
252
|
+
return
|
|
253
|
+
if installed_tuple < latest_tuple:
|
|
254
|
+
print(
|
|
255
|
+
f"aq: a newer version is available ({latest}, you have {installed}) - "
|
|
256
|
+
f"upgrade with: pip install --upgrade {PACKAGE_NAME}",
|
|
257
|
+
file=sys.stderr,
|
|
258
|
+
)
|
|
259
|
+
except Exception:
|
|
260
|
+
pass
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def cmd_train(args: argparse.Namespace) -> int:
|
|
264
|
+
seed = getattr(args, "seed", None)
|
|
265
|
+
ranking_objective = getattr(args, "ranking_objective", None)
|
|
266
|
+
if (seed is not None or ranking_objective is not None) and not (args.multitask_only or args.sequence_only):
|
|
267
|
+
print(
|
|
268
|
+
"aq train: --seed/--ranking-objective only apply to --multitask-only/--sequence-only - ignored here.",
|
|
269
|
+
file=sys.stderr,
|
|
270
|
+
)
|
|
271
|
+
# V5.1 Phase 4 (item 4) - same not-silently-ignored precedent as
|
|
272
|
+
# --seed/--ranking-objective above.
|
|
273
|
+
if (
|
|
274
|
+
getattr(args, "include_multitask", False)
|
|
275
|
+
or getattr(args, "include_sequence", False)
|
|
276
|
+
or getattr(args, "metrics", None) is not None
|
|
277
|
+
) and not args.walk_forward:
|
|
278
|
+
print(
|
|
279
|
+
"aq train: --include-multitask/--include-sequence/--metrics only apply to --walk-forward - ignored here.",
|
|
280
|
+
file=sys.stderr,
|
|
281
|
+
)
|
|
282
|
+
if args.gating_only:
|
|
283
|
+
return _train_gating_only()
|
|
284
|
+
if args.multitask_only:
|
|
285
|
+
return _train_multitask_only(seed=seed, ranking_objective=ranking_objective)
|
|
286
|
+
if args.sequence_only:
|
|
287
|
+
return _train_sequence_only(seed=seed, ranking_objective=ranking_objective)
|
|
288
|
+
if args.topology_only:
|
|
289
|
+
return _train_topology_only()
|
|
290
|
+
if args.strategy_selector_only:
|
|
291
|
+
return _train_strategy_selector_only()
|
|
292
|
+
if args.rl_sizing_only:
|
|
293
|
+
return _train_rl_sizing_only()
|
|
294
|
+
cmd = [sys.executable, "train.py"]
|
|
295
|
+
# A fully bare `aq train` (no scope flag at all) is the only case that
|
|
296
|
+
# chains gating/multitask/sequence training on top of train.py's own
|
|
297
|
+
# baseline+experts run below - --dataset-only/--init-only/--experts-only/
|
|
298
|
+
# --walk-forward all still mean exactly what they say and stop there.
|
|
299
|
+
bare_run = not (args.dataset_only or args.init_only or args.experts_only or args.walk_forward)
|
|
300
|
+
if args.dataset_only:
|
|
301
|
+
cmd.append("--dataset-only")
|
|
302
|
+
elif args.init_only:
|
|
303
|
+
cmd.append("--init-only")
|
|
304
|
+
elif args.experts_only:
|
|
305
|
+
cmd.append("--experts-only")
|
|
306
|
+
elif args.walk_forward:
|
|
307
|
+
cmd.append("--walk-forward")
|
|
308
|
+
if args.step_days is not None:
|
|
309
|
+
cmd += ["--step-days", str(args.step_days)]
|
|
310
|
+
if args.mode is not None:
|
|
311
|
+
cmd += ["--mode", args.mode]
|
|
312
|
+
if getattr(args, "include_multitask", False):
|
|
313
|
+
cmd.append("--include-multitask")
|
|
314
|
+
if getattr(args, "include_sequence", False):
|
|
315
|
+
cmd.append("--include-sequence")
|
|
316
|
+
if getattr(args, "metrics", None) is not None:
|
|
317
|
+
cmd += ["--metrics", args.metrics]
|
|
318
|
+
|
|
319
|
+
returncode = _run(cmd)
|
|
320
|
+
if returncode != 0 or not bare_run:
|
|
321
|
+
return returncode
|
|
322
|
+
|
|
323
|
+
# Each of these installs straight into active ml/ the same way its own
|
|
324
|
+
# --gating-only/--multitask-only/--sequence-only flag does - a bare
|
|
325
|
+
# `aq train` genuinely trains baseline + experts + gating + multitask +
|
|
326
|
+
# sequence, not just the first two. Stops at the first failure rather
|
|
327
|
+
# than silently continuing with a partially-updated active model.
|
|
328
|
+
for stage_name, stage_fn in (
|
|
329
|
+
("gating", _train_gating_only),
|
|
330
|
+
("multitask", _train_multitask_only),
|
|
331
|
+
("sequence", _train_sequence_only),
|
|
332
|
+
):
|
|
333
|
+
returncode = stage_fn()
|
|
334
|
+
if returncode != 0:
|
|
335
|
+
print(f"aq train: stopped after {stage_name} training failed (exit {returncode}).", file=sys.stderr)
|
|
336
|
+
return returncode
|
|
337
|
+
return 0
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _train_gating_only() -> int:
|
|
341
|
+
"""`aq train --gating-only`: trains the learned gating blend
|
|
342
|
+
(train_gating.py) and installs it straight into active ml/, mirroring
|
|
343
|
+
what `train.py --experts-only` already does for the expert models.
|
|
344
|
+
|
|
345
|
+
train_gating.py always writes to ml/versions/<version_id>/ (same
|
|
346
|
+
versioned-candidate convention every other trainer in this project
|
|
347
|
+
uses), so this generates a throwaway version-id, runs the trainer,
|
|
348
|
+
then copies the 3 resulting artifacts into active ml/ - the same
|
|
349
|
+
manual promotion-simulation step already documented for verifying this
|
|
350
|
+
trainer, skipping the full retraining/validate/backtest/commit/promote
|
|
351
|
+
pipeline since this is an ad-hoc, user-triggered run, not a scheduled
|
|
352
|
+
candidate."""
|
|
353
|
+
version_id = f"gating-only-{uuid.uuid4()}"
|
|
354
|
+
returncode = _run([sys.executable, "train_gating.py", "--version-id", version_id])
|
|
355
|
+
if returncode != 0:
|
|
356
|
+
return returncode
|
|
357
|
+
|
|
358
|
+
version_dir = ROOT_DIR / "ml" / "versions" / version_id
|
|
359
|
+
artifact_names = ("gating_model.json", "gating_feature_schema.json", "gating_training_metrics.json")
|
|
360
|
+
if any(not (version_dir / name).exists() for name in artifact_names):
|
|
361
|
+
print(
|
|
362
|
+
"aq train --gating-only: train_gating.py exited 0 but skipped writing artifacts "
|
|
363
|
+
"(likely insufficient validation/backtest rows) - active ml/ left unchanged.",
|
|
364
|
+
file=sys.stderr,
|
|
365
|
+
)
|
|
366
|
+
return 0
|
|
367
|
+
|
|
368
|
+
ml_dir = ROOT_DIR / "ml"
|
|
369
|
+
for name in artifact_names:
|
|
370
|
+
shutil.copy2(version_dir / name, ml_dir / name)
|
|
371
|
+
print(f"aq train --gating-only: copied {', '.join(artifact_names)} into active ml/.")
|
|
372
|
+
return 0
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _train_multitask_only(seed: int | None = None, ranking_objective: str | None = None) -> int:
|
|
376
|
+
"""`aq train --multitask-only`: trains the joint direction+magnitude+
|
|
377
|
+
volatility model (train_multitask.py) and installs it straight into
|
|
378
|
+
active ml/ - identical shape to _train_gating_only() above, including
|
|
379
|
+
the throwaway version-id / manual promotion-simulation / "skipped must
|
|
380
|
+
never look like failed" handling.
|
|
381
|
+
|
|
382
|
+
seed/ranking_objective (V5.1 Phase 3, item 1) - passed straight
|
|
383
|
+
through to train_multitask.py's own --seed/--ranking-objective flags
|
|
384
|
+
when provided, the `aq`-level entry point for the seed-ensembling/
|
|
385
|
+
ranking-objective-A/B workflow that previously required calling the
|
|
386
|
+
script directly."""
|
|
387
|
+
version_id = f"multitask-only-{uuid.uuid4()}"
|
|
388
|
+
cmd = [sys.executable, "train_multitask.py", "--version-id", version_id]
|
|
389
|
+
if seed is not None:
|
|
390
|
+
cmd += ["--seed", str(seed)]
|
|
391
|
+
if ranking_objective is not None:
|
|
392
|
+
cmd += ["--ranking-objective", ranking_objective]
|
|
393
|
+
returncode = _run(cmd)
|
|
394
|
+
if returncode != 0:
|
|
395
|
+
return returncode
|
|
396
|
+
|
|
397
|
+
version_dir = ROOT_DIR / "ml" / "versions" / version_id
|
|
398
|
+
artifact_names = ("multitask_model.json", "multitask_feature_schema.json", "multitask_training_metrics.json")
|
|
399
|
+
if any(not (version_dir / name).exists() for name in artifact_names):
|
|
400
|
+
print(
|
|
401
|
+
"aq train --multitask-only: train_multitask.py exited 0 but skipped writing artifacts "
|
|
402
|
+
"(likely insufficient train/validation/backtest rows) - active ml/ left unchanged.",
|
|
403
|
+
file=sys.stderr,
|
|
404
|
+
)
|
|
405
|
+
return 0
|
|
406
|
+
|
|
407
|
+
ml_dir = ROOT_DIR / "ml"
|
|
408
|
+
for name in artifact_names:
|
|
409
|
+
shutil.copy2(version_dir / name, ml_dir / name)
|
|
410
|
+
print(f"aq train --multitask-only: copied {', '.join(artifact_names)} into active ml/.")
|
|
411
|
+
return 0
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _train_sequence_only(seed: int | None = None, ranking_objective: str | None = None) -> int:
|
|
415
|
+
"""`aq train --sequence-only`: trains the Phase 2 causal-TCN sequence
|
|
416
|
+
encoder (train_sequence.py) and installs it straight into active ml/ -
|
|
417
|
+
identical shape to _train_multitask_only()/_train_gating_only() above.
|
|
418
|
+
|
|
419
|
+
seed/ranking_objective - see _train_multitask_only()'s identical
|
|
420
|
+
docstring."""
|
|
421
|
+
version_id = f"sequence-only-{uuid.uuid4()}"
|
|
422
|
+
cmd = [sys.executable, "train_sequence.py", "--version-id", version_id]
|
|
423
|
+
if seed is not None:
|
|
424
|
+
cmd += ["--seed", str(seed)]
|
|
425
|
+
if ranking_objective is not None:
|
|
426
|
+
cmd += ["--ranking-objective", ranking_objective]
|
|
427
|
+
returncode = _run(cmd)
|
|
428
|
+
if returncode != 0:
|
|
429
|
+
return returncode
|
|
430
|
+
|
|
431
|
+
version_dir = ROOT_DIR / "ml" / "versions" / version_id
|
|
432
|
+
artifact_names = ("sequence_model.json", "sequence_feature_schema.json", "sequence_training_metrics.json")
|
|
433
|
+
if any(not (version_dir / name).exists() for name in artifact_names):
|
|
434
|
+
print(
|
|
435
|
+
"aq train --sequence-only: train_sequence.py exited 0 but skipped writing artifacts "
|
|
436
|
+
"(likely insufficient train/validation/backtest rows) - active ml/ left unchanged.",
|
|
437
|
+
file=sys.stderr,
|
|
438
|
+
)
|
|
439
|
+
return 0
|
|
440
|
+
|
|
441
|
+
ml_dir = ROOT_DIR / "ml"
|
|
442
|
+
for name in artifact_names:
|
|
443
|
+
shutil.copy2(version_dir / name, ml_dir / name)
|
|
444
|
+
print(f"aq train --sequence-only: copied {', '.join(artifact_names)} into active ml/.")
|
|
445
|
+
return 0
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _train_topology_only() -> int:
|
|
449
|
+
"""`aq train --topology-only`: trains the learned topology overlay
|
|
450
|
+
(train_topology.py) and installs it straight into active ml/ -
|
|
451
|
+
identical shape to _train_multitask_only()/_train_gating_only()/
|
|
452
|
+
_train_sequence_only() above.
|
|
453
|
+
|
|
454
|
+
Different data source and a real prerequisite the other three
|
|
455
|
+
trainers don't have: train_topology.py fits over realized trading
|
|
456
|
+
outcomes pulled from Postgres (AETHER_POSTGRES_DSN env var, read by
|
|
457
|
+
train_topology.py itself - no DSN plumbing needed here), and needs
|
|
458
|
+
at least phase_v2.topology_learning.training.min_training_events
|
|
459
|
+
(default 500) usable events from a lookback_days (default 90) window.
|
|
460
|
+
That data only accumulates from runs with the full stack (Postgres +
|
|
461
|
+
the audit worker) up - on a fresh checkout, or before Postgres has
|
|
462
|
+
been running long enough, this is expected to hit the "skipped"
|
|
463
|
+
branch below, not the "artifacts written" one. development/
|
|
464
|
+
Problems.md #56 has the full offset-scale story this trainer feeds."""
|
|
465
|
+
version_id = f"topology-only-{uuid.uuid4()}"
|
|
466
|
+
returncode = _run([sys.executable, "train_topology.py", "--version-id", version_id])
|
|
467
|
+
if returncode != 0:
|
|
468
|
+
return returncode
|
|
469
|
+
|
|
470
|
+
version_dir = ROOT_DIR / "ml" / "versions" / version_id
|
|
471
|
+
artifact_names = ("topology_model.json", "topology_feature_schema.json", "topology_training_metrics.json")
|
|
472
|
+
if any(not (version_dir / name).exists() for name in artifact_names):
|
|
473
|
+
print(
|
|
474
|
+
"aq train --topology-only: train_topology.py exited 0 but skipped writing artifacts "
|
|
475
|
+
"(fewer than min_training_events realized-outcome events in the lookback window - needs "
|
|
476
|
+
"Postgres up and the audit worker running long enough to accumulate them) - active ml/ left "
|
|
477
|
+
"unchanged.",
|
|
478
|
+
file=sys.stderr,
|
|
479
|
+
)
|
|
480
|
+
return 0
|
|
481
|
+
|
|
482
|
+
ml_dir = ROOT_DIR / "ml"
|
|
483
|
+
for name in artifact_names:
|
|
484
|
+
shutil.copy2(version_dir / name, ml_dir / name)
|
|
485
|
+
print(f"aq train --topology-only: copied {', '.join(artifact_names)} into active ml/.")
|
|
486
|
+
return 0
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _train_strategy_selector_only() -> int:
|
|
490
|
+
"""`aq train --strategy-selector-only`: trains the learned multi-leg
|
|
491
|
+
strategy-selector model (train_strategy_selector.py, V4.7,
|
|
492
|
+
development/Problems.md #29's own framing) and installs it straight
|
|
493
|
+
into active ml/ - identical shape to _train_topology_only()/
|
|
494
|
+
_train_multitask_only()/_train_gating_only()/_train_sequence_only()
|
|
495
|
+
above.
|
|
496
|
+
|
|
497
|
+
Different, and much harder to satisfy, prerequisite than any of the
|
|
498
|
+
other four: train_strategy_selector.py fits over
|
|
499
|
+
"option_strategy_outcome" experience events (Postgres,
|
|
500
|
+
AETHER_POSTGRES_DSN env var), which only main.py::
|
|
501
|
+
_emit_option_strategy_outcome_if_pending() can ever emit - and that
|
|
502
|
+
only fires for a multi-leg OPTION position that was actually opened
|
|
503
|
+
AND closed. Unlike train_topology.py (dormant only until enough
|
|
504
|
+
Postgres VOLUME accumulates from an already-running system), this
|
|
505
|
+
trainer has NO data source at all until real option positions
|
|
506
|
+
actually trade - confirmed during scoping research that every options
|
|
507
|
+
code path in this repo is still IB-unverified with zero option assets
|
|
508
|
+
in the live universe. Expect the "skipped" branch below on every run
|
|
509
|
+
in this environment, indefinitely - see train_strategy_selector.py's
|
|
510
|
+
own module docstring for the full story."""
|
|
511
|
+
version_id = f"strategy-selector-only-{uuid.uuid4()}"
|
|
512
|
+
returncode = _run([sys.executable, "train_strategy_selector.py", "--version-id", version_id])
|
|
513
|
+
if returncode != 0:
|
|
514
|
+
return returncode
|
|
515
|
+
|
|
516
|
+
version_dir = ROOT_DIR / "ml" / "versions" / version_id
|
|
517
|
+
artifact_names = (
|
|
518
|
+
"strategy_selector_model.json",
|
|
519
|
+
"strategy_selector_feature_schema.json",
|
|
520
|
+
"strategy_selector_training_metrics.json",
|
|
521
|
+
)
|
|
522
|
+
if any(not (version_dir / name).exists() for name in artifact_names):
|
|
523
|
+
print(
|
|
524
|
+
"aq train --strategy-selector-only: train_strategy_selector.py exited 0 but skipped writing "
|
|
525
|
+
"artifacts (no option_strategy_outcome events yet - needs real option positions to have actually "
|
|
526
|
+
"traded and closed) - active ml/ left unchanged.",
|
|
527
|
+
file=sys.stderr,
|
|
528
|
+
)
|
|
529
|
+
return 0
|
|
530
|
+
|
|
531
|
+
ml_dir = ROOT_DIR / "ml"
|
|
532
|
+
for name in artifact_names:
|
|
533
|
+
shutil.copy2(version_dir / name, ml_dir / name)
|
|
534
|
+
print(f"aq train --strategy-selector-only: copied {', '.join(artifact_names)} into active ml/.")
|
|
535
|
+
return 0
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _train_rl_sizing_only() -> int:
|
|
539
|
+
"""`aq train --rl-sizing-only`: trains the offline contextual-bandit
|
|
540
|
+
sizing overlay (train_rl_sizing.py, development/Problems.md #71) and
|
|
541
|
+
installs it straight into active ml/ - identical shape to
|
|
542
|
+
_train_topology_only()/_train_strategy_selector_only() above.
|
|
543
|
+
|
|
544
|
+
Different, but much more tractable, prerequisite than either of those
|
|
545
|
+
two: train_rl_sizing.py reads ml/datasets/{validation,backtest}_dataset.csv
|
|
546
|
+
- NOT Postgres, NOT real option trades - so this can run any time a
|
|
547
|
+
normal `python train.py` has already produced a multitask model and
|
|
548
|
+
the standard dataset files. Requires phase1.features.input_set to
|
|
549
|
+
already include Component D's 3 alt-data feature names (a retrain
|
|
550
|
+
with those in the schema) - see train_rl_sizing.py's own module
|
|
551
|
+
docstring for the full honest framing (full-information contextual
|
|
552
|
+
bandit, NOT off-policy RL) and risk/rl_sizing.py for the abandon
|
|
553
|
+
criteria this feature should be judged against before ever being
|
|
554
|
+
enabled by default."""
|
|
555
|
+
version_id = f"rl-sizing-only-{uuid.uuid4()}"
|
|
556
|
+
returncode = _run([sys.executable, "train_rl_sizing.py", "--version-id", version_id])
|
|
557
|
+
if returncode != 0:
|
|
558
|
+
return returncode
|
|
559
|
+
|
|
560
|
+
version_dir = ROOT_DIR / "ml" / "versions" / version_id
|
|
561
|
+
artifact_names = ("rl_sizing_model.json", "rl_sizing_feature_schema.json", "rl_sizing_training_metrics.json")
|
|
562
|
+
if any(not (version_dir / name).exists() for name in artifact_names):
|
|
563
|
+
print(
|
|
564
|
+
"aq train --rl-sizing-only: train_rl_sizing.py exited 0 but skipped writing artifacts "
|
|
565
|
+
"(fewer than min_training_rows usable rows, or the multitask model/dataset files are missing) - "
|
|
566
|
+
"active ml/ left unchanged.",
|
|
567
|
+
file=sys.stderr,
|
|
568
|
+
)
|
|
569
|
+
return 0
|
|
570
|
+
|
|
571
|
+
ml_dir = ROOT_DIR / "ml"
|
|
572
|
+
for name in artifact_names:
|
|
573
|
+
shutil.copy2(version_dir / name, ml_dir / name)
|
|
574
|
+
print(f"aq train --rl-sizing-only: copied {', '.join(artifact_names)} into active ml/.")
|
|
575
|
+
return 0
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _update_readme_test_badge(passed: int, failed: int) -> None:
|
|
579
|
+
"""Atomically rewrites the shields.io test-count badge AND every
|
|
580
|
+
AQ:TEST_COUNT-marked "N tests" prose mention (Test Suite section,
|
|
581
|
+
Module Documentation table's tests/ row) in README.md, so neither ever
|
|
582
|
+
drifts from the real collected-test total. Mirrors the equivalent
|
|
583
|
+
mechanism in the sibling Aether-Vault project's `av test`. Never
|
|
584
|
+
raises - a badge-update bug must never fail `aq test` itself."""
|
|
585
|
+
total = passed + failed
|
|
586
|
+
if total == 0:
|
|
587
|
+
return # nothing collected - leave the badge/count alone rather than zero them out
|
|
588
|
+
if not README_PATH.is_file():
|
|
589
|
+
return
|
|
590
|
+
text = README_PATH.read_text(encoding="utf-8")
|
|
591
|
+
|
|
592
|
+
if _TEST_BADGE_MARKER_START in text and _TEST_BADGE_MARKER_END in text:
|
|
593
|
+
color = "brightgreen" if failed == 0 else "red"
|
|
594
|
+
badge = (
|
|
595
|
+
f'<img src="https://img.shields.io/badge/tests-{passed}%2F{total}%20passing-{color}'
|
|
596
|
+
f'?style=flat-square&labelColor=1A1A1A" alt="{passed} of {total} tests passing">'
|
|
597
|
+
)
|
|
598
|
+
badge_pattern = re.compile(
|
|
599
|
+
re.escape(_TEST_BADGE_MARKER_START) + r".*?" + re.escape(_TEST_BADGE_MARKER_END), re.DOTALL
|
|
600
|
+
)
|
|
601
|
+
text = badge_pattern.sub(f"{_TEST_BADGE_MARKER_START}{badge}{_TEST_BADGE_MARKER_END}", text, count=1)
|
|
602
|
+
|
|
603
|
+
if _TEST_COUNT_MARKER_START in text and _TEST_COUNT_MARKER_END in text:
|
|
604
|
+
count_pattern = re.compile(
|
|
605
|
+
re.escape(_TEST_COUNT_MARKER_START) + r".*?" + re.escape(_TEST_COUNT_MARKER_END), re.DOTALL
|
|
606
|
+
)
|
|
607
|
+
text = count_pattern.sub(f"{_TEST_COUNT_MARKER_START}{total}{_TEST_COUNT_MARKER_END}", text)
|
|
608
|
+
|
|
609
|
+
original_text = README_PATH.read_text(encoding="utf-8")
|
|
610
|
+
if text == original_text:
|
|
611
|
+
return
|
|
612
|
+
|
|
613
|
+
tmp_path = README_PATH.with_suffix(README_PATH.suffix + ".tmp")
|
|
614
|
+
tmp_path.write_text(text, encoding="utf-8")
|
|
615
|
+
tmp_path.replace(README_PATH)
|
|
616
|
+
print(f"Updated README.md test badge/count: {passed}/{total} passing")
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _run_captured(cmd: list[str], cwd: Path = ROOT_DIR) -> tuple[int, str]:
|
|
620
|
+
"""Like _run(), but also captures combined stdout+stderr while still
|
|
621
|
+
streaming it live to the terminal - used only by cmd_test, which needs
|
|
622
|
+
the captured text afterward to parse the real pass/fail count for the
|
|
623
|
+
README badge. Kept as its own function, separate from _run() (every
|
|
624
|
+
other subprocess-wrapping command's single choke point - see this
|
|
625
|
+
module's test file docstring), specifically so tests can mock this one
|
|
626
|
+
choke point without silently falling through to a real subprocess call
|
|
627
|
+
the way mocking only `_run` would (that exact gap previously let
|
|
628
|
+
`aq test`'s own test recursively spawn a real, full pytest run on every
|
|
629
|
+
invocation of the suite)."""
|
|
630
|
+
process = subprocess.Popen(
|
|
631
|
+
cmd,
|
|
632
|
+
cwd=str(cwd),
|
|
633
|
+
stdout=subprocess.PIPE,
|
|
634
|
+
stderr=subprocess.STDOUT,
|
|
635
|
+
text=True,
|
|
636
|
+
bufsize=1,
|
|
637
|
+
)
|
|
638
|
+
output_lines = []
|
|
639
|
+
for line in process.stdout:
|
|
640
|
+
print(line, end="")
|
|
641
|
+
output_lines.append(line)
|
|
642
|
+
process.wait()
|
|
643
|
+
return process.returncode, "".join(output_lines)
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
# Subsystem -> tests/*.py filenames, used only when the user passes one or
|
|
647
|
+
# more --<subsystem> flags to filter the run. Not required to be exhaustive
|
|
648
|
+
# (the default, flag-less `aq test` just runs tests/ directly - marker-based
|
|
649
|
+
# exclusion, not this mapping, is what makes that complete) but kept
|
|
650
|
+
# reasonably complete so the flags are actually useful for "test every
|
|
651
|
+
# subsystem" one at a time. test_lean_backtest_ml_coverage.py is
|
|
652
|
+
# deliberately absent from every bucket - it's gated by the lean_backtest
|
|
653
|
+
# marker (see --lean/--full below), never by a subsystem flag.
|
|
654
|
+
#
|
|
655
|
+
# Hand-maintained, not derived from the filesystem - real drift found and
|
|
656
|
+
# fixed once already (~14 files, including a whole missing "audit" bucket,
|
|
657
|
+
# had silently fallen off every group). tests/test_aq_cli.py::
|
|
658
|
+
# test_subsystem_test_files_maps_every_real_test_file_to_exactly_one_bucket()
|
|
659
|
+
# now locks in "every real test file (except the one deliberate exclusion
|
|
660
|
+
# above) is in exactly one bucket" so a new test file missing from every
|
|
661
|
+
# bucket fails CI instead of quietly working only via the flag-less
|
|
662
|
+
# default run.
|
|
663
|
+
_SUBSYSTEM_TEST_FILES: dict[str, list[str]] = {
|
|
664
|
+
"cli": [
|
|
665
|
+
"test_aq_cli.py", "test_generate_backtest_report.py", "test_generate_evaluation_report.py",
|
|
666
|
+
"test_lean_config_render.py", "test_dockerignore_secrets.py", "test_secret_scan.py",
|
|
667
|
+
"test_profile_inference.py", "test_profile_subsystems.py", "test_lean_runtime_imports.py",
|
|
668
|
+
"test_order_events_audit.py",
|
|
669
|
+
# V5.4.7 - json_safety.py + the CLI help-surface/README-sync guards.
|
|
670
|
+
"test_json_safety.py", "test_cli_help_surface.py",
|
|
671
|
+
],
|
|
672
|
+
"audit": [
|
|
673
|
+
"test_hash_chain.py", "test_audit_queue.py", "test_postgres_audit.py",
|
|
674
|
+
"test_audit_postgres_worker.py", "test_audit_status_export.py",
|
|
675
|
+
],
|
|
676
|
+
"risk": [
|
|
677
|
+
"test_risk_controls.py", "test_asset_class_router.py", "test_futures_risk.py",
|
|
678
|
+
"test_forex_risk.py",
|
|
679
|
+
"test_order_gate.py", "test_position_sizing.py", "test_backtest_gate.py",
|
|
680
|
+
"test_validation_gate.py", "test_manual_override.py", "test_rl_sizing.py",
|
|
681
|
+
# V5.1 Phase 6 (production safety) - risk/kill_switch.py,
|
|
682
|
+
# execution/reconciliation.py (grouped here, not a dedicated
|
|
683
|
+
# "execution" bucket, alongside every other production-safety
|
|
684
|
+
# risk mechanism in this list).
|
|
685
|
+
"test_kill_switch.py", "test_reconciliation.py",
|
|
686
|
+
],
|
|
687
|
+
"portfolio": [
|
|
688
|
+
"test_portfolio_book_construction.py", "test_options_strategy.py",
|
|
689
|
+
"test_options_strategy_multileg.py", "test_options_arbitrage_detector.py",
|
|
690
|
+
"test_options_margin_sizing.py",
|
|
691
|
+
"test_options_greeks.py", "test_simulated_portfolio.py", "test_options_assignment_risk.py",
|
|
692
|
+
# V5.1 Phase 1 (development/Problems.md #73) - portfolio/rank_signal.py.
|
|
693
|
+
"test_rank_signal.py",
|
|
694
|
+
# V5.3.5 (development/Problems.md #102) - portfolio/rolling_ic_gate.py.
|
|
695
|
+
"test_rolling_ic_gate.py",
|
|
696
|
+
],
|
|
697
|
+
"features": [
|
|
698
|
+
"test_bond_features.py", "test_derivatives_macro_features.py", "test_macro_features.py",
|
|
699
|
+
"test_technical_indicators.py", "test_train_bond_features.py",
|
|
700
|
+
"test_train_derivatives_macro_features.py", "test_train_macro_features.py",
|
|
701
|
+
"test_train_asset_class_context_features.py", "test_train_cross_sectional_features.py",
|
|
702
|
+
"test_train_indicators.py", "test_alt_data_features.py", "test_train_alt_data_features.py",
|
|
703
|
+
# V5.1 Phase 2 (item 8 / F2) - features/cross_asset_sensitivity.py.
|
|
704
|
+
"test_cross_asset_sensitivity.py", "test_train_cross_asset_sensitivity.py",
|
|
705
|
+
],
|
|
706
|
+
"data-pipeline": [
|
|
707
|
+
"test_fetch.py", "test_ib_backfill.py", "test_fred_backfill.py", "test_yfinance_backfill.py",
|
|
708
|
+
"test_dividend_backfill.py", "test_factor_file_backfill.py",
|
|
709
|
+
],
|
|
710
|
+
"webui": [
|
|
711
|
+
"test_neural_network_state.py", "test_assets_status.py", "test_status_export.py",
|
|
712
|
+
"test_rank_ic_monitor.py", "test_observation_metrics.py", "test_strategy_catalog.py",
|
|
713
|
+
"test_api_server.py",
|
|
714
|
+
],
|
|
715
|
+
"ml": [
|
|
716
|
+
"test_expert_models.py", "test_expert_datasets.py", "test_gating_network.py", "test_train_gating.py",
|
|
717
|
+
"test_train_multitask.py", "test_train_multitask_architecture.py", "test_train_sequence.py",
|
|
718
|
+
"test_train_sequence_architecture.py", "test_train_pipeline.py", "test_train_ranking_validation.py",
|
|
719
|
+
"test_train_walk_forward_windows.py", "test_train_topology.py", "test_learned_topology.py",
|
|
720
|
+
"test_exported_model.py", "test_market_topology.py", "test_market_regime.py",
|
|
721
|
+
"test_market_analyzer.py", "test_market_liquidity.py",
|
|
722
|
+
"test_train_strategy_selector.py", "test_strategy_selector_inference.py",
|
|
723
|
+
"test_parallel_inference.py", "test_train_threshold_and_early_stop.py",
|
|
724
|
+
"test_train_select_model_context_columns.py", "test_train_rl_sizing.py",
|
|
725
|
+
# V5.1 Phase 2 (item 5 of the roadmap) - train.py::build_residual_rank_targets().
|
|
726
|
+
"test_train_residual_rank_targets.py",
|
|
727
|
+
# V5.1 Phase 3 (items 1, 10, 11) - cross-sectional batching, ranking
|
|
728
|
+
# losses, optimizer/schedule/SWA/smoothing helpers.
|
|
729
|
+
"test_train_cross_sectional_batching.py", "test_train_ranking_loss.py", "test_train_optimizer_and_swa.py",
|
|
730
|
+
# V5.1 Phase 4 (item 4) - multi-model walk-forward subprocess/net-
|
|
731
|
+
# performance-simulation helpers.
|
|
732
|
+
"test_walk_forward_multimodel.py",
|
|
733
|
+
# V5.2.8 (Problems.md #94) - train.py::compute_gate_friendliness_weight_by_date().
|
|
734
|
+
"test_train_gate_friendliness.py",
|
|
735
|
+
],
|
|
736
|
+
"retraining": [
|
|
737
|
+
"test_retraining_artifacts.py", "test_retraining_orchestrator.py", "test_retraining_planning.py",
|
|
738
|
+
"test_retraining_postgres_registry.py", "test_retraining_worker.py", "test_trigger_worker.py",
|
|
739
|
+
"test_triggers.py", "test_vault_client.py", "test_vault_commands.py", "test_lean_backtest.py",
|
|
740
|
+
"test_v2_pipeline_manifest.py",
|
|
741
|
+
# V5.1 Phase 6 (production safety) - retraining/auto_rollback.py.
|
|
742
|
+
"test_auto_rollback.py",
|
|
743
|
+
],
|
|
744
|
+
"notifications": ["test_telegram_alerts.py", "test_telegram_client.py", "test_telegram_worker.py", "test_postgres_telegram.py"],
|
|
745
|
+
"storage": ["test_postgres_triggers.py", "test_postgres_worker.py", "test_config_cache.py", "test_runtime_config_io.py", "test_experience_queue.py"],
|
|
746
|
+
"live": [
|
|
747
|
+
"test_live_credentials.py", "test_live_credentials_io.py", "test_paper_readiness.py",
|
|
748
|
+
"test_paper_readiness_io.py", "test_paper_readiness_report.py", "test_paper_readiness_scheduler.py",
|
|
749
|
+
],
|
|
750
|
+
"evaluation": [
|
|
751
|
+
"test_rank_book_simulator.py", "test_book_neutrality.py", "test_cost_model.py",
|
|
752
|
+
"test_model_predictions.py", "test_evaluation_state.py", "test_sector_map.py",
|
|
753
|
+
# V5.1 Phase 5 (item 9) - evaluation/ablation.py.
|
|
754
|
+
"test_ablation.py",
|
|
755
|
+
# V5.1 (Problems.md) - evaluation/rank_signal_calibration.py.
|
|
756
|
+
"test_rank_signal_calibration.py",
|
|
757
|
+
# V5.2.6 (Problems.md) - evaluation/confidence_threshold_calibration.py.
|
|
758
|
+
"test_confidence_threshold_calibration.py",
|
|
759
|
+
# V5.2.8 (Problems.md #94) - evaluation/kill_switch_replay.py.
|
|
760
|
+
"test_kill_switch_replay.py",
|
|
761
|
+
# V5.3.1 (Problems.md #34/#96) - evaluation/limit_fill_simulator.py.
|
|
762
|
+
"test_limit_fill_simulator.py",
|
|
763
|
+
# V5.3.5 (Problems.md #102) - evaluation/rank_ic_core.py,
|
|
764
|
+
# evaluation/rolling_ic_gate_calibration.py, evaluation/
|
|
765
|
+
# rolling_ic_gate_replay.py.
|
|
766
|
+
"test_rank_ic_core.py", "test_rolling_ic_gate_calibration.py", "test_rolling_ic_gate_replay.py",
|
|
767
|
+
# V5.3.5.3 (Problems.md #91/#100) - evaluation/feature_reconciliation.py.
|
|
768
|
+
"test_feature_reconciliation.py",
|
|
769
|
+
"test_feature_parity.py",
|
|
770
|
+
"test_promotion_gate_era_rule.py",
|
|
771
|
+
"test_aq_test_ruff_flag.py",
|
|
772
|
+
"test_ci_workflows.py",
|
|
773
|
+
"test_impact_model.py",
|
|
774
|
+
"test_v543_new_features.py",
|
|
775
|
+
"test_bar_synthesis_parity.py",
|
|
776
|
+
"test_rollback_hardening.py",
|
|
777
|
+
"test_rl_sizing_v541.py",
|
|
778
|
+
"test_monte_carlo.py",
|
|
779
|
+
# V5.4.4 - evaluation/public_benchmarks.py (SP500 + 60-40 public
|
|
780
|
+
# baselines) and the HRP live-book-sizing integration tests.
|
|
781
|
+
"test_public_benchmarks.py",
|
|
782
|
+
"test_hrp_book_sizing.py",
|
|
783
|
+
# V5.4.7 - full-system bug-hunt round file (#120-#125).
|
|
784
|
+
"test_v547_bug_hunt.py",
|
|
785
|
+
],
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def cmd_test(args: argparse.Namespace) -> int:
|
|
790
|
+
"""Runs pytest with live-streamed output (same UX as a plain
|
|
791
|
+
subprocess.run), while also capturing it so the real pass/fail count can
|
|
792
|
+
be parsed afterward and used to refresh README.md's test badge - mirrors
|
|
793
|
+
the sibling Aether-Vault project's `av test` exactly.
|
|
794
|
+
|
|
795
|
+
Default (no flags): excludes tests/test_lean_backtest_ml_coverage.py's
|
|
796
|
+
lean_backtest-marked tests - a real `lean backtest .` run there takes
|
|
797
|
+
over an hour wall-clock, and this repo's own .venv happens to have a
|
|
798
|
+
real Lean CLI installed, so that file's skipif alone never actually
|
|
799
|
+
skipped it. --lean/--full opts back in. --parallel adds pytest-xdist's
|
|
800
|
+
-n auto (off by default - multiple workers each importing torch is a
|
|
801
|
+
real OOM risk on memory-constrained dev machines). One or more
|
|
802
|
+
--<subsystem> flags restrict the run to _SUBSYSTEM_TEST_FILES'
|
|
803
|
+
filenames for those subsystems instead of the whole tree.
|
|
804
|
+
|
|
805
|
+
V5.3.10 - --ruff: runs `ruff check .` BEFORE pytest so a lint failure
|
|
806
|
+
fails fast without paying the full suite's wall-clock first (mirrors
|
|
807
|
+
ci.yml's python-lint job; single source of truth for the invocation)."""
|
|
808
|
+
if getattr(args, "ruff", False):
|
|
809
|
+
import subprocess as _sp
|
|
810
|
+
|
|
811
|
+
ruff_exit = _sp.run(["ruff", "check", "."]).returncode
|
|
812
|
+
if ruff_exit != 0:
|
|
813
|
+
print(f"ruff check failed (exit {ruff_exit}) - fix lint findings before running the suite.")
|
|
814
|
+
return ruff_exit
|
|
815
|
+
|
|
816
|
+
cmd = [sys.executable, "-m", "pytest", "--color=yes", "--durations=15"]
|
|
817
|
+
|
|
818
|
+
subsystem_files: list[str] = []
|
|
819
|
+
for name in _SUBSYSTEM_TEST_FILES:
|
|
820
|
+
if getattr(args, name.replace("-", "_"), False):
|
|
821
|
+
subsystem_files.extend(_SUBSYSTEM_TEST_FILES[name])
|
|
822
|
+
is_filtered_run = bool(subsystem_files)
|
|
823
|
+
|
|
824
|
+
if is_filtered_run:
|
|
825
|
+
cmd.extend(f"tests/{name}" for name in dict.fromkeys(subsystem_files))
|
|
826
|
+
else:
|
|
827
|
+
cmd.append("tests/")
|
|
828
|
+
|
|
829
|
+
if getattr(args, "lean", False) or getattr(args, "full", False):
|
|
830
|
+
if is_filtered_run:
|
|
831
|
+
cmd.append("tests/test_lean_backtest_ml_coverage.py")
|
|
832
|
+
else:
|
|
833
|
+
cmd.extend(["-m", "not lean_backtest"])
|
|
834
|
+
|
|
835
|
+
if getattr(args, "parallel", False):
|
|
836
|
+
cmd.extend(["-n", "auto"])
|
|
837
|
+
|
|
838
|
+
exit_code, output = _run_captured(cmd)
|
|
839
|
+
|
|
840
|
+
captured = _ANSI_ESCAPE_PATTERN.sub("", output)
|
|
841
|
+
passed_match = re.search(r"(\d+) passed", captured)
|
|
842
|
+
failed_match = re.search(r"(\d+) failed", captured)
|
|
843
|
+
error_match = re.search(r"(\d+) error", captured)
|
|
844
|
+
# Only the full, unfiltered default run's pass/fail count reflects the
|
|
845
|
+
# whole suite - updating the badge from a --cli-only or --lean-only
|
|
846
|
+
# partial run would make it silently report a subset as if it were
|
|
847
|
+
# everything.
|
|
848
|
+
if passed_match and not is_filtered_run:
|
|
849
|
+
passed = int(passed_match.group(1))
|
|
850
|
+
failed = (int(failed_match.group(1)) if failed_match else 0) + (int(error_match.group(1)) if error_match else 0)
|
|
851
|
+
_update_readme_test_badge(passed, failed)
|
|
852
|
+
|
|
853
|
+
return exit_code
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def cmd_backtest(args: argparse.Namespace) -> int:
|
|
857
|
+
lean_binary = _find_quantconnect_lean_binary()
|
|
858
|
+
if lean_binary is None:
|
|
859
|
+
print("error: QuantConnect Lean CLI not found (checked .venv and PATH).", file=sys.stderr)
|
|
860
|
+
return 1
|
|
861
|
+
# The default is a small local derivative of the pinned official image.
|
|
862
|
+
# It contains redis in the image itself, so Lean CLI does not need to
|
|
863
|
+
# create and Windows-bind-mount a generated requirements.txt file.
|
|
864
|
+
# --image remains an escape hatch for deliberately selecting another
|
|
865
|
+
# engine image without editing this file.
|
|
866
|
+
if args.image:
|
|
867
|
+
engine_image = args.image
|
|
868
|
+
else:
|
|
869
|
+
if not _ensure_local_lean_engine_image():
|
|
870
|
+
return 1
|
|
871
|
+
engine_image = LOCAL_LEAN_ENGINE_IMAGE
|
|
872
|
+
lean_command = [lean_binary, "backtest", ".", "--image", engine_image]
|
|
873
|
+
# Lean CLI creates config/startup files with tempfile.mkdtemp(). On some
|
|
874
|
+
# Windows Docker Desktop installations those Python-created directories
|
|
875
|
+
# are not readable by Docker, while ordinary host files are. Run through
|
|
876
|
+
# the project wrapper so every generated temp directory receives
|
|
877
|
+
# Docker-readable inherited permissions before the container is created.
|
|
878
|
+
windows_wrapper = ROOT_DIR / "scripts" / "run_lean_cli_windows.py"
|
|
879
|
+
if sys.platform == "win32" and windows_wrapper.is_file():
|
|
880
|
+
lean_command = [sys.executable, str(windows_wrapper), *lean_command[1:]]
|
|
881
|
+
exit_code = _run(lean_command)
|
|
882
|
+
# Attempt the README update regardless of exit_code, NOT only on == 0.
|
|
883
|
+
# On a resource-constrained machine Lean's Python-interpreter teardown
|
|
884
|
+
# can exceed its own hardcoded 10-second shutdown-isolator budget and
|
|
885
|
+
# make `lean backtest` return a NON-zero exit code even though the
|
|
886
|
+
# backtest itself fully completed and wrote valid results - a cosmetic
|
|
887
|
+
# "PythonInitializer.Shutdown() ... Operation timed out" error that
|
|
888
|
+
# fires AFTER statistics are already saved (see development/Problems.md).
|
|
889
|
+
# update_readme_from_latest_backtest() self-guards: it only ever picks a
|
|
890
|
+
# backtest folder whose result JSON has BOTH non-empty statistics and a
|
|
891
|
+
# non-empty equity curve, so calling it after a genuinely-failed run
|
|
892
|
+
# (e.g. an init-timeout that produced no statistics) is a safe no-op.
|
|
893
|
+
# Decoupling the README update from Lean's flaky shutdown-phase exit
|
|
894
|
+
# code is the robust fix - the exit code is still returned to the caller.
|
|
895
|
+
try:
|
|
896
|
+
from generate_backtest_report import update_readme_from_latest_backtest
|
|
897
|
+
|
|
898
|
+
if update_readme_from_latest_backtest():
|
|
899
|
+
print("Updated README.md's Backtest Results section.")
|
|
900
|
+
except Exception as error: # noqa: BLE001 - must never fail the backtest command itself
|
|
901
|
+
print(f"warning: failed to update README.md's backtest results ({error})", file=sys.stderr)
|
|
902
|
+
# Also refreshes Other Metrics (Lean-vs-offline comparison) with this
|
|
903
|
+
# backtest's fresh Sharpe, rather than leaving it stale until the next
|
|
904
|
+
# `aq evaluate` run - same self-guarding, never-fail contract as above.
|
|
905
|
+
_refresh_readme_evaluation_sections()
|
|
906
|
+
if exit_code != 0:
|
|
907
|
+
print(
|
|
908
|
+
"note: `lean backtest` returned a non-zero exit code. If the run reached the end date and "
|
|
909
|
+
"statistics were written, this is most likely the benign Python-shutdown timeout on a "
|
|
910
|
+
"resource-constrained machine (development/Problems.md) - the README was still updated from "
|
|
911
|
+
"the completed results above. Check the backtest log if unsure the run actually finished.",
|
|
912
|
+
file=sys.stderr,
|
|
913
|
+
)
|
|
914
|
+
return exit_code
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
# One flag per scripts/profile_subsystems.py subsystem - same
|
|
918
|
+
# established loop-generated-flags convention as _SUBSYSTEM_TEST_FILES
|
|
919
|
+
# above (`aq test --cli --risk` etc.), applied to `aq profile` instead of
|
|
920
|
+
# `aq test`. Values aren't used (dispatch is by attribute presence, same
|
|
921
|
+
# as cmd_test()) - a plain tuple would do, kept as a dict of Nones so the
|
|
922
|
+
# iteration pattern below reads identically to _SUBSYSTEM_TEST_FILES's.
|
|
923
|
+
_PROFILE_SUBSYSTEM_FLAGS: dict[str, None] = {
|
|
924
|
+
"regime": None, "topology": None, "topology-cached": None, "learned-topology": None, "liquidity": None,
|
|
925
|
+
"gating": None, "analyzer": None, "indicators": None, "options": None,
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def cmd_profile(args: argparse.Namespace) -> int:
|
|
930
|
+
"""Wraps scripts/profile_inference.py (default) - the cProfile+wall-
|
|
931
|
+
clock harness for main.py's per-bar inference hot path (see
|
|
932
|
+
development/Problems.md for what it found: weight-array/batched-stack
|
|
933
|
+
caching, expert-loop batching, and _conv1d_causal vectorization,
|
|
934
|
+
-89.2% total profiled cost) - or, when any --<subsystem> flag is set,
|
|
935
|
+
scripts/profile_subsystems.py instead (regime/topology/topology-cached/
|
|
936
|
+
liquidity/gating/analyzer/indicators/options - everything else main.py
|
|
937
|
+
calls per bar that inference profiling never covered; --topology-cached
|
|
938
|
+
exercises development/Problems.md#36's correlation-stability cache
|
|
939
|
+
against slowly-drifting synthetic data, since --topology's fully
|
|
940
|
+
independent per-iteration returns can never show that cache's
|
|
941
|
+
benefit; --options exercises risk/asset_class_router.py::
|
|
942
|
+
route_multi_leg_option_sizing() end-to-end, V4.9 Priority 7). Same
|
|
943
|
+
subprocess-wrapper convention every other non-`trade-lock`/`fetch`
|
|
944
|
+
command follows (_run(), not an in-process import).
|
|
945
|
+
|
|
946
|
+
--batched/--no-gc/--bucket-report/--parallel/--pool-workers/
|
|
947
|
+
--symbols-per-bar only have meaning for the inference path (no batched
|
|
948
|
+
variant, GC-isolation/bucketing diagnostic, or process-pool benchmark
|
|
949
|
+
exists for these pure subsystem functions) - combining any of them
|
|
950
|
+
with a subsystem flag is a user error, rejected loudly rather than
|
|
951
|
+
silently ignored. --parallel (V4.9 Priority 6) runs a real
|
|
952
|
+
ProcessPoolExecutor benchmark of inference/parallel_inference.py's
|
|
953
|
+
run_symbol_inference() against a sequential baseline, answering that
|
|
954
|
+
module's own never-measured IPC/pickling break-even warning."""
|
|
955
|
+
subsystem_flags = [name for name in _PROFILE_SUBSYSTEM_FLAGS if getattr(args, name.replace("-", "_"), False)]
|
|
956
|
+
inference_only_flags = args.batched or args.no_gc or args.bucket_report or args.parallel
|
|
957
|
+
if subsystem_flags and inference_only_flags:
|
|
958
|
+
print(
|
|
959
|
+
"error: --batched/--no-gc/--bucket-report/--parallel only apply to inference profiling, not --<subsystem> flags",
|
|
960
|
+
file=sys.stderr,
|
|
961
|
+
)
|
|
962
|
+
return 1
|
|
963
|
+
|
|
964
|
+
if subsystem_flags:
|
|
965
|
+
cmd = [sys.executable, "scripts/profile_subsystems.py"]
|
|
966
|
+
if args.iterations is not None:
|
|
967
|
+
cmd.extend(["--iterations", str(args.iterations)])
|
|
968
|
+
cmd.extend(["--sort", args.sort])
|
|
969
|
+
cmd.extend(f"--{name}" for name in subsystem_flags)
|
|
970
|
+
return _run(cmd)
|
|
971
|
+
|
|
972
|
+
cmd = [sys.executable, "scripts/profile_inference.py"]
|
|
973
|
+
if args.iterations is not None:
|
|
974
|
+
cmd.extend(["--iterations", str(args.iterations)])
|
|
975
|
+
cmd.extend(["--sort", args.sort])
|
|
976
|
+
if args.batched:
|
|
977
|
+
cmd.append("--batched")
|
|
978
|
+
if args.no_gc:
|
|
979
|
+
cmd.append("--no-gc")
|
|
980
|
+
if args.bucket_report:
|
|
981
|
+
cmd.append("--bucket-report")
|
|
982
|
+
if args.parallel:
|
|
983
|
+
cmd.append("--parallel")
|
|
984
|
+
if args.pool_workers is not None:
|
|
985
|
+
cmd.extend(["--pool-workers", str(args.pool_workers)])
|
|
986
|
+
if args.symbols_per_bar is not None:
|
|
987
|
+
cmd.extend(["--symbols-per-bar", str(args.symbols_per_bar)])
|
|
988
|
+
return _run(cmd)
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
def cmd_report(args: argparse.Namespace) -> int:
|
|
992
|
+
lean_binary = _find_quantconnect_lean_binary()
|
|
993
|
+
if lean_binary is None:
|
|
994
|
+
print("error: QuantConnect Lean CLI not found (checked .venv and PATH).", file=sys.stderr)
|
|
995
|
+
return 1
|
|
996
|
+
backtest_dir = ROOT_DIR / "backtests" / args.backtest_dir
|
|
997
|
+
return _run(
|
|
998
|
+
[
|
|
999
|
+
lean_binary,
|
|
1000
|
+
"report",
|
|
1001
|
+
"--backtest-results",
|
|
1002
|
+
str(backtest_dir / f"{args.result_id}.json"),
|
|
1003
|
+
"--report-destination",
|
|
1004
|
+
str(backtest_dir / "report.html"),
|
|
1005
|
+
"--overwrite",
|
|
1006
|
+
]
|
|
1007
|
+
)
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def cmd_api(_args: argparse.Namespace) -> int:
|
|
1011
|
+
return _run([sys.executable, "-m", "uvicorn", "monitoring.api_server:app", "--port", "8001", "--reload"])
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def cmd_webui(_args: argparse.Namespace) -> int:
|
|
1015
|
+
npm = "npm.cmd" if sys.platform == "win32" else "npm"
|
|
1016
|
+
return _run([npm, "run", "dev"], cwd=WEBUI_DIR)
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def cmd_docker_up(args: argparse.Namespace) -> int:
|
|
1020
|
+
if args.all:
|
|
1021
|
+
# "engine" is the one consolidated build (aether-quant-engine) that
|
|
1022
|
+
# backs the app AND every worker below - see docker-compose.yml's
|
|
1023
|
+
# own comment on the `engine` service and requirements/README.md.
|
|
1024
|
+
services = [
|
|
1025
|
+
"redis",
|
|
1026
|
+
"postgres",
|
|
1027
|
+
"engine",
|
|
1028
|
+
"experience-worker",
|
|
1029
|
+
"audit-worker",
|
|
1030
|
+
"performance-trigger-worker",
|
|
1031
|
+
"retraining-worker",
|
|
1032
|
+
"telegram-worker",
|
|
1033
|
+
]
|
|
1034
|
+
return _run(["docker", "compose", "up", "-d", *services])
|
|
1035
|
+
if args.lean:
|
|
1036
|
+
return _run(["docker", "compose", "--profile", "lean", "up", "-d"])
|
|
1037
|
+
return _run(["docker", "compose", "up", "-d", "redis", "postgres"])
|
|
1038
|
+
|
|
1039
|
+
|
|
1040
|
+
def cmd_docker_build(_args: argparse.Namespace) -> int:
|
|
1041
|
+
return _run(["docker", "compose", "build", "engine"])
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def cmd_render_lean_config(args: argparse.Namespace) -> int:
|
|
1045
|
+
"""Render the gitignored lean.live.json from .env.live / AETHER_* env vars.
|
|
1046
|
+
The tracked lean.json stays all-empty; live/paper deploy uses the rendered
|
|
1047
|
+
file via `--lean-config`. See execution/lean_config_render.py."""
|
|
1048
|
+
from execution.lean_config_render import build_render_environment, write_rendered_config
|
|
1049
|
+
|
|
1050
|
+
base = Path(args.base) if args.base else LEAN_JSON_PATH
|
|
1051
|
+
out = Path(args.out) if args.out else ROOT_DIR / "lean.live.json"
|
|
1052
|
+
env_file = Path(args.env_file) if args.env_file else ROOT_DIR / ".env.live"
|
|
1053
|
+
|
|
1054
|
+
try:
|
|
1055
|
+
filled = write_rendered_config(base, out, build_render_environment(env_file=env_file))
|
|
1056
|
+
except FileNotFoundError as exc:
|
|
1057
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
1058
|
+
return 1
|
|
1059
|
+
|
|
1060
|
+
# Credential-load audit event (development/Problems.md #42) - field
|
|
1061
|
+
# NAMES only, matching the print statement below. A short-lived local
|
|
1062
|
+
# AuditQueue (this is a one-shot CLI invocation, not a long-running
|
|
1063
|
+
# process like main.py's) - never blocks/raises on a Redis hiccup.
|
|
1064
|
+
try:
|
|
1065
|
+
from audit import AuditQueue, CREDENTIAL_LOAD, build_audit_event
|
|
1066
|
+
|
|
1067
|
+
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) if CONFIG_PATH.exists() else {}
|
|
1068
|
+
audit_config = config.get("phase_v2", {}).get("audit_log", {})
|
|
1069
|
+
AuditQueue(
|
|
1070
|
+
enabled=bool(audit_config.get("enabled", True)),
|
|
1071
|
+
stream_name=str(audit_config.get("redis_stream", "aether:audit")),
|
|
1072
|
+
).push(build_audit_event(CREDENTIAL_LOAD, {"loaded_fields": filled}, actor="cli"))
|
|
1073
|
+
except Exception: # noqa: BLE001 - audit logging must never fail this command
|
|
1074
|
+
pass
|
|
1075
|
+
|
|
1076
|
+
if filled:
|
|
1077
|
+
# Field NAMES only - never the secret values.
|
|
1078
|
+
print(f"Rendered {out} with {len(filled)} field(s): {', '.join(filled)}")
|
|
1079
|
+
else:
|
|
1080
|
+
print(
|
|
1081
|
+
f"Rendered {out}, but NO secret fields were populated - check that "
|
|
1082
|
+
f"{env_file} exists and its AETHER_* values are set."
|
|
1083
|
+
)
|
|
1084
|
+
print(f"Deploy against it with Lean's --lean-config {out}")
|
|
1085
|
+
return 0
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
def cmd_secrets_check(_args: argparse.Namespace) -> int:
|
|
1089
|
+
"""Fail (non-zero) if secrets are about to be committed: a populated
|
|
1090
|
+
secret field in the tracked lean.json, or a tracked real `.env` file.
|
|
1091
|
+
Backs .githooks/pre-commit. Pure detection lives in
|
|
1092
|
+
execution/secret_scan.py."""
|
|
1093
|
+
from execution.secret_scan import find_populated_secret_fields, is_tracked_env_secret
|
|
1094
|
+
|
|
1095
|
+
problems: list[str] = []
|
|
1096
|
+
|
|
1097
|
+
if LEAN_JSON_PATH.exists():
|
|
1098
|
+
try:
|
|
1099
|
+
lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
|
|
1100
|
+
except (json.JSONDecodeError, OSError) as exc:
|
|
1101
|
+
print(f"Error reading lean.json: {exc}", file=sys.stderr)
|
|
1102
|
+
return 1
|
|
1103
|
+
populated = find_populated_secret_fields(lean_config)
|
|
1104
|
+
if populated:
|
|
1105
|
+
problems.append(
|
|
1106
|
+
"lean.json has populated secret field(s) that must not be committed "
|
|
1107
|
+
f"(render them into gitignored lean.live.json instead): {', '.join(populated)}"
|
|
1108
|
+
)
|
|
1109
|
+
|
|
1110
|
+
# Ask git which files are tracked; flag any real .env among them. Degrades
|
|
1111
|
+
# gracefully (skips this check) outside a git repo or without git on PATH.
|
|
1112
|
+
try:
|
|
1113
|
+
tracked = subprocess.run(
|
|
1114
|
+
["git", "ls-files"], cwd=str(ROOT_DIR), capture_output=True, text=True
|
|
1115
|
+
)
|
|
1116
|
+
if tracked.returncode == 0:
|
|
1117
|
+
for path in tracked.stdout.splitlines():
|
|
1118
|
+
if is_tracked_env_secret(path):
|
|
1119
|
+
problems.append(f"tracked secret file must not be committed: {path}")
|
|
1120
|
+
except (OSError, FileNotFoundError):
|
|
1121
|
+
pass
|
|
1122
|
+
|
|
1123
|
+
if problems:
|
|
1124
|
+
print("aq secrets-check FAILED:", file=sys.stderr)
|
|
1125
|
+
for problem in problems:
|
|
1126
|
+
print(f" - {problem}", file=sys.stderr)
|
|
1127
|
+
return 1
|
|
1128
|
+
|
|
1129
|
+
print("aq secrets-check passed: no committed secrets detected.")
|
|
1130
|
+
return 0
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
def cmd_audit_log(args: argparse.Namespace) -> int:
|
|
1134
|
+
"""Query the tamper-evident audit log (development/Problems.md #42) -
|
|
1135
|
+
order placement, credential loads, live-mode transitions. Requires
|
|
1136
|
+
AETHER_POSTGRES_DSN (same var every other Postgres-backed `aq` command
|
|
1137
|
+
uses) - the audit-worker service must have drained at least one batch
|
|
1138
|
+
from Redis into Postgres for anything to show up here (see
|
|
1139
|
+
docker-compose.yml's audit-worker service / `python -m audit.postgres_worker`)."""
|
|
1140
|
+
import psycopg
|
|
1141
|
+
|
|
1142
|
+
from audit import fetch_all_events_ordered, fetch_recent_events, verify_chain
|
|
1143
|
+
|
|
1144
|
+
dsn = os.environ.get("AETHER_POSTGRES_DSN", "")
|
|
1145
|
+
if not dsn:
|
|
1146
|
+
print("error: AETHER_POSTGRES_DSN is not set.", file=sys.stderr)
|
|
1147
|
+
return 1
|
|
1148
|
+
|
|
1149
|
+
try:
|
|
1150
|
+
conn = psycopg.connect(dsn, autocommit=False)
|
|
1151
|
+
except Exception as exc:
|
|
1152
|
+
print(f"error: could not connect to Postgres: {exc}", file=sys.stderr)
|
|
1153
|
+
return 1
|
|
1154
|
+
|
|
1155
|
+
try:
|
|
1156
|
+
if args.verify:
|
|
1157
|
+
rows = fetch_all_events_ordered(conn)
|
|
1158
|
+
valid, broken_index = verify_chain(rows)
|
|
1159
|
+
if not rows:
|
|
1160
|
+
print("aq audit-log --verify: table is empty — trivially valid.")
|
|
1161
|
+
return 0
|
|
1162
|
+
if valid:
|
|
1163
|
+
print(f"aq audit-log --verify: chain intact across {len(rows)} entries.")
|
|
1164
|
+
return 0
|
|
1165
|
+
broken_row = rows[broken_index]
|
|
1166
|
+
print(
|
|
1167
|
+
f"aq audit-log --verify: CHAIN BROKEN at entry {broken_index} "
|
|
1168
|
+
f"(event_id={broken_row['event_id']}, event_type={broken_row['event_type']}, "
|
|
1169
|
+
f"created_at={broken_row['created_at']}) — tampering or data loss detected.",
|
|
1170
|
+
file=sys.stderr,
|
|
1171
|
+
)
|
|
1172
|
+
return 1
|
|
1173
|
+
|
|
1174
|
+
since = None
|
|
1175
|
+
if args.since:
|
|
1176
|
+
from datetime import datetime as _datetime, timezone as _timezone
|
|
1177
|
+
|
|
1178
|
+
since = _datetime.fromisoformat(args.since).replace(tzinfo=_timezone.utc)
|
|
1179
|
+
rows = fetch_recent_events(conn, event_type=args.event_type, since=since, limit=args.limit)
|
|
1180
|
+
if not rows:
|
|
1181
|
+
print("aq audit-log: no matching entries.")
|
|
1182
|
+
return 0
|
|
1183
|
+
for row in rows:
|
|
1184
|
+
print(f"{row['created_at']} {row['event_type']:<20} {row['actor']:<8} {json.dumps(row['payload'])}")
|
|
1185
|
+
return 0
|
|
1186
|
+
finally:
|
|
1187
|
+
conn.close()
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def cmd_retrain(args: argparse.Namespace) -> int:
|
|
1191
|
+
return _run([sys.executable, "-m", "retraining.orchestrator", args.stage, *args.retrain_args])
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
# Phase 4.8 - whole-universe bulk-backfill scripts, each its own standalone
|
|
1195
|
+
# argparse/__main__ module with a PLURAL filter (--tickers/--series nargs="*",
|
|
1196
|
+
# --apply) - genuinely different from cmd_fetch()'s single-ticker/ad-hoc IB
|
|
1197
|
+
# tool below, which does NOT wire to or overlap with any of these three.
|
|
1198
|
+
# None of them was reachable via `aq` before this - purely a discoverability
|
|
1199
|
+
# wrapper, zero new backfill logic (each script's own main()/argparse is
|
|
1200
|
+
# unchanged and still works standalone via `python -m data_pipeline.<module>`).
|
|
1201
|
+
_BACKFILL_MODULE_BY_TARGET: dict[str, str] = {
|
|
1202
|
+
"dividends": "data_pipeline.dividend_backfill",
|
|
1203
|
+
"fred": "data_pipeline.fred_backfill",
|
|
1204
|
+
"yfinance": "data_pipeline.yfinance_backfill",
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def cmd_backfill(args: argparse.Namespace) -> int:
|
|
1209
|
+
return _run([sys.executable, "-m", _BACKFILL_MODULE_BY_TARGET[args.target], *args.backfill_args])
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def cmd_paper_readiness(_args: argparse.Namespace) -> int:
|
|
1213
|
+
return _run([sys.executable, "-m", "execution.paper_readiness_report"])
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def cmd_trade_lock(args: argparse.Namespace) -> int:
|
|
1217
|
+
if args.on:
|
|
1218
|
+
write_manual_trade_lock_override(True, CONFIG_PATH)
|
|
1219
|
+
print("Trade lock override: ON (trading forced paused).")
|
|
1220
|
+
elif args.off:
|
|
1221
|
+
write_manual_trade_lock_override(False, CONFIG_PATH)
|
|
1222
|
+
print("Trade lock override: OFF (trading forced resumed, even past a sticky total-drawdown lock).")
|
|
1223
|
+
elif args.auto:
|
|
1224
|
+
write_manual_trade_lock_override(None, CONFIG_PATH)
|
|
1225
|
+
print("Trade lock override: AUTO (back to today's default automatic behavior).")
|
|
1226
|
+
else: # status
|
|
1227
|
+
override = read_manual_trade_lock_override(CONFIG_PATH)
|
|
1228
|
+
label = {True: "ON (forced paused)", False: "OFF (forced resumed)", None: "AUTO (automatic behavior)"}[override]
|
|
1229
|
+
print(f"Trade lock override: {label}")
|
|
1230
|
+
return 0
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
def _cmd_kill_switch_history(args: argparse.Namespace) -> int:
|
|
1234
|
+
"""Queries the SAME tamper-evident audit log as `aq audit-log`
|
|
1235
|
+
(development/Problems.md #42), filtering for kill-switch trips. main.py's
|
|
1236
|
+
Phase 6 wiring pushes a trip as a LIVE_MODE_TRANSITION event with
|
|
1237
|
+
payload {"event": "kill_switch_tripped", ...} - LIVE_MODE_TRANSITION is
|
|
1238
|
+
reused rather than registering a new EVENT_TYPES entry (audit/redis_queue.py
|
|
1239
|
+
fixes that tuple; build_audit_event() raises ValueError on anything else)."""
|
|
1240
|
+
import psycopg
|
|
1241
|
+
|
|
1242
|
+
from audit import LIVE_MODE_TRANSITION, fetch_recent_events
|
|
1243
|
+
|
|
1244
|
+
dsn = os.environ.get("AETHER_POSTGRES_DSN", "")
|
|
1245
|
+
if not dsn:
|
|
1246
|
+
print("error: AETHER_POSTGRES_DSN is not set.", file=sys.stderr)
|
|
1247
|
+
return 1
|
|
1248
|
+
|
|
1249
|
+
try:
|
|
1250
|
+
conn = psycopg.connect(dsn, autocommit=False)
|
|
1251
|
+
except Exception as exc:
|
|
1252
|
+
print(f"error: could not connect to Postgres: {exc}", file=sys.stderr)
|
|
1253
|
+
return 1
|
|
1254
|
+
|
|
1255
|
+
try:
|
|
1256
|
+
rows = fetch_recent_events(conn, event_type=LIVE_MODE_TRANSITION, limit=args.limit)
|
|
1257
|
+
trip_rows = [row for row in rows if row["payload"].get("event") == "kill_switch_tripped"]
|
|
1258
|
+
if not trip_rows:
|
|
1259
|
+
print("aq kill-switch --history: no kill-switch trips recorded.")
|
|
1260
|
+
return 0
|
|
1261
|
+
for row in trip_rows:
|
|
1262
|
+
print(f"{row['created_at']} {json.dumps(row['payload'])}")
|
|
1263
|
+
return 0
|
|
1264
|
+
finally:
|
|
1265
|
+
conn.close()
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
def cmd_kill_switch(args: argparse.Namespace) -> int:
|
|
1269
|
+
if args.arm:
|
|
1270
|
+
write_kill_switch_manual_override(True, CONFIG_PATH)
|
|
1271
|
+
print("Kill switch override: ARMED (forced evaluation active, even if phase_v2.risk.kill_switch.enabled is false).")
|
|
1272
|
+
elif args.disarm:
|
|
1273
|
+
write_kill_switch_manual_override(False, CONFIG_PATH)
|
|
1274
|
+
print("Kill switch override: DISARMED (forced off, even if phase_v2.risk.kill_switch.enabled is true).")
|
|
1275
|
+
elif args.auto:
|
|
1276
|
+
write_kill_switch_manual_override(None, CONFIG_PATH)
|
|
1277
|
+
print("Kill switch override: AUTO (defers to phase_v2.risk.kill_switch.enabled).")
|
|
1278
|
+
elif args.history:
|
|
1279
|
+
return _cmd_kill_switch_history(args)
|
|
1280
|
+
else: # status
|
|
1281
|
+
override = read_kill_switch_manual_override(CONFIG_PATH)
|
|
1282
|
+
label = {True: "ARMED (forced on)", False: "DISARMED (forced off)", None: "AUTO (defers to config)"}[override]
|
|
1283
|
+
print(f"Kill switch override: {label}")
|
|
1284
|
+
return 0
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
class ConfigPathError(Exception):
|
|
1288
|
+
"""Raised by _get_config_value/_set_config_value for a bad dotted path."""
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
def _get_config_value(config: dict, dotted_path: str) -> object:
|
|
1292
|
+
node = config
|
|
1293
|
+
walked: list[str] = []
|
|
1294
|
+
for segment in dotted_path.split("."):
|
|
1295
|
+
walked.append(segment)
|
|
1296
|
+
if not isinstance(node, dict) or segment not in node:
|
|
1297
|
+
raise ConfigPathError(f"no such config key: {'.'.join(walked)!r}")
|
|
1298
|
+
node = node[segment]
|
|
1299
|
+
return node
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
def _coerce_config_value(raw: str) -> object:
|
|
1303
|
+
"""JSON-first parsing so true/false/123/0.5/[...]/{...} all become their
|
|
1304
|
+
real type automatically; anything that isn't valid JSON on its own
|
|
1305
|
+
(e.g. a bare word) is kept as a plain string."""
|
|
1306
|
+
try:
|
|
1307
|
+
return json.loads(raw)
|
|
1308
|
+
except json.JSONDecodeError:
|
|
1309
|
+
return raw
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
def _set_config_value(config: dict, dotted_path: str, raw_value: str) -> tuple[object, object, bool]:
|
|
1313
|
+
"""Mutates `config` in place. Returns (old_value, new_value, type_changed) -
|
|
1314
|
+
deliberately does not refuse to overwrite a list/dict: the caller wants
|
|
1315
|
+
full read/write access to every key, not just scalars. Safety instead
|
|
1316
|
+
comes from always reporting old -> new (and a type-change warning) to
|
|
1317
|
+
the caller, plus the automatic config.json.bak snapshot cmd_config()
|
|
1318
|
+
writes before every set."""
|
|
1319
|
+
*parents, leaf = dotted_path.split(".")
|
|
1320
|
+
node = config
|
|
1321
|
+
for segment in parents:
|
|
1322
|
+
if not isinstance(node, dict) or segment not in node:
|
|
1323
|
+
raise ConfigPathError(f"no such config key: {dotted_path!r}")
|
|
1324
|
+
node = node[segment]
|
|
1325
|
+
if not isinstance(node, dict) or leaf not in node:
|
|
1326
|
+
raise ConfigPathError(f"no such config key: {dotted_path!r}")
|
|
1327
|
+
old_value = node[leaf]
|
|
1328
|
+
new_value = _coerce_config_value(raw_value)
|
|
1329
|
+
node[leaf] = new_value
|
|
1330
|
+
return old_value, new_value, type(old_value) is not type(new_value)
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
def _iter_leaf_paths(node: object, prefix: str = "") -> Iterator[str]:
|
|
1334
|
+
"""Recursively yields every dot-joined leaf path under `node`. A "leaf"
|
|
1335
|
+
is any non-dict value (or an empty dict) - list-valued keys show up as
|
|
1336
|
+
one leaf, never expanded per-element."""
|
|
1337
|
+
if isinstance(node, dict) and node:
|
|
1338
|
+
for key, value in node.items():
|
|
1339
|
+
yield from _iter_leaf_paths(value, f"{prefix}.{key}" if prefix else key)
|
|
1340
|
+
else:
|
|
1341
|
+
yield prefix
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def _dispatch_json_config_command(args: argparse.Namespace, json_path: Path, command_attr: str) -> int:
|
|
1345
|
+
"""Shared dispatch for `aq config`/`aq lean` - both are the same
|
|
1346
|
+
dump/get/set/keys tool over a single flat JSON file, just pointed at a
|
|
1347
|
+
different path. See cmd_config()/cmd_lean()."""
|
|
1348
|
+
data = json.loads(json_path.read_text(encoding="utf-8"))
|
|
1349
|
+
command = getattr(args, command_attr, None)
|
|
1350
|
+
|
|
1351
|
+
try:
|
|
1352
|
+
if command is None:
|
|
1353
|
+
print(json.dumps(data, indent=2))
|
|
1354
|
+
return 0
|
|
1355
|
+
|
|
1356
|
+
if command == "get":
|
|
1357
|
+
value = _get_config_value(data, args.dotted_path)
|
|
1358
|
+
print(value if isinstance(value, str) else json.dumps(value, indent=2))
|
|
1359
|
+
return 0
|
|
1360
|
+
|
|
1361
|
+
if command == "keys":
|
|
1362
|
+
root = _get_config_value(data, args.dotted_prefix) if args.dotted_prefix else data
|
|
1363
|
+
for path in _iter_leaf_paths(root, args.dotted_prefix or ""):
|
|
1364
|
+
print(path)
|
|
1365
|
+
return 0
|
|
1366
|
+
|
|
1367
|
+
if command == "set":
|
|
1368
|
+
shutil.copy2(json_path, json_path.with_suffix(".json.bak"))
|
|
1369
|
+
old_value, new_value, type_changed = _set_config_value(data, args.dotted_path, args.value)
|
|
1370
|
+
json_path.write_text(json.dumps(data, indent=4) + "\n", encoding="utf-8")
|
|
1371
|
+
print(f"{args.dotted_path}: {old_value!r} -> {new_value!r}")
|
|
1372
|
+
if type_changed:
|
|
1373
|
+
print(
|
|
1374
|
+
f"WARNING: type changed from {type(old_value).__name__} to {type(new_value).__name__} "
|
|
1375
|
+
f"for {args.dotted_path}",
|
|
1376
|
+
file=sys.stderr,
|
|
1377
|
+
)
|
|
1378
|
+
if json_path == CONFIG_PATH and args.dotted_path == "phase_v2.inference_parallelism.enabled" and new_value:
|
|
1379
|
+
print(
|
|
1380
|
+
"WARNING: phase_v2.inference_parallelism.enabled=true - development/Problems.md #65 "
|
|
1381
|
+
"measured this ProcessPoolExecutor pool as dramatically slower than sequential on "
|
|
1382
|
+
"Windows' spawn start method (never measured on Linux/fork). main.py will also log "
|
|
1383
|
+
"this at startup on Windows if the pool actually starts.",
|
|
1384
|
+
file=sys.stderr,
|
|
1385
|
+
)
|
|
1386
|
+
return 0
|
|
1387
|
+
|
|
1388
|
+
if command == "preset":
|
|
1389
|
+
return _dispatch_config_preset_command(args, data, json_path)
|
|
1390
|
+
except ConfigPathError as error:
|
|
1391
|
+
print(f"error: {error}", file=sys.stderr)
|
|
1392
|
+
return 1
|
|
1393
|
+
|
|
1394
|
+
return 1
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
def _dispatch_config_preset_command(args: argparse.Namespace, data: dict, json_path: Path) -> int:
|
|
1398
|
+
"""`aq config preset` (V5.1 Phase 1) - list/show/apply a named
|
|
1399
|
+
dotted-key-override block from phase_v2.presets. Reuses
|
|
1400
|
+
_get_config_value()/_set_config_value() so backup/type-coercion/old->new
|
|
1401
|
+
printing are IDENTICAL to a manual `aq config set` per key - a preset
|
|
1402
|
+
is just several `set` calls applied together, never a separate write
|
|
1403
|
+
path. Only registered under `config` (not `lean` - presets are a
|
|
1404
|
+
config.json-only concept)."""
|
|
1405
|
+
presets_root = data.get("phase_v2", {}).get("presets", {})
|
|
1406
|
+
preset_names = sorted(name for name in presets_root if name != "active")
|
|
1407
|
+
|
|
1408
|
+
if args.preset_list:
|
|
1409
|
+
active = presets_root.get("active")
|
|
1410
|
+
for name in preset_names:
|
|
1411
|
+
print(f"{name}{' (active)' if name == active else ''}")
|
|
1412
|
+
return 0
|
|
1413
|
+
|
|
1414
|
+
if args.preset_show:
|
|
1415
|
+
preset_values = presets_root.get(args.preset_show)
|
|
1416
|
+
if preset_values is None:
|
|
1417
|
+
print(f"error: no such preset {args.preset_show!r} - available: {preset_names}", file=sys.stderr)
|
|
1418
|
+
return 1
|
|
1419
|
+
print(json.dumps(preset_values, indent=2))
|
|
1420
|
+
return 0
|
|
1421
|
+
|
|
1422
|
+
if args.preset_apply:
|
|
1423
|
+
preset_values = presets_root.get(args.preset_apply)
|
|
1424
|
+
if preset_values is None:
|
|
1425
|
+
print(f"error: no such preset {args.preset_apply!r} - available: {preset_names}", file=sys.stderr)
|
|
1426
|
+
return 1
|
|
1427
|
+
|
|
1428
|
+
# Validate every dotted path resolves BEFORE writing anything -
|
|
1429
|
+
# apply the whole preset or none. A partial apply that failed
|
|
1430
|
+
# halfway through would leave config.json in a mixed state no
|
|
1431
|
+
# preset actually describes.
|
|
1432
|
+
for dotted_path in preset_values:
|
|
1433
|
+
_get_config_value(data, dotted_path)
|
|
1434
|
+
|
|
1435
|
+
if args.preset_dry_run:
|
|
1436
|
+
for dotted_path, new_value in preset_values.items():
|
|
1437
|
+
old_value = _get_config_value(data, dotted_path)
|
|
1438
|
+
print(f"{dotted_path}: {old_value!r} -> {new_value!r} (dry run)")
|
|
1439
|
+
return 0
|
|
1440
|
+
|
|
1441
|
+
shutil.copy2(json_path, json_path.with_suffix(".json.bak"))
|
|
1442
|
+
for dotted_path, new_value in preset_values.items():
|
|
1443
|
+
old_value, applied_value, type_changed = _set_config_value(data, dotted_path, json.dumps(new_value))
|
|
1444
|
+
print(f"{dotted_path}: {old_value!r} -> {applied_value!r}")
|
|
1445
|
+
if type_changed:
|
|
1446
|
+
print(f"WARNING: type changed for {dotted_path}", file=sys.stderr)
|
|
1447
|
+
if "active" in presets_root:
|
|
1448
|
+
_set_config_value(data, "phase_v2.presets.active", json.dumps(args.preset_apply))
|
|
1449
|
+
json_path.write_text(json.dumps(data, indent=4) + "\n", encoding="utf-8")
|
|
1450
|
+
return 0
|
|
1451
|
+
|
|
1452
|
+
# No flag given - print the whole presets block, same "bare command
|
|
1453
|
+
# dumps everything" convention as `aq config` itself.
|
|
1454
|
+
print(json.dumps(presets_root, indent=2))
|
|
1455
|
+
return 0
|
|
1456
|
+
|
|
1457
|
+
|
|
1458
|
+
def cmd_config(args: argparse.Namespace) -> int:
|
|
1459
|
+
return _dispatch_json_config_command(args, CONFIG_PATH, "config_command")
|
|
1460
|
+
|
|
1461
|
+
|
|
1462
|
+
def cmd_lean(args: argparse.Namespace) -> int:
|
|
1463
|
+
return _dispatch_json_config_command(args, LEAN_JSON_PATH, "lean_command")
|
|
1464
|
+
|
|
1465
|
+
return 1
|
|
1466
|
+
|
|
1467
|
+
|
|
1468
|
+
def cmd_fetch(args: argparse.Namespace) -> int:
|
|
1469
|
+
ib = None
|
|
1470
|
+
fetch_fn = None
|
|
1471
|
+
extra_metadata = None
|
|
1472
|
+
if args.asset_class in IB_ASSET_CLASSES:
|
|
1473
|
+
if args.expiry is None:
|
|
1474
|
+
print(f"error: --expiry is required for asset_class={args.asset_class!r}", file=sys.stderr)
|
|
1475
|
+
return 1
|
|
1476
|
+
if args.asset_class == "options" and (args.strike is None or args.right is None):
|
|
1477
|
+
print("error: --strike and --right are required for asset_class='options'", file=sys.stderr)
|
|
1478
|
+
return 1
|
|
1479
|
+
|
|
1480
|
+
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
1481
|
+
lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
|
|
1482
|
+
try:
|
|
1483
|
+
ib = connect_ib(config, lean_config)
|
|
1484
|
+
except IBNotConfiguredError as error:
|
|
1485
|
+
print(f"error: {error}", file=sys.stderr)
|
|
1486
|
+
return 1
|
|
1487
|
+
|
|
1488
|
+
family_ticker = args.family_ticker or args.ticker.upper()
|
|
1489
|
+
if args.asset_class == "futures":
|
|
1490
|
+
contract_spec = load_futures_contract_specs().get(args.ticker.upper(), {})
|
|
1491
|
+
fetch_fn = lambda symbol, start, end: fetch_future_historical_bars( # noqa: E731
|
|
1492
|
+
ib, symbol, contract_spec, start, end, contract_month=args.contract_month
|
|
1493
|
+
)
|
|
1494
|
+
extra_metadata = {"family_ticker": family_ticker}
|
|
1495
|
+
if args.contract_month:
|
|
1496
|
+
extra_metadata["contract_month"] = args.contract_month
|
|
1497
|
+
else:
|
|
1498
|
+
fetch_fn = lambda symbol, start, end: fetch_option_historical_bars( # noqa: E731
|
|
1499
|
+
ib, symbol, args.expiry, args.strike, args.right, start, end
|
|
1500
|
+
)
|
|
1501
|
+
extra_metadata = {
|
|
1502
|
+
"family_ticker": family_ticker,
|
|
1503
|
+
"strike": args.strike,
|
|
1504
|
+
"expiry": args.expiry,
|
|
1505
|
+
"right": args.right,
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
try:
|
|
1509
|
+
fetch_kwargs = {"fetch_fn": fetch_fn} if fetch_fn is not None else {}
|
|
1510
|
+
report = fetch_adhoc_asset(
|
|
1511
|
+
args.asset_class, args.ticker, args.start, args.end, apply=args.apply,
|
|
1512
|
+
extra_metadata=extra_metadata, **fetch_kwargs,
|
|
1513
|
+
)
|
|
1514
|
+
finally:
|
|
1515
|
+
if ib is not None:
|
|
1516
|
+
disconnect_ib(ib)
|
|
1517
|
+
|
|
1518
|
+
label = "APPLY" if args.apply else "DRY RUN"
|
|
1519
|
+
print(f"{label} — {report['ticker']} ({report['yahoo_symbol']}): {report['action']}, rows_fetched={report['rows_fetched']}")
|
|
1520
|
+
if report["suggested_available_from"]:
|
|
1521
|
+
print(f" date range fetched: {report['suggested_available_from']} .. {report['suggested_available_to']}")
|
|
1522
|
+
print(f" data_path: {report['data_path']}")
|
|
1523
|
+
|
|
1524
|
+
if report["config_status"] == "added":
|
|
1525
|
+
print(f" config.json: added a new {report['ticker']} asset block to phase1.universe.assets[]")
|
|
1526
|
+
elif report["config_status"] == "already_exists":
|
|
1527
|
+
print(
|
|
1528
|
+
f" config.json: {report['ticker']} is already configured - left untouched. "
|
|
1529
|
+
"Use data_pipeline/yfinance_backfill.py to extend an existing asset's date range instead."
|
|
1530
|
+
)
|
|
1531
|
+
|
|
1532
|
+
if not args.apply:
|
|
1533
|
+
print("\nDry run only — nothing was written. Re-run with --apply to write the zip file and update config.json.")
|
|
1534
|
+
elif report["action"] == "written":
|
|
1535
|
+
print("\nReady to prepare training: run `python train.py --dataset-only` to confirm this ticker's asset quality, then `python train.py` when ready.")
|
|
1536
|
+
|
|
1537
|
+
return 1 if report["action"] == "no_data_returned" else 0
|
|
1538
|
+
|
|
1539
|
+
|
|
1540
|
+
def cmd_ib(_args: argparse.Namespace) -> int:
|
|
1541
|
+
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
1542
|
+
lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
|
|
1543
|
+
status = ib_readiness_status(config, lean_config)
|
|
1544
|
+
|
|
1545
|
+
if status == "disabled":
|
|
1546
|
+
print("IB: disabled (phase_v2.ib.enabled is false)")
|
|
1547
|
+
print(" Enable with: aq config set phase_v2.ib.enabled true")
|
|
1548
|
+
return 0
|
|
1549
|
+
|
|
1550
|
+
if status == "enabled_but_lean_credentials_missing":
|
|
1551
|
+
print("IB: enabled in config.json, but lean.json's IB credentials are not filled in")
|
|
1552
|
+
print(" Set them with: aq lean set ib-account <ACCOUNT> and aq lean set ib-user-name <USERNAME>")
|
|
1553
|
+
return 1
|
|
1554
|
+
|
|
1555
|
+
reachable, detail = attempt_connection(config, lean_config)
|
|
1556
|
+
if reachable:
|
|
1557
|
+
print(f"IB: {detail} (connected to {config['phase_v2']['ib'].get('host')}:{config['phase_v2']['ib'].get('port')})")
|
|
1558
|
+
return 0
|
|
1559
|
+
|
|
1560
|
+
print(f"IB: enabled and credentialed, but not reachable — {detail}")
|
|
1561
|
+
print(" Check that TWS or IB Gateway is running and logged in on the configured host/port.")
|
|
1562
|
+
return 1
|
|
1563
|
+
|
|
1564
|
+
|
|
1565
|
+
def cmd_assets(_args: argparse.Namespace) -> int:
|
|
1566
|
+
"""`aq assets status`: one command reporting full multi-asset-class
|
|
1567
|
+
readiness at a glance - IB, the futures_risk/options_risk feature
|
|
1568
|
+
flags, how many futures contract specs are loaded, how much of the
|
|
1569
|
+
FRED yield-curve cache is populated, and how many futures/options
|
|
1570
|
+
assets are actually configured in config.json's universe. Read-only
|
|
1571
|
+
reporting only - basic enable/disable of any of these already works
|
|
1572
|
+
today via the generic
|
|
1573
|
+
`aq config set phase_v2.{ib,futures_risk,options_risk}.enabled true|false`
|
|
1574
|
+
(_dispatch_json_config_command), same as cmd_ib's own "Enable with"
|
|
1575
|
+
hint.
|
|
1576
|
+
|
|
1577
|
+
The actual report is built by monitoring/assets_status.py::
|
|
1578
|
+
build_assets_status() - shared with the webui's `/api/assets-status`
|
|
1579
|
+
endpoint so the readiness logic is defined exactly once."""
|
|
1580
|
+
from monitoring.assets_status import build_assets_status
|
|
1581
|
+
|
|
1582
|
+
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
1583
|
+
lean_config = json.loads(LEAN_JSON_PATH.read_text(encoding="utf-8"))
|
|
1584
|
+
report = build_assets_status(config, lean_config)
|
|
1585
|
+
|
|
1586
|
+
print(f"IB: {report['ib_status']}")
|
|
1587
|
+
print(f"futures_risk.enabled: {report['futures_risk_enabled']}")
|
|
1588
|
+
print(f"options_risk.enabled: {report['options_risk_enabled']}")
|
|
1589
|
+
tickers = ", ".join(report["futures_contract_specs_tickers"]) or "none"
|
|
1590
|
+
print(f"Futures contract specs loaded: {report['futures_contract_specs_loaded']} ({tickers})")
|
|
1591
|
+
most_recent = report["fred_cache_most_recent_date"] or "never populated"
|
|
1592
|
+
print(f"FRED cache: {report['fred_cache_series_count']} series populated, most recent date: {most_recent}")
|
|
1593
|
+
print(f"Configured futures assets: {report['configured_futures_assets']}")
|
|
1594
|
+
print(f"Configured options assets: {report['configured_options_assets']}")
|
|
1595
|
+
# V5.1 Phase 2, Step 2.5 - development/Problems.md: derivatives macro
|
|
1596
|
+
# features are computed every dataset build, but deliberately left out
|
|
1597
|
+
# of phase1.features.input_set (constant 0.0 with no futures/options
|
|
1598
|
+
# contract subscribed - see build_dataset_manifest()'s
|
|
1599
|
+
# computed_but_unused_features list for the code-enforced version of
|
|
1600
|
+
# this same fact).
|
|
1601
|
+
derivatives_in_input_set = config.get("phase1", {}).get("features", {}).get("derivatives_in_input_set", False)
|
|
1602
|
+
status = "in input_set" if derivatives_in_input_set else "computed, NOT in input_set"
|
|
1603
|
+
print(
|
|
1604
|
+
"Derivatives macro features (futures_term_structure_slope, options_put_call_ratio, "
|
|
1605
|
+
f"options_implied_vol_skew): {status} - needs a futures/options contract subscribed "
|
|
1606
|
+
"(IB-gated) to carry real signal; flip phase1.features.derivatives_in_input_set "
|
|
1607
|
+
"once one is."
|
|
1608
|
+
)
|
|
1609
|
+
|
|
1610
|
+
return 0
|
|
1611
|
+
|
|
1612
|
+
|
|
1613
|
+
def _apply_preset_in_memory(config: dict, preset: dict) -> dict:
|
|
1614
|
+
"""Applies a phase_v2.presets.<name> dotted-key overlay to an IN-MEMORY
|
|
1615
|
+
deep copy of `config` only - `aq evaluate --preset` must never write
|
|
1616
|
+
config.json (that is `aq config preset --apply`'s job, a separate
|
|
1617
|
+
command). Reuses _set_config_value()'s own dotted-path resolution -
|
|
1618
|
+
json.dumps() round-trips each already-typed preset value through the
|
|
1619
|
+
same string-coercion path `aq config set` uses, so both entry points
|
|
1620
|
+
share one implementation and can never silently disagree on how a
|
|
1621
|
+
dotted path resolves."""
|
|
1622
|
+
overlaid = copy.deepcopy(config)
|
|
1623
|
+
for dotted_path, value in preset.items():
|
|
1624
|
+
_set_config_value(overlaid, dotted_path, json.dumps(value))
|
|
1625
|
+
return overlaid
|
|
1626
|
+
|
|
1627
|
+
|
|
1628
|
+
def _write_evaluation_json(path: Path, payload: object) -> None:
|
|
1629
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1630
|
+
path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
def _refresh_readme_evaluation_sections() -> None:
|
|
1634
|
+
"""Best-effort call to generate_evaluation_report.py after any `aq
|
|
1635
|
+
evaluate` flag that could have changed what's on ml/evaluation/ or
|
|
1636
|
+
ml/versions/walk-forward-*/ - mirrors cmd_backtest()'s own "a report-
|
|
1637
|
+
generation bug must never fail the real command" contract. Lazily
|
|
1638
|
+
imported for the same reason as this function's pandas/torch imports
|
|
1639
|
+
above: every other `aq` subcommand's startup must stay unaffected."""
|
|
1640
|
+
try:
|
|
1641
|
+
from generate_evaluation_report import update_readme_evaluation_sections
|
|
1642
|
+
|
|
1643
|
+
update_readme_evaluation_sections()
|
|
1644
|
+
except Exception as exc: # noqa: BLE001 - report generation must never break `aq evaluate`
|
|
1645
|
+
print(f"warning: README evaluation-section refresh failed: {exc}", file=sys.stderr)
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
_EVALUATE_MODEL_ARTIFACTS = {
|
|
1649
|
+
"sequence": ("sequence_model.json", "sequence_feature_schema.json"),
|
|
1650
|
+
"multitask": ("multitask_model.json", "multitask_feature_schema.json"),
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
|
|
1654
|
+
def cmd_evaluate(args: argparse.Namespace) -> int:
|
|
1655
|
+
"""`aq evaluate`: offline, cost-aware evaluation of the cross-sectional
|
|
1656
|
+
rank book (V5.1 Phase 0, development/Problems.md) - turns "how good is
|
|
1657
|
+
this rank prediction" into "what would this have earned net of fees",
|
|
1658
|
+
the question nothing else in this codebase answers
|
|
1659
|
+
(train.py::compute_strategy_metrics() simulates a long-only
|
|
1660
|
+
1-day-direction strategy gross of costs, not the rank book, not net of
|
|
1661
|
+
cost). Runs every dataset row through inference/exported_model.py (the
|
|
1662
|
+
SAME torch-free interpreter main.py's live decision path uses) and
|
|
1663
|
+
evaluation/rank_book_simulator.py's simulate_rank_book() (which itself
|
|
1664
|
+
reuses portfolio/book_construction.py's build_rank_based_book() and
|
|
1665
|
+
portfolio/book_neutrality.py's apply_book_neutrality()) - so every
|
|
1666
|
+
number this prints is the offline mirror of live behavior, not a
|
|
1667
|
+
separately re-derived approximation.
|
|
1668
|
+
|
|
1669
|
+
Heavy imports (pandas/numpy/evaluation/inference) are deliberately
|
|
1670
|
+
LAZY, inside this function body, not at module top - the same
|
|
1671
|
+
convention cmd_assets() already follows for monitoring.assets_status -
|
|
1672
|
+
so every other `aq` subcommand's startup stays exactly as fast as
|
|
1673
|
+
before this command existed.
|
|
1674
|
+
"""
|
|
1675
|
+
import pandas as pd
|
|
1676
|
+
|
|
1677
|
+
from evaluation import capacity_curve, predict_head, simulate_rank_book, stress_test_costs
|
|
1678
|
+
from features import load_sector_mapping
|
|
1679
|
+
|
|
1680
|
+
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
1681
|
+
|
|
1682
|
+
if getattr(args, "preset", None):
|
|
1683
|
+
presets = config.get("phase_v2", {}).get("presets", {})
|
|
1684
|
+
preset = presets.get(args.preset)
|
|
1685
|
+
if preset is None:
|
|
1686
|
+
available = sorted(name for name in presets if name != "active")
|
|
1687
|
+
print(f"error: unknown preset {args.preset!r} - available: {available or 'none configured yet'}", file=sys.stderr)
|
|
1688
|
+
return 1
|
|
1689
|
+
config = _apply_preset_in_memory(config, preset)
|
|
1690
|
+
|
|
1691
|
+
# V5.1 Phase 4 (item 4) - `aq evaluate --walk-forward-summary` reads an
|
|
1692
|
+
# already-written ml/versions/walk-forward-*/walk_forward_summary.json
|
|
1693
|
+
# (train.py::_run_walk_forward()'s output) rather than running anything
|
|
1694
|
+
# itself - a completely separate, lightweight code path from every
|
|
1695
|
+
# other --rank-book/--capacity/--stress/--calibrate-edge flag below,
|
|
1696
|
+
# so it returns early before any of that dataset/model loading.
|
|
1697
|
+
if getattr(args, "walk_forward_summary", False):
|
|
1698
|
+
versions_dir = ML_DIR / "versions"
|
|
1699
|
+
run_id = getattr(args, "run_id", None)
|
|
1700
|
+
if run_id:
|
|
1701
|
+
summary_path = versions_dir / run_id / "walk_forward_summary.json"
|
|
1702
|
+
else:
|
|
1703
|
+
candidates = sorted(versions_dir.glob("walk-forward-*/walk_forward_summary.json"), key=lambda p: p.stat().st_mtime)
|
|
1704
|
+
summary_path = candidates[-1] if candidates else None
|
|
1705
|
+
if summary_path is None or not summary_path.exists():
|
|
1706
|
+
print("error: no walk-forward summary found - run `aq train --walk-forward` first.", file=sys.stderr)
|
|
1707
|
+
return 1
|
|
1708
|
+
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
|
1709
|
+
if args.json:
|
|
1710
|
+
print(json.dumps(summary, indent=2))
|
|
1711
|
+
else:
|
|
1712
|
+
print(f"Walk-forward summary ({summary_path.parent.name}): {summary.get('num_windows', 0)} windows")
|
|
1713
|
+
for name, stats in (summary.get("summary_by_metric") or {}).items():
|
|
1714
|
+
bootstrap = stats.get("cross_window_bootstrap", {})
|
|
1715
|
+
print(
|
|
1716
|
+
f" {name}: mean={bootstrap.get('mean_ic', 0.0):.4f} "
|
|
1717
|
+
f"95% CI=[{bootstrap.get('lower_bound', 0.0):.4f}, {bootstrap.get('upper_bound', 0.0):.4f}]"
|
|
1718
|
+
)
|
|
1719
|
+
for name, stability in (summary.get("stability_by_metric") or {}).items():
|
|
1720
|
+
status = "stable" if stability.get("stable") else "UNSTABLE"
|
|
1721
|
+
print(
|
|
1722
|
+
f" {name} stability: {status} "
|
|
1723
|
+
f"(sign_flip_fraction={stability.get('sign_flip_fraction', 0.0):.2f}, "
|
|
1724
|
+
f"num_windows={stability.get('num_windows', 0)})"
|
|
1725
|
+
)
|
|
1726
|
+
net_performance_windows = summary.get("net_performance_by_window") or []
|
|
1727
|
+
if net_performance_windows:
|
|
1728
|
+
print(f" net_performance: {len(net_performance_windows)} window(s) with a simulated book")
|
|
1729
|
+
_refresh_readme_evaluation_sections()
|
|
1730
|
+
return 0
|
|
1731
|
+
|
|
1732
|
+
# V5.3.5.3 Workstream B (development/Problems.md #91/#100) -
|
|
1733
|
+
# `aq evaluate --reconcile-features --symbol XOM` diffs the RAW feature
|
|
1734
|
+
# values the live path logged for one symbol (the optional
|
|
1735
|
+
# "feature_snapshot" key on book_history.jsonl records, written when
|
|
1736
|
+
# phase_v2.diagnostics.book_history.include_feature_snapshot is on and
|
|
1737
|
+
# the symbol is in feature_snapshot_symbols' allowlist - see main.py's
|
|
1738
|
+
# write site) against the SAME (ticker, date) row of full_dataset.csv -
|
|
1739
|
+
# per-feature, not a single mismatch number: XOM is the one tracked
|
|
1740
|
+
# ticker whose live-vs-offline divergence never resolved across
|
|
1741
|
+
# #91/#97/#99/#100, and this is the tool that either names the exact
|
|
1742
|
+
# diverging feature(s) or rules the lead out cleanly.
|
|
1743
|
+
# Early-return, same precedent as --reconcile-book-history below, and
|
|
1744
|
+
# deliberately MUCH lighter: no model artifacts, no re-inference at all
|
|
1745
|
+
# - a pure log-vs-dataset diff. Not included in --all,
|
|
1746
|
+
# investigation-only like --replay-kill-switch.
|
|
1747
|
+
if getattr(args, "reconcile_features", False):
|
|
1748
|
+
from evaluation import segment_logged_records_by_run
|
|
1749
|
+
from evaluation.feature_reconciliation import (
|
|
1750
|
+
reconcile_feature_snapshot,
|
|
1751
|
+
summarize_feature_reconciliation,
|
|
1752
|
+
)
|
|
1753
|
+
|
|
1754
|
+
target_symbol = str(getattr(args, "symbol", "") or "").strip().upper()
|
|
1755
|
+
if not target_symbol:
|
|
1756
|
+
print(
|
|
1757
|
+
"error: --reconcile-features requires --symbol <TICKER> "
|
|
1758
|
+
"(e.g. --symbol XOM).",
|
|
1759
|
+
file=sys.stderr,
|
|
1760
|
+
)
|
|
1761
|
+
return 1
|
|
1762
|
+
|
|
1763
|
+
book_history_path_arg = getattr(args, "book_history_path", None)
|
|
1764
|
+
diagnostics_config = config.get("phase_v2", {}).get("diagnostics", {}).get("book_history", {})
|
|
1765
|
+
book_history_path = (
|
|
1766
|
+
Path(book_history_path_arg)
|
|
1767
|
+
if book_history_path_arg
|
|
1768
|
+
else ROOT_DIR / diagnostics_config.get("output_path", "visualization/book_history.jsonl")
|
|
1769
|
+
)
|
|
1770
|
+
if not book_history_path.exists():
|
|
1771
|
+
print(
|
|
1772
|
+
f"error: {book_history_path} not found - run a backtest with "
|
|
1773
|
+
"phase_v2.diagnostics.book_history.enabled=true first.",
|
|
1774
|
+
file=sys.stderr,
|
|
1775
|
+
)
|
|
1776
|
+
return 1
|
|
1777
|
+
|
|
1778
|
+
logged_records: list[dict] = []
|
|
1779
|
+
with open(book_history_path, "r", encoding="utf-8") as book_history_file:
|
|
1780
|
+
for line_number, raw_line in enumerate(book_history_file, start=1):
|
|
1781
|
+
stripped_line = raw_line.strip()
|
|
1782
|
+
if not stripped_line:
|
|
1783
|
+
continue
|
|
1784
|
+
try:
|
|
1785
|
+
logged_records.append(json.loads(stripped_line))
|
|
1786
|
+
except json.JSONDecodeError as error:
|
|
1787
|
+
print(
|
|
1788
|
+
f"warning: skipping malformed line {line_number} in {book_history_path}: {error}",
|
|
1789
|
+
file=sys.stderr,
|
|
1790
|
+
)
|
|
1791
|
+
if not logged_records:
|
|
1792
|
+
print(f"error: {book_history_path} contains no usable records.", file=sys.stderr)
|
|
1793
|
+
return 1
|
|
1794
|
+
|
|
1795
|
+
# Same cumulative-log discipline as --reconcile-book-history
|
|
1796
|
+
# (#91/#97/#99): segment FIRST, default to the most recent run only.
|
|
1797
|
+
run_segments = segment_logged_records_by_run(logged_records)
|
|
1798
|
+
if not run_segments:
|
|
1799
|
+
print(f"error: {book_history_path} contains no dated records.", file=sys.stderr)
|
|
1800
|
+
return 1
|
|
1801
|
+
reconcile_all_runs = bool(getattr(args, "reconcile_all_runs", False))
|
|
1802
|
+
reconcile_run_index_arg = getattr(args, "reconcile_run_index", None)
|
|
1803
|
+
requested_indices = (
|
|
1804
|
+
list(range(len(run_segments)))
|
|
1805
|
+
if reconcile_all_runs
|
|
1806
|
+
else [reconcile_run_index_arg if reconcile_run_index_arg is not None else -1]
|
|
1807
|
+
)
|
|
1808
|
+
selected_run_indices: list[int] = []
|
|
1809
|
+
for raw_index in requested_indices:
|
|
1810
|
+
normalized_index = raw_index if raw_index >= 0 else raw_index + len(run_segments)
|
|
1811
|
+
if not (0 <= normalized_index < len(run_segments)):
|
|
1812
|
+
print(
|
|
1813
|
+
f"error: --reconcile-run-index {raw_index} out of range - {len(run_segments)} run(s) "
|
|
1814
|
+
f"detected in {book_history_path}.", file=sys.stderr,
|
|
1815
|
+
)
|
|
1816
|
+
return 1
|
|
1817
|
+
selected_run_indices.append(normalized_index)
|
|
1818
|
+
|
|
1819
|
+
recon_dataset_path = ML_DIR / "datasets" / "full_dataset.csv"
|
|
1820
|
+
if not recon_dataset_path.exists():
|
|
1821
|
+
print(f"error: {recon_dataset_path} not found - run `aq train --dataset-only` first.", file=sys.stderr)
|
|
1822
|
+
return 1
|
|
1823
|
+
full_dataset = pd.read_csv(recon_dataset_path)
|
|
1824
|
+
dataset_has_ticker_column = "ticker" in full_dataset.columns
|
|
1825
|
+
rows_by_date_ticker: dict[tuple[str, str], dict] = {}
|
|
1826
|
+
if dataset_has_ticker_column:
|
|
1827
|
+
# (date, ticker) -> row dict, built vectorized - O(rows) once,
|
|
1828
|
+
# then O(1) lookups per logged snapshot below.
|
|
1829
|
+
rows_by_date_ticker = {
|
|
1830
|
+
(str(date), str(ticker)): row
|
|
1831
|
+
for (date, ticker), row in full_dataset.set_index(["date", "ticker"]).to_dict("index").items()
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
per_run_payloads = []
|
|
1835
|
+
total_compared_dates = 0
|
|
1836
|
+
for run_index in selected_run_indices:
|
|
1837
|
+
run_records = run_segments[run_index]
|
|
1838
|
+
records_with_snapshot = [
|
|
1839
|
+
record for record in run_records if isinstance(record.get("feature_snapshot"), dict)
|
|
1840
|
+
]
|
|
1841
|
+
matching_snapshots = []
|
|
1842
|
+
snapshot_keys_seen: set[str] = set()
|
|
1843
|
+
num_missing_offline_rows = 0
|
|
1844
|
+
per_date_results = []
|
|
1845
|
+
for record in records_with_snapshot:
|
|
1846
|
+
snapshot_entry = record["feature_snapshot"].get(target_symbol)
|
|
1847
|
+
if snapshot_entry is None:
|
|
1848
|
+
# Allowlist may cover several symbols; only this one's
|
|
1849
|
+
# entries reconcile. Track near-miss keys for the error
|
|
1850
|
+
# message so a ticker/symbol_key spelling mismatch is
|
|
1851
|
+
# immediately visible instead of silently zero.
|
|
1852
|
+
snapshot_keys_seen.update(str(key).upper() for key in record["feature_snapshot"])
|
|
1853
|
+
continue
|
|
1854
|
+
if not isinstance(snapshot_entry, Mapping):
|
|
1855
|
+
continue
|
|
1856
|
+
date_str = str(record.get("date", ""))
|
|
1857
|
+
offline_row = rows_by_date_ticker.get((date_str, target_symbol)) if dataset_has_ticker_column else None
|
|
1858
|
+
if offline_row is None:
|
|
1859
|
+
num_missing_offline_rows += 1
|
|
1860
|
+
continue
|
|
1861
|
+
per_date_results.append(reconcile_feature_snapshot(snapshot_entry, offline_row))
|
|
1862
|
+
matching_snapshots.append(date_str)
|
|
1863
|
+
|
|
1864
|
+
summary = summarize_feature_reconciliation(per_date_results)
|
|
1865
|
+
total_compared_dates += len(per_date_results)
|
|
1866
|
+
run_dates = [record["date"] for record in run_records if record.get("date")]
|
|
1867
|
+
per_run_payloads.append({
|
|
1868
|
+
"run_metadata": {
|
|
1869
|
+
"num_runs_detected": len(run_segments),
|
|
1870
|
+
"selected_run_index": run_index,
|
|
1871
|
+
"start_date": run_dates[0] if run_dates else None,
|
|
1872
|
+
"end_date": run_dates[-1] if run_dates else None,
|
|
1873
|
+
"num_records": len(run_records),
|
|
1874
|
+
"num_records_scanned": len(records_with_snapshot),
|
|
1875
|
+
"num_dates_reconciled": len(per_date_results),
|
|
1876
|
+
"num_dates_missing_offline_row": num_missing_offline_rows,
|
|
1877
|
+
"snapshot_symbol_keys_seen": sorted(snapshot_keys_seen),
|
|
1878
|
+
},
|
|
1879
|
+
"summary": summary,
|
|
1880
|
+
"per_date": [
|
|
1881
|
+
{"date": date_str, **result}
|
|
1882
|
+
for date_str, result in zip(matching_snapshots, per_date_results)
|
|
1883
|
+
],
|
|
1884
|
+
})
|
|
1885
|
+
|
|
1886
|
+
payload = dict(per_run_payloads[-1]) if per_run_payloads else {}
|
|
1887
|
+
if len(per_run_payloads) > 1:
|
|
1888
|
+
payload["all_runs"] = per_run_payloads
|
|
1889
|
+
|
|
1890
|
+
_write_evaluation_json(ML_DIR / "evaluation" / "feature_reconciliation.json", payload)
|
|
1891
|
+
|
|
1892
|
+
if args.json:
|
|
1893
|
+
print(json.dumps(payload, indent=2, default=str))
|
|
1894
|
+
else:
|
|
1895
|
+
last_meta = payload.get("run_metadata", {})
|
|
1896
|
+
print(
|
|
1897
|
+
f"Feature reconciliation for {target_symbol} ({book_history_path}): "
|
|
1898
|
+
f"{last_meta.get('num_records_scanned', 0)} record(s) carried a feature_snapshot, "
|
|
1899
|
+
f"{last_meta.get('num_dates_reconciled', 0)} reconciled against full_dataset.csv."
|
|
1900
|
+
)
|
|
1901
|
+
if last_meta.get("num_dates_missing_offline_row"):
|
|
1902
|
+
print(
|
|
1903
|
+
f" {last_meta['num_dates_missing_offline_row']} snapshot date(s) had no matching "
|
|
1904
|
+
"(date, ticker) row in the dataset - skipped, not counted as divergences."
|
|
1905
|
+
)
|
|
1906
|
+
summary = payload.get("summary", {})
|
|
1907
|
+
if summary.get("num_dates", 0) == 0 or summary.get("num_dates_with_comparison", 0) == 0:
|
|
1908
|
+
keys_hint = ", ".join(last_meta.get("snapshot_symbol_keys_seen", [])[:10])
|
|
1909
|
+
print(
|
|
1910
|
+
f" NO comparable snapshots found for {target_symbol}. Either the diagnostic flags were "
|
|
1911
|
+
"off during that run (set phase_v2.diagnostics.book_history.include_feature_snapshot=true "
|
|
1912
|
+
f"+ add \"{target_symbol}\" to feature_snapshot_symbols, then re-run a backtest)"
|
|
1913
|
+
+ (f"; snapshot keys seen in the log: [{keys_hint}]" if keys_hint else "")
|
|
1914
|
+
+ ".",
|
|
1915
|
+
file=sys.stderr,
|
|
1916
|
+
)
|
|
1917
|
+
else:
|
|
1918
|
+
print(
|
|
1919
|
+
f" compared={summary['num_features_compared_per_date']['mean']:.1f} features/date avg "
|
|
1920
|
+
f"divergent_dates={summary['dates_with_any_divergence']}/{summary['num_dates']} "
|
|
1921
|
+
f"total_divergences={summary['total_divergences']}"
|
|
1922
|
+
)
|
|
1923
|
+
top_offenders = summary.get("features_diverged_most_often", [])
|
|
1924
|
+
if top_offenders:
|
|
1925
|
+
print(" Worst offenders by divergence count:")
|
|
1926
|
+
for entry in top_offenders:
|
|
1927
|
+
print(
|
|
1928
|
+
f" {entry['feature']}: diverged on {entry['diverged_count']} date(s) "
|
|
1929
|
+
f"({entry['diverged_fraction']:.1%}), max_abs_delta={entry['max_abs_delta']:.6g}, "
|
|
1930
|
+
f"mean_abs_delta={entry['mean_abs_delta']:.6g}"
|
|
1931
|
+
)
|
|
1932
|
+
never_compared = summary.get("features_never_compared", [])
|
|
1933
|
+
if never_compared:
|
|
1934
|
+
print(f" Features present live but NEVER in the offline row: {never_compared}")
|
|
1935
|
+
return 0
|
|
1936
|
+
|
|
1937
|
+
# V5.2.2 (development/Problems.md) - `aq evaluate --reconcile-book-history`
|
|
1938
|
+
# compares a real Lean backtest's actual per-date book selections
|
|
1939
|
+
# (logged via phase_v2.diagnostics.book_history, see main.py's write
|
|
1940
|
+
# site) against a fresh offline re-derivation of the same raw scores -
|
|
1941
|
+
# true ground truth instead of more indirect offline-vs-live hypothesis
|
|
1942
|
+
# testing. Early-return, same precedent as --walk-forward-summary above:
|
|
1943
|
+
# needs its own UNFILTERED-by-split dataset load, because a recorded
|
|
1944
|
+
# date's required lookback context can reach into rows from a different
|
|
1945
|
+
# split boundary (see select_context_date_range()'s own docstring for
|
|
1946
|
+
# why naively reusing the split-filtered dataset below would be wrong).
|
|
1947
|
+
if getattr(args, "reconcile_book_history", False):
|
|
1948
|
+
from evaluation import (
|
|
1949
|
+
compute_blended_raw_scores,
|
|
1950
|
+
reconcile_book_history_date,
|
|
1951
|
+
replay_book_history_reconciliation,
|
|
1952
|
+
segment_logged_records_by_run,
|
|
1953
|
+
select_context_date_range,
|
|
1954
|
+
summarize_book_history_reconciliation,
|
|
1955
|
+
summarize_book_member_diversion,
|
|
1956
|
+
summarize_universe_presence_by_symbol,
|
|
1957
|
+
summarize_universe_snapshot_by_security_type,
|
|
1958
|
+
)
|
|
1959
|
+
from portfolio.rank_signal import resolve_rank_signal_policy
|
|
1960
|
+
|
|
1961
|
+
book_history_path_arg = getattr(args, "book_history_path", None)
|
|
1962
|
+
diagnostics_config = config.get("phase_v2", {}).get("diagnostics", {}).get("book_history", {})
|
|
1963
|
+
book_history_path = (
|
|
1964
|
+
Path(book_history_path_arg)
|
|
1965
|
+
if book_history_path_arg
|
|
1966
|
+
else ROOT_DIR / diagnostics_config.get("output_path", "visualization/book_history.jsonl")
|
|
1967
|
+
)
|
|
1968
|
+
if not book_history_path.exists():
|
|
1969
|
+
print(
|
|
1970
|
+
f"error: {book_history_path} not found - run a backtest with "
|
|
1971
|
+
"phase_v2.diagnostics.book_history.enabled=true first.",
|
|
1972
|
+
file=sys.stderr,
|
|
1973
|
+
)
|
|
1974
|
+
return 1
|
|
1975
|
+
|
|
1976
|
+
logged_records: list[dict] = []
|
|
1977
|
+
with open(book_history_path, "r", encoding="utf-8") as book_history_file:
|
|
1978
|
+
for line_number, raw_line in enumerate(book_history_file, start=1):
|
|
1979
|
+
stripped_line = raw_line.strip()
|
|
1980
|
+
if not stripped_line:
|
|
1981
|
+
continue
|
|
1982
|
+
try:
|
|
1983
|
+
logged_records.append(json.loads(stripped_line))
|
|
1984
|
+
except json.JSONDecodeError as error:
|
|
1985
|
+
print(
|
|
1986
|
+
f"warning: skipping malformed line {line_number} in {book_history_path}: {error}",
|
|
1987
|
+
file=sys.stderr,
|
|
1988
|
+
)
|
|
1989
|
+
|
|
1990
|
+
if not logged_records:
|
|
1991
|
+
print(f"error: {book_history_path} contains no usable records.", file=sys.stderr)
|
|
1992
|
+
return 1
|
|
1993
|
+
|
|
1994
|
+
# V5.3.2 (development/Problems.md #91/#97/#99) - book_history.jsonl
|
|
1995
|
+
# is a cumulative, NEVER-rotated log: every real backtest run's
|
|
1996
|
+
# records are appended to the same file forever. Reconciling
|
|
1997
|
+
# against it unsegmented (the pre-V5.3.2 behavior) silently merges
|
|
1998
|
+
# unrelated runs together - confirmed against the real file: 160 of
|
|
1999
|
+
# 174 unique dates recur across more than one run, one date in as
|
|
2000
|
+
# many as 8 - and a --replay-hysteresis walk carries held
|
|
2001
|
+
# allocations across a real run boundary as if it were one
|
|
2002
|
+
# continuous backtest. Segment FIRST; default to the MOST RECENT
|
|
2003
|
+
# run only, never a silent cross-run merge.
|
|
2004
|
+
run_segments = segment_logged_records_by_run(logged_records)
|
|
2005
|
+
if not run_segments:
|
|
2006
|
+
print(f"error: {book_history_path} contains no dated records.", file=sys.stderr)
|
|
2007
|
+
return 1
|
|
2008
|
+
|
|
2009
|
+
reconcile_all_runs = bool(getattr(args, "reconcile_all_runs", False))
|
|
2010
|
+
reconcile_run_index_arg = getattr(args, "reconcile_run_index", None)
|
|
2011
|
+
requested_indices = (
|
|
2012
|
+
list(range(len(run_segments)))
|
|
2013
|
+
if reconcile_all_runs
|
|
2014
|
+
else [reconcile_run_index_arg if reconcile_run_index_arg is not None else -1]
|
|
2015
|
+
)
|
|
2016
|
+
selected_run_indices: list[int] = []
|
|
2017
|
+
for raw_index in requested_indices:
|
|
2018
|
+
normalized_index = raw_index if raw_index >= 0 else raw_index + len(run_segments)
|
|
2019
|
+
if not (0 <= normalized_index < len(run_segments)):
|
|
2020
|
+
print(
|
|
2021
|
+
f"error: --reconcile-run-index {raw_index} out of range - {len(run_segments)} run(s) "
|
|
2022
|
+
f"detected in {book_history_path}.", file=sys.stderr,
|
|
2023
|
+
)
|
|
2024
|
+
return 1
|
|
2025
|
+
selected_run_indices.append(normalized_index)
|
|
2026
|
+
|
|
2027
|
+
run_metadata_by_index: dict[int, dict] = {}
|
|
2028
|
+
for i, run_records in enumerate(run_segments):
|
|
2029
|
+
run_dates = [record["date"] for record in run_records if record.get("date")]
|
|
2030
|
+
run_metadata_by_index[i] = {
|
|
2031
|
+
"num_runs_detected": len(run_segments),
|
|
2032
|
+
"selected_run_index": i,
|
|
2033
|
+
"start_date": run_dates[0] if run_dates else None,
|
|
2034
|
+
"end_date": run_dates[-1] if run_dates else None,
|
|
2035
|
+
"num_records": len(run_records),
|
|
2036
|
+
}
|
|
2037
|
+
|
|
2038
|
+
recon_dataset_path = ML_DIR / "datasets" / "full_dataset.csv"
|
|
2039
|
+
if not recon_dataset_path.exists():
|
|
2040
|
+
print(f"error: {recon_dataset_path} not found - run `aq train --dataset-only` first.", file=sys.stderr)
|
|
2041
|
+
return 1
|
|
2042
|
+
full_dataset = pd.read_csv(recon_dataset_path)
|
|
2043
|
+
if "training_eligible" in full_dataset.columns:
|
|
2044
|
+
full_dataset = full_dataset[full_dataset["training_eligible"]].reset_index(drop=True)
|
|
2045
|
+
|
|
2046
|
+
sequence_window_default = config.get("phase_v2", {}).get("sequence_model", {}).get("window_size", 30)
|
|
2047
|
+
training_metrics_by_model: dict[str, dict | None] = {}
|
|
2048
|
+
for model_name, metrics_filename in (
|
|
2049
|
+
("sequence", "sequence_training_metrics.json"),
|
|
2050
|
+
("multitask", "multitask_training_metrics.json"),
|
|
2051
|
+
):
|
|
2052
|
+
metrics_path = ML_DIR / metrics_filename
|
|
2053
|
+
training_metrics_by_model[model_name] = (
|
|
2054
|
+
json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
|
|
2055
|
+
)
|
|
2056
|
+
policy = resolve_rank_signal_policy(training_metrics_by_model, config)
|
|
2057
|
+
active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
|
|
2058
|
+
|
|
2059
|
+
# V5.3.2 (development/Problems.md #91/#97/#99) - the SAME symbol
|
|
2060
|
+
# order main.py's self.symbols uses (config.json's
|
|
2061
|
+
# phase1.universe.assets, in file order) - cross_sectional_rank_scores()/
|
|
2062
|
+
# _select_book_group() are both Python-stable sorts keyed only on
|
|
2063
|
+
# score, so an exact tie between two symbols resolves by dict
|
|
2064
|
+
# insertion order. Live builds its raw-scores dict in self.symbols
|
|
2065
|
+
# order; this reconciliation tool previously built it from a pandas
|
|
2066
|
+
# groupby (dataset row order) - a different, uncorrelated order -
|
|
2067
|
+
# so a tied pair near a top/bottom-N boundary could pick a
|
|
2068
|
+
# different winner offline than live with byte-identical scores.
|
|
2069
|
+
# Re-inserting in this canonical order before it ever reaches
|
|
2070
|
+
# cross_sectional_rank_scores() fixes that, with zero changes to
|
|
2071
|
+
# the live decision path itself.
|
|
2072
|
+
canonical_symbol_order = [
|
|
2073
|
+
str(asset.get("ticker")) for asset in config.get("phase1", {}).get("universe", {}).get("assets", [])
|
|
2074
|
+
]
|
|
2075
|
+
|
|
2076
|
+
book_config = config.get("phase_v2", {}).get("portfolio_book", {})
|
|
2077
|
+
strategy_mode = config.get("phase5", {}).get("backtest", {}).get("strategy_mode", "long_flat")
|
|
2078
|
+
book_top_n = int(book_config.get("top_n", 6))
|
|
2079
|
+
book_bottom_n = int(book_config.get("bottom_n", 6)) if strategy_mode == "long_short" else 0
|
|
2080
|
+
# V5.2.3 (development/Problems.md #91) - --replay-hysteresis
|
|
2081
|
+
# switches from independently reconciling each date (the
|
|
2082
|
+
# V5.2.2 default - can't tell a real divergence apart from
|
|
2083
|
+
# hysteresis correctly holding an incumbent) to a walk-forward
|
|
2084
|
+
# replay of offline's OWN selection history, carrying hysteresis
|
|
2085
|
+
# forward the same way main.py's live book does.
|
|
2086
|
+
replay_hysteresis = bool(getattr(args, "replay_hysteresis", False))
|
|
2087
|
+
hysteresis_rank_margin = float(book_config.get("hysteresis_rank_margin", 0.0))
|
|
2088
|
+
|
|
2089
|
+
def _reconcile_one_run(run_records: list[dict]) -> dict:
|
|
2090
|
+
# V5.3.2: each run gets its OWN context date range/predictions -
|
|
2091
|
+
# previously computed once over the entire cumulative file's
|
|
2092
|
+
# date span even when only one small run was being reconciled.
|
|
2093
|
+
recorded_dates = sorted({record["date"] for record in run_records if record.get("date")})
|
|
2094
|
+
logged_records_by_date = {record["date"]: record for record in run_records if record.get("date")}
|
|
2095
|
+
|
|
2096
|
+
recon_min_date, recon_max_date = select_context_date_range(
|
|
2097
|
+
full_dataset, recorded_dates, window_size=int(sequence_window_default)
|
|
2098
|
+
)
|
|
2099
|
+
context_dataset = full_dataset[
|
|
2100
|
+
(full_dataset["date"] >= recon_min_date) & (full_dataset["date"] <= recon_max_date)
|
|
2101
|
+
].reset_index(drop=True)
|
|
2102
|
+
|
|
2103
|
+
predictions_by_model_head: dict[str, dict[str, object]] = {}
|
|
2104
|
+
for model_kind_for_head in policy["model_priority"]:
|
|
2105
|
+
recon_model_filename, recon_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
|
|
2106
|
+
recon_model_path = ML_DIR / recon_model_filename
|
|
2107
|
+
recon_schema_path = ML_DIR / recon_schema_filename
|
|
2108
|
+
if not recon_model_path.exists() or not recon_schema_path.exists():
|
|
2109
|
+
continue # best-effort - same convention as --calibrate-book-spread above
|
|
2110
|
+
recon_model_export = json.loads(recon_model_path.read_text(encoding="utf-8"))
|
|
2111
|
+
recon_feature_schema = json.loads(recon_schema_path.read_text(encoding="utf-8"))
|
|
2112
|
+
recon_feature_names = recon_feature_schema["model_input_names"]
|
|
2113
|
+
head_predictions: dict[str, object] = {}
|
|
2114
|
+
for head_name in active_heads:
|
|
2115
|
+
head_predictions[head_name] = predict_head(
|
|
2116
|
+
context_dataset, recon_model_export, recon_feature_names, head_name,
|
|
2117
|
+
model_kind=model_kind_for_head,
|
|
2118
|
+
sequence_feature_schema=recon_feature_schema if model_kind_for_head == "sequence" else None,
|
|
2119
|
+
configured_window_size=int(sequence_window_default),
|
|
2120
|
+
)
|
|
2121
|
+
predictions_by_model_head[model_kind_for_head] = head_predictions
|
|
2122
|
+
|
|
2123
|
+
context_dataset["raw_blended_score"] = compute_blended_raw_scores(
|
|
2124
|
+
context_dataset, predictions_by_model_head, policy
|
|
2125
|
+
)
|
|
2126
|
+
|
|
2127
|
+
recorded_subset = context_dataset[context_dataset["date"].isin(recorded_dates)]
|
|
2128
|
+
raw_scores_by_date: dict[str, dict[str, float]] = {}
|
|
2129
|
+
for recon_date, group in recorded_subset.groupby("date", sort=True):
|
|
2130
|
+
if recon_date not in logged_records_by_date:
|
|
2131
|
+
continue
|
|
2132
|
+
scores_lookup = {
|
|
2133
|
+
str(ticker): float(score)
|
|
2134
|
+
for ticker, score in zip(group["ticker"], group["raw_blended_score"])
|
|
2135
|
+
if score is not None and not (isinstance(score, float) and pd.isna(score))
|
|
2136
|
+
}
|
|
2137
|
+
ordered_scores = {
|
|
2138
|
+
ticker: scores_lookup[ticker] for ticker in canonical_symbol_order if ticker in scores_lookup
|
|
2139
|
+
}
|
|
2140
|
+
ordered_scores.update({t: s for t, s in scores_lookup.items() if t not in ordered_scores})
|
|
2141
|
+
raw_scores_by_date[recon_date] = ordered_scores
|
|
2142
|
+
|
|
2143
|
+
if replay_hysteresis:
|
|
2144
|
+
ordered_logged_records = [
|
|
2145
|
+
logged_records_by_date[d] for d in recorded_dates if d in raw_scores_by_date
|
|
2146
|
+
]
|
|
2147
|
+
per_date_results = replay_book_history_reconciliation(
|
|
2148
|
+
ordered_logged_records, raw_scores_by_date,
|
|
2149
|
+
top_n=book_top_n, bottom_n=book_bottom_n, hysteresis_rank_margin=hysteresis_rank_margin,
|
|
2150
|
+
)
|
|
2151
|
+
else:
|
|
2152
|
+
per_date_results = [
|
|
2153
|
+
reconcile_book_history_date(
|
|
2154
|
+
logged_records_by_date[recon_date], raw_scores_by_symbol,
|
|
2155
|
+
top_n=book_top_n, bottom_n=book_bottom_n,
|
|
2156
|
+
)
|
|
2157
|
+
for recon_date, raw_scores_by_symbol in raw_scores_by_date.items()
|
|
2158
|
+
]
|
|
2159
|
+
|
|
2160
|
+
summary = summarize_book_history_reconciliation(per_date_results)
|
|
2161
|
+
# Cheap and always attempted (pure, reads straight off the log's
|
|
2162
|
+
# own "universe" keys - no re-inference) - degrades to
|
|
2163
|
+
# num_dates_with_universe_data=0 for a log written without
|
|
2164
|
+
# include_full_universe, never raises. V5.3.2: computed over
|
|
2165
|
+
# this RUN's own records only, not the whole cumulative file.
|
|
2166
|
+
universe_summary = summarize_universe_snapshot_by_security_type(run_records)
|
|
2167
|
+
# V5.2.6 (development/Problems.md) - same "cheap, always attempted,
|
|
2168
|
+
# degrades to zeros" contract as universe_summary above - reads
|
|
2169
|
+
# straight off the log's own optional "book_member_decisions" key.
|
|
2170
|
+
diversion_summary = summarize_book_member_diversion(run_records)
|
|
2171
|
+
# V5.3.1 (development/Problems.md #91/#97) - summarize_universe_presence_by_symbol()
|
|
2172
|
+
# does its own internal run-segmentation too, but since run_records
|
|
2173
|
+
# is already a single run here, it always reports exactly one.
|
|
2174
|
+
universe_presence_summary = summarize_universe_presence_by_symbol(run_records)
|
|
2175
|
+
return {
|
|
2176
|
+
"mode": "replay_hysteresis" if replay_hysteresis else "independent",
|
|
2177
|
+
"per_date": per_date_results,
|
|
2178
|
+
"summary": summary,
|
|
2179
|
+
"universe_summary": universe_summary,
|
|
2180
|
+
"diversion_summary": diversion_summary,
|
|
2181
|
+
"universe_presence_summary": universe_presence_summary,
|
|
2182
|
+
}
|
|
2183
|
+
|
|
2184
|
+
if reconcile_all_runs:
|
|
2185
|
+
all_runs_payloads = []
|
|
2186
|
+
for i in selected_run_indices:
|
|
2187
|
+
run_payload = _reconcile_one_run(run_segments[i])
|
|
2188
|
+
run_payload["run_metadata"] = run_metadata_by_index[i]
|
|
2189
|
+
all_runs_payloads.append(run_payload)
|
|
2190
|
+
# Top-level default keys mirror the most-recent run (last in
|
|
2191
|
+
# selected_run_indices, since run_segments is chronological) -
|
|
2192
|
+
# --reconcile-all-runs is additive on top of the normal default
|
|
2193
|
+
# output, never a replacement for it.
|
|
2194
|
+
payload = dict(all_runs_payloads[-1])
|
|
2195
|
+
payload["all_runs"] = all_runs_payloads
|
|
2196
|
+
else:
|
|
2197
|
+
selected_index = selected_run_indices[0]
|
|
2198
|
+
payload = _reconcile_one_run(run_segments[selected_index])
|
|
2199
|
+
payload["run_metadata"] = run_metadata_by_index[selected_index]
|
|
2200
|
+
|
|
2201
|
+
_write_evaluation_json(ML_DIR / "evaluation" / "book_history_reconciliation.json", payload)
|
|
2202
|
+
|
|
2203
|
+
if args.json:
|
|
2204
|
+
print(json.dumps(payload, indent=2, default=str))
|
|
2205
|
+
else:
|
|
2206
|
+
def _print_run_result(run_payload: dict, label: str) -> None:
|
|
2207
|
+
run_summary = run_payload["summary"]
|
|
2208
|
+
run_universe_summary = run_payload["universe_summary"]
|
|
2209
|
+
run_diversion_summary = run_payload["diversion_summary"]
|
|
2210
|
+
run_universe_presence_summary = run_payload["universe_presence_summary"]
|
|
2211
|
+
run_meta = run_payload["run_metadata"]
|
|
2212
|
+
print(f"{label}{run_meta['start_date']}..{run_meta['end_date']}, {run_summary['num_dates']} dates:")
|
|
2213
|
+
print(
|
|
2214
|
+
f" exact_match={run_summary['num_dates_exact_match']}/{run_summary['num_dates']} "
|
|
2215
|
+
f"mean_overlap_fraction={run_summary['mean_overlap_fraction']}"
|
|
2216
|
+
)
|
|
2217
|
+
print(
|
|
2218
|
+
f" mean_raw_score_delta_abs={run_summary['mean_raw_score_delta_abs']} "
|
|
2219
|
+
f"mean_weight_delta_abs={run_summary['mean_weight_delta_abs']} "
|
|
2220
|
+
f"({run_summary['num_dates_with_weight_logged']}/{run_summary['num_dates']} dates had a "
|
|
2221
|
+
"logged weight)"
|
|
2222
|
+
)
|
|
2223
|
+
print(
|
|
2224
|
+
f" symbols_only_logged_total={run_summary['num_symbols_only_logged_total']} "
|
|
2225
|
+
f"symbols_only_offline_total={run_summary['num_symbols_only_offline_total']}"
|
|
2226
|
+
)
|
|
2227
|
+
if run_universe_summary["num_dates_with_universe_data"] > 0:
|
|
2228
|
+
print(
|
|
2229
|
+
f" Universe snapshot ({run_universe_summary['num_dates_with_universe_data']}/"
|
|
2230
|
+
f"{run_summary['num_dates']} dates carry full-universe data):"
|
|
2231
|
+
)
|
|
2232
|
+
for security_type, stats in sorted(run_universe_summary["by_security_type"].items()):
|
|
2233
|
+
print(
|
|
2234
|
+
f" {security_type}: mean_raw_rank_score={stats['mean_raw_rank_score']} "
|
|
2235
|
+
f"feature_ready_rate={stats['feature_ready_rate']} "
|
|
2236
|
+
f"trading_eligible_rate={stats['trading_eligible_rate']} "
|
|
2237
|
+
f"(n={stats['num_symbol_dates']})"
|
|
2238
|
+
)
|
|
2239
|
+
if run_diversion_summary["num_records_with_decisions"] > 0:
|
|
2240
|
+
print(
|
|
2241
|
+
f" Book-member diversion ({run_diversion_summary['num_records_with_decisions']}/"
|
|
2242
|
+
f"{run_summary['num_dates']} dates carry decision data, "
|
|
2243
|
+
f"{run_diversion_summary['total_book_member_dates']} book-member-dates total):"
|
|
2244
|
+
)
|
|
2245
|
+
for action, count in sorted(
|
|
2246
|
+
run_diversion_summary["action_counts"].items(), key=lambda item: -item[1]
|
|
2247
|
+
):
|
|
2248
|
+
print(f" action={action}: {count}")
|
|
2249
|
+
for reason, count in sorted(
|
|
2250
|
+
run_diversion_summary["reason_counts"].items(), key=lambda item: -item[1]
|
|
2251
|
+
):
|
|
2252
|
+
print(f" reason={reason}: {count}")
|
|
2253
|
+
if run_universe_presence_summary["num_runs_detected"] > 0:
|
|
2254
|
+
top_absent = sorted(
|
|
2255
|
+
run_universe_presence_summary["runs"][0]["absence_rate_by_symbol"].items(),
|
|
2256
|
+
key=lambda item: -item[1],
|
|
2257
|
+
)[:10]
|
|
2258
|
+
for symbol, absence_rate in top_absent:
|
|
2259
|
+
if absence_rate > 0:
|
|
2260
|
+
print(f" {symbol}: absent from universe on {absence_rate:.1%} of this run's dates")
|
|
2261
|
+
|
|
2262
|
+
print(
|
|
2263
|
+
f"Book-history reconciliation ({book_history_path}): "
|
|
2264
|
+
f"{payload['run_metadata']['num_runs_detected']} run(s) detected total "
|
|
2265
|
+
"(cumulative, never-rotated log)."
|
|
2266
|
+
)
|
|
2267
|
+
if replay_hysteresis:
|
|
2268
|
+
print(
|
|
2269
|
+
" NOTE: this reconciliation REPLAYS hysteresis (--replay-hysteresis) - offline's own "
|
|
2270
|
+
"held allocations are carried forward date-by-date, the same way main.py's live book "
|
|
2271
|
+
"does, WITHIN each run only (V5.3.2: never across a run boundary). The first reconciled "
|
|
2272
|
+
"date of each run still starts from a COLD (empty) held-allocations state regardless of "
|
|
2273
|
+
"the book's true earlier history, so early dates may show a colder-start mismatch than "
|
|
2274
|
+
"mid-series dates."
|
|
2275
|
+
)
|
|
2276
|
+
else:
|
|
2277
|
+
print(
|
|
2278
|
+
" NOTE: this reconciliation replays NO hysteresis (previous_allocations=None) - a "
|
|
2279
|
+
"mismatch can mean either a real divergence OR the live book correctly holding an "
|
|
2280
|
+
"incumbent that day's natural ranking alone wouldn't pick. Read per-date role_mismatches "
|
|
2281
|
+
"before concluding anything is actually wrong. Pass --replay-hysteresis for the "
|
|
2282
|
+
"hysteresis-aware alternative."
|
|
2283
|
+
)
|
|
2284
|
+
if reconcile_all_runs:
|
|
2285
|
+
num_runs = len(payload["all_runs"])
|
|
2286
|
+
for position, run_payload in enumerate(payload["all_runs"]):
|
|
2287
|
+
_print_run_result(run_payload, f"Run {position + 1}/{num_runs} (")
|
|
2288
|
+
else:
|
|
2289
|
+
_print_run_result(payload, "Reconciled run (")
|
|
2290
|
+
|
|
2291
|
+
_refresh_readme_evaluation_sections()
|
|
2292
|
+
return 0
|
|
2293
|
+
|
|
2294
|
+
ranking_config = config.get("phase1", {}).get("target", {}).get("ranking", {})
|
|
2295
|
+
net_perf_config = ranking_config.get("net_performance", {})
|
|
2296
|
+
if not net_perf_config:
|
|
2297
|
+
print("error: phase1.target.ranking.net_performance is not configured.", file=sys.stderr)
|
|
2298
|
+
return 1
|
|
2299
|
+
|
|
2300
|
+
dataset_path = ML_DIR / "datasets" / "full_dataset.csv"
|
|
2301
|
+
if not dataset_path.exists():
|
|
2302
|
+
print(f"error: {dataset_path} not found - run `aq train --dataset-only` first.", file=sys.stderr)
|
|
2303
|
+
return 1
|
|
2304
|
+
|
|
2305
|
+
model_kind = args.model or "sequence"
|
|
2306
|
+
head = args.head or "rank_20d"
|
|
2307
|
+
split = args.split or "backtest"
|
|
2308
|
+
|
|
2309
|
+
model_filename, schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind]
|
|
2310
|
+
model_path = ML_DIR / model_filename
|
|
2311
|
+
schema_path = ML_DIR / schema_filename
|
|
2312
|
+
if not model_path.exists() or not schema_path.exists():
|
|
2313
|
+
print(f"error: {model_path} or {schema_path} not found - run `aq train` first.", file=sys.stderr)
|
|
2314
|
+
return 1
|
|
2315
|
+
|
|
2316
|
+
model_export = json.loads(model_path.read_text(encoding="utf-8"))
|
|
2317
|
+
feature_schema = json.loads(schema_path.read_text(encoding="utf-8"))
|
|
2318
|
+
feature_names = feature_schema["model_input_names"]
|
|
2319
|
+
|
|
2320
|
+
dataset = pd.read_csv(dataset_path)
|
|
2321
|
+
if "training_eligible" in dataset.columns:
|
|
2322
|
+
dataset = dataset[dataset["training_eligible"]].reset_index(drop=True)
|
|
2323
|
+
if split != "all":
|
|
2324
|
+
dataset = dataset[dataset["split"] == split].reset_index(drop=True)
|
|
2325
|
+
if dataset.empty:
|
|
2326
|
+
print(f"error: no rows for split={split!r} after filtering - check `aq train` has run for this window.", file=sys.stderr)
|
|
2327
|
+
return 1
|
|
2328
|
+
|
|
2329
|
+
sequence_window_default = config.get("phase_v2", {}).get("sequence_model", {}).get("window_size", 30)
|
|
2330
|
+
predictions = predict_head(
|
|
2331
|
+
dataset, model_export, feature_names, head,
|
|
2332
|
+
model_kind=model_kind,
|
|
2333
|
+
sequence_feature_schema=feature_schema if model_kind == "sequence" else None,
|
|
2334
|
+
configured_window_size=int(sequence_window_default),
|
|
2335
|
+
)
|
|
2336
|
+
dataset = dataset.copy()
|
|
2337
|
+
dataset["predicted_head"] = predictions
|
|
2338
|
+
|
|
2339
|
+
sector_by_ticker = load_sector_mapping(config)
|
|
2340
|
+
base_kwargs = dict(
|
|
2341
|
+
prediction_column="predicted_head",
|
|
2342
|
+
forward_return_column="target_return_1d",
|
|
2343
|
+
ticker_column="ticker",
|
|
2344
|
+
date_column="date",
|
|
2345
|
+
top_n=int(net_perf_config.get("top_n", 6)),
|
|
2346
|
+
bottom_n=int(net_perf_config.get("bottom_n", 6)),
|
|
2347
|
+
rebalance_every_bars=int(net_perf_config.get("rebalance_every_bars", 10)),
|
|
2348
|
+
cost_bps_per_side=float(net_perf_config.get("cost_bps_per_side", 5.0)),
|
|
2349
|
+
commission_bps=float(net_perf_config.get("commission_bps", 1.0)),
|
|
2350
|
+
gross_exposure=float(net_perf_config.get("gross_exposure", 1.0)),
|
|
2351
|
+
dollar_neutral=bool(net_perf_config.get("dollar_neutral", True)),
|
|
2352
|
+
sector_neutral=bool(net_perf_config.get("sector_neutral", True)),
|
|
2353
|
+
sector_by_ticker=sector_by_ticker,
|
|
2354
|
+
hysteresis_rank_margin=float(net_perf_config.get("hysteresis_rank_margin", 0.0)),
|
|
2355
|
+
max_weight_per_name=float(net_perf_config.get("max_weight_per_name", 0.12)),
|
|
2356
|
+
min_universe_size=int(ranking_config.get("min_universe_size", 20)),
|
|
2357
|
+
# V5.2.1 (development/Problems.md) - both no-op by default (0/0.0/0.0),
|
|
2358
|
+
# matching simulate_rank_book()'s own defaults exactly.
|
|
2359
|
+
entry_lag_bars=int(net_perf_config.get("entry_lag_bars", 0)),
|
|
2360
|
+
min_commission_usd=float(net_perf_config.get("min_commission_usd", 0.0)),
|
|
2361
|
+
assumed_portfolio_value_usd=float(net_perf_config.get("assumed_portfolio_value_usd", 0.0)),
|
|
2362
|
+
)
|
|
2363
|
+
|
|
2364
|
+
run_rank_book = bool(args.rank_book or args.all)
|
|
2365
|
+
# V5.3.8 - Monte Carlo layer rides on top of the base rank-book run's
|
|
2366
|
+
# daily net returns, so requesting it implicitly requests the sim.
|
|
2367
|
+
run_monte_carlo = bool(getattr(args, "monte_carlo", False))
|
|
2368
|
+
if run_monte_carlo:
|
|
2369
|
+
run_rank_book = True
|
|
2370
|
+
# V5.4.4 - benchmark comparison: cheap (pure pandas over the dataset's
|
|
2371
|
+
# own close pivot, no model inference), so it rides in --all and the
|
|
2372
|
+
# README's benchmark section auto-refreshes on every full evaluate run.
|
|
2373
|
+
run_benchmarks = bool(getattr(args, "benchmarks", False) or args.all)
|
|
2374
|
+
run_capacity = bool(args.capacity or args.all)
|
|
2375
|
+
run_stress = bool(args.stress or args.all)
|
|
2376
|
+
run_calibrate = bool(args.calibrate_edge or args.all)
|
|
2377
|
+
# V5.1 Phase 5 (item 9) - deliberately NOT bundled into --all (whose
|
|
2378
|
+
# bundle is rank-book/capacity/stress/calibrate-edge + the V5.4.4
|
|
2379
|
+
# benchmarks) - ablation is a heavier, opt-in-only report.
|
|
2380
|
+
run_ablation_flag = bool(getattr(args, "ablation", False))
|
|
2381
|
+
# V5.1 (Problems.md) - also deliberately NOT bundled into --all, same
|
|
2382
|
+
# reasoning as --ablation: a heavier, two-model, opt-in-only report.
|
|
2383
|
+
run_calibrate_book_spread = bool(getattr(args, "calibrate_book_spread", False))
|
|
2384
|
+
# V5.2.6 (development/Problems.md) - same reasoning as
|
|
2385
|
+
# run_calibrate_book_spread immediately above: a heavier, opt-in-only
|
|
2386
|
+
# report, not bundled into --all.
|
|
2387
|
+
run_calibrate_confidence_threshold = bool(getattr(args, "calibrate_confidence_threshold", False))
|
|
2388
|
+
# V5.3.5 (development/Problems.md #102) - same reasoning as
|
|
2389
|
+
# run_calibrate_book_spread/run_calibrate_confidence_threshold above: a
|
|
2390
|
+
# heavier, opt-in-only report, not bundled into --all.
|
|
2391
|
+
run_calibrate_rolling_ic_floor = bool(getattr(args, "calibrate_rolling_ic_floor", False))
|
|
2392
|
+
# V5.2.8 (development/Problems.md #94) - same reasoning as --ablation/
|
|
2393
|
+
# --calibrate-book-spread above: a heavier, investigation-only report,
|
|
2394
|
+
# not bundled into --all.
|
|
2395
|
+
run_replay_kill_switch = bool(getattr(args, "replay_kill_switch", False))
|
|
2396
|
+
# V5.3.5 (development/Problems.md #102) - same reasoning as
|
|
2397
|
+
# --replay-kill-switch above: a heavier, investigation-only report,
|
|
2398
|
+
# not bundled into --all.
|
|
2399
|
+
run_replay_rolling_ic_gate = bool(getattr(args, "replay_rolling_ic_gate", False))
|
|
2400
|
+
# V5.3.1 (development/Problems.md #34/#96) - same reasoning as
|
|
2401
|
+
# --replay-kill-switch above: a heavier, investigation-only report,
|
|
2402
|
+
# not bundled into --all.
|
|
2403
|
+
run_simulate_limit_fills = bool(getattr(args, "simulate_limit_fills", False))
|
|
2404
|
+
# Bare `aq evaluate` with no flags at all defaults to --rank-book - the
|
|
2405
|
+
# single most useful number ("is the fee drag fixed"), matching every
|
|
2406
|
+
# other `aq` command's "sane default when no scope flag is given"
|
|
2407
|
+
# convention (e.g. `aq test` with no subsystem flags runs everything).
|
|
2408
|
+
# Every opt-in-only flag (not just the --all-bundled ones) must be
|
|
2409
|
+
# included here - Problems.md bug: --calibrate-book-spread alone used
|
|
2410
|
+
# to also silently run a whole extra --rank-book simulation because it
|
|
2411
|
+
# wasn't counted as "a flag was given".
|
|
2412
|
+
if not (
|
|
2413
|
+
run_rank_book or run_capacity or run_stress or run_calibrate or run_ablation_flag
|
|
2414
|
+
or run_calibrate_book_spread or run_calibrate_confidence_threshold or run_calibrate_rolling_ic_floor
|
|
2415
|
+
or run_replay_kill_switch or run_replay_rolling_ic_gate or run_simulate_limit_fills
|
|
2416
|
+
or run_monte_carlo or run_benchmarks
|
|
2417
|
+
):
|
|
2418
|
+
run_rank_book = True
|
|
2419
|
+
|
|
2420
|
+
report: dict = {}
|
|
2421
|
+
evaluation_dir = ML_DIR / "evaluation"
|
|
2422
|
+
|
|
2423
|
+
if run_rank_book:
|
|
2424
|
+
result = simulate_rank_book(dataset, **base_kwargs)
|
|
2425
|
+
report["rank_book"] = result.to_dict()
|
|
2426
|
+
_write_evaluation_json(evaluation_dir / "rank_book_simulation.json", report["rank_book"])
|
|
2427
|
+
# Additive, per-model copy alongside the unsuffixed "last run" file
|
|
2428
|
+
# above (untouched - monitoring/evaluation_state.py's webui feed
|
|
2429
|
+
# and existing tests depend on that exact fixed name) - lets a
|
|
2430
|
+
# multitask run and a sequence run coexist on disk instead of the
|
|
2431
|
+
# second silently clobbering the first, for callers (e.g.
|
|
2432
|
+
# generate_evaluation_report.py) that want to compare both models
|
|
2433
|
+
# at once rather than only "whichever ran last".
|
|
2434
|
+
_write_evaluation_json(evaluation_dir / f"rank_book_simulation_{model_kind}.json", report["rank_book"])
|
|
2435
|
+
if not args.json:
|
|
2436
|
+
print(f"Rank book ({model_kind}/{head}, split={split}, entry_lag_bars={base_kwargs['entry_lag_bars']}):")
|
|
2437
|
+
print(f" gross_sharpe={result.gross_sharpe:.4f} net_sharpe={result.net_sharpe:.4f}")
|
|
2438
|
+
print(f" gross_total_return={result.gross_total_return:.4%} net_total_return={result.net_total_return:.4%}")
|
|
2439
|
+
print(f" net_max_drawdown={result.net_max_drawdown:.4%}")
|
|
2440
|
+
print(f" annualized_turnover={result.annualized_turnover:.2f}x cost_drag_annual_bps={result.cost_drag_annual_bps:.2f}")
|
|
2441
|
+
print(f" num_rebalances={result.num_rebalances} num_dates_used={result.num_dates_used}")
|
|
2442
|
+
print(f" mean_names_long={result.mean_names_long:.2f} mean_names_short={result.mean_names_short:.2f}")
|
|
2443
|
+
|
|
2444
|
+
# V5.2.1 (development/Problems.md) - a Daily-resolution market
|
|
2445
|
+
# order decided off bar N's close fills at bar N+1's open, not
|
|
2446
|
+
# bar N's close; this simulator's own return accrual previously
|
|
2447
|
+
# had no way to model that. ALWAYS compute and show both
|
|
2448
|
+
# entry_lag_bars=0 (same-bar fill, what this simulator has always
|
|
2449
|
+
# assumed) and =1 (the honest, next-bar-fill "lag tax") here,
|
|
2450
|
+
# regardless of what net_perf_config.entry_lag_bars is configured
|
|
2451
|
+
# to - so the gap this investigation found is never silently
|
|
2452
|
+
# invisible again, whichever value base_kwargs happens to carry.
|
|
2453
|
+
lag_kwargs = dict(base_kwargs)
|
|
2454
|
+
lag_kwargs["entry_lag_bars"] = 1
|
|
2455
|
+
lagged_result = simulate_rank_book(dataset, **lag_kwargs)
|
|
2456
|
+
report["rank_book_entry_lag_1"] = lagged_result.to_dict()
|
|
2457
|
+
_write_evaluation_json(evaluation_dir / "rank_book_simulation_entry_lag_1.json", report["rank_book_entry_lag_1"])
|
|
2458
|
+
_write_evaluation_json(
|
|
2459
|
+
evaluation_dir / f"rank_book_simulation_entry_lag_1_{model_kind}.json", report["rank_book_entry_lag_1"]
|
|
2460
|
+
)
|
|
2461
|
+
if run_monte_carlo:
|
|
2462
|
+
# V5.3.8 - Monte Carlo layer: stationary-block bootstrap of this
|
|
2463
|
+
# base run's daily NET returns into n_runs alternative equity
|
|
2464
|
+
# histories. Pure core in evaluation/monte_carlo.py; heavy runs
|
|
2465
|
+
# matrix persisted as .npz next to the JSON (README renderer owns
|
|
2466
|
+
# the PNG), summary + avg/band curves go in the JSON.
|
|
2467
|
+
from evaluation.monte_carlo import run_monte_carlo
|
|
2468
|
+
|
|
2469
|
+
import numpy as _np_mc
|
|
2470
|
+
|
|
2471
|
+
mc = run_monte_carlo(
|
|
2472
|
+
result.per_date_net_return,
|
|
2473
|
+
n_runs=int(getattr(args, "mc_runs", 1000)),
|
|
2474
|
+
block_size=int(getattr(args, "mc_block_size", 20)),
|
|
2475
|
+
seed=int(getattr(args, "mc_seed", 42)),
|
|
2476
|
+
method=str(getattr(args, "mc_method", "block")),
|
|
2477
|
+
)
|
|
2478
|
+
mc_payload = {k: v for k, v in mc.items() if k != "runs_curves"}
|
|
2479
|
+
mc_payload["model_kind"] = model_kind
|
|
2480
|
+
mc_payload["head"] = head
|
|
2481
|
+
mc_payload["split"] = split
|
|
2482
|
+
_write_evaluation_json(evaluation_dir / "monte_carlo.json", mc_payload)
|
|
2483
|
+
_np_mc.savez_compressed(evaluation_dir / "monte_carlo_curves.npz", runs_curves=_np_mc.asarray(mc["runs_curves"]))
|
|
2484
|
+
if not args.json:
|
|
2485
|
+
fr = mc["final_return_pct"]
|
|
2486
|
+
md = mc["max_drawdown"]
|
|
2487
|
+
sh = mc["sharpe"]
|
|
2488
|
+
print(f"Monte Carlo ({mc['status']}): {mc['config']['n_runs']} runs "
|
|
2489
|
+
f"(method={mc['config']['method']}, block={mc['config']['block_size']}, seed={mc['config']['seed']})")
|
|
2490
|
+
print(f" final_return p5/p50/p95 = {fr['p5']}% / {fr['p50']}% / {fr['p95']}% | P(neg)={mc['prob_negative_return']}")
|
|
2491
|
+
print(f" sharpe p5/p95 = {sh['p5']} / {sh['p95']} | maxDD p95 = {md['p95']}%")
|
|
2492
|
+
if not args.json:
|
|
2493
|
+
print("Rank book (entry_lag_bars=1, the 'lag tax' - see development/Problems.md):")
|
|
2494
|
+
print(f" gross_sharpe={lagged_result.gross_sharpe:.4f} net_sharpe={lagged_result.net_sharpe:.4f}")
|
|
2495
|
+
print(f" delta_net_sharpe vs entry_lag_bars=0: {lagged_result.net_sharpe - result.net_sharpe:+.4f}")
|
|
2496
|
+
|
|
2497
|
+
# V5.4.4 - benchmark comparison over the SAME dataset window/split the
|
|
2498
|
+
# rank book just ran on (the `dataset` frame is already split-filtered
|
|
2499
|
+
# above, so the pivot - and every baseline number - is automatically
|
|
2500
|
+
# like-for-like with the rank book's own window). Pure pandas over the
|
|
2501
|
+
# dataset's own close columns; zero new data dependencies, zero model
|
|
2502
|
+
# inference. Missing close column or missing SPY/TLT degrades to
|
|
2503
|
+
# SKIPPED entries inside the report, never a crash.
|
|
2504
|
+
if run_benchmarks:
|
|
2505
|
+
from evaluation.benchmark_comparison import close_pivot_from_frame, run_all_baselines
|
|
2506
|
+
|
|
2507
|
+
benchmark_close_pivot = close_pivot_from_frame(dataset)
|
|
2508
|
+
benchmark_report = run_all_baselines(benchmark_close_pivot)
|
|
2509
|
+
benchmark_report["model_kind"] = model_kind
|
|
2510
|
+
benchmark_report["head"] = head
|
|
2511
|
+
benchmark_report["split"] = split
|
|
2512
|
+
report["benchmarks"] = benchmark_report
|
|
2513
|
+
_write_evaluation_json(evaluation_dir / "benchmark_comparison.json", benchmark_report)
|
|
2514
|
+
if not args.json:
|
|
2515
|
+
print(f"Benchmarks (split={split}, {len(benchmark_close_pivot)} dates x {benchmark_close_pivot.shape[1]} tickers):")
|
|
2516
|
+
for baseline in benchmark_report["baselines"]:
|
|
2517
|
+
sharpe = baseline["net_sharpe"]
|
|
2518
|
+
sharpe_str = f"{sharpe:+.4f}" if sharpe is not None else " n/a "
|
|
2519
|
+
print(f" {baseline['strategy']:<14} net_sharpe={sharpe_str} total_return={baseline['total_return_pct']}% [{baseline.get('status', 'OK')}]")
|
|
2520
|
+
|
|
2521
|
+
# V5.2.8 (development/Problems.md #94) - reuses --rank-book's own
|
|
2522
|
+
# `result` (per_date/per_date_net_return) when that flag was also
|
|
2523
|
+
# given, rather than recomputing simulate_rank_book() a second time;
|
|
2524
|
+
# computes it fresh (entry_lag_bars=0, the same default --rank-book
|
|
2525
|
+
# uses) when --replay-kill-switch is passed on its own.
|
|
2526
|
+
if run_replay_kill_switch:
|
|
2527
|
+
from evaluation.kill_switch_replay import (
|
|
2528
|
+
replay_kill_switch_over_dataset,
|
|
2529
|
+
summarize_kill_switch_replay,
|
|
2530
|
+
)
|
|
2531
|
+
|
|
2532
|
+
kill_switch_replay_result = result if run_rank_book else simulate_rank_book(dataset, **base_kwargs)
|
|
2533
|
+
kill_switch_config = config.get("phase_v2", {}).get("risk", {}).get("kill_switch", {})
|
|
2534
|
+
phase6_risk = config.get("phase6", {}).get("risk", {})
|
|
2535
|
+
daily_portfolio_returns = dict(
|
|
2536
|
+
zip(kill_switch_replay_result.per_date, kill_switch_replay_result.per_date_net_return)
|
|
2537
|
+
)
|
|
2538
|
+
replay_records = replay_kill_switch_over_dataset(
|
|
2539
|
+
kill_switch_replay_result.per_date,
|
|
2540
|
+
daily_portfolio_returns,
|
|
2541
|
+
kill_switch_config,
|
|
2542
|
+
max_daily_drawdown_pct=float(phase6_risk.get("max_daily_drawdown_pct", 0.03)),
|
|
2543
|
+
max_total_drawdown_pct=float(phase6_risk.get("max_total_drawdown_pct", 0.12)),
|
|
2544
|
+
)
|
|
2545
|
+
replay_summary = summarize_kill_switch_replay(replay_records)
|
|
2546
|
+
report["kill_switch_replay"] = {"summary": replay_summary, "per_date": replay_records}
|
|
2547
|
+
_write_evaluation_json(evaluation_dir / "kill_switch_replay.json", report["kill_switch_replay"])
|
|
2548
|
+
if not args.json:
|
|
2549
|
+
print(
|
|
2550
|
+
f"Kill-switch replay (APPROXIMATION - see development/Problems.md #94): "
|
|
2551
|
+
f"{replay_summary['trip_count']} trips across {replay_summary['total_dates']} dates, "
|
|
2552
|
+
f"{replay_summary['locked_days']} locked days ({replay_summary['locked_day_fraction']:.2%})"
|
|
2553
|
+
)
|
|
2554
|
+
|
|
2555
|
+
# V5.3.1 (development/Problems.md #34/#96) - offline counterfactual:
|
|
2556
|
+
# "how often would a real limit order have filled" without needing a
|
|
2557
|
+
# live Lean run. Deliberately independent of --rank-book above - it
|
|
2558
|
+
# simulates a signal firing on every row, not just book-selected ones
|
|
2559
|
+
# (see evaluation/limit_fill_simulator.py's own docstring for why
|
|
2560
|
+
# that's a different, complementary question, not a replay of the
|
|
2561
|
+
# real order-events.json evidence).
|
|
2562
|
+
if run_simulate_limit_fills:
|
|
2563
|
+
from evaluation.limit_fill_simulator import simulate_limit_fills, sweep_limit_fill_offsets
|
|
2564
|
+
|
|
2565
|
+
limit_orders_config = config.get("phase_v2", {}).get("limit_orders", {})
|
|
2566
|
+
timeout_bars = int(limit_orders_config.get("unfilled_timeout_bars", 3))
|
|
2567
|
+
offset_sweep_arg = getattr(args, "limit_fill_offset_sweep", None)
|
|
2568
|
+
if offset_sweep_arg:
|
|
2569
|
+
offsets = [float(value) for value in offset_sweep_arg.split(",")]
|
|
2570
|
+
limit_fill_result = sweep_limit_fill_offsets(dataset, unfilled_timeout_bars=timeout_bars, offset_multipliers=offsets)
|
|
2571
|
+
else:
|
|
2572
|
+
configured_offset = float(limit_orders_config.get("offset_multiplier", 1.0))
|
|
2573
|
+
limit_fill_result = simulate_limit_fills(dataset, unfilled_timeout_bars=timeout_bars, offset_multiplier=configured_offset)
|
|
2574
|
+
report["limit_fill_simulation"] = limit_fill_result
|
|
2575
|
+
_write_evaluation_json(evaluation_dir / "limit_fill_simulation.json", limit_fill_result)
|
|
2576
|
+
if not args.json:
|
|
2577
|
+
print(f"Limit-fill simulation (APPROXIMATION, unfilled_timeout_bars={timeout_bars}):")
|
|
2578
|
+
if offset_sweep_arg:
|
|
2579
|
+
for multiplier, result_for_multiplier in limit_fill_result.items():
|
|
2580
|
+
overall = result_for_multiplier["overall"]
|
|
2581
|
+
print(
|
|
2582
|
+
f" offset_multiplier={multiplier}: fill_rate={overall['fill_rate']:.2%} "
|
|
2583
|
+
f"timeout_rate={overall['timeout_rate']:.2%} num_signals={overall['num_signals']}"
|
|
2584
|
+
)
|
|
2585
|
+
else:
|
|
2586
|
+
overall = limit_fill_result["overall"]
|
|
2587
|
+
print(
|
|
2588
|
+
f" overall: fill_rate={overall['fill_rate']:.2%} timeout_rate={overall['timeout_rate']:.2%} "
|
|
2589
|
+
f"num_signals={overall['num_signals']}"
|
|
2590
|
+
)
|
|
2591
|
+
for security_type, stats in sorted(limit_fill_result["by_asset_class"].items()):
|
|
2592
|
+
print(f" {security_type}: fill_rate={stats['fill_rate']:.2%} num_signals={stats['num_signals']}")
|
|
2593
|
+
|
|
2594
|
+
if run_capacity:
|
|
2595
|
+
cap = capacity_curve(
|
|
2596
|
+
dataset,
|
|
2597
|
+
participation_cap=float(net_perf_config.get("capacity_participation_cap", 0.01)),
|
|
2598
|
+
base_kwargs=base_kwargs,
|
|
2599
|
+
top_n_sweep=[int(n) for n in net_perf_config.get("capacity_top_n_sweep", [3, 6, 10, 15])],
|
|
2600
|
+
)
|
|
2601
|
+
report["capacity"] = cap
|
|
2602
|
+
_write_evaluation_json(evaluation_dir / "capacity_report.json", cap)
|
|
2603
|
+
_write_evaluation_json(evaluation_dir / f"capacity_report_{model_kind}.json", cap)
|
|
2604
|
+
if not args.json:
|
|
2605
|
+
print(f"Capacity: ${cap['capacity_usd']:,.0f} (binding: {cap['binding_ticker']})")
|
|
2606
|
+
for row in cap["per_top_n"]:
|
|
2607
|
+
print(f" top_n={row['top_n']}: net_sharpe={row['net_sharpe']:.4f}")
|
|
2608
|
+
|
|
2609
|
+
if run_stress:
|
|
2610
|
+
stress = stress_test_costs(
|
|
2611
|
+
dataset, base_kwargs=base_kwargs,
|
|
2612
|
+
cost_multipliers=tuple(float(m) for m in net_perf_config.get("stress_cost_multipliers", [1.0, 2.0, 3.0])),
|
|
2613
|
+
)
|
|
2614
|
+
report["stress"] = stress
|
|
2615
|
+
_write_evaluation_json(evaluation_dir / "cost_stress_report.json", {"stress": stress})
|
|
2616
|
+
_write_evaluation_json(evaluation_dir / f"cost_stress_report_{model_kind}.json", {"stress": stress})
|
|
2617
|
+
if not args.json:
|
|
2618
|
+
print("Cost stress test:")
|
|
2619
|
+
for row in stress:
|
|
2620
|
+
print(f" {row['cost_multiplier']}x: net_sharpe={row['net_sharpe']:.4f} net_total_return={row['net_total_return']:.4%}")
|
|
2621
|
+
|
|
2622
|
+
if run_calibrate:
|
|
2623
|
+
# A simple OLS slope-through-the-origin of forward return on
|
|
2624
|
+
# (rank - 0.5)*2 - matches execution/cost_model.py::expected_edge_bps()'s
|
|
2625
|
+
# own linear-in-rank-deviation model. That function documents its
|
|
2626
|
+
# edge_bps_per_rank_unit as the FULL predicted move over
|
|
2627
|
+
# phase_v2.costs.horizon_days (it then scales down by
|
|
2628
|
+
# holding_bars/horizon_days itself) - so the regression must use
|
|
2629
|
+
# the SAME horizon's forward return, not a fixed 1-day one, or the
|
|
2630
|
+
# calibrated number is off by roughly horizon_days/1 (a ~20x
|
|
2631
|
+
# understatement of edge at the default horizon_days=20, since a
|
|
2632
|
+
# 1-day slope was previously being treated as a 20-day move and
|
|
2633
|
+
# then halved again downstream).
|
|
2634
|
+
costs_config = config.get("phase_v2", {}).get("costs", {})
|
|
2635
|
+
horizon_days = int(costs_config.get("horizon_days", 20))
|
|
2636
|
+
forward_return_column = f"target_return_{horizon_days}d"
|
|
2637
|
+
if forward_return_column not in dataset.columns:
|
|
2638
|
+
print(
|
|
2639
|
+
f"warning: {forward_return_column!r} not in dataset (horizon_days={horizon_days}); "
|
|
2640
|
+
"falling back to target_return_1d - the calibrated value will then represent a "
|
|
2641
|
+
"1-day move, not the horizon_days move expected_edge_bps() assumes.",
|
|
2642
|
+
file=sys.stderr,
|
|
2643
|
+
)
|
|
2644
|
+
forward_return_column = "target_return_1d"
|
|
2645
|
+
rank_deviation = (dataset["predicted_head"] - 0.5) * 2.0
|
|
2646
|
+
forward_return = dataset[forward_return_column]
|
|
2647
|
+
valid = rank_deviation.notna() & forward_return.notna()
|
|
2648
|
+
x = rank_deviation[valid].to_numpy()
|
|
2649
|
+
y = forward_return[valid].to_numpy()
|
|
2650
|
+
denominator = float((x * x).sum()) if len(x) >= 2 else 0.0
|
|
2651
|
+
slope = float((x * y).sum() / denominator) if denominator > 0 else 0.0
|
|
2652
|
+
calibrated_edge_bps = slope * 10_000.0
|
|
2653
|
+
report["calibrated_edge_bps_per_rank_unit"] = calibrated_edge_bps
|
|
2654
|
+
report["calibrated_edge_forward_return_column"] = forward_return_column
|
|
2655
|
+
if not args.json:
|
|
2656
|
+
print(f"Calibrated edge_bps_per_rank_unit ({forward_return_column}): {calibrated_edge_bps:.4f}")
|
|
2657
|
+
print(f" Apply with: aq config set phase_v2.costs.edge_bps_per_rank_unit {calibrated_edge_bps:.4f}")
|
|
2658
|
+
print(" Then enable the gate: aq config set phase_v2.costs.enabled true")
|
|
2659
|
+
|
|
2660
|
+
if run_calibrate_book_spread:
|
|
2661
|
+
# V5.1 (development/Problems.md) - min_rank_confidence_spread was a
|
|
2662
|
+
# guessed constant (0.2) never checked against this codebase's
|
|
2663
|
+
# actual raw-score scale; evaluation/rank_book_simulator.py's own
|
|
2664
|
+
# book simulation hardcodes min_rank_confidence_spread=0.0 (see
|
|
2665
|
+
# that module's comment), so nothing ever exercised the real gate
|
|
2666
|
+
# before it shipped live. This mirrors --calibrate-edge's own
|
|
2667
|
+
# discipline: derive the number from data, don't guess it.
|
|
2668
|
+
from evaluation import calibrate_book_confidence_spread, compute_blended_raw_scores
|
|
2669
|
+
from portfolio.rank_signal import resolve_rank_signal_policy
|
|
2670
|
+
|
|
2671
|
+
training_metrics_by_model: dict[str, dict | None] = {}
|
|
2672
|
+
for model_name, metrics_filename in (
|
|
2673
|
+
("sequence", "sequence_training_metrics.json"),
|
|
2674
|
+
("multitask", "multitask_training_metrics.json"),
|
|
2675
|
+
):
|
|
2676
|
+
metrics_path = ML_DIR / metrics_filename
|
|
2677
|
+
training_metrics_by_model[model_name] = (
|
|
2678
|
+
json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
|
|
2679
|
+
)
|
|
2680
|
+
policy = resolve_rank_signal_policy(training_metrics_by_model, config)
|
|
2681
|
+
active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
|
|
2682
|
+
|
|
2683
|
+
predictions_by_model_head: dict[str, dict[str, object]] = {}
|
|
2684
|
+
for model_kind_for_head in policy["model_priority"]:
|
|
2685
|
+
calib_model_filename, calib_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
|
|
2686
|
+
calib_model_path = ML_DIR / calib_model_filename
|
|
2687
|
+
calib_schema_path = ML_DIR / calib_schema_filename
|
|
2688
|
+
if not calib_model_path.exists() or not calib_schema_path.exists():
|
|
2689
|
+
continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
|
|
2690
|
+
calib_model_export = json.loads(calib_model_path.read_text(encoding="utf-8"))
|
|
2691
|
+
calib_feature_schema = json.loads(calib_schema_path.read_text(encoding="utf-8"))
|
|
2692
|
+
calib_feature_names = calib_feature_schema["model_input_names"]
|
|
2693
|
+
head_predictions: dict[str, object] = {}
|
|
2694
|
+
for head_name in active_heads:
|
|
2695
|
+
head_predictions[head_name] = predict_head(
|
|
2696
|
+
dataset, calib_model_export, calib_feature_names, head_name,
|
|
2697
|
+
model_kind=model_kind_for_head,
|
|
2698
|
+
sequence_feature_schema=calib_feature_schema if model_kind_for_head == "sequence" else None,
|
|
2699
|
+
configured_window_size=int(sequence_window_default),
|
|
2700
|
+
)
|
|
2701
|
+
predictions_by_model_head[model_kind_for_head] = head_predictions
|
|
2702
|
+
|
|
2703
|
+
dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
|
|
2704
|
+
|
|
2705
|
+
book_config = config.get("phase_v2", {}).get("portfolio_book", {})
|
|
2706
|
+
strategy_mode = config.get("phase5", {}).get("backtest", {}).get("strategy_mode", "long_flat")
|
|
2707
|
+
book_top_n = int(book_config.get("top_n", 6))
|
|
2708
|
+
book_bottom_n = int(book_config.get("bottom_n", 6)) if strategy_mode == "long_short" else 0
|
|
2709
|
+
|
|
2710
|
+
book_spread_result = calibrate_book_confidence_spread(
|
|
2711
|
+
dataset,
|
|
2712
|
+
raw_score_column="raw_blended_score",
|
|
2713
|
+
top_n=book_top_n,
|
|
2714
|
+
bottom_n=book_bottom_n,
|
|
2715
|
+
percentile=float(getattr(args, "book_spread_percentile", 0.10)),
|
|
2716
|
+
)
|
|
2717
|
+
report["book_spread_calibration"] = book_spread_result
|
|
2718
|
+
_write_evaluation_json(evaluation_dir / "book_spread_calibration.json", book_spread_result)
|
|
2719
|
+
if not args.json:
|
|
2720
|
+
calibrated_spread = book_spread_result["calibrated_min_rank_confidence_spread"]
|
|
2721
|
+
distribution = book_spread_result["spread_distribution"]
|
|
2722
|
+
print(
|
|
2723
|
+
f"Calibrated min_rank_confidence_spread (p{book_spread_result['percentile']*100:.0f}, "
|
|
2724
|
+
f"{book_spread_result['num_dates_used']} dates used, "
|
|
2725
|
+
f"{book_spread_result['num_dates_skipped_thin_universe']} skipped thin-universe): "
|
|
2726
|
+
f"{calibrated_spread:.4f}"
|
|
2727
|
+
)
|
|
2728
|
+
print(
|
|
2729
|
+
f" Distribution: min={distribution['min']}, p10={distribution['p10']}, "
|
|
2730
|
+
f"median={distribution['median']}, p75={distribution['p75']}, max={distribution['max']}"
|
|
2731
|
+
)
|
|
2732
|
+
print(f" Apply with: aq config set phase_v2.portfolio_book.min_rank_confidence_spread {calibrated_spread:.4f}")
|
|
2733
|
+
print(
|
|
2734
|
+
" NOTE: this recalibrates the gate's threshold to this model's real achievable dispersion - "
|
|
2735
|
+
"it does not certify the underlying rank heads are skillful. Check ml/*_training_metrics.json's "
|
|
2736
|
+
"quality_status before trusting the resulting book."
|
|
2737
|
+
)
|
|
2738
|
+
|
|
2739
|
+
if run_calibrate_rolling_ic_floor:
|
|
2740
|
+
# V5.3.5 (development/Problems.md #102) - min_rolling_mean_ic was
|
|
2741
|
+
# about to be shipped as a guessed constant, the same
|
|
2742
|
+
# "guessed-then-calibrated" gap min_rank_confidence_spread had
|
|
2743
|
+
# (Problems.md #89) before --calibrate-book-spread fixed it.
|
|
2744
|
+
# Mirrors that tool's exact structure, including recomputing the
|
|
2745
|
+
# blended raw score fresh rather than reusing a value from a
|
|
2746
|
+
# different block - see run_calibrate_book_spread above for why
|
|
2747
|
+
# that duplication is this file's own established convention.
|
|
2748
|
+
from evaluation import compute_blended_raw_scores
|
|
2749
|
+
from evaluation.rolling_ic_gate_calibration import calibrate_rolling_ic_floor
|
|
2750
|
+
from portfolio.rank_signal import resolve_rank_signal_policy
|
|
2751
|
+
|
|
2752
|
+
training_metrics_by_model: dict[str, dict | None] = {}
|
|
2753
|
+
for model_name, metrics_filename in (
|
|
2754
|
+
("sequence", "sequence_training_metrics.json"),
|
|
2755
|
+
("multitask", "multitask_training_metrics.json"),
|
|
2756
|
+
):
|
|
2757
|
+
metrics_path = ML_DIR / metrics_filename
|
|
2758
|
+
training_metrics_by_model[model_name] = (
|
|
2759
|
+
json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
|
|
2760
|
+
)
|
|
2761
|
+
policy = resolve_rank_signal_policy(training_metrics_by_model, config)
|
|
2762
|
+
active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
|
|
2763
|
+
|
|
2764
|
+
predictions_by_model_head: dict[str, dict[str, object]] = {}
|
|
2765
|
+
for model_kind_for_head in policy["model_priority"]:
|
|
2766
|
+
calib_model_filename, calib_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
|
|
2767
|
+
calib_model_path = ML_DIR / calib_model_filename
|
|
2768
|
+
calib_schema_path = ML_DIR / calib_schema_filename
|
|
2769
|
+
if not calib_model_path.exists() or not calib_schema_path.exists():
|
|
2770
|
+
continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
|
|
2771
|
+
calib_model_export = json.loads(calib_model_path.read_text(encoding="utf-8"))
|
|
2772
|
+
calib_feature_schema = json.loads(calib_schema_path.read_text(encoding="utf-8"))
|
|
2773
|
+
calib_feature_names = calib_feature_schema["model_input_names"]
|
|
2774
|
+
head_predictions: dict[str, object] = {}
|
|
2775
|
+
for head_name in active_heads:
|
|
2776
|
+
head_predictions[head_name] = predict_head(
|
|
2777
|
+
dataset, calib_model_export, calib_feature_names, head_name,
|
|
2778
|
+
model_kind=model_kind_for_head,
|
|
2779
|
+
sequence_feature_schema=calib_feature_schema if model_kind_for_head == "sequence" else None,
|
|
2780
|
+
configured_window_size=int(sequence_window_default),
|
|
2781
|
+
)
|
|
2782
|
+
predictions_by_model_head[model_kind_for_head] = head_predictions
|
|
2783
|
+
|
|
2784
|
+
dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
|
|
2785
|
+
|
|
2786
|
+
rolling_ic_gate_config = config.get("phase_v2", {}).get("rolling_ic_gate", {})
|
|
2787
|
+
rolling_ic_result = calibrate_rolling_ic_floor(
|
|
2788
|
+
dataset,
|
|
2789
|
+
raw_score_column="raw_blended_score",
|
|
2790
|
+
horizon_days=int(rolling_ic_gate_config.get("horizon_days", 20)),
|
|
2791
|
+
rolling_window_days=int(rolling_ic_gate_config.get("rolling_window_days", 40)),
|
|
2792
|
+
percentile=float(getattr(args, "rolling_ic_floor_percentile", 0.10)),
|
|
2793
|
+
min_names_per_date=int(rolling_ic_gate_config.get("min_names_per_date", 10)),
|
|
2794
|
+
)
|
|
2795
|
+
report["rolling_ic_floor_calibration"] = rolling_ic_result
|
|
2796
|
+
_write_evaluation_json(evaluation_dir / "rolling_ic_floor_calibration.json", rolling_ic_result)
|
|
2797
|
+
if not args.json:
|
|
2798
|
+
calibrated_floor = rolling_ic_result["calibrated_min_rolling_mean_ic"]
|
|
2799
|
+
distribution = rolling_ic_result["rolling_ic_distribution"]
|
|
2800
|
+
print(
|
|
2801
|
+
f"Calibrated min_rolling_mean_ic (p{rolling_ic_result['percentile']*100:.0f}, "
|
|
2802
|
+
f"{rolling_ic_result['num_samples_used']} samples used, "
|
|
2803
|
+
f"{rolling_ic_result['num_samples_skipped_insufficient_history']} skipped insufficient-history): "
|
|
2804
|
+
f"{calibrated_floor:.4f}"
|
|
2805
|
+
)
|
|
2806
|
+
print(
|
|
2807
|
+
f" Distribution: min={distribution['min']}, p10={distribution['p10']}, "
|
|
2808
|
+
f"median={distribution['median']}, p75={distribution['p75']}, max={distribution['max']}"
|
|
2809
|
+
)
|
|
2810
|
+
print(f" Apply with: aq config set phase_v2.rolling_ic_gate.min_rolling_mean_ic {calibrated_floor:.4f}")
|
|
2811
|
+
print(
|
|
2812
|
+
" NOTE: only sets the threshold - phase_v2.rolling_ic_gate.enabled still defaults to false. "
|
|
2813
|
+
"Check --replay-rolling-ic-gate's per-era breakdown before flipping it on live."
|
|
2814
|
+
)
|
|
2815
|
+
|
|
2816
|
+
if run_replay_rolling_ic_gate:
|
|
2817
|
+
# V5.3.5 (development/Problems.md #102) - the critical pre-main.py
|
|
2818
|
+
# checkpoint: recomputes the blended raw score fresh (same
|
|
2819
|
+
# duplication convention as run_calibrate_rolling_ic_floor above),
|
|
2820
|
+
# then replays the gate day-by-day at the same rebalance cadence
|
|
2821
|
+
# phase_v2.portfolio_book.rebalance_every_bars uses live, and
|
|
2822
|
+
# reports engagement broken down by the three known bad eras.
|
|
2823
|
+
#
|
|
2824
|
+
# Always evaluated as ENABLED regardless of
|
|
2825
|
+
# phase_v2.rolling_ic_gate.enabled's live default (False) - this
|
|
2826
|
+
# is a "what would happen IF this were live" diagnostic, the same
|
|
2827
|
+
# relationship evaluation/kill_switch_replay.py's own replay has
|
|
2828
|
+
# to the live kill switch (which has no master on/off flag to
|
|
2829
|
+
# begin with). Everything else - crucially min_rolling_mean_ic -
|
|
2830
|
+
# is read from config as-is, so this reports honestly against
|
|
2831
|
+
# whatever floor is currently configured (the shipped -1e12
|
|
2832
|
+
# sentinel gives a trivial always-engaged result until
|
|
2833
|
+
# --calibrate-rolling-ic-floor's value has actually been applied
|
|
2834
|
+
# with `aq config set`).
|
|
2835
|
+
from evaluation import compute_blended_raw_scores
|
|
2836
|
+
from evaluation.rolling_ic_gate_replay import (
|
|
2837
|
+
replay_rolling_ic_gate_over_dataset,
|
|
2838
|
+
summarize_rolling_ic_gate_replay,
|
|
2839
|
+
)
|
|
2840
|
+
from portfolio.rank_signal import resolve_rank_signal_policy
|
|
2841
|
+
|
|
2842
|
+
training_metrics_by_model: dict[str, dict | None] = {}
|
|
2843
|
+
for model_name, metrics_filename in (
|
|
2844
|
+
("sequence", "sequence_training_metrics.json"),
|
|
2845
|
+
("multitask", "multitask_training_metrics.json"),
|
|
2846
|
+
):
|
|
2847
|
+
metrics_path = ML_DIR / metrics_filename
|
|
2848
|
+
training_metrics_by_model[model_name] = (
|
|
2849
|
+
json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
|
|
2850
|
+
)
|
|
2851
|
+
policy = resolve_rank_signal_policy(training_metrics_by_model, config)
|
|
2852
|
+
active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
|
|
2853
|
+
|
|
2854
|
+
predictions_by_model_head: dict[str, dict[str, object]] = {}
|
|
2855
|
+
for model_kind_for_head in policy["model_priority"]:
|
|
2856
|
+
replay_model_filename, replay_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
|
|
2857
|
+
replay_model_path = ML_DIR / replay_model_filename
|
|
2858
|
+
replay_schema_path = ML_DIR / replay_schema_filename
|
|
2859
|
+
if not replay_model_path.exists() or not replay_schema_path.exists():
|
|
2860
|
+
continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
|
|
2861
|
+
replay_model_export = json.loads(replay_model_path.read_text(encoding="utf-8"))
|
|
2862
|
+
replay_feature_schema = json.loads(replay_schema_path.read_text(encoding="utf-8"))
|
|
2863
|
+
replay_feature_names = replay_feature_schema["model_input_names"]
|
|
2864
|
+
head_predictions: dict[str, object] = {}
|
|
2865
|
+
for head_name in active_heads:
|
|
2866
|
+
head_predictions[head_name] = predict_head(
|
|
2867
|
+
dataset, replay_model_export, replay_feature_names, head_name,
|
|
2868
|
+
model_kind=model_kind_for_head,
|
|
2869
|
+
sequence_feature_schema=replay_feature_schema if model_kind_for_head == "sequence" else None,
|
|
2870
|
+
configured_window_size=int(sequence_window_default),
|
|
2871
|
+
)
|
|
2872
|
+
predictions_by_model_head[model_kind_for_head] = head_predictions
|
|
2873
|
+
|
|
2874
|
+
dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
|
|
2875
|
+
|
|
2876
|
+
rolling_ic_gate_config = dict(config.get("phase_v2", {}).get("rolling_ic_gate", {}))
|
|
2877
|
+
rolling_ic_gate_config["enabled"] = True
|
|
2878
|
+
rebalance_every_bars = int(config.get("phase_v2", {}).get("portfolio_book", {}).get("rebalance_every_bars", 10))
|
|
2879
|
+
rebalance_dates = sorted(dataset["date"].unique())[::max(1, rebalance_every_bars)]
|
|
2880
|
+
|
|
2881
|
+
replay_records = replay_rolling_ic_gate_over_dataset(
|
|
2882
|
+
dataset,
|
|
2883
|
+
raw_score_column="raw_blended_score",
|
|
2884
|
+
rebalance_dates=rebalance_dates,
|
|
2885
|
+
horizon_days=int(rolling_ic_gate_config.get("horizon_days", 20)),
|
|
2886
|
+
rolling_window_days=int(rolling_ic_gate_config.get("rolling_window_days", 40)),
|
|
2887
|
+
gate_config=rolling_ic_gate_config,
|
|
2888
|
+
min_names_per_date=int(rolling_ic_gate_config.get("min_names_per_date", 10)),
|
|
2889
|
+
)
|
|
2890
|
+
replay_summary = summarize_rolling_ic_gate_replay(replay_records)
|
|
2891
|
+
report["rolling_ic_gate_replay"] = {"summary": replay_summary, "per_date": replay_records}
|
|
2892
|
+
_write_evaluation_json(evaluation_dir / "rolling_ic_gate_replay.json", report["rolling_ic_gate_replay"])
|
|
2893
|
+
if not args.json:
|
|
2894
|
+
overall = replay_summary["overall"]
|
|
2895
|
+
print(
|
|
2896
|
+
f"Rolling-IC gate replay (floor={rolling_ic_gate_config.get('min_rolling_mean_ic')}): "
|
|
2897
|
+
f"{overall['disengaged_days']}/{overall['total_dates']} rebalance dates disengaged "
|
|
2898
|
+
f"({overall['disengaged_day_fraction']:.2%})"
|
|
2899
|
+
)
|
|
2900
|
+
for era_name, era_stats in replay_summary["by_era"].items():
|
|
2901
|
+
fraction = era_stats["disengaged_day_fraction"]
|
|
2902
|
+
fraction_display = f"{fraction:.2%}" if fraction is not None else "n/a (no dates in era)"
|
|
2903
|
+
print(f" {era_name}: {era_stats['disengaged_days']}/{era_stats['total_dates']} disengaged ({fraction_display})")
|
|
2904
|
+
|
|
2905
|
+
if run_calibrate_confidence_threshold:
|
|
2906
|
+
# V5.2.6 (development/Problems.md) - min_confidence_to_trade was a
|
|
2907
|
+
# guessed constant (0.12), never checked against this codebase's
|
|
2908
|
+
# actual confidence-vs-forward-return relationship - the same
|
|
2909
|
+
# "guessed constant, never calibrated" problem
|
|
2910
|
+
# min_rank_confidence_spread had (Problems.md #89) before
|
|
2911
|
+
# --calibrate-book-spread fixed it. Mirrors that tool's exact
|
|
2912
|
+
# structure.
|
|
2913
|
+
from evaluation import calibrate_confidence_threshold, compute_blended_raw_scores
|
|
2914
|
+
from portfolio.rank_signal import resolve_rank_signal_policy
|
|
2915
|
+
|
|
2916
|
+
training_metrics_by_model: dict[str, dict | None] = {}
|
|
2917
|
+
for model_name, metrics_filename in (
|
|
2918
|
+
("sequence", "sequence_training_metrics.json"),
|
|
2919
|
+
("multitask", "multitask_training_metrics.json"),
|
|
2920
|
+
):
|
|
2921
|
+
metrics_path = ML_DIR / metrics_filename
|
|
2922
|
+
training_metrics_by_model[model_name] = (
|
|
2923
|
+
json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else None
|
|
2924
|
+
)
|
|
2925
|
+
policy = resolve_rank_signal_policy(training_metrics_by_model, config)
|
|
2926
|
+
active_heads = [head_name for head_name, weight in policy["heads"].items() if weight > 0.0]
|
|
2927
|
+
|
|
2928
|
+
predictions_by_model_head: dict[str, dict[str, object]] = {}
|
|
2929
|
+
for model_kind_for_head in policy["model_priority"]:
|
|
2930
|
+
calib_model_filename, calib_schema_filename = _EVALUATE_MODEL_ARTIFACTS[model_kind_for_head]
|
|
2931
|
+
calib_model_path = ML_DIR / calib_model_filename
|
|
2932
|
+
calib_schema_path = ML_DIR / calib_schema_filename
|
|
2933
|
+
if not calib_model_path.exists() or not calib_schema_path.exists():
|
|
2934
|
+
continue # best-effort - a missing model just means its heads fall through to the next model_priority entry
|
|
2935
|
+
calib_model_export = json.loads(calib_model_path.read_text(encoding="utf-8"))
|
|
2936
|
+
calib_feature_schema = json.loads(calib_schema_path.read_text(encoding="utf-8"))
|
|
2937
|
+
calib_feature_names = calib_feature_schema["model_input_names"]
|
|
2938
|
+
head_predictions: dict[str, object] = {}
|
|
2939
|
+
for head_name in active_heads:
|
|
2940
|
+
head_predictions[head_name] = predict_head(
|
|
2941
|
+
dataset, calib_model_export, calib_feature_names, head_name,
|
|
2942
|
+
model_kind=model_kind_for_head,
|
|
2943
|
+
sequence_feature_schema=calib_feature_schema if model_kind_for_head == "sequence" else None,
|
|
2944
|
+
configured_window_size=int(sequence_window_default),
|
|
2945
|
+
)
|
|
2946
|
+
predictions_by_model_head[model_kind_for_head] = head_predictions
|
|
2947
|
+
|
|
2948
|
+
dataset["raw_blended_score"] = compute_blended_raw_scores(dataset, predictions_by_model_head, policy)
|
|
2949
|
+
|
|
2950
|
+
book_config = config.get("phase_v2", {}).get("portfolio_book", {})
|
|
2951
|
+
strategy_mode = config.get("phase5", {}).get("backtest", {}).get("strategy_mode", "long_flat")
|
|
2952
|
+
book_top_n = int(book_config.get("top_n", 6))
|
|
2953
|
+
book_bottom_n = int(book_config.get("bottom_n", 6)) if strategy_mode == "long_short" else 0
|
|
2954
|
+
|
|
2955
|
+
confidence_forward_return_column = "target_return_20d"
|
|
2956
|
+
if confidence_forward_return_column not in dataset.columns:
|
|
2957
|
+
print(
|
|
2958
|
+
f"warning: {confidence_forward_return_column!r} not in dataset; falling back to "
|
|
2959
|
+
"target_return_1d - the calibrated value will then represent a 1-day move, not the "
|
|
2960
|
+
"20-day move predicted_rank_20d assumes.",
|
|
2961
|
+
file=sys.stderr,
|
|
2962
|
+
)
|
|
2963
|
+
confidence_forward_return_column = "target_return_1d"
|
|
2964
|
+
|
|
2965
|
+
confidence_result = calibrate_confidence_threshold(
|
|
2966
|
+
dataset,
|
|
2967
|
+
raw_score_column="raw_blended_score",
|
|
2968
|
+
forward_return_column=confidence_forward_return_column,
|
|
2969
|
+
top_n=book_top_n,
|
|
2970
|
+
bottom_n=book_bottom_n,
|
|
2971
|
+
round_trip_cost_fraction=float(
|
|
2972
|
+
config.get("phase_v2", {}).get("liquidity", {}).get("max_round_trip_cost_fraction") or 0.001
|
|
2973
|
+
),
|
|
2974
|
+
percentile=float(getattr(args, "confidence_threshold_percentile", 0.10)),
|
|
2975
|
+
)
|
|
2976
|
+
report["confidence_threshold_calibration"] = confidence_result
|
|
2977
|
+
_write_evaluation_json(evaluation_dir / "confidence_threshold_calibration.json", confidence_result)
|
|
2978
|
+
if not args.json:
|
|
2979
|
+
calibrated_general = confidence_result["calibrated_min_confidence_to_trade"]
|
|
2980
|
+
calibrated_book_selected = confidence_result["calibrated_min_confidence_to_trade_book_selected"]
|
|
2981
|
+
print(
|
|
2982
|
+
f"Calibrated min_confidence_to_trade (p{confidence_result['percentile']*100:.0f}, "
|
|
2983
|
+
f"{confidence_result['num_rows_used']} paying rows used, "
|
|
2984
|
+
f"{confidence_result['num_dates_skipped_thin_universe']} dates skipped thin-universe): "
|
|
2985
|
+
f"{calibrated_general:.4f}"
|
|
2986
|
+
)
|
|
2987
|
+
print(
|
|
2988
|
+
f"Calibrated min_confidence_to_trade_book_selected ({confidence_result['num_rows_used_book_selected']} "
|
|
2989
|
+
f"paying book-selected rows used): {calibrated_book_selected:.4f}"
|
|
2990
|
+
)
|
|
2991
|
+
print(f" Apply with: aq config set phase6.risk.min_confidence_to_trade {calibrated_general:.4f}")
|
|
2992
|
+
print(
|
|
2993
|
+
" Apply with: aq config set phase6.risk.min_confidence_to_trade_book_selected "
|
|
2994
|
+
f"{calibrated_book_selected:.4f}"
|
|
2995
|
+
)
|
|
2996
|
+
print(
|
|
2997
|
+
" NOTE: this recalibrates the gate's threshold to this model's real achievable confidence "
|
|
2998
|
+
"among historically-paying trades - it does not certify the underlying rank heads are "
|
|
2999
|
+
"skillful. Check ml/*_training_metrics.json's quality_status before trusting the result."
|
|
3000
|
+
)
|
|
3001
|
+
|
|
3002
|
+
if run_ablation_flag:
|
|
3003
|
+
from evaluation import run_ablation
|
|
3004
|
+
|
|
3005
|
+
ablation_config = config.get("phase_v2", {}).get("evaluation", {}).get("ablation", {})
|
|
3006
|
+
requested_variants = (
|
|
3007
|
+
[name.strip() for name in args.variants.split(",") if name.strip()]
|
|
3008
|
+
if getattr(args, "variants", None)
|
|
3009
|
+
else list(ablation_config.get("variants", ["static_baseline", "no_neutrality", "no_hysteresis", "no_cost_model"]))
|
|
3010
|
+
)
|
|
3011
|
+
ablation_results = run_ablation(dataset, base_kwargs, requested_variants)
|
|
3012
|
+
report["ablation"] = ablation_results
|
|
3013
|
+
_write_evaluation_json(
|
|
3014
|
+
ROOT_DIR / ablation_config.get("report_path", "ml/evaluation/ablation_report.json"), ablation_results
|
|
3015
|
+
)
|
|
3016
|
+
if not args.json:
|
|
3017
|
+
print(f"Ablation ({model_kind}/{head}, split={split}):")
|
|
3018
|
+
for name, entry in ablation_results.items():
|
|
3019
|
+
if entry.get("status") in {"not_offline_measurable", "unknown_variant", "insufficient_windows"}:
|
|
3020
|
+
print(f" {name}: [{entry['status']}] {entry.get('reason', '')}")
|
|
3021
|
+
else:
|
|
3022
|
+
print(f" {name}: net_sharpe={entry['net_sharpe']:.4f} delta_vs_static_baseline={entry['delta_vs_static_baseline']:+.4f}")
|
|
3023
|
+
|
|
3024
|
+
if args.json:
|
|
3025
|
+
print(json.dumps(report, indent=2, default=str))
|
|
3026
|
+
|
|
3027
|
+
_refresh_readme_evaluation_sections()
|
|
3028
|
+
return 0
|
|
3029
|
+
|
|
3030
|
+
|
|
3031
|
+
def cmd_status(_args: argparse.Namespace) -> int:
|
|
3032
|
+
return _run(["git", "status"])
|
|
3033
|
+
|
|
3034
|
+
|
|
3035
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
3036
|
+
parser = argparse.ArgumentParser(prog="aq", description="Aether Quant convenience CLI")
|
|
3037
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
3038
|
+
|
|
3039
|
+
train_parser = subparsers.add_parser("train", help="Run the training pipeline (wraps python train.py)")
|
|
3040
|
+
train_group = train_parser.add_mutually_exclusive_group()
|
|
3041
|
+
train_group.add_argument("--dataset-only", action="store_true", help="Build dataset/scaler/manifest only")
|
|
3042
|
+
train_group.add_argument("--init-only", action="store_true", help="Refresh the data inventory only")
|
|
3043
|
+
train_group.add_argument("--experts-only", action="store_true", help="Train the 4 expert models only")
|
|
3044
|
+
train_group.add_argument(
|
|
3045
|
+
"--gating-only", action="store_true", help="Train the learned gating blend only (wraps python train_gating.py)"
|
|
3046
|
+
)
|
|
3047
|
+
train_group.add_argument(
|
|
3048
|
+
"--multitask-only",
|
|
3049
|
+
action="store_true",
|
|
3050
|
+
help="Train the joint direction+magnitude+volatility model only (wraps python train_multitask.py)",
|
|
3051
|
+
)
|
|
3052
|
+
train_group.add_argument(
|
|
3053
|
+
"--sequence-only",
|
|
3054
|
+
action="store_true",
|
|
3055
|
+
help="Train the Phase 2 causal-TCN sequence encoder only (wraps python train_sequence.py)",
|
|
3056
|
+
)
|
|
3057
|
+
train_group.add_argument(
|
|
3058
|
+
"--topology-only",
|
|
3059
|
+
action="store_true",
|
|
3060
|
+
help=(
|
|
3061
|
+
"Train the learned topology overlay only (wraps python train_topology.py). Needs Postgres up "
|
|
3062
|
+
"and enough accumulated realized-outcome events - see the command's own output if it skips."
|
|
3063
|
+
),
|
|
3064
|
+
)
|
|
3065
|
+
train_group.add_argument(
|
|
3066
|
+
"--strategy-selector-only",
|
|
3067
|
+
action="store_true",
|
|
3068
|
+
help=(
|
|
3069
|
+
"Train the learned multi-leg strategy-selector model only (wraps python "
|
|
3070
|
+
"train_strategy_selector.py, V4.7, development/Problems.md #29's own framing). Needs real "
|
|
3071
|
+
"option_strategy_outcome events (real option positions traded and closed) - expect this to "
|
|
3072
|
+
"skip indefinitely in this environment, see the command's own output."
|
|
3073
|
+
),
|
|
3074
|
+
)
|
|
3075
|
+
train_group.add_argument(
|
|
3076
|
+
"--rl-sizing-only",
|
|
3077
|
+
action="store_true",
|
|
3078
|
+
help=(
|
|
3079
|
+
"Train the offline contextual-bandit sizing overlay only (wraps python train_rl_sizing.py, "
|
|
3080
|
+
"development/Problems.md #71). Reads ml/datasets/*.csv, NOT Postgres - needs a completed "
|
|
3081
|
+
"`aq train` and Component D's alt-data features in phase1.features.input_set."
|
|
3082
|
+
),
|
|
3083
|
+
)
|
|
3084
|
+
train_group.add_argument(
|
|
3085
|
+
"--walk-forward",
|
|
3086
|
+
action="store_true",
|
|
3087
|
+
help=(
|
|
3088
|
+
"Phase 4 of the 5/10 -> 9/10 roadmap: run the dataset-build + training pipeline once per "
|
|
3089
|
+
"walk-forward window (wraps python train.py --walk-forward). Never touches active ml/ - "
|
|
3090
|
+
"diagnostic only, writes to ml/versions/<run-id>/window_<i>/."
|
|
3091
|
+
),
|
|
3092
|
+
)
|
|
3093
|
+
train_parser.add_argument(
|
|
3094
|
+
"--step-days", type=int, default=None, help="Walk-forward step size in days (only with --walk-forward)."
|
|
3095
|
+
)
|
|
3096
|
+
train_parser.add_argument(
|
|
3097
|
+
"--mode",
|
|
3098
|
+
type=str,
|
|
3099
|
+
choices=("rolling", "expanding"),
|
|
3100
|
+
default=None,
|
|
3101
|
+
help="Walk-forward mode: rolling or expanding (only with --walk-forward).",
|
|
3102
|
+
)
|
|
3103
|
+
# V5.1 Phase 4 (item 4) - one-run overrides of config.json's
|
|
3104
|
+
# phase_v2.retraining.walk_forward.{train_multitask,train_sequence,
|
|
3105
|
+
# tracked_metrics}, only meaningful with --walk-forward - passthrough
|
|
3106
|
+
# to train.py --walk-forward's own identical flags.
|
|
3107
|
+
train_parser.add_argument(
|
|
3108
|
+
"--include-multitask",
|
|
3109
|
+
action="store_true",
|
|
3110
|
+
help="Force multitask training per walk-forward window on for this run (only with --walk-forward).",
|
|
3111
|
+
)
|
|
3112
|
+
train_parser.add_argument(
|
|
3113
|
+
"--include-sequence",
|
|
3114
|
+
action="store_true",
|
|
3115
|
+
help="Force sequence training per walk-forward window on for this run (only with --walk-forward).",
|
|
3116
|
+
)
|
|
3117
|
+
train_parser.add_argument(
|
|
3118
|
+
"--metrics",
|
|
3119
|
+
type=str,
|
|
3120
|
+
default=None,
|
|
3121
|
+
help=(
|
|
3122
|
+
"Comma-separated tracked-metric names to restrict walk-forward tracking to for this run "
|
|
3123
|
+
"(only with --walk-forward)."
|
|
3124
|
+
),
|
|
3125
|
+
)
|
|
3126
|
+
# V5.1 Phase 3 (item 1) - passthrough to --multitask-only/--sequence-only's
|
|
3127
|
+
# own train_multitask.py/train_sequence.py --seed/--ranking-objective
|
|
3128
|
+
# flags (development/Problems.md - previously the seed-ensembling
|
|
3129
|
+
# workflow required calling those scripts directly, bypassing `aq`
|
|
3130
|
+
# entirely). Ignored (with a warning) by every other --*-only mode and
|
|
3131
|
+
# by the default full-pipeline train.py invocation, neither of which
|
|
3132
|
+
# accepts these flags.
|
|
3133
|
+
train_parser.add_argument(
|
|
3134
|
+
"--seed", type=int, default=None, help="Override config.json's seed (only with --multitask-only/--sequence-only)"
|
|
3135
|
+
)
|
|
3136
|
+
train_parser.add_argument(
|
|
3137
|
+
"--ranking-objective",
|
|
3138
|
+
type=str,
|
|
3139
|
+
default=None,
|
|
3140
|
+
choices=["mse", "soft_spearman", "listnet"],
|
|
3141
|
+
help="Override config.json's ranking_loss.objective for this run only (only with --multitask-only/--sequence-only)",
|
|
3142
|
+
)
|
|
3143
|
+
train_parser.set_defaults(func=cmd_train)
|
|
3144
|
+
|
|
3145
|
+
test_parser = subparsers.add_parser("test", help="Run the test suite (wraps pytest tests/)")
|
|
3146
|
+
test_parser.add_argument(
|
|
3147
|
+
"--lean", "--full", dest="lean", action="store_true",
|
|
3148
|
+
help="Include the real `lean backtest .` integration test (tests/test_lean_backtest_ml_coverage.py, over an hour wall-clock) - excluded by default",
|
|
3149
|
+
)
|
|
3150
|
+
test_parser.add_argument(
|
|
3151
|
+
"--parallel", action="store_true",
|
|
3152
|
+
help="Run via pytest-xdist (-n auto) - off by default, multiple workers importing torch risk OOM on memory-constrained machines",
|
|
3153
|
+
)
|
|
3154
|
+
test_parser.add_argument(
|
|
3155
|
+
"--ruff", action="store_true",
|
|
3156
|
+
help="V5.3.10: run `ruff check .` BEFORE pytest - a lint failure fails fast without paying the full suite's wall-clock first (mirrors ci.yml's python-lint job).",
|
|
3157
|
+
)
|
|
3158
|
+
for _subsystem_name in _SUBSYSTEM_TEST_FILES:
|
|
3159
|
+
test_parser.add_argument(
|
|
3160
|
+
f"--{_subsystem_name}",
|
|
3161
|
+
action="store_true",
|
|
3162
|
+
help=f"Run only the {_subsystem_name} subsystem's tests ({', '.join(_SUBSYSTEM_TEST_FILES[_subsystem_name][:2])}, ...)",
|
|
3163
|
+
)
|
|
3164
|
+
test_parser.set_defaults(func=cmd_test)
|
|
3165
|
+
|
|
3166
|
+
backtest_parser = subparsers.add_parser("backtest", help="Run a Lean backtest (wraps lean backtest .)")
|
|
3167
|
+
backtest_parser.add_argument(
|
|
3168
|
+
"--image",
|
|
3169
|
+
default=None,
|
|
3170
|
+
help=f"Override the Lean engine Docker image (default: pinned {PINNED_LEAN_ENGINE_IMAGE}, "
|
|
3171
|
+
"never the mutable :latest tag - see PINNED_LEAN_ENGINE_IMAGE's comment in aq_cli.py)",
|
|
3172
|
+
)
|
|
3173
|
+
backtest_parser.set_defaults(func=cmd_backtest)
|
|
3174
|
+
|
|
3175
|
+
profile_parser = subparsers.add_parser(
|
|
3176
|
+
"profile",
|
|
3177
|
+
help="Profile the per-bar hot path (wraps scripts/profile_inference.py / scripts/profile_subsystems.py)",
|
|
3178
|
+
)
|
|
3179
|
+
# default=None (not 10_000/200) so cmd_profile() can tell "user didn't
|
|
3180
|
+
# pass --iterations" apart from "user explicitly passed the same
|
|
3181
|
+
# number" and let whichever script actually runs use ITS OWN default -
|
|
3182
|
+
# profile_inference.py's 10,000 (cheap ~5ms/call) and
|
|
3183
|
+
# profile_subsystems.py's 200 (build_market_topology() alone costs
|
|
3184
|
+
# ~500-600ms/call at this project's real universe size - 10,000 there
|
|
3185
|
+
# would take over an hour) are deliberately very different, and
|
|
3186
|
+
# hardcoding either one here would silently override the other.
|
|
3187
|
+
profile_parser.add_argument(
|
|
3188
|
+
"--iterations", type=int, default=None,
|
|
3189
|
+
help="Iterations to profile (default: 10000 for inference, 200 for --<subsystem> flags)",
|
|
3190
|
+
)
|
|
3191
|
+
profile_parser.add_argument("--sort", default="cumulative", help="pstats sort key (default: cumulative)")
|
|
3192
|
+
profile_parser.add_argument(
|
|
3193
|
+
"--batched", action="store_true",
|
|
3194
|
+
help="Use the batched expert-inference path (with its precomputed stack caches) instead of a per-expert loop",
|
|
3195
|
+
)
|
|
3196
|
+
profile_parser.add_argument(
|
|
3197
|
+
"--no-gc", action="store_true",
|
|
3198
|
+
help="Disable the GC around the profiled region, to isolate whether GC pauses drive tail latency (inference only)",
|
|
3199
|
+
)
|
|
3200
|
+
profile_parser.add_argument(
|
|
3201
|
+
"--bucket-report", action="store_true",
|
|
3202
|
+
help="Print a 10-bucket-by-iteration-index duration breakdown, to check for a warmup effect (inference only)",
|
|
3203
|
+
)
|
|
3204
|
+
profile_parser.add_argument(
|
|
3205
|
+
"--parallel", action="store_true",
|
|
3206
|
+
help="V4.9 Priority 6: run the ProcessPoolExecutor IPC-overhead benchmark against run_symbol_inference() (inference only)",
|
|
3207
|
+
)
|
|
3208
|
+
profile_parser.add_argument(
|
|
3209
|
+
"--pool-workers", type=int, default=None,
|
|
3210
|
+
help="Worker process count for --parallel (default: profile_inference.py's own default, 4)",
|
|
3211
|
+
)
|
|
3212
|
+
profile_parser.add_argument(
|
|
3213
|
+
"--symbols-per-bar", type=int, default=None,
|
|
3214
|
+
help="Symbols-per-bar grouping for --batched's sequence-batching comparison and --parallel's benchmark (default: profile_inference.py's own default, 74)",
|
|
3215
|
+
)
|
|
3216
|
+
for _profile_subsystem_name in _PROFILE_SUBSYSTEM_FLAGS:
|
|
3217
|
+
profile_parser.add_argument(
|
|
3218
|
+
f"--{_profile_subsystem_name}",
|
|
3219
|
+
action="store_true",
|
|
3220
|
+
help=f"Profile only the {_profile_subsystem_name} subsystem (scripts/profile_subsystems.py)",
|
|
3221
|
+
)
|
|
3222
|
+
profile_parser.set_defaults(func=cmd_profile)
|
|
3223
|
+
|
|
3224
|
+
report_parser = subparsers.add_parser("report", help="Generate a Lean HTML report for a finished backtest")
|
|
3225
|
+
report_parser.add_argument("backtest_dir", help="Folder name under backtests/, e.g. 2026-07-04_13-06-51")
|
|
3226
|
+
report_parser.add_argument("result_id", help="Result JSON id, e.g. 1366365999")
|
|
3227
|
+
report_parser.set_defaults(func=cmd_report)
|
|
3228
|
+
|
|
3229
|
+
api_parser = subparsers.add_parser("api", help="Start the FastAPI monitoring server on :8001")
|
|
3230
|
+
api_parser.set_defaults(func=cmd_api)
|
|
3231
|
+
|
|
3232
|
+
webui_parser = subparsers.add_parser("webui", help="Start the webui dev server (npm run dev)")
|
|
3233
|
+
webui_parser.set_defaults(func=cmd_webui)
|
|
3234
|
+
|
|
3235
|
+
docker_parser = subparsers.add_parser("docker", help="Docker Compose shortcuts")
|
|
3236
|
+
docker_subparsers = docker_parser.add_subparsers(dest="docker_command", required=True)
|
|
3237
|
+
|
|
3238
|
+
docker_up_parser = docker_subparsers.add_parser("up", help="Start infra services")
|
|
3239
|
+
docker_up_group = docker_up_parser.add_mutually_exclusive_group()
|
|
3240
|
+
docker_up_group.add_argument("--lean", action="store_true", help="Start via the lean Compose profile")
|
|
3241
|
+
docker_up_group.add_argument("--all", action="store_true", help="Start the full stack, including all workers")
|
|
3242
|
+
docker_up_parser.set_defaults(func=cmd_docker_up)
|
|
3243
|
+
|
|
3244
|
+
docker_build_parser = docker_subparsers.add_parser(
|
|
3245
|
+
"build", help="Rebuild the consolidated aether-quant-engine image (app + every worker)"
|
|
3246
|
+
)
|
|
3247
|
+
docker_build_parser.set_defaults(func=cmd_docker_build)
|
|
3248
|
+
|
|
3249
|
+
config_parser = subparsers.add_parser("config", help="Show or edit config.json")
|
|
3250
|
+
config_parser.set_defaults(func=cmd_config)
|
|
3251
|
+
config_subparsers = config_parser.add_subparsers(dest="config_command")
|
|
3252
|
+
|
|
3253
|
+
config_get_parser = config_subparsers.add_parser("get", help="Print a config.json value")
|
|
3254
|
+
config_get_parser.add_argument("dotted_path")
|
|
3255
|
+
|
|
3256
|
+
config_keys_parser = config_subparsers.add_parser("keys", help="List leaf key paths (optionally scoped to a prefix)")
|
|
3257
|
+
config_keys_parser.add_argument("dotted_prefix", nargs="?", default=None)
|
|
3258
|
+
|
|
3259
|
+
config_set_parser = config_subparsers.add_parser("set", help="Set a config.json value (JSON-parsed, string fallback)")
|
|
3260
|
+
config_set_parser.add_argument("dotted_path")
|
|
3261
|
+
config_set_parser.add_argument("value")
|
|
3262
|
+
|
|
3263
|
+
# V5.1 Phase 1 - phase_v2.presets, named dotted-key-override blocks
|
|
3264
|
+
# (e.g. "aggressive"/"moderate" turnover profiles). config-only, no
|
|
3265
|
+
# `aq lean preset` equivalent.
|
|
3266
|
+
config_preset_parser = config_subparsers.add_parser(
|
|
3267
|
+
"preset", help="List/show/apply a named config preset (phase_v2.presets)"
|
|
3268
|
+
)
|
|
3269
|
+
config_preset_group = config_preset_parser.add_mutually_exclusive_group()
|
|
3270
|
+
config_preset_group.add_argument("--list", dest="preset_list", action="store_true", help="List available presets")
|
|
3271
|
+
config_preset_group.add_argument("--show", dest="preset_show", metavar="NAME", help="Print one preset's overrides")
|
|
3272
|
+
config_preset_group.add_argument(
|
|
3273
|
+
"--apply", dest="preset_apply", metavar="NAME", help="Apply one preset's overrides to config.json"
|
|
3274
|
+
)
|
|
3275
|
+
config_preset_parser.add_argument(
|
|
3276
|
+
"--dry-run", dest="preset_dry_run", action="store_true", help="With --apply: print old -> new, write nothing"
|
|
3277
|
+
)
|
|
3278
|
+
|
|
3279
|
+
lean_parser = subparsers.add_parser("lean", help="Show or edit lean.json (same shape as `aq config`)")
|
|
3280
|
+
lean_parser.set_defaults(func=cmd_lean)
|
|
3281
|
+
lean_subparsers = lean_parser.add_subparsers(dest="lean_command")
|
|
3282
|
+
|
|
3283
|
+
lean_get_parser = lean_subparsers.add_parser("get", help="Print a lean.json value")
|
|
3284
|
+
lean_get_parser.add_argument("dotted_path")
|
|
3285
|
+
|
|
3286
|
+
lean_keys_parser = lean_subparsers.add_parser("keys", help="List leaf key paths (optionally scoped to a prefix)")
|
|
3287
|
+
lean_keys_parser.add_argument("dotted_prefix", nargs="?", default=None)
|
|
3288
|
+
|
|
3289
|
+
lean_set_parser = lean_subparsers.add_parser("set", help="Set a lean.json value (JSON-parsed, string fallback)")
|
|
3290
|
+
lean_set_parser.add_argument("dotted_path")
|
|
3291
|
+
lean_set_parser.add_argument("value")
|
|
3292
|
+
|
|
3293
|
+
retrain_parser = subparsers.add_parser(
|
|
3294
|
+
"retrain", help="Thin dispatcher to python -m retraining.orchestrator <stage> ..."
|
|
3295
|
+
)
|
|
3296
|
+
retrain_parser.add_argument(
|
|
3297
|
+
"stage",
|
|
3298
|
+
choices=[
|
|
3299
|
+
"plan",
|
|
3300
|
+
"train",
|
|
3301
|
+
"train_topology",
|
|
3302
|
+
"train_gating",
|
|
3303
|
+
"train_multitask",
|
|
3304
|
+
"train_sequence",
|
|
3305
|
+
"train_strategy_selector",
|
|
3306
|
+
"validate",
|
|
3307
|
+
"backtest",
|
|
3308
|
+
"commit",
|
|
3309
|
+
"promote",
|
|
3310
|
+
"rollback",
|
|
3311
|
+
"status",
|
|
3312
|
+
# V5.1 Phase 6 (production safety) - read-only diagnostic,
|
|
3313
|
+
# never itself calls rollback() - see retraining/orchestrator.py::
|
|
3314
|
+
# auto_rollback_status()'s own docstring.
|
|
3315
|
+
"auto-rollback",
|
|
3316
|
+
],
|
|
3317
|
+
)
|
|
3318
|
+
retrain_parser.add_argument("retrain_args", nargs=argparse.REMAINDER, help="Passed through verbatim, e.g. --version-id <uuid>")
|
|
3319
|
+
retrain_parser.set_defaults(func=cmd_retrain)
|
|
3320
|
+
|
|
3321
|
+
backfill_parser = subparsers.add_parser(
|
|
3322
|
+
"backfill",
|
|
3323
|
+
help=(
|
|
3324
|
+
"Thin dispatcher to python -m data_pipeline.<target>_backfill ... (whole-universe bulk "
|
|
3325
|
+
"backfills). Different from `aq fetch`, which is for one ad-hoc ticker."
|
|
3326
|
+
),
|
|
3327
|
+
)
|
|
3328
|
+
backfill_parser.add_argument("target", choices=["dividends", "fred", "yfinance"])
|
|
3329
|
+
backfill_parser.add_argument(
|
|
3330
|
+
"backfill_args",
|
|
3331
|
+
nargs=argparse.REMAINDER,
|
|
3332
|
+
help="Passed through verbatim, e.g. --apply --tickers AAPL MSFT (dividends/yfinance), or --series DGS10 (fred)",
|
|
3333
|
+
)
|
|
3334
|
+
backfill_parser.set_defaults(func=cmd_backfill)
|
|
3335
|
+
|
|
3336
|
+
paper_readiness_parser = subparsers.add_parser(
|
|
3337
|
+
"paper-readiness", help="Check whether the system is ready for phase_v2.runtime.mode='paper'"
|
|
3338
|
+
)
|
|
3339
|
+
paper_readiness_parser.set_defaults(func=cmd_paper_readiness)
|
|
3340
|
+
|
|
3341
|
+
trade_lock_parser = subparsers.add_parser(
|
|
3342
|
+
"trade-lock", help="Manually override the sticky total-drawdown trade lock"
|
|
3343
|
+
)
|
|
3344
|
+
trade_lock_group = trade_lock_parser.add_mutually_exclusive_group(required=True)
|
|
3345
|
+
trade_lock_group.add_argument("--on", action="store_true", help="Force trading paused")
|
|
3346
|
+
trade_lock_group.add_argument("--off", action="store_true", help="Force trading resumed")
|
|
3347
|
+
trade_lock_group.add_argument("--auto", action="store_true", help="Return to fully automatic behavior")
|
|
3348
|
+
trade_lock_group.add_argument("--status", dest="status", action="store_true", help="Print the current override state")
|
|
3349
|
+
trade_lock_parser.set_defaults(func=cmd_trade_lock)
|
|
3350
|
+
|
|
3351
|
+
# V5.1 Phase 6 (production safety) - deliberately the same
|
|
3352
|
+
# override-file/mutually-exclusive-group shape as `trade-lock` above (see
|
|
3353
|
+
# risk/manual_override.py's kill_switch_manual_override, the exact same
|
|
3354
|
+
# read/write/cache convention as manual_trade_lock_override) so the two
|
|
3355
|
+
# switches can never disagree about their CLI surface. --history is the
|
|
3356
|
+
# one addition, querying the tamper-evident audit log instead.
|
|
3357
|
+
kill_switch_parser = subparsers.add_parser(
|
|
3358
|
+
"kill-switch", help="Manually override / inspect the automated production kill switch"
|
|
3359
|
+
)
|
|
3360
|
+
kill_switch_group = kill_switch_parser.add_mutually_exclusive_group(required=True)
|
|
3361
|
+
kill_switch_group.add_argument("--arm", action="store_true", help="Force kill-switch evaluation on")
|
|
3362
|
+
kill_switch_group.add_argument("--disarm", action="store_true", help="Force kill-switch evaluation off")
|
|
3363
|
+
kill_switch_group.add_argument("--auto", action="store_true", help="Defer to phase_v2.risk.kill_switch.enabled")
|
|
3364
|
+
kill_switch_group.add_argument("--status", dest="status", action="store_true", help="Print the current override state")
|
|
3365
|
+
kill_switch_group.add_argument("--history", action="store_true", help="List recorded kill-switch trips from the audit log")
|
|
3366
|
+
kill_switch_parser.add_argument("--limit", type=int, default=50, help="Max --history rows (default 50)")
|
|
3367
|
+
kill_switch_parser.set_defaults(func=cmd_kill_switch)
|
|
3368
|
+
|
|
3369
|
+
fetch_parser = subparsers.add_parser(
|
|
3370
|
+
"fetch",
|
|
3371
|
+
help=(
|
|
3372
|
+
"Ad-hoc fetch of historical OHLCV from Yahoo Finance for a ticker not yet in config.json. "
|
|
3373
|
+
"For a whole-universe bulk refresh instead (dividends/FRED series/Yahoo gaps), see `aq backfill`."
|
|
3374
|
+
),
|
|
3375
|
+
)
|
|
3376
|
+
fetch_parser.add_argument(
|
|
3377
|
+
"asset_class", choices=list(ASSET_CLASSES), help="Asset class (picks the Lean data_path/market convention)"
|
|
3378
|
+
)
|
|
3379
|
+
fetch_parser.add_argument("--ticker", required=True, help="Internal ticker, e.g. AAPL or BTCUSD")
|
|
3380
|
+
fetch_parser.add_argument("--start", required=True, type=_iso_date, help="Start date, ISO 8601 YYYY-MM-DD")
|
|
3381
|
+
fetch_parser.add_argument("--end", required=True, type=_iso_date, help="End date, ISO 8601 YYYY-MM-DD")
|
|
3382
|
+
fetch_parser.add_argument(
|
|
3383
|
+
"--apply", action="store_true", help="Actually write the zip file and update config.json (default: dry run, report only)"
|
|
3384
|
+
)
|
|
3385
|
+
fetch_parser.add_argument(
|
|
3386
|
+
"--expiry", default=None, help="Contract expiry, YYYY-MM-DD (required for asset_class futures/options; requires IB, see 'aq ib status')"
|
|
3387
|
+
)
|
|
3388
|
+
fetch_parser.add_argument("--strike", type=float, default=None, help="Strike price (required for asset_class options)")
|
|
3389
|
+
fetch_parser.add_argument("--right", choices=["call", "put"], default=None, help="Option right (required for asset_class options)")
|
|
3390
|
+
fetch_parser.add_argument(
|
|
3391
|
+
"--contract-month", default=None,
|
|
3392
|
+
help="Futures only: YYYYMM - fetch a specific dated contract instead of the continuous front-month "
|
|
3393
|
+
"(e.g. for building real historical term structure with a second, later --contract-month fetch under "
|
|
3394
|
+
"the same --family-ticker)",
|
|
3395
|
+
)
|
|
3396
|
+
fetch_parser.add_argument(
|
|
3397
|
+
"--family-ticker", default=None,
|
|
3398
|
+
help="Groups multiple fetched contracts under one root for offline training's derivatives-macro features "
|
|
3399
|
+
"(train.py::build_derivatives_macro_features_by_date()) - e.g. two futures/options fetches sharing "
|
|
3400
|
+
"--family-ticker ES. Defaults to --ticker itself (a single, ungrouped contract).",
|
|
3401
|
+
)
|
|
3402
|
+
fetch_parser.set_defaults(func=cmd_fetch)
|
|
3403
|
+
|
|
3404
|
+
ib_parser = subparsers.add_parser("ib", help="Check Interactive Brokers configuration/connectivity")
|
|
3405
|
+
ib_subparsers = ib_parser.add_subparsers(dest="ib_command", required=True)
|
|
3406
|
+
ib_subparsers.add_parser("status", help="Report disabled / credentials-missing / reachable")
|
|
3407
|
+
ib_parser.set_defaults(func=cmd_ib)
|
|
3408
|
+
|
|
3409
|
+
assets_parser = subparsers.add_parser("assets", help="Report multi-asset-class (futures/options/FRED) readiness")
|
|
3410
|
+
assets_subparsers = assets_parser.add_subparsers(dest="assets_command", required=True)
|
|
3411
|
+
assets_subparsers.add_parser("status", help="Report IB/futures/options/FRED readiness at a glance")
|
|
3412
|
+
assets_parser.set_defaults(func=cmd_assets)
|
|
3413
|
+
|
|
3414
|
+
evaluate_parser = subparsers.add_parser(
|
|
3415
|
+
"evaluate",
|
|
3416
|
+
help="Offline, cost-aware evaluation of the cross-sectional rank book (V5.1) - net Sharpe/turnover/"
|
|
3417
|
+
"capacity/cost-stress from the active model + dataset, without a Lean backtest",
|
|
3418
|
+
)
|
|
3419
|
+
evaluate_parser.add_argument("--rank-book", action="store_true", help="Simulate the rank book net of costs")
|
|
3420
|
+
evaluate_parser.add_argument("--capacity", action="store_true", help="Breadth (top_n sweep) + capacity estimate")
|
|
3421
|
+
evaluate_parser.add_argument("--stress", action="store_true", help="Re-run at 1x/2x/3x the configured cost")
|
|
3422
|
+
evaluate_parser.add_argument(
|
|
3423
|
+
"--calibrate-edge", action="store_true",
|
|
3424
|
+
help="Print an edge_bps_per_rank_unit calibrated from this split's realized rank-vs-return relationship",
|
|
3425
|
+
)
|
|
3426
|
+
evaluate_parser.add_argument("--all", action="store_true", help="Run --rank-book, --capacity, --stress, --calibrate-edge and --benchmarks together")
|
|
3427
|
+
evaluate_parser.add_argument(
|
|
3428
|
+
"--walk-forward-summary", action="store_true",
|
|
3429
|
+
help="Print an already-written ml/versions/walk-forward-*/walk_forward_summary.json (V5.1 Phase 4) - "
|
|
3430
|
+
"never runs training itself; run `aq train --walk-forward` first.",
|
|
3431
|
+
)
|
|
3432
|
+
evaluate_parser.add_argument(
|
|
3433
|
+
"--run-id", default=None,
|
|
3434
|
+
help="Specific walk-forward run-id to read with --walk-forward-summary (default: the most recent run).",
|
|
3435
|
+
)
|
|
3436
|
+
evaluate_parser.add_argument(
|
|
3437
|
+
"--ablation", action="store_true",
|
|
3438
|
+
help="V5.1 Phase 5: run the adaptive-machinery ablation harness - runtime-only mechanisms "
|
|
3439
|
+
"(gating, topology sizing, the net-edge gate, ...) honestly report not_offline_measurable "
|
|
3440
|
+
"rather than a fabricated number. Not included in --all.",
|
|
3441
|
+
)
|
|
3442
|
+
evaluate_parser.add_argument(
|
|
3443
|
+
"--variants", default=None,
|
|
3444
|
+
help="Comma-separated ablation variant names to run with --ablation (default: "
|
|
3445
|
+
"phase_v2.evaluation.ablation.variants from config.json).",
|
|
3446
|
+
)
|
|
3447
|
+
evaluate_parser.add_argument(
|
|
3448
|
+
"--calibrate-book-spread", action="store_true",
|
|
3449
|
+
help="Print a min_rank_confidence_spread calibrated from this split's actual raw-score dispersion "
|
|
3450
|
+
"(the same resolved rank-signal blend and book top_n/bottom_n main.py uses live). Not included in "
|
|
3451
|
+
"--all - loads both models' predictions, a heavier run like --ablation.",
|
|
3452
|
+
)
|
|
3453
|
+
evaluate_parser.add_argument(
|
|
3454
|
+
"--book-spread-percentile", type=float, default=0.10,
|
|
3455
|
+
help="Percentile (0-1) of the per-date confidence-spread distribution to use as the calibrated "
|
|
3456
|
+
"floor with --calibrate-book-spread (default: 0.10).",
|
|
3457
|
+
)
|
|
3458
|
+
evaluate_parser.add_argument(
|
|
3459
|
+
"--calibrate-confidence-threshold", action="store_true",
|
|
3460
|
+
help="Print a min_confidence_to_trade (and, when book-selection data is available, a separate "
|
|
3461
|
+
"min_confidence_to_trade_book_selected) calibrated from this split's real confidence-vs-forward-"
|
|
3462
|
+
"return relationship, mirroring --calibrate-book-spread's discipline. Not included in --all.",
|
|
3463
|
+
)
|
|
3464
|
+
evaluate_parser.add_argument(
|
|
3465
|
+
"--confidence-threshold-percentile", type=float, default=0.10,
|
|
3466
|
+
help="Percentile (0-1) of the paying-trade confidence distribution to use as the calibrated floor "
|
|
3467
|
+
"with --calibrate-confidence-threshold (default: 0.10).",
|
|
3468
|
+
)
|
|
3469
|
+
evaluate_parser.add_argument(
|
|
3470
|
+
"--calibrate-rolling-ic-floor", action="store_true",
|
|
3471
|
+
help="V5.3.5 (development/Problems.md #102): print a phase_v2.rolling_ic_gate.min_rolling_mean_ic "
|
|
3472
|
+
"calibrated from this split's real walk-forward rolling trailing-IC distribution (portfolio/"
|
|
3473
|
+
"rolling_ic_gate.py's own aggregation, no-lookahead - see evaluation/rolling_ic_gate_calibration.py's "
|
|
3474
|
+
"docstring), mirroring --calibrate-book-spread's discipline. Not included in --all - loads both "
|
|
3475
|
+
"models' predictions, a heavier run like --ablation.",
|
|
3476
|
+
)
|
|
3477
|
+
evaluate_parser.add_argument(
|
|
3478
|
+
"--rolling-ic-floor-percentile", type=float, default=0.10,
|
|
3479
|
+
help="Percentile (0-1) of the rolling-IC distribution to use as the calibrated floor with "
|
|
3480
|
+
"--calibrate-rolling-ic-floor (default: 0.10).",
|
|
3481
|
+
)
|
|
3482
|
+
evaluate_parser.add_argument(
|
|
3483
|
+
"--reconcile-features", action="store_true",
|
|
3484
|
+
help="V5.3.5.3 Workstream B (development/Problems.md #91/#100): diff ONE symbol's logged live "
|
|
3485
|
+
"feature values (book_history.jsonl's optional 'feature_snapshot' key, written when "
|
|
3486
|
+
"phase_v2.diagnostics.book_history.include_feature_snapshot is on and the symbol is in "
|
|
3487
|
+
"feature_snapshot_symbols) against full_dataset.csv's same (ticker, date) row - per-feature "
|
|
3488
|
+
"deltas, worst offenders first. Requires --symbol. Not included in --all - investigation-only.",
|
|
3489
|
+
)
|
|
3490
|
+
evaluate_parser.add_argument(
|
|
3491
|
+
"--symbol", default=None,
|
|
3492
|
+
help="Ticker to reconcile with --reconcile-features (e.g. --symbol XOM). Matched "
|
|
3493
|
+
"case-insensitively against each record's feature_snapshot keys.",
|
|
3494
|
+
)
|
|
3495
|
+
evaluate_parser.add_argument(
|
|
3496
|
+
"--reconcile-book-history", action="store_true",
|
|
3497
|
+
help="V5.2.2: compare a real Lean backtest's logged book selections "
|
|
3498
|
+
"(phase_v2.diagnostics.book_history) against a fresh offline re-derivation of the same raw "
|
|
3499
|
+
"scores on those same dates - true ground truth for diagnosing offline-vs-live divergence. "
|
|
3500
|
+
"Not included in --all - loads both models' predictions like --calibrate-book-spread.",
|
|
3501
|
+
)
|
|
3502
|
+
evaluate_parser.add_argument(
|
|
3503
|
+
"--book-history-path", default=None,
|
|
3504
|
+
help="Path to the book_history.jsonl log to reconcile with --reconcile-book-history "
|
|
3505
|
+
"(default: phase_v2.diagnostics.book_history.output_path from config.json).",
|
|
3506
|
+
)
|
|
3507
|
+
evaluate_parser.add_argument(
|
|
3508
|
+
"--replay-hysteresis", action="store_true",
|
|
3509
|
+
help="V5.2.3: with --reconcile-book-history, replay offline's own hysteresis-aware selection "
|
|
3510
|
+
"walk-forward across the log's dates (carrying held allocations forward, same as main.py's live "
|
|
3511
|
+
"book) instead of reconciling each date independently - tells a real divergence apart from the "
|
|
3512
|
+
"live book correctly holding an incumbent a from-scratch reselection wouldn't naturally pick.",
|
|
3513
|
+
)
|
|
3514
|
+
reconcile_run_group = evaluate_parser.add_mutually_exclusive_group()
|
|
3515
|
+
reconcile_run_group.add_argument(
|
|
3516
|
+
"--reconcile-run-index", type=int, default=None,
|
|
3517
|
+
help="V5.3.2: with --reconcile-book-history, reconcile only the run at this 0-indexed position "
|
|
3518
|
+
"within the log's own run-segmented history (negative indices count from the end, -1 == most "
|
|
3519
|
+
"recent - the default). book_history.jsonl is a cumulative, never-rotated log - every historical "
|
|
3520
|
+
"real backtest's records are appended forever - so reconciling without this flag defaults to the "
|
|
3521
|
+
"LATEST run only, never a silent cross-run merge (development/Problems.md #91/#97/#99).",
|
|
3522
|
+
)
|
|
3523
|
+
reconcile_run_group.add_argument(
|
|
3524
|
+
"--reconcile-all-runs", action="store_true",
|
|
3525
|
+
help="V5.3.2: with --reconcile-book-history, reconcile EVERY run segment independently (each its "
|
|
3526
|
+
"own held-allocations/summary, never merged across a run boundary) instead of just the most "
|
|
3527
|
+
"recent - for investigating an older run's own numbers, not for combining them into one figure.",
|
|
3528
|
+
)
|
|
3529
|
+
evaluate_parser.add_argument(
|
|
3530
|
+
"--replay-kill-switch", action="store_true",
|
|
3531
|
+
help="V5.2.8: day-by-day OFFLINE replay of the kill-switch + sticky trade-lock state machine "
|
|
3532
|
+
"(evaluation/kill_switch_replay.py) against the rank book's own simulated return series - an "
|
|
3533
|
+
"explicitly approximate estimate of how much of the run would have been locked out, without "
|
|
3534
|
+
"needing a real Lean backtest. Not included in --all - investigation-only, see development/"
|
|
3535
|
+
"Problems.md #94 for the caveats.",
|
|
3536
|
+
)
|
|
3537
|
+
evaluate_parser.add_argument(
|
|
3538
|
+
"--replay-rolling-ic-gate", action="store_true",
|
|
3539
|
+
help="V5.3.5 (development/Problems.md #102): day-by-day, no-lookahead OFFLINE replay of "
|
|
3540
|
+
"portfolio/rolling_ic_gate.py's engagement decision (evaluation/rolling_ic_gate_replay.py) at "
|
|
3541
|
+
"phase_v2.net_performance's rebalance cadence, with a per-era breakdown against the three known "
|
|
3542
|
+
"bad eras this round's research found - the critical pre-main.py checkpoint before flipping "
|
|
3543
|
+
"phase_v2.rolling_ic_gate.enabled live. Not included in --all - investigation-only.",
|
|
3544
|
+
)
|
|
3545
|
+
evaluate_parser.add_argument(
|
|
3546
|
+
"--simulate-limit-fills", action="store_true",
|
|
3547
|
+
help="V5.3.1: offline counterfactual (evaluation/limit_fill_simulator.py) estimating how often a "
|
|
3548
|
+
"real limit order would fill vs. time out, using the existing dataset's own high/low bars and "
|
|
3549
|
+
"phase_v2.limit_orders' pricing/timeout config - without needing a real Lean backtest. Not "
|
|
3550
|
+
"included in --all - investigation-only, see development/Problems.md #34/#96 for the caveats.",
|
|
3551
|
+
)
|
|
3552
|
+
evaluate_parser.add_argument(
|
|
3553
|
+
"--limit-fill-offset-sweep", default=None,
|
|
3554
|
+
help="Comma-separated offset_multiplier values to sweep with --simulate-limit-fills (default: a "
|
|
3555
|
+
"single run at the configured phase_v2.limit_orders.offset_multiplier).",
|
|
3556
|
+
)
|
|
3557
|
+
evaluate_parser.add_argument(
|
|
3558
|
+
"--monte-carlo", action="store_true",
|
|
3559
|
+
help="V5.3.8: Monte Carlo simulation testing layer (evaluation/monte_carlo.py) - stationary-block "
|
|
3560
|
+
"bootstraps the base rank-book run's daily net returns into --mc-runs alternative equity histories, "
|
|
3561
|
+
"writing ml/evaluation/monte_carlo.json + monte_carlo_curves.npz and refreshing the README's Monte "
|
|
3562
|
+
"Carlo section (curves chart + red average line + foldable deep stats). Requires/implies the base "
|
|
3563
|
+
"--rank-book simulation. Not included in --all.",
|
|
3564
|
+
)
|
|
3565
|
+
evaluate_parser.add_argument("--mc-runs", type=int, default=1000,
|
|
3566
|
+
help="Number of Monte Carlo resampled equity runs (default: 1000).")
|
|
3567
|
+
evaluate_parser.add_argument("--mc-block-size", type=int, default=20,
|
|
3568
|
+
help="Stationary block bootstrap block length in days (default: 20).")
|
|
3569
|
+
evaluate_parser.add_argument("--mc-seed", type=int, default=42,
|
|
3570
|
+
help="RNG seed - pinned by default so the README stays stable unless results change.")
|
|
3571
|
+
evaluate_parser.add_argument("--mc-method", choices=["block", "iid"], default="block",
|
|
3572
|
+
help="Resampling method (default: block = stationary block bootstrap).")
|
|
3573
|
+
evaluate_parser.add_argument(
|
|
3574
|
+
"--benchmarks", action="store_true",
|
|
3575
|
+
help="V5.4.4: benchmark comparison (evaluation/benchmark_comparison.py + "
|
|
3576
|
+
"evaluation/public_benchmarks.py) - runs the momentum/mean-reversion/random-entry strategy "
|
|
3577
|
+
"baselines AND the SP500 buy-and-hold + daily-rebalanced 60/40 SPY-TLT public baselines over "
|
|
3578
|
+
"the SAME dataset window/split as the rank book, writing ml/evaluation/benchmark_comparison.json "
|
|
3579
|
+
"and refreshing the README's Benchmark Comparison section. Included in --all.",
|
|
3580
|
+
)
|
|
3581
|
+
evaluate_parser.add_argument("--model", choices=["sequence", "multitask"], default=None, help="Default: sequence")
|
|
3582
|
+
evaluate_parser.add_argument("--head", default=None, help="Model head to evaluate, e.g. rank_20d/rank_5d (default: rank_20d)")
|
|
3583
|
+
evaluate_parser.add_argument("--split", default=None, help="Dataset split to evaluate: train/validation/backtest/all (default: backtest)")
|
|
3584
|
+
evaluate_parser.add_argument(
|
|
3585
|
+
"--preset", default=None,
|
|
3586
|
+
help="Overlay phase_v2.presets.<name> in memory only (never writes config.json) - see `aq config preset --list`",
|
|
3587
|
+
)
|
|
3588
|
+
evaluate_parser.add_argument("--json", action="store_true", help="Print the full report as JSON instead of a summary")
|
|
3589
|
+
evaluate_parser.set_defaults(func=cmd_evaluate)
|
|
3590
|
+
|
|
3591
|
+
render_lean_parser = subparsers.add_parser(
|
|
3592
|
+
"render-lean-config",
|
|
3593
|
+
help="Render gitignored lean.live.json from .env.live / AETHER_* env vars (secrets for live/paper)",
|
|
3594
|
+
)
|
|
3595
|
+
render_lean_parser.add_argument("--base", default=None, help="Empty tracked template (default: lean.json)")
|
|
3596
|
+
render_lean_parser.add_argument("--out", default=None, help="Rendered secret-bearing output (default: lean.live.json)")
|
|
3597
|
+
render_lean_parser.add_argument("--env-file", dest="env_file", default=None, help="Secrets file (default: .env.live)")
|
|
3598
|
+
render_lean_parser.set_defaults(func=cmd_render_lean_config)
|
|
3599
|
+
|
|
3600
|
+
secrets_check_parser = subparsers.add_parser(
|
|
3601
|
+
"secrets-check",
|
|
3602
|
+
help="Fail if a populated secret field in lean.json or a tracked .env is about to be committed (backs the pre-commit hook)",
|
|
3603
|
+
)
|
|
3604
|
+
secrets_check_parser.set_defaults(func=cmd_secrets_check)
|
|
3605
|
+
|
|
3606
|
+
audit_log_parser = subparsers.add_parser(
|
|
3607
|
+
"audit-log",
|
|
3608
|
+
help="Query the tamper-evident audit log (order placement, credential loads, live-mode transitions)",
|
|
3609
|
+
)
|
|
3610
|
+
audit_log_parser.add_argument("--event-type", dest="event_type", default=None,
|
|
3611
|
+
choices=["order_placement", "credential_load", "live_mode_transition"],
|
|
3612
|
+
help="Filter to one event type (default: all)")
|
|
3613
|
+
audit_log_parser.add_argument("--since", default=None, type=_iso_date, help="Only entries after this date, ISO 8601 YYYY-MM-DD")
|
|
3614
|
+
audit_log_parser.add_argument("--limit", type=int, default=100, help="Max entries to show (default: 100)")
|
|
3615
|
+
audit_log_parser.add_argument("--verify", action="store_true", help="Walk the whole hash chain and report the first break, if any")
|
|
3616
|
+
audit_log_parser.set_defaults(func=cmd_audit_log)
|
|
3617
|
+
|
|
3618
|
+
status_parser = subparsers.add_parser("status", help="Show git status")
|
|
3619
|
+
status_parser.set_defaults(func=cmd_status)
|
|
3620
|
+
|
|
3621
|
+
return parser
|
|
3622
|
+
|
|
3623
|
+
|
|
3624
|
+
def main() -> None:
|
|
3625
|
+
parser = build_parser()
|
|
3626
|
+
args = parser.parse_args()
|
|
3627
|
+
exit_code = args.func(args)
|
|
3628
|
+
check_for_update()
|
|
3629
|
+
sys.exit(exit_code)
|
|
3630
|
+
|
|
3631
|
+
|
|
3632
|
+
if __name__ == "__main__":
|
|
3633
|
+
main()
|