aether-quant 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aether_quant-0.0.1.dist-info/METADATA +7 -0
- aether_quant-0.0.1.dist-info/RECORD +99 -0
- aether_quant-0.0.1.dist-info/WHEEL +5 -0
- aether_quant-0.0.1.dist-info/entry_points.txt +2 -0
- aether_quant-0.0.1.dist-info/licenses/LICENSE +133 -0
- aether_quant-0.0.1.dist-info/top_level.txt +13 -0
- analyzer/README.md +69 -0
- analyzer/__init__.py +17 -0
- analyzer/market_analyzer.py +521 -0
- aq_cli.py +3633 -0
- audit/README.md +47 -0
- audit/__init__.py +38 -0
- audit/hash_chain.py +52 -0
- audit/postgres_audit.py +61 -0
- audit/postgres_worker.py +402 -0
- audit/redis_queue.py +124 -0
- audit/status_export.py +62 -0
- data_pipeline/README.md +257 -0
- data_pipeline/__init__.py +6 -0
- data_pipeline/bar_synthesis.py +97 -0
- data_pipeline/dividend_backfill.py +243 -0
- data_pipeline/factor_file_backfill.py +276 -0
- data_pipeline/fetch.py +206 -0
- data_pipeline/fred_backfill.py +435 -0
- data_pipeline/ib_backfill.py +287 -0
- data_pipeline/v2_manifest.py +75 -0
- data_pipeline/yfinance_backfill.py +501 -0
- evaluation/README.md +152 -0
- evaluation/__init__.py +73 -0
- evaluation/ablation.py +310 -0
- evaluation/benchmark_comparison.py +195 -0
- evaluation/confidence_threshold_calibration.py +157 -0
- evaluation/factor_neutralization_check.py +94 -0
- evaluation/feature_parity.py +321 -0
- evaluation/feature_reconciliation.py +286 -0
- evaluation/impact_model.py +77 -0
- evaluation/kill_switch_replay.py +186 -0
- evaluation/limit_fill_simulator.py +152 -0
- evaluation/model_predictions.py +192 -0
- evaluation/monte_carlo.py +161 -0
- evaluation/public_benchmarks.py +117 -0
- evaluation/rank_book_simulator.py +494 -0
- evaluation/rank_ic_core.py +204 -0
- evaluation/rank_signal_calibration.py +690 -0
- evaluation/rolling_ic_gate_calibration.py +190 -0
- evaluation/rolling_ic_gate_replay.py +138 -0
- execution/README.md +281 -0
- execution/__init__.py +80 -0
- execution/config_cache.py +48 -0
- execution/cost_model.py +213 -0
- execution/lean_config_render.py +131 -0
- execution/live_credentials.py +38 -0
- execution/live_credentials_io.py +52 -0
- execution/order_gate.py +388 -0
- execution/paper_readiness.py +181 -0
- execution/paper_readiness_io.py +54 -0
- execution/paper_readiness_report.py +103 -0
- execution/paper_readiness_scheduler.py +137 -0
- execution/reconciliation.py +160 -0
- execution/runtime_config_io.py +38 -0
- execution/secret_scan.py +73 -0
- features/README.md +77 -0
- features/__init__.py +137 -0
- features/alt_data_features.py +154 -0
- features/bond_features.py +275 -0
- features/cross_asset_sensitivity.py +79 -0
- features/derivatives_macro_features.py +72 -0
- features/macro_features.py +79 -0
- features/options_greeks.py +329 -0
- features/sector_map.py +53 -0
- features/technical_indicators.py +185 -0
- generate_backtest_report.py +311 -0
- generate_evaluation_report.py +712 -0
- inference/README.md +105 -0
- inference/__init__.py +36 -0
- inference/exported_model.py +929 -0
- inference/parallel_inference.py +160 -0
- inference/strategy_selector_inference.py +75 -0
- json_safety.py +130 -0
- portfolio/README.md +447 -0
- portfolio/__init__.py +41 -0
- portfolio/book_construction.py +786 -0
- portfolio/book_neutrality.py +178 -0
- portfolio/hrp_allocation.py +159 -0
- portfolio/options_arbitrage_detector.py +232 -0
- portfolio/options_assignment_risk.py +119 -0
- portfolio/options_margin_sizing.py +333 -0
- portfolio/options_strategy.py +1999 -0
- portfolio/rank_signal.py +219 -0
- portfolio/rolling_ic_gate.py +183 -0
- risk/README.md +275 -0
- risk/__init__.py +38 -0
- risk/asset_class_router.py +584 -0
- risk/forex_risk.py +158 -0
- risk/futures_risk.py +202 -0
- risk/kill_switch.py +261 -0
- risk/manual_override.py +118 -0
- risk/position_sizing.py +303 -0
- risk/rl_sizing.py +189 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Offline paper-trading readiness report (Phase V2-21).
|
|
2
|
+
|
|
3
|
+
main.py cannot compute this itself - it never opens its own Postgres
|
|
4
|
+
connection (same invariant retraining/status_export.py's docstring
|
|
5
|
+
describes). This module is the sole writer of
|
|
6
|
+
visualization/grafana/paper_readiness_report.json; monitoring/api_server.py
|
|
7
|
+
merges that file into /api/state server-side, and `aq paper-readiness`
|
|
8
|
+
(aq_cli.py) runs this as a human-invoked gate before flipping
|
|
9
|
+
phase_v2.runtime.mode to "paper".
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
|
|
16
|
+
from json_safety import atomic_write_json
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from execution.paper_readiness import evaluate_observation_readiness, evaluate_paper_broker_config
|
|
24
|
+
from execution.paper_readiness_io import fetch_observation_mode_events
|
|
25
|
+
from experience.observation_metrics import compute_observation_summary
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
ROOT_DIR = Path(__file__).resolve().parent.parent
|
|
30
|
+
DEFAULT_CONFIG_PATH = ROOT_DIR / "config.json"
|
|
31
|
+
DEFAULT_REPORT_PATH = ROOT_DIR / "visualization" / "grafana" / "paper_readiness_report.json"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_paper_readiness_view(conn, config: dict) -> dict:
|
|
35
|
+
"""Returns the full paper_readiness_report.json payload.
|
|
36
|
+
|
|
37
|
+
{generated_at, ready, checks, blocking_reasons, broker_config_present,
|
|
38
|
+
broker_config_reason, observation_summary}
|
|
39
|
+
"""
|
|
40
|
+
paper_trading_config = config.get("phase_v2", {}).get("paper_trading", {})
|
|
41
|
+
thresholds = paper_trading_config.get("readiness_thresholds", {})
|
|
42
|
+
|
|
43
|
+
events = fetch_observation_mode_events(conn)
|
|
44
|
+
summary = compute_observation_summary(events)
|
|
45
|
+
observation_readiness = evaluate_observation_readiness(summary, thresholds)
|
|
46
|
+
broker_config_present, broker_config_reason = evaluate_paper_broker_config(paper_trading_config)
|
|
47
|
+
|
|
48
|
+
blocking_reasons = list(observation_readiness["blocking_reasons"])
|
|
49
|
+
if not broker_config_present:
|
|
50
|
+
blocking_reasons.append(broker_config_reason)
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
54
|
+
"ready": observation_readiness["ready"] and broker_config_present,
|
|
55
|
+
"checks": observation_readiness["checks"],
|
|
56
|
+
"blocking_reasons": blocking_reasons,
|
|
57
|
+
"broker_config_present": broker_config_present,
|
|
58
|
+
"broker_config_reason": broker_config_reason,
|
|
59
|
+
"observation_summary": summary,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def write_paper_readiness_file(view: dict, path: Path = DEFAULT_REPORT_PATH) -> None:
|
|
64
|
+
# V5.4.7 (Problems.md #120): atomic + finite-safe.
|
|
65
|
+
atomic_write_json(path, view, indent=2)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _print_summary(view: dict) -> None:
|
|
69
|
+
status = "READY" if view["ready"] else "NOT READY"
|
|
70
|
+
print(f"Paper trading readiness: {status}")
|
|
71
|
+
for name, check in view["checks"].items():
|
|
72
|
+
mark = "PASS" if check["pass"] else "FAIL"
|
|
73
|
+
print(f" [{mark}] {name}: value={check['value']} threshold={check['threshold']}")
|
|
74
|
+
broker_mark = "PASS" if view["broker_config_present"] else "FAIL"
|
|
75
|
+
print(f" [{broker_mark}] broker_config: {view['broker_config_reason']}")
|
|
76
|
+
if view["blocking_reasons"]:
|
|
77
|
+
print("Blocking reasons: " + ", ".join(view["blocking_reasons"]))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def main() -> int:
|
|
81
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s — %(message)s")
|
|
82
|
+
|
|
83
|
+
with DEFAULT_CONFIG_PATH.open("r", encoding="utf-8") as f:
|
|
84
|
+
config = json.load(f)
|
|
85
|
+
|
|
86
|
+
import psycopg
|
|
87
|
+
|
|
88
|
+
dsn = os.environ.get("AETHER_POSTGRES_DSN", "")
|
|
89
|
+
conn = psycopg.connect(dsn, autocommit=False)
|
|
90
|
+
try:
|
|
91
|
+
view = build_paper_readiness_view(conn, config)
|
|
92
|
+
finally:
|
|
93
|
+
conn.close()
|
|
94
|
+
|
|
95
|
+
write_paper_readiness_file(view)
|
|
96
|
+
_print_summary(view)
|
|
97
|
+
return 0 if view["ready"] else 1
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
if __name__ == "__main__":
|
|
101
|
+
sys.exit(main())
|
|
102
|
+
|
|
103
|
+
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Scheduled paper-trading readiness reporting (Phase 7 of the 5/10 -> 9/10
|
|
2
|
+
roadmap).
|
|
3
|
+
|
|
4
|
+
execution/paper_readiness_report.py's evaluation logic and dashboard wiring
|
|
5
|
+
(monitoring/api_server.py already merges visualization/grafana/
|
|
6
|
+
paper_readiness_report.json into /api/state, with a dedicated
|
|
7
|
+
get_paper_readiness() endpoint) were already correct and already
|
|
8
|
+
dashboard-visible before this module existed - the one real gap this
|
|
9
|
+
closes is CADENCE: previously the report only regenerated when a human ran
|
|
10
|
+
`aq paper-readiness` by hand, so the dashboard tile could silently go
|
|
11
|
+
stale between manual runs. This wraps the exact same
|
|
12
|
+
build_paper_readiness_view()/write_paper_readiness_file() calls in a
|
|
13
|
+
periodic loop, mirroring performance/trigger_worker.py::TriggerWorker's
|
|
14
|
+
shape (sync-only, DSN resolution via AETHER_POSTGRES_DSN, --once CLI flag,
|
|
15
|
+
_pg_conn constructor injection for tests) - purely additive reporting.
|
|
16
|
+
|
|
17
|
+
Hard boundary (Phase 7's scope, per the user's explicit instruction): this
|
|
18
|
+
module NEVER touches phase_v2.paper_trading's config flags
|
|
19
|
+
(live_data_provider_configured/manual_review_confirmed) and never changes
|
|
20
|
+
main.py's actual order-routing behavior - it only keeps the readiness
|
|
21
|
+
REPORT fresh. See execution/README.md's "Activating real paper-trading
|
|
22
|
+
fills — manual step" section for what a human must still do to actually
|
|
23
|
+
go live.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import argparse
|
|
29
|
+
import json
|
|
30
|
+
import logging
|
|
31
|
+
import os
|
|
32
|
+
import time
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
from execution.paper_readiness_report import (
|
|
36
|
+
DEFAULT_CONFIG_PATH,
|
|
37
|
+
DEFAULT_REPORT_PATH,
|
|
38
|
+
build_paper_readiness_view,
|
|
39
|
+
write_paper_readiness_file,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger(__name__)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class PaperReadinessScheduler:
|
|
46
|
+
"""Periodically regenerates paper_readiness_report.json.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
postgres_dsn : psycopg3 DSN (overridden by AETHER_POSTGRES_DSN env)
|
|
51
|
+
config : full config.json dict (build_paper_readiness_view()
|
|
52
|
+
reads phase_v2.paper_trading from it directly)
|
|
53
|
+
poll_interval : seconds to sleep between regenerations in run()
|
|
54
|
+
report_path : where to write the report (test injection point)
|
|
55
|
+
_pg_conn : injected psycopg3 connection (skips real connection — tests only)
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
*,
|
|
61
|
+
postgres_dsn: str = "",
|
|
62
|
+
config: dict,
|
|
63
|
+
poll_interval: int = 3600,
|
|
64
|
+
report_path: Path = DEFAULT_REPORT_PATH,
|
|
65
|
+
_pg_conn=None,
|
|
66
|
+
) -> None:
|
|
67
|
+
self.config = config
|
|
68
|
+
self.poll_interval = poll_interval
|
|
69
|
+
self.report_path = report_path
|
|
70
|
+
|
|
71
|
+
if _pg_conn is not None:
|
|
72
|
+
self._conn = _pg_conn
|
|
73
|
+
self._owns_connection = False
|
|
74
|
+
else:
|
|
75
|
+
import psycopg
|
|
76
|
+
|
|
77
|
+
dsn = os.environ.get("AETHER_POSTGRES_DSN", postgres_dsn)
|
|
78
|
+
self._conn = psycopg.connect(dsn, autocommit=False)
|
|
79
|
+
self._owns_connection = True
|
|
80
|
+
logger.info("PaperReadinessScheduler: PostgreSQL connected.")
|
|
81
|
+
|
|
82
|
+
def run_once(self) -> dict:
|
|
83
|
+
"""Regenerates the report once. Returns the written view dict."""
|
|
84
|
+
view = build_paper_readiness_view(self._conn, self.config)
|
|
85
|
+
write_paper_readiness_file(view, path=self.report_path)
|
|
86
|
+
logger.info(
|
|
87
|
+
"PaperReadinessScheduler: report regenerated - ready=%s, blocking_reasons=%s.",
|
|
88
|
+
view["ready"],
|
|
89
|
+
view["blocking_reasons"],
|
|
90
|
+
)
|
|
91
|
+
return view
|
|
92
|
+
|
|
93
|
+
def run(self) -> None:
|
|
94
|
+
logger.info("PaperReadinessScheduler: entering run loop (interval=%ds).", self.poll_interval)
|
|
95
|
+
while True:
|
|
96
|
+
try:
|
|
97
|
+
self.run_once()
|
|
98
|
+
time.sleep(self.poll_interval)
|
|
99
|
+
except KeyboardInterrupt:
|
|
100
|
+
logger.info("PaperReadinessScheduler: shutdown requested.")
|
|
101
|
+
break
|
|
102
|
+
except Exception as exc:
|
|
103
|
+
logger.error("PaperReadinessScheduler error — %s. Retrying in %ds.", exc, self.poll_interval)
|
|
104
|
+
time.sleep(self.poll_interval)
|
|
105
|
+
|
|
106
|
+
def close(self) -> None:
|
|
107
|
+
if not self._owns_connection:
|
|
108
|
+
return
|
|
109
|
+
try:
|
|
110
|
+
self._conn.close()
|
|
111
|
+
except Exception:
|
|
112
|
+
pass
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def main() -> None:
|
|
116
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s — %(message)s")
|
|
117
|
+
parser = argparse.ArgumentParser(description="Aether Quant paper-trading readiness scheduler")
|
|
118
|
+
parser.add_argument("--once", action="store_true", help="Regenerate the report once and exit")
|
|
119
|
+
parser.add_argument("--poll-interval", type=int, default=3600, help="Seconds between regenerations")
|
|
120
|
+
args = parser.parse_args()
|
|
121
|
+
|
|
122
|
+
with DEFAULT_CONFIG_PATH.open("r", encoding="utf-8") as f:
|
|
123
|
+
config = json.load(f)
|
|
124
|
+
|
|
125
|
+
postgres_dsn = os.environ.get("AETHER_POSTGRES_DSN", "")
|
|
126
|
+
scheduler = PaperReadinessScheduler(postgres_dsn=postgres_dsn, config=config, poll_interval=args.poll_interval)
|
|
127
|
+
try:
|
|
128
|
+
if args.once:
|
|
129
|
+
scheduler.run_once()
|
|
130
|
+
else:
|
|
131
|
+
scheduler.run()
|
|
132
|
+
finally:
|
|
133
|
+
scheduler.close()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
if __name__ == "__main__":
|
|
137
|
+
main()
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""V5.1 Phase 6 (production safety): position reconciliation - compares
|
|
2
|
+
the book's INTENDED per-symbol weights against what the broker (Lean's
|
|
3
|
+
own `self.Portfolio`, or an IB snapshot once Problems.md #72 clears)
|
|
4
|
+
actually holds, and classifies every discrepancy.
|
|
5
|
+
|
|
6
|
+
Pure, Lean-free - no QCAlgorithm/self.Portfolio dependency here at all, so
|
|
7
|
+
this is fully unit-testable with plain dicts (same "compute the math in a
|
|
8
|
+
pure module, let the live caller supply real data" split every other
|
|
9
|
+
risk/execution module in this codebase already follows) and directly
|
|
10
|
+
reusable against a future IB portfolio snapshot with zero changes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import math
|
|
16
|
+
from dataclasses import asdict, dataclass, field
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class ReconciliationReport:
|
|
21
|
+
matched: list[str] = field(default_factory=list)
|
|
22
|
+
drifted: list[dict] = field(default_factory=list)
|
|
23
|
+
orphan_broker: list[dict] = field(default_factory=list)
|
|
24
|
+
missing_broker: list[dict] = field(default_factory=list)
|
|
25
|
+
max_abs_weight_drift: float = 0.0
|
|
26
|
+
breach: bool = False
|
|
27
|
+
|
|
28
|
+
def to_dict(self) -> dict:
|
|
29
|
+
return asdict(self)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def reconcile_positions(
|
|
33
|
+
expected_by_symbol: dict[str, float],
|
|
34
|
+
actual_by_symbol: dict[str, float],
|
|
35
|
+
*,
|
|
36
|
+
weight_tolerance: float,
|
|
37
|
+
value_tolerance_usd: float,
|
|
38
|
+
portfolio_value: float,
|
|
39
|
+
max_tolerated_drift: float | None = None,
|
|
40
|
+
) -> ReconciliationReport:
|
|
41
|
+
"""Classifies every symbol appearing in EITHER dict:
|
|
42
|
+
- "matched": both expected and actual are present and within
|
|
43
|
+
weight_tolerance (as a fraction of portfolio weight) AND within
|
|
44
|
+
value_tolerance_usd (the dollar-value equivalent of that weight
|
|
45
|
+
difference) - both must hold, since a large-value portfolio can
|
|
46
|
+
have a tiny weight_tolerance breach that is still a huge dollar
|
|
47
|
+
drift, and a small-value portfolio can have a huge weight_tolerance
|
|
48
|
+
breach that is a trivial dollar amount.
|
|
49
|
+
- "drifted": both present, but outside tolerance - full per-symbol
|
|
50
|
+
delta detail (expected_weight, actual_weight, delta_weight,
|
|
51
|
+
delta_usd).
|
|
52
|
+
- "orphan_broker": held by the broker but not in the expected book at
|
|
53
|
+
all (expected_weight treated as 0.0) - e.g. a position main.py's
|
|
54
|
+
book no longer wants but the broker never closed.
|
|
55
|
+
- "missing_broker": expected by the book but the broker holds nothing
|
|
56
|
+
for it (actual_weight treated as 0.0) - e.g. an order that never
|
|
57
|
+
filled.
|
|
58
|
+
|
|
59
|
+
max_abs_weight_drift is the single largest |delta_weight| across every
|
|
60
|
+
NON-matched symbol (drifted + orphan_broker + missing_broker combined -
|
|
61
|
+
a symbol classified "matched" is, by construction, already within both
|
|
62
|
+
tolerances, so including it could only ever pull the max down toward a
|
|
63
|
+
value the genuine drifts already exceed - excluding it keeps this
|
|
64
|
+
field's value exactly matching what its name and this docstring claim).
|
|
65
|
+
breach is that value compared against max_tolerated_drift (typically
|
|
66
|
+
phase_v2.reconciliation.max_tolerated_drift) - the one field a caller
|
|
67
|
+
checks to decide whether this reconciliation should trip the kill
|
|
68
|
+
switch (risk/kill_switch.py's "reconciliation_breach" trigger) without
|
|
69
|
+
re-deriving the comparison itself. max_tolerated_drift=None (the
|
|
70
|
+
default) always reports breach=False - an unconfigured caller gets a
|
|
71
|
+
strict no-op, matching every other fail-open contract in this codebase.
|
|
72
|
+
|
|
73
|
+
Tolerance is checked BEFORE presence: a symbol held/expected only on
|
|
74
|
+
one side but within weight_tolerance/value_tolerance_usd of the other
|
|
75
|
+
side's implicit 0.0 (a dust position - a fractional share leftover, a
|
|
76
|
+
rounding remainder) is "matched", never "orphan_broker"/"missing_broker".
|
|
77
|
+
Without this, every dust amount was a permanent, unfixable "orphan" no
|
|
78
|
+
matter how negligible, even though it could never move max_abs_weight_drift
|
|
79
|
+
past a real drift either - pure report-quality noise.
|
|
80
|
+
|
|
81
|
+
Pure dict-in/dict-out - result does not depend on either dict's
|
|
82
|
+
insertion order (both are iterated via a sorted union of keys). Empty
|
|
83
|
+
inputs (both dicts empty) return an all-empty, non-breaching report,
|
|
84
|
+
never raise.
|
|
85
|
+
|
|
86
|
+
V5.4.5 (development/Problems.md #117): a non-finite weight on either
|
|
87
|
+
side previously slipped through every comparison silently - `abs(nan)
|
|
88
|
+
<= tol` is False (so the symbol still classified drifted/orphan), but
|
|
89
|
+
`max(0.0, abs(nan))` kept 0.0, leaving max_abs_weight_drift at 0.0 and
|
|
90
|
+
breach False: a real blow-up reported as NaN was a false-clean. A
|
|
91
|
+
non-finite weight now forces max_abs_weight_drift to +inf, so any
|
|
92
|
+
configured max_tolerated_drift reports breach=True (fail-closed for a
|
|
93
|
+
safety trigger). An explicit None value no longer raises float(None)
|
|
94
|
+
either - it degrades through this same non-finite path."""
|
|
95
|
+
all_symbols = sorted(set(expected_by_symbol) | set(actual_by_symbol))
|
|
96
|
+
|
|
97
|
+
def _finite_weight(value) -> float | None:
|
|
98
|
+
if value is None:
|
|
99
|
+
return None
|
|
100
|
+
try:
|
|
101
|
+
as_float = float(value)
|
|
102
|
+
except (TypeError, ValueError):
|
|
103
|
+
return None
|
|
104
|
+
return as_float if math.isfinite(as_float) else None
|
|
105
|
+
|
|
106
|
+
matched: list[str] = []
|
|
107
|
+
drifted: list[dict] = []
|
|
108
|
+
orphan_broker: list[dict] = []
|
|
109
|
+
missing_broker: list[dict] = []
|
|
110
|
+
max_abs_weight_drift = 0.0
|
|
111
|
+
|
|
112
|
+
for symbol in all_symbols:
|
|
113
|
+
expected_weight = _finite_weight(expected_by_symbol.get(symbol, 0.0))
|
|
114
|
+
actual_weight = _finite_weight(actual_by_symbol.get(symbol, 0.0))
|
|
115
|
+
if expected_weight is None or actual_weight is None:
|
|
116
|
+
# Unreadable weight: count it as infinite drift (breach-worthy)
|
|
117
|
+
# rather than silently clean.
|
|
118
|
+
max_abs_weight_drift = math.inf
|
|
119
|
+
delta_usd: float | None = None
|
|
120
|
+
drifted.append(
|
|
121
|
+
{"symbol": symbol, "expected_weight": expected_weight, "actual_weight": actual_weight,
|
|
122
|
+
"delta_weight": None, "delta_usd": None}
|
|
123
|
+
)
|
|
124
|
+
continue
|
|
125
|
+
delta_weight = actual_weight - expected_weight
|
|
126
|
+
delta_usd = delta_weight * portfolio_value
|
|
127
|
+
|
|
128
|
+
within_tolerance = abs(delta_weight) <= weight_tolerance and abs(delta_usd) <= value_tolerance_usd
|
|
129
|
+
|
|
130
|
+
if within_tolerance:
|
|
131
|
+
matched.append(symbol)
|
|
132
|
+
elif symbol not in expected_by_symbol:
|
|
133
|
+
max_abs_weight_drift = max(max_abs_weight_drift, abs(delta_weight))
|
|
134
|
+
orphan_broker.append(
|
|
135
|
+
{"symbol": symbol, "expected_weight": 0.0, "actual_weight": actual_weight,
|
|
136
|
+
"delta_weight": delta_weight, "delta_usd": delta_usd}
|
|
137
|
+
)
|
|
138
|
+
elif symbol not in actual_by_symbol:
|
|
139
|
+
max_abs_weight_drift = max(max_abs_weight_drift, abs(delta_weight))
|
|
140
|
+
missing_broker.append(
|
|
141
|
+
{"symbol": symbol, "expected_weight": expected_weight, "actual_weight": 0.0,
|
|
142
|
+
"delta_weight": delta_weight, "delta_usd": delta_usd}
|
|
143
|
+
)
|
|
144
|
+
else:
|
|
145
|
+
max_abs_weight_drift = max(max_abs_weight_drift, abs(delta_weight))
|
|
146
|
+
drifted.append(
|
|
147
|
+
{"symbol": symbol, "expected_weight": expected_weight, "actual_weight": actual_weight,
|
|
148
|
+
"delta_weight": delta_weight, "delta_usd": delta_usd}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
breach = max_tolerated_drift is not None and max_abs_weight_drift > max_tolerated_drift
|
|
152
|
+
|
|
153
|
+
return ReconciliationReport(
|
|
154
|
+
matched=matched,
|
|
155
|
+
drifted=drifted,
|
|
156
|
+
orphan_broker=orphan_broker,
|
|
157
|
+
missing_broker=missing_broker,
|
|
158
|
+
max_abs_weight_drift=max_abs_weight_drift,
|
|
159
|
+
breach=breach,
|
|
160
|
+
)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""IO layer for reading phase_v2.runtime.mode from config.json.
|
|
2
|
+
|
|
3
|
+
Mirrors risk/manual_override.py::read_manual_trade_lock_override()'s
|
|
4
|
+
pattern exactly, including the mtime-gated cache (see
|
|
5
|
+
execution/config_cache.py). Used by retraining/worker.py's
|
|
6
|
+
auto-promote-in-live safety net (Phase V2-22), whose poll loop calls this
|
|
7
|
+
far more often than config.json actually changes - a process outside
|
|
8
|
+
main.py (which has its own copy of this same read, see main.py's
|
|
9
|
+
runtime_mode resolution in initialize()/_refresh_risk_state()) needs its
|
|
10
|
+
own read since it never shares main.py's in-memory state.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from execution.config_cache import read_cached
|
|
19
|
+
from execution.order_gate import resolve_runtime_mode
|
|
20
|
+
|
|
21
|
+
_PHASE_V2_KEY = "phase_v2"
|
|
22
|
+
_RUNTIME_KEY = "runtime"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def read_runtime_mode(config_path: Path) -> str:
|
|
26
|
+
"""Mtime-gated read of phase_v2.runtime.mode, normalized via
|
|
27
|
+
resolve_runtime_mode() (fails safe to 'observation' if the file is
|
|
28
|
+
missing or the value is absent/unknown) - never raises."""
|
|
29
|
+
return read_cached(config_path, _read_runtime_mode_uncached)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _read_runtime_mode_uncached(config_path: Path) -> str:
|
|
33
|
+
if not config_path.exists():
|
|
34
|
+
return resolve_runtime_mode(None)
|
|
35
|
+
with config_path.open("r", encoding="utf-8") as f:
|
|
36
|
+
config = json.load(f)
|
|
37
|
+
raw_mode = config.get(_PHASE_V2_KEY, {}).get(_RUNTIME_KEY, {}).get("mode")
|
|
38
|
+
return resolve_runtime_mode(raw_mode)
|
execution/secret_scan.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Detect secrets that must never be committed (Phase V2-22 security guard).
|
|
2
|
+
|
|
3
|
+
Backs `aq secrets-check` and the `.githooks/pre-commit` hook. Two independent
|
|
4
|
+
checks, both pure over their inputs so they are trivially testable:
|
|
5
|
+
|
|
6
|
+
1. `find_populated_secret_fields()` - flags any field in a parsed `lean.json`
|
|
7
|
+
whose NAME looks like a credential (`*-api-key`, `*-password`, `ib-account`,
|
|
8
|
+
…) and whose VALUE is non-empty. The tracked `lean.json` is meant to ship
|
|
9
|
+
with every such field empty (see execution/lean_config_render.py); a
|
|
10
|
+
populated one means real keys are about to be committed.
|
|
11
|
+
2. `is_tracked_env_secret()` - flags a `.env`-style filename that is NOT one of
|
|
12
|
+
the committed `*.example` templates.
|
|
13
|
+
|
|
14
|
+
Nothing here touches disk or git - the IO/glue (reading files, asking git what
|
|
15
|
+
is tracked) lives in aq_cli.py so this module stays pure and unit-testable.
|
|
16
|
+
Only field NAMES are ever returned, never the secret values.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
# A lean.json key is treated as a secret if it ends with any of these. Covers
|
|
22
|
+
# the whole stock template's credential surface (api keys/secrets, passwords,
|
|
23
|
+
# access/refresh tokens, private keys) plus the IB identity fields, which are
|
|
24
|
+
# not "*-password" suffixed but still must not be published.
|
|
25
|
+
SECRET_FIELD_SUFFIXES: tuple[str, ...] = (
|
|
26
|
+
"-api-key",
|
|
27
|
+
"-api-secret",
|
|
28
|
+
"-secret",
|
|
29
|
+
"-password",
|
|
30
|
+
"-access-token",
|
|
31
|
+
"-refresh-token",
|
|
32
|
+
"-auth-token",
|
|
33
|
+
"-shared-key",
|
|
34
|
+
"-private-key-hex",
|
|
35
|
+
"-session-password",
|
|
36
|
+
"-rest-app-key",
|
|
37
|
+
"-rest-app-secret",
|
|
38
|
+
"-app-key",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# Exact-match secret fields that don't follow the suffix pattern above.
|
|
42
|
+
SECRET_FIELD_EXACT: frozenset[str] = frozenset(
|
|
43
|
+
{
|
|
44
|
+
"ib-account",
|
|
45
|
+
"ib-user-name",
|
|
46
|
+
"ib-password",
|
|
47
|
+
}
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def is_secret_field(field: str) -> bool:
|
|
52
|
+
return field in SECRET_FIELD_EXACT or field.endswith(SECRET_FIELD_SUFFIXES)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def find_populated_secret_fields(lean_config: dict) -> list[str]:
|
|
56
|
+
"""Return the sorted names of every secret-looking lean.json field that has
|
|
57
|
+
a non-empty string value. Empty template -> empty list."""
|
|
58
|
+
populated = [
|
|
59
|
+
field
|
|
60
|
+
for field, value in lean_config.items()
|
|
61
|
+
if is_secret_field(field) and isinstance(value, str) and value.strip()
|
|
62
|
+
]
|
|
63
|
+
return sorted(populated)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def is_tracked_env_secret(filename: str) -> bool:
|
|
67
|
+
"""True if `filename` is a real `.env` secret file (must not be committed)
|
|
68
|
+
rather than one of the committed `*.example` templates. Accepts a bare
|
|
69
|
+
filename or a path - only the final component matters."""
|
|
70
|
+
name = filename.replace("\\", "/").rsplit("/", 1)[-1]
|
|
71
|
+
if name.endswith(".example"):
|
|
72
|
+
return False
|
|
73
|
+
return name == ".env" or name.startswith(".env.")
|
features/README.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# features
|
|
2
|
+
|
|
3
|
+
Pure, dependency-light feature-computation functions shared between the
|
|
4
|
+
offline training pipeline (`train.py`) and the Lean runtime (`main.py`) —
|
|
5
|
+
every function here is called from both places for train/inference parity
|
|
6
|
+
by construction, never re-derived independently in either one.
|
|
7
|
+
|
|
8
|
+
- `technical_indicators.py` — RSI, ATR%, Bollinger %B, volume z-score,
|
|
9
|
+
MACD histogram, distance-from-52w-high, cross-sectional momentum rank.
|
|
10
|
+
- `macro_features.py` — cross-asset "macro" signals derived from bond-ETF/
|
|
11
|
+
crypto price momentum (yield-curve slope proxy, credit spread proxy,
|
|
12
|
+
crypto risk appetite proxy), computed once per date/bar and broadcast
|
|
13
|
+
identically to every asset's model input.
|
|
14
|
+
- `bond_features.py` — the real-data sibling of `macro_features.py`'s
|
|
15
|
+
proxies: actual FRED Treasury-yield/credit-spread series
|
|
16
|
+
(`data_pipeline/fred_backfill.py`, no API key required) instead of
|
|
17
|
+
price-momentum proxies, plus an empirically-regressed per-ETF duration
|
|
18
|
+
beta (`empirical_duration_beta()`). All 5 features broadcast to every
|
|
19
|
+
asset, not just bonds — the mechanism making real yield-curve signal
|
|
20
|
+
usable for equity/crypto predictions too.
|
|
21
|
+
- `options_greeks.py` — real Black-Scholes-Merton pricing, greeks (delta/
|
|
22
|
+
gamma/theta/vega/rho), and implied volatility (safeguarded Newton-
|
|
23
|
+
Raphson + bisection). Verified against Hull's textbook example and
|
|
24
|
+
put-call parity (`tests/test_options_greeks.py`). Consumed by
|
|
25
|
+
`portfolio/options_strategy.py` for greeks-sized options position
|
|
26
|
+
construction — see `risk/README.md`/`portfolio/README.md`.
|
|
27
|
+
- `alt_data_features.py` (Phase 4.12, `development/Problems.md` #71) —
|
|
28
|
+
options-implied volatility level/term-structure (`VIXCLS`/`VXVCLS`, via
|
|
29
|
+
`data_pipeline/fred_backfill.py`, same no-API-key FRED endpoint as the
|
|
30
|
+
bond series) and a 4-week financial-conditions change (`NFCI`),
|
|
31
|
+
broadcast to every asset like `bond_features.py`/`macro_features.py`
|
|
32
|
+
above. Added specifically to target the COVID-crash era in the
|
|
33
|
+
cross-sectional rank-IC promotion gate's era-sign-instability diagnosis
|
|
34
|
+
— VIX *is* options-implied volatility, the most direct available
|
|
35
|
+
discriminator of that regime. Publication-lag-aware (`series_value_asof()`/
|
|
36
|
+
`series_change_asof()` in `data_pipeline/fred_backfill.py` — NFCI is
|
|
37
|
+
Friday-dated but not released until the following Wednesday; a naive
|
|
38
|
+
same-day join would be lookahead), with a dedicated test asserting a
|
|
39
|
+
feature value at date T only ever uses observations dated ≤ T.
|
|
40
|
+
- `derivatives_macro_features.py` — futures term-structure slope and
|
|
41
|
+
options put/call-ratio (a bounded skew, not a raw ratio)/IV-skew,
|
|
42
|
+
broadcast to every asset like the macro/bond features above. Fully
|
|
43
|
+
wired into both pipelines but always neutral (`0.0`) until
|
|
44
|
+
`config.json`'s universe contains futures/options assets shaped for a
|
|
45
|
+
real front/next-month or chain-aggregate lookup — see
|
|
46
|
+
`train.py::build_derivatives_macro_features_by_date()`'s docstring and
|
|
47
|
+
`development/Problems.md` #29.
|
|
48
|
+
- `cross_asset_sensitivity.py` (V5.1 Phase 2/F2) — per-asset macro
|
|
49
|
+
**sensitivity betas** (rolling regression of each asset's return against
|
|
50
|
+
ΔVIX/Δreal-rate/Δcredit/Δdollar) so macro actually varies
|
|
51
|
+
cross-sectionally instead of shifting every asset equally — broadcast-
|
|
52
|
+
constant macro features are rank-invariant and can never move the
|
|
53
|
+
cross-sectional ranking. Imported by BOTH `train.py` and `main.py` for
|
|
54
|
+
train/inference parity by construction.
|
|
55
|
+
- `sector_map.py` (V5.1 Phase 0, `development/Problems.md` #75) —
|
|
56
|
+
`load_sector_mapping()`, reading `data/reference/sector_mapping.json`
|
|
57
|
+
(a `"Forex"` pseudo-sector covers all 15 forex pairs). `train.py::load_sector_mapping()`
|
|
58
|
+
is a thin delegate
|
|
59
|
+
to this module — `main.py` cannot import `train.py` (torch), so the
|
|
60
|
+
file-read/comment-stripping logic lives here where both callers can
|
|
61
|
+
share it. Consumed by `portfolio/book_neutrality.py` and
|
|
62
|
+
`evaluation/rank_book_simulator.py` for sector-neutral book
|
|
63
|
+
construction (offline and, from Phase 1, live).
|
|
64
|
+
|
|
65
|
+
## Neutral-default convention
|
|
66
|
+
|
|
67
|
+
Every function here returns a documented neutral default (almost always
|
|
68
|
+
`0.0`) when its inputs are missing — a reference ticker not yet trading, a
|
|
69
|
+
FRED series never backfilled, no futures/options data configured — rather
|
|
70
|
+
than raising. This is deliberate: "no data available" must be
|
|
71
|
+
indistinguishable from "the signal happens to be neutral today" in the
|
|
72
|
+
model's input, never a crash. The one exception is
|
|
73
|
+
`bond_features.py::empirical_duration_beta()`, which returns `None` (not
|
|
74
|
+
`0.0`) below its minimum-observation floor — an unknown duration
|
|
75
|
+
sensitivity must stay distinguishable from a genuinely-zero one; callers
|
|
76
|
+
are responsible for substituting the neutral default at the point they
|
|
77
|
+
assign a model-input feature value.
|