backtest2live 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- backtest2live/__init__.py +12 -0
- backtest2live/__main__.py +5 -0
- backtest2live/cli.py +68 -0
- backtest2live/convergence.py +567 -0
- backtest2live/fillgap/__init__.py +57 -0
- backtest2live/fillgap/data.py +289 -0
- backtest2live/fillgap/engine.py +169 -0
- backtest2live/fillgap/report.py +101 -0
- backtest2live/fillgap/sim.py +192 -0
- backtest2live-0.2.0.dist-info/METADATA +218 -0
- backtest2live-0.2.0.dist-info/RECORD +14 -0
- backtest2live-0.2.0.dist-info/WHEEL +4 -0
- backtest2live-0.2.0.dist-info/entry_points.txt +2 -0
- backtest2live-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""backtest2live — check a backtest against what live execution would do.
|
|
2
|
+
|
|
3
|
+
Evaluators run locally on your files and public data. Each is a `bt2live`
|
|
4
|
+
subcommand and an importable module.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
9
|
+
|
|
10
|
+
__version__ = version("backtest2live")
|
|
11
|
+
except PackageNotFoundError: # running from a source tree without install
|
|
12
|
+
__version__ = "0.0.0"
|
backtest2live/cli.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""`bt2live` command line.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0/1/2 carry the verdict of the evaluator that ran (see each
|
|
4
|
+
subcommand's help); 3 means the run failed (bad input or usage), so a failed
|
|
5
|
+
run never reads as a verdict in CI.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from backtest2live import __version__, convergence, fillgap
|
|
12
|
+
|
|
13
|
+
EXIT_ERROR = 3
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class _Parser(argparse.ArgumentParser):
|
|
17
|
+
# argparse exits 2 on a usage error, which would read as KILL.
|
|
18
|
+
def error(self, message):
|
|
19
|
+
self.print_usage(sys.stderr)
|
|
20
|
+
self.exit(EXIT_ERROR, f"{self.prog}: error: {message}\n")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def build_parser():
|
|
24
|
+
p = _Parser(
|
|
25
|
+
prog="bt2live",
|
|
26
|
+
description="Check a backtest against what live execution would do.",
|
|
27
|
+
)
|
|
28
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
29
|
+
sub = p.add_subparsers(dest="command", metavar="<command>", parser_class=_Parser)
|
|
30
|
+
sub.required = True
|
|
31
|
+
|
|
32
|
+
c = sub.add_parser(
|
|
33
|
+
"convergence",
|
|
34
|
+
help="Variant-sweep convergence: CONVERGED / ITERATE / KILL",
|
|
35
|
+
description="Convergence analysis for variant backtest sweeps. "
|
|
36
|
+
"Exit code: 0 CONVERGED, 1 ITERATE, 2 KILL, 3 error.",
|
|
37
|
+
)
|
|
38
|
+
convergence.add_arguments(c)
|
|
39
|
+
c.set_defaults(run=convergence.run)
|
|
40
|
+
|
|
41
|
+
f = sub.add_parser(
|
|
42
|
+
"fill-gap",
|
|
43
|
+
help="Your backtest's passive fills vs a queue proxy on market data",
|
|
44
|
+
description="Re-run your backtest's resting orders under a queue proxy on top-of-book and trades. "
|
|
45
|
+
"Exit code: 0 CONSISTENT, 1 OVERSTATED, 3 error.",
|
|
46
|
+
)
|
|
47
|
+
fillgap.add_arguments(f)
|
|
48
|
+
f.set_defaults(run=fillgap.run)
|
|
49
|
+
return p
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def main(argv=None):
|
|
53
|
+
args = build_parser().parse_args(argv)
|
|
54
|
+
try:
|
|
55
|
+
return args.run(args)
|
|
56
|
+
except SystemExit as e:
|
|
57
|
+
# Evaluators raise SystemExit("error: ...") on bad input.
|
|
58
|
+
if isinstance(e.code, str):
|
|
59
|
+
print(e.code, file=sys.stderr)
|
|
60
|
+
return EXIT_ERROR
|
|
61
|
+
raise
|
|
62
|
+
except Exception as e: # a crash must not read as a verdict
|
|
63
|
+
print(f"error: {type(e).__name__}: {e}", file=sys.stderr)
|
|
64
|
+
return EXIT_ERROR
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
sys.exit(main())
|
|
@@ -0,0 +1,567 @@
|
|
|
1
|
+
"""
|
|
2
|
+
backtest2live · convergence — variant-sweep convergence analyzer.
|
|
3
|
+
|
|
4
|
+
Reads a table of variant backtest results (one row per variant), measures
|
|
5
|
+
convergence on a chosen metric, and emits a verdict: CONVERGED / ITERATE / KILL.
|
|
6
|
+
|
|
7
|
+
Pure Python 3 standard library — no third-party dependencies, no pip install.
|
|
8
|
+
CSV in, Markdown report out. For Parquet/SQLite inputs, export to CSV first.
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
bt2live convergence results.csv --metric sharpe_oos
|
|
12
|
+
bt2live convergence results.csv --metric sharpe --top-k 5 --save
|
|
13
|
+
bt2live convergence results.csv --json
|
|
14
|
+
|
|
15
|
+
The math is deterministic and lives here. The qualitative framing of an
|
|
16
|
+
ITERATE result (which next variants to try) is left to the calling agent.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import csv
|
|
20
|
+
import json
|
|
21
|
+
import math
|
|
22
|
+
import statistics
|
|
23
|
+
import sys
|
|
24
|
+
from datetime import datetime
|
|
25
|
+
|
|
26
|
+
# --- column-role heuristics ------------------------------------------------
|
|
27
|
+
|
|
28
|
+
ID_KEYWORDS = ("variant", "run", "config", "params", "param", "id", "name", "trial")
|
|
29
|
+
METRIC_KEYWORDS = (
|
|
30
|
+
"sharpe", "sortino", "calmar", "pnl", "profit", "return", "ret", "win",
|
|
31
|
+
"hit", "drawdown", "dd", "mdd", "loss", "score", "metric", "alpha", "ir",
|
|
32
|
+
"pf", "expectancy", "cagr",
|
|
33
|
+
)
|
|
34
|
+
SAMPLE_KEYWORDS = (
|
|
35
|
+
"n_trades", "num_trades", "trades", "n_samples", "samples", "n_obs",
|
|
36
|
+
"nobs", "count", "n",
|
|
37
|
+
)
|
|
38
|
+
LOWER_BETTER_KEYWORDS = ("drawdown", "dd", "mdd", "loss", "vol", "variance", "stdev", "std")
|
|
39
|
+
|
|
40
|
+
OOS_SUFFIXES = ("_oos", "_out", "_test", "_val", "_holdout", "_oss")
|
|
41
|
+
IS_SUFFIXES = ("_is", "_in", "_train", "_insample", "_ins")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _to_float(s):
|
|
45
|
+
if s is None:
|
|
46
|
+
return None
|
|
47
|
+
s = s.strip()
|
|
48
|
+
if s == "":
|
|
49
|
+
return None
|
|
50
|
+
try:
|
|
51
|
+
v = float(s)
|
|
52
|
+
except ValueError:
|
|
53
|
+
return None
|
|
54
|
+
return v if math.isfinite(v) else None # inf/nan cells count as missing
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def read_table(path):
|
|
58
|
+
"""Return (headers, rows) where rows is a list of dict[str, str]."""
|
|
59
|
+
try:
|
|
60
|
+
with open(path, newline="", encoding="utf-8-sig") as fh:
|
|
61
|
+
reader = csv.DictReader(fh)
|
|
62
|
+
headers = reader.fieldnames
|
|
63
|
+
if not headers:
|
|
64
|
+
raise SystemExit(f"error: no header row found in {path!r}")
|
|
65
|
+
rows = [r for r in reader]
|
|
66
|
+
except FileNotFoundError:
|
|
67
|
+
raise SystemExit(f"error: file not found: {path!r}")
|
|
68
|
+
except OSError as e:
|
|
69
|
+
raise SystemExit(f"error: could not read {path!r}: {e}")
|
|
70
|
+
if not rows:
|
|
71
|
+
raise SystemExit(f"error: no data rows in {path!r}")
|
|
72
|
+
return headers, rows
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def numeric_columns(headers, rows):
|
|
76
|
+
"""Columns where every non-empty cell parses as a float."""
|
|
77
|
+
out = []
|
|
78
|
+
for h in headers:
|
|
79
|
+
vals = [_to_float(r.get(h)) for r in rows]
|
|
80
|
+
present = [v for v in vals if v is not None]
|
|
81
|
+
if present and all(_to_float(r.get(h)) is not None or (r.get(h) or "").strip() == "" for r in rows):
|
|
82
|
+
out.append(h)
|
|
83
|
+
return out
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def pick_id_column(headers, numeric):
|
|
87
|
+
non_numeric = [h for h in headers if h not in numeric]
|
|
88
|
+
for h in headers:
|
|
89
|
+
if any(k in h.lower() for k in ID_KEYWORDS):
|
|
90
|
+
return h
|
|
91
|
+
if non_numeric:
|
|
92
|
+
return non_numeric[0]
|
|
93
|
+
return headers[0]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def pick_sample_column(numeric):
|
|
97
|
+
# Prefer the most specific match (longest keyword hit) to avoid grabbing a
|
|
98
|
+
# bare param named "n" before a real "n_trades".
|
|
99
|
+
best, best_len = None, 0
|
|
100
|
+
for h in numeric:
|
|
101
|
+
for k in SAMPLE_KEYWORDS:
|
|
102
|
+
if k in h.lower() and len(k) > best_len:
|
|
103
|
+
best, best_len = h, len(k)
|
|
104
|
+
return best
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _strip_suffix(name):
|
|
108
|
+
low = name.lower()
|
|
109
|
+
for suf in OOS_SUFFIXES + IS_SUFFIXES:
|
|
110
|
+
if low.endswith(suf):
|
|
111
|
+
return name[: -len(suf)], suf
|
|
112
|
+
return name, ""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def resolve_metrics(numeric, requested):
|
|
116
|
+
"""Return (rank_col, is_col, oos_col, base)."""
|
|
117
|
+
if not numeric:
|
|
118
|
+
raise SystemExit("error: no numeric metric columns detected")
|
|
119
|
+
|
|
120
|
+
if requested:
|
|
121
|
+
if requested in numeric:
|
|
122
|
+
rank_col = requested
|
|
123
|
+
else:
|
|
124
|
+
cand = [c for c in numeric if c == requested or c.lower().startswith(requested.lower() + "_")]
|
|
125
|
+
oos = [c for c in cand if c.lower().endswith(OOS_SUFFIXES)]
|
|
126
|
+
if not cand:
|
|
127
|
+
raise SystemExit(
|
|
128
|
+
f"error: metric {requested!r} not found. Numeric columns: {', '.join(numeric)}"
|
|
129
|
+
)
|
|
130
|
+
rank_col = oos[0] if oos else cand[0]
|
|
131
|
+
else:
|
|
132
|
+
metric_like = [c for c in numeric if any(k in c.lower() for k in METRIC_KEYWORDS)]
|
|
133
|
+
oos = [c for c in metric_like if c.lower().endswith(OOS_SUFFIXES)]
|
|
134
|
+
rank_col = oos[0] if oos else (metric_like[0] if metric_like else None)
|
|
135
|
+
if rank_col is None:
|
|
136
|
+
raise SystemExit(
|
|
137
|
+
"error: could not auto-detect a metric. Pass --metric. "
|
|
138
|
+
f"Numeric columns: {', '.join(numeric)}"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
base, _ = _strip_suffix(rank_col)
|
|
142
|
+
is_col = oos_col = None
|
|
143
|
+
for c in numeric:
|
|
144
|
+
b, suf = _strip_suffix(c)
|
|
145
|
+
if b != base:
|
|
146
|
+
continue
|
|
147
|
+
if suf in IS_SUFFIXES:
|
|
148
|
+
is_col = c
|
|
149
|
+
elif suf in OOS_SUFFIXES:
|
|
150
|
+
oos_col = c
|
|
151
|
+
# If rank_col itself carries an IS/OOS suffix, slot it in.
|
|
152
|
+
_, rank_suf = _strip_suffix(rank_col)
|
|
153
|
+
if rank_suf in OOS_SUFFIXES:
|
|
154
|
+
oos_col = rank_col
|
|
155
|
+
elif rank_suf in IS_SUFFIXES:
|
|
156
|
+
is_col = rank_col
|
|
157
|
+
return rank_col, is_col, oos_col, base
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def spearman(xs, ys):
|
|
161
|
+
"""Spearman rank correlation with average-rank tie handling. None if undefined."""
|
|
162
|
+
pairs = [(x, y) for x, y in zip(xs, ys) if x is not None and y is not None]
|
|
163
|
+
n = len(pairs)
|
|
164
|
+
if n < 2:
|
|
165
|
+
return None
|
|
166
|
+
rx = _avg_ranks([p[0] for p in pairs])
|
|
167
|
+
ry = _avg_ranks([p[1] for p in pairs])
|
|
168
|
+
return _pearson(rx, ry)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _avg_ranks(vals):
|
|
172
|
+
order = sorted(range(len(vals)), key=lambda i: vals[i])
|
|
173
|
+
ranks = [0.0] * len(vals)
|
|
174
|
+
i = 0
|
|
175
|
+
while i < len(order):
|
|
176
|
+
j = i
|
|
177
|
+
while j + 1 < len(order) and vals[order[j + 1]] == vals[order[i]]:
|
|
178
|
+
j += 1
|
|
179
|
+
avg = (i + j) / 2.0 + 1.0 # ranks are 1-based
|
|
180
|
+
for k in range(i, j + 1):
|
|
181
|
+
ranks[order[k]] = avg
|
|
182
|
+
i = j + 1
|
|
183
|
+
return ranks
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _pearson(xs, ys):
|
|
187
|
+
n = len(xs)
|
|
188
|
+
mx, my = sum(xs) / n, sum(ys) / n
|
|
189
|
+
num = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
|
|
190
|
+
dx = math.sqrt(sum((x - mx) ** 2 for x in xs))
|
|
191
|
+
dy = math.sqrt(sum((y - my) ** 2 for y in ys))
|
|
192
|
+
if dx == 0 or dy == 0:
|
|
193
|
+
return None
|
|
194
|
+
return num / (dx * dy)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def relative_dispersion(vals):
|
|
198
|
+
"""std / |mean| over the values. None if < 2 values or mean ~ 0."""
|
|
199
|
+
vals = [v for v in vals if v is not None]
|
|
200
|
+
if len(vals) < 2:
|
|
201
|
+
return None
|
|
202
|
+
mean = statistics.mean(vals)
|
|
203
|
+
sd = statistics.stdev(vals)
|
|
204
|
+
if abs(mean) < 1e-12:
|
|
205
|
+
return None
|
|
206
|
+
return sd / abs(mean)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def parameter_check(top_rows, all_rows, param_cols, getf):
|
|
210
|
+
"""For each param, see whether the top-K occupy a contiguous slice of the
|
|
211
|
+
swept grid (plateau) or are scattered (isolated). Also flag edge peaks."""
|
|
212
|
+
findings = {}
|
|
213
|
+
for p in param_cols:
|
|
214
|
+
grid = sorted({getf(r, p) for r in all_rows if getf(r, p) is not None})
|
|
215
|
+
if len(grid) <= 1:
|
|
216
|
+
findings[p] = {"grid": grid, "contiguous": True, "edge": False, "occupied": grid}
|
|
217
|
+
continue
|
|
218
|
+
idx = {v: i for i, v in enumerate(grid)}
|
|
219
|
+
occ_vals = sorted({getf(r, p) for r in top_rows if getf(r, p) is not None})
|
|
220
|
+
occ_idx = sorted(idx[v] for v in occ_vals)
|
|
221
|
+
span = occ_idx[-1] - occ_idx[0]
|
|
222
|
+
contiguous = span <= (len(occ_idx) - 1) + 1 # allow one gap
|
|
223
|
+
top1_val = getf(top_rows[0], p)
|
|
224
|
+
edge = top1_val in (grid[0], grid[-1])
|
|
225
|
+
findings[p] = {
|
|
226
|
+
"grid": grid,
|
|
227
|
+
"occupied": occ_vals,
|
|
228
|
+
"contiguous": contiguous,
|
|
229
|
+
"edge": edge,
|
|
230
|
+
"top1": top1_val,
|
|
231
|
+
}
|
|
232
|
+
return findings
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def fmt_num(x, nd=2):
|
|
236
|
+
if x is None:
|
|
237
|
+
return "n/a"
|
|
238
|
+
return f"{x:.{nd}f}"
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def fmt_pct(x, nd=1):
|
|
242
|
+
if x is None:
|
|
243
|
+
return "n/a"
|
|
244
|
+
return f"{x * 100:.{nd}f}%"
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def analyze(path, args):
|
|
248
|
+
if args.top_k < 1:
|
|
249
|
+
raise SystemExit("error: --top-k must be at least 1")
|
|
250
|
+
headers, rows = read_table(path)
|
|
251
|
+
numeric = numeric_columns(headers, rows)
|
|
252
|
+
id_col = args.id_column or pick_id_column(headers, numeric)
|
|
253
|
+
sample_col = args.sample_column or pick_sample_column(numeric)
|
|
254
|
+
rank_col, is_col, oos_col, base = resolve_metrics(numeric, args.metric)
|
|
255
|
+
|
|
256
|
+
lower_better = (
|
|
257
|
+
args.lower_is_better
|
|
258
|
+
or (not args.higher_is_better and any(k in rank_col.lower() for k in LOWER_BETTER_KEYWORDS))
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
def getf(r, col):
|
|
262
|
+
return _to_float(r.get(col)) if col else None
|
|
263
|
+
|
|
264
|
+
ranked = [r for r in rows if getf(r, rank_col) is not None]
|
|
265
|
+
if len(ranked) < 2:
|
|
266
|
+
raise SystemExit(f"error: need >=2 variants with a value for {rank_col!r}; got {len(ranked)}")
|
|
267
|
+
ranked.sort(key=lambda r: getf(r, rank_col), reverse=not lower_better)
|
|
268
|
+
|
|
269
|
+
k = min(args.top_k, len(ranked))
|
|
270
|
+
top = ranked[:k]
|
|
271
|
+
|
|
272
|
+
# --- metrics ---
|
|
273
|
+
disp_rank = relative_dispersion([getf(r, rank_col) for r in top])
|
|
274
|
+
disp_is = relative_dispersion([getf(r, is_col) for r in top]) if is_col else None
|
|
275
|
+
disp_oos = relative_dispersion([getf(r, oos_col) for r in top]) if oos_col else None
|
|
276
|
+
|
|
277
|
+
rho = None
|
|
278
|
+
if is_col and oos_col:
|
|
279
|
+
rho = spearman([getf(r, is_col) for r in ranked], [getf(r, oos_col) for r in ranked])
|
|
280
|
+
|
|
281
|
+
all_rank_vals = [getf(r, rank_col) for r in ranked]
|
|
282
|
+
median_all = statistics.median(all_rank_vals)
|
|
283
|
+
stdev_all = statistics.pstdev(all_rank_vals) if len(all_rank_vals) > 1 else 0.0
|
|
284
|
+
best = getf(top[0], rank_col)
|
|
285
|
+
if stdev_all > 0:
|
|
286
|
+
separation = (median_all - best) / stdev_all if lower_better else (best - median_all) / stdev_all
|
|
287
|
+
else:
|
|
288
|
+
separation = 0.0
|
|
289
|
+
|
|
290
|
+
param_cols = [
|
|
291
|
+
c for c in numeric
|
|
292
|
+
if c not in (rank_col, is_col, oos_col, sample_col, id_col)
|
|
293
|
+
and not any(kw in c.lower() for kw in METRIC_KEYWORDS)
|
|
294
|
+
]
|
|
295
|
+
params = parameter_check(top, ranked, param_cols, getf)
|
|
296
|
+
any_isolated = any(not f["contiguous"] for f in params.values())
|
|
297
|
+
any_edge = any(f["edge"] for f in params.values())
|
|
298
|
+
plateau = bool(params) and not any_isolated
|
|
299
|
+
|
|
300
|
+
samples_ok = None
|
|
301
|
+
low_sample_variants = []
|
|
302
|
+
if sample_col:
|
|
303
|
+
samples_ok = True
|
|
304
|
+
for r in top:
|
|
305
|
+
n = getf(r, sample_col)
|
|
306
|
+
if n is not None and n < args.min_samples:
|
|
307
|
+
samples_ok = False
|
|
308
|
+
low_sample_variants.append((r.get(id_col), n))
|
|
309
|
+
|
|
310
|
+
# --- verdict ---
|
|
311
|
+
disp = disp_rank
|
|
312
|
+
kill = False
|
|
313
|
+
kill_reasons = []
|
|
314
|
+
if rho is not None and rho <= 0.0:
|
|
315
|
+
kill = True
|
|
316
|
+
kill_reasons.append(f"IS/OOS rank correlation {fmt_num(rho)} ≤ 0 — in-sample ordering does not survive out-of-sample")
|
|
317
|
+
if any_isolated and disp is not None and disp > 2 * args.dispersion_threshold:
|
|
318
|
+
kill = True
|
|
319
|
+
kill_reasons.append("top variants are scattered points in parameter space with high dispersion — no stable region")
|
|
320
|
+
if separation < 0.5:
|
|
321
|
+
kill = True
|
|
322
|
+
kill_reasons.append(f"best variant is ~indistinguishable from the median ({fmt_num(separation)}σ separation)")
|
|
323
|
+
|
|
324
|
+
converged = (
|
|
325
|
+
not kill
|
|
326
|
+
and disp is not None and disp < args.dispersion_threshold
|
|
327
|
+
and (rho is None or rho > args.rank_threshold)
|
|
328
|
+
and plateau
|
|
329
|
+
and (samples_ok is None or samples_ok)
|
|
330
|
+
and not any_edge
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
if kill:
|
|
334
|
+
verdict = "KILL"
|
|
335
|
+
elif converged:
|
|
336
|
+
verdict = "CONVERGED"
|
|
337
|
+
else:
|
|
338
|
+
verdict = "ITERATE"
|
|
339
|
+
|
|
340
|
+
ctx = dict(
|
|
341
|
+
path=path, id_col=id_col, rank_col=rank_col, is_col=is_col, oos_col=oos_col,
|
|
342
|
+
sample_col=sample_col, k=k, n=len(ranked), top=top, lower_better=lower_better,
|
|
343
|
+
disp_rank=disp_rank, disp_is=disp_is, disp_oos=disp_oos, rho=rho,
|
|
344
|
+
params=params, plateau=plateau, any_edge=any_edge, any_isolated=any_isolated,
|
|
345
|
+
samples_ok=samples_ok, low_sample_variants=low_sample_variants,
|
|
346
|
+
verdict=verdict, kill_reasons=kill_reasons, separation=separation,
|
|
347
|
+
best=best, median_all=median_all, getf=getf, args=args,
|
|
348
|
+
)
|
|
349
|
+
return ctx
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def build_report(ctx):
|
|
353
|
+
a = ctx["args"]
|
|
354
|
+
run_name = a.run_name or ctx["path"].split("/")[-1]
|
|
355
|
+
date = a.date or datetime.now().strftime("%Y-%m-%d")
|
|
356
|
+
getf = ctx["getf"]
|
|
357
|
+
rank_col, is_col, oos_col = ctx["rank_col"], ctx["is_col"], ctx["oos_col"]
|
|
358
|
+
thr = a.dispersion_threshold
|
|
359
|
+
|
|
360
|
+
L = []
|
|
361
|
+
L.append(f"# Iteration Check | {run_name} | {date}")
|
|
362
|
+
L.append("")
|
|
363
|
+
L.append(f"## Verdict: {ctx['verdict']}")
|
|
364
|
+
L.append("")
|
|
365
|
+
L.append(f"**{_summary_line(ctx)}**")
|
|
366
|
+
L.append("")
|
|
367
|
+
L.append("---")
|
|
368
|
+
L.append("")
|
|
369
|
+
L.append("## Convergence Analysis")
|
|
370
|
+
L.append("")
|
|
371
|
+
L.append(f"Ranking metric: `{rank_col}` ({'lower is better' if ctx['lower_better'] else 'higher is better'}) · "
|
|
372
|
+
f"Top-{ctx['k']} of {ctx['n']} variants")
|
|
373
|
+
L.append("")
|
|
374
|
+
L.append("| Metric | Top-K Mean | Top-K Std | Dispersion | Threshold |")
|
|
375
|
+
L.append("|--------|-----------|-----------|------------|-----------|")
|
|
376
|
+
seen = set()
|
|
377
|
+
for col, disp in [(rank_col, ctx["disp_rank"]), (is_col, ctx["disp_is"]), (oos_col, ctx["disp_oos"])]:
|
|
378
|
+
if not col or col in seen:
|
|
379
|
+
continue
|
|
380
|
+
seen.add(col)
|
|
381
|
+
vals = [getf(r, col) for r in ctx["top"] if getf(r, col) is not None]
|
|
382
|
+
if not vals:
|
|
383
|
+
continue
|
|
384
|
+
mean = statistics.mean(vals)
|
|
385
|
+
sd = statistics.stdev(vals) if len(vals) > 1 else 0.0
|
|
386
|
+
flag = "—" if disp is None else ("✓" if disp < thr else "✗")
|
|
387
|
+
label = col + (" *(rank)*" if col == rank_col else "")
|
|
388
|
+
L.append(f"| {label} | {fmt_num(mean)} | {fmt_num(sd)} | {fmt_pct(disp)} | < {fmt_pct(thr,0)} {flag} |")
|
|
389
|
+
L.append("")
|
|
390
|
+
if ctx["rho"] is not None:
|
|
391
|
+
ok = "✓" if ctx["rho"] > a.rank_threshold else "✗"
|
|
392
|
+
L.append(f"**IS→OOS rank stability (Spearman):** {fmt_num(ctx['rho'])} "
|
|
393
|
+
f"(threshold > {fmt_num(a.rank_threshold)} {ok})")
|
|
394
|
+
L.append("")
|
|
395
|
+
L.append(f"**Read:** {_read_line(ctx)}")
|
|
396
|
+
L.append("")
|
|
397
|
+
L.append("---")
|
|
398
|
+
L.append("")
|
|
399
|
+
L.append("## Parameter-Space Check")
|
|
400
|
+
L.append("")
|
|
401
|
+
if not ctx["params"]:
|
|
402
|
+
L.append("No parameter columns detected — cannot assess clustering. "
|
|
403
|
+
"Pass parameter columns or check column inference.")
|
|
404
|
+
else:
|
|
405
|
+
clusters = []
|
|
406
|
+
for p, f in ctx["params"].items():
|
|
407
|
+
occ = ", ".join(fmt_num(v, 4).rstrip("0").rstrip(".") for v in f["occupied"])
|
|
408
|
+
edge = " **(edge of swept range)**" if f["edge"] else ""
|
|
409
|
+
clusters.append(f"- `{p}`: top-K at [{occ}]{edge} — {'contiguous' if f['contiguous'] else 'scattered'}")
|
|
410
|
+
L.extend(clusters)
|
|
411
|
+
L.append("")
|
|
412
|
+
L.append(f"[{'x' if not ctx['plateau'] else ' '}] Single isolated peak — fragile, retest with more samples")
|
|
413
|
+
L.append(f"[{'x' if ctx['plateau'] else ' '}] Plateau region — robust, safe to deploy")
|
|
414
|
+
L.append("")
|
|
415
|
+
if ctx["sample_col"]:
|
|
416
|
+
if ctx["samples_ok"]:
|
|
417
|
+
L.append(f"Sample sizes OK — all top-{ctx['k']} variants ≥ {a.min_samples} on `{ctx['sample_col']}`.")
|
|
418
|
+
else:
|
|
419
|
+
offenders = ", ".join(f"{vid} ({fmt_num(n,0)})" for vid, n in ctx["low_sample_variants"])
|
|
420
|
+
L.append(f"⚠ Below `--min-samples` ({a.min_samples}) on `{ctx['sample_col']}`: {offenders}")
|
|
421
|
+
L.append("")
|
|
422
|
+
L.append("---")
|
|
423
|
+
L.append("")
|
|
424
|
+
L.append("## Recommendation")
|
|
425
|
+
L.append("")
|
|
426
|
+
L.extend(_recommendation(ctx))
|
|
427
|
+
L.append("")
|
|
428
|
+
return "\n".join(L)
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _summary_line(ctx):
|
|
432
|
+
v = ctx["verdict"]
|
|
433
|
+
if v == "CONVERGED":
|
|
434
|
+
top_id = ctx["top"][0].get(ctx["id_col"])
|
|
435
|
+
# Nested same-quote f-strings are Python 3.12+ only; keep 3.9 parsing.
|
|
436
|
+
stable = "" if ctx["rho"] is None else f", OOS-stable (ρ={fmt_num(ctx['rho'])})"
|
|
437
|
+
return (f"Top-{ctx['k']} cluster tightly ({fmt_pct(ctx['disp_rank'])} dispersion) on a parameter plateau"
|
|
438
|
+
f"{stable} — ship `{top_id}`.")
|
|
439
|
+
if v == "KILL":
|
|
440
|
+
return ctx["kill_reasons"][0] if ctx["kill_reasons"] else "No stable edge in the swept parameter space."
|
|
441
|
+
return "Convergence not yet achieved — some criteria unmet (see below)."
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _read_line(ctx):
|
|
445
|
+
bits = []
|
|
446
|
+
d = ctx["disp_rank"]
|
|
447
|
+
if d is not None:
|
|
448
|
+
tight = d < ctx["args"].dispersion_threshold
|
|
449
|
+
bits.append(f"top-{ctx['k']} are {'clustered tightly' if tight else 'spread out'} on the ranking metric ({fmt_pct(d)})")
|
|
450
|
+
if ctx["rho"] is not None:
|
|
451
|
+
bits.append(f"IS↔OOS rank correlation is {fmt_num(ctx['rho'])}")
|
|
452
|
+
if not bits:
|
|
453
|
+
return "Limited signal — too few variants or near-zero mean for a dispersion read."
|
|
454
|
+
return "; ".join(bits).capitalize() + "."
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _recommendation(ctx):
|
|
458
|
+
v = ctx["verdict"]
|
|
459
|
+
if v == "CONVERGED":
|
|
460
|
+
top_id = ctx["top"][0].get(ctx["id_col"])
|
|
461
|
+
return [f"**[CONVERGED]** Top variant `{top_id}` is representative — its neighbors perform similarly "
|
|
462
|
+
f"and the winning region is a plateau, not a lucky point. Safe to advance this variant to the next stage."]
|
|
463
|
+
if v == "KILL":
|
|
464
|
+
out = ["**[KILL]** No region of the swept parameter space shows a stable edge. Specifically:"]
|
|
465
|
+
out += [f"- {r}" for r in ctx["kill_reasons"]]
|
|
466
|
+
out.append("Recommend abandoning this strategy line, or rethinking the metric/feature set before re-sweeping.")
|
|
467
|
+
return out
|
|
468
|
+
# ITERATE
|
|
469
|
+
out = ["**[ITERATE]** Convergence not yet achieved. Suggested next sweep:"]
|
|
470
|
+
sugg = []
|
|
471
|
+
for p, f in ctx["params"].items():
|
|
472
|
+
if f["edge"]:
|
|
473
|
+
sugg.append(f"- Expand `{p}` beyond its current edge (top variant sits at {fmt_num(f['top1'],4).rstrip('0').rstrip('.')}, "
|
|
474
|
+
f"the boundary of the swept grid)")
|
|
475
|
+
elif not f["contiguous"]:
|
|
476
|
+
sugg.append(f"- Refine the `{p}` grid around the top cluster — winners are currently scattered")
|
|
477
|
+
if ctx["disp_rank"] is not None and ctx["disp_rank"] >= ctx["args"].dispersion_threshold:
|
|
478
|
+
sugg.append(f"- Tighten dispersion: top-K spread is {fmt_pct(ctx['disp_rank'])} "
|
|
479
|
+
f"(> {fmt_pct(ctx['args'].dispersion_threshold,0)}) — narrow the grid or add samples")
|
|
480
|
+
if ctx["rho"] is not None and ctx["rho"] <= ctx["args"].rank_threshold:
|
|
481
|
+
sugg.append(f"- IS/OOS rank stability is weak (ρ={fmt_num(ctx['rho'])}) — likely undersampled; "
|
|
482
|
+
f"increase trades per variant or lengthen the OOS window")
|
|
483
|
+
if ctx["samples_ok"] is False:
|
|
484
|
+
sugg.append(f"- Rerun low-sample variants with ≥ {ctx['args'].min_samples} trades before trusting their rank")
|
|
485
|
+
if not sugg:
|
|
486
|
+
sugg.append("- Add samples and re-run; the signal is borderline on at least one criterion")
|
|
487
|
+
out += sugg
|
|
488
|
+
return out
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
VERDICT_EXIT = {"CONVERGED": 0, "ITERATE": 1, "KILL": 2}
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def add_arguments(p):
|
|
495
|
+
p.add_argument("input", help="CSV file: one row per variant")
|
|
496
|
+
p.add_argument("--metric", help="Ranking metric column (e.g. sharpe_oos). Auto-detected if omitted.")
|
|
497
|
+
p.add_argument("--top-k", type=int, default=5, help="Number of top variants to assess (default 5)")
|
|
498
|
+
p.add_argument("--dispersion-threshold", type=float, default=0.05,
|
|
499
|
+
help="Max relative dispersion for CONVERGED (default 0.05 = 5%%)")
|
|
500
|
+
p.add_argument("--min-samples", type=int, default=200, help="Min samples/variant (default 200)")
|
|
501
|
+
p.add_argument("--rank-threshold", type=float, default=0.6,
|
|
502
|
+
help="Min IS/OOS Spearman for CONVERGED (default 0.6)")
|
|
503
|
+
p.add_argument("--id-column", help="Override the variant-id column")
|
|
504
|
+
p.add_argument("--sample-column", help="Override the sample-size column")
|
|
505
|
+
p.add_argument("--lower-is-better", action="store_true", help="Treat the metric as lower-is-better")
|
|
506
|
+
p.add_argument("--higher-is-better", action="store_true", help="Force higher-is-better (override name heuristic)")
|
|
507
|
+
p.add_argument("--run-name", help="Label for the report header")
|
|
508
|
+
p.add_argument("--date", help="Date for the report header (default: today)")
|
|
509
|
+
p.add_argument("--save", action="store_true",
|
|
510
|
+
help="Also write iteration_check_<timestamp>.md in the current directory")
|
|
511
|
+
p.add_argument("--json", action="store_true", help="Print a machine-readable verdict instead of Markdown")
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def to_json(ctx):
|
|
515
|
+
"""Machine-readable verdict: the numbers behind the Markdown report."""
|
|
516
|
+
getf = ctx["getf"]
|
|
517
|
+
return {
|
|
518
|
+
"schema": "backtest2live.convergence/1",
|
|
519
|
+
"input": ctx["path"],
|
|
520
|
+
"verdict": ctx["verdict"],
|
|
521
|
+
"exit_code": VERDICT_EXIT[ctx["verdict"]],
|
|
522
|
+
"metric": ctx["rank_col"],
|
|
523
|
+
"lower_is_better": ctx["lower_better"],
|
|
524
|
+
"is_column": ctx["is_col"],
|
|
525
|
+
"oos_column": ctx["oos_col"],
|
|
526
|
+
"sample_column": ctx["sample_col"],
|
|
527
|
+
"top_k": ctx["k"],
|
|
528
|
+
"n_variants": ctx["n"],
|
|
529
|
+
"top": [{"id": r.get(ctx["id_col"]), "value": getf(r, ctx["rank_col"])} for r in ctx["top"]],
|
|
530
|
+
"dispersion": {"rank": ctx["disp_rank"], "is": ctx["disp_is"], "oos": ctx["disp_oos"]},
|
|
531
|
+
"is_oos_spearman": ctx["rho"],
|
|
532
|
+
"separation_sigma": ctx["separation"],
|
|
533
|
+
"plateau": ctx["plateau"],
|
|
534
|
+
"edge_peak": ctx["any_edge"],
|
|
535
|
+
"parameters": {
|
|
536
|
+
p: {"occupied": f["occupied"], "contiguous": f["contiguous"], "edge": f["edge"]}
|
|
537
|
+
for p, f in ctx["params"].items()
|
|
538
|
+
},
|
|
539
|
+
"samples_ok": ctx["samples_ok"],
|
|
540
|
+
"low_sample_variants": [{"id": vid, "n": n} for vid, n in ctx["low_sample_variants"]],
|
|
541
|
+
"kill_reasons": ctx["kill_reasons"],
|
|
542
|
+
"thresholds": {
|
|
543
|
+
"dispersion": ctx["args"].dispersion_threshold,
|
|
544
|
+
"rank": ctx["args"].rank_threshold,
|
|
545
|
+
"min_samples": ctx["args"].min_samples,
|
|
546
|
+
},
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def run(args):
|
|
551
|
+
"""CLI entry for `bt2live convergence`. Returns the verdict exit code."""
|
|
552
|
+
ctx = analyze(args.input, args)
|
|
553
|
+
report = build_report(ctx)
|
|
554
|
+
if args.json:
|
|
555
|
+
print(json.dumps(to_json(ctx), indent=2, ensure_ascii=False, allow_nan=False))
|
|
556
|
+
else:
|
|
557
|
+
print(report)
|
|
558
|
+
|
|
559
|
+
if args.save:
|
|
560
|
+
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
561
|
+
out_path = f"iteration_check_{stamp}.md"
|
|
562
|
+
with open(out_path, "w", encoding="utf-8") as fh:
|
|
563
|
+
fh.write(report + "\n")
|
|
564
|
+
print(f"\n[saved report → {out_path}]", file=sys.stderr)
|
|
565
|
+
|
|
566
|
+
# Exit code encodes the verdict for scripting: 0 CONVERGED, 1 ITERATE, 2 KILL.
|
|
567
|
+
return VERDICT_EXIT[ctx["verdict"]]
|