backtest2live 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,12 @@
1
+ """backtest2live — check a backtest against what live execution would do.
2
+
3
+ Evaluators run locally on your files and public data. Each is a `bt2live`
4
+ subcommand and an importable module.
5
+ """
6
+
7
+ try:
8
+ from importlib.metadata import PackageNotFoundError, version
9
+
10
+ __version__ = version("backtest2live")
11
+ except PackageNotFoundError: # running from a source tree without install
12
+ __version__ = "0.0.0"
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from backtest2live.cli import main
4
+
5
+ sys.exit(main())
backtest2live/cli.py ADDED
@@ -0,0 +1,68 @@
1
+ """`bt2live` command line.
2
+
3
+ Exit codes: 0/1/2 carry the verdict of the evaluator that ran (see each
4
+ subcommand's help); 3 means the run failed (bad input or usage), so a failed
5
+ run never reads as a verdict in CI.
6
+ """
7
+
8
+ import argparse
9
+ import sys
10
+
11
+ from backtest2live import __version__, convergence, fillgap
12
+
13
+ EXIT_ERROR = 3
14
+
15
+
16
+ class _Parser(argparse.ArgumentParser):
17
+ # argparse exits 2 on a usage error, which would read as KILL.
18
+ def error(self, message):
19
+ self.print_usage(sys.stderr)
20
+ self.exit(EXIT_ERROR, f"{self.prog}: error: {message}\n")
21
+
22
+
23
+ def build_parser():
24
+ p = _Parser(
25
+ prog="bt2live",
26
+ description="Check a backtest against what live execution would do.",
27
+ )
28
+ p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
29
+ sub = p.add_subparsers(dest="command", metavar="<command>", parser_class=_Parser)
30
+ sub.required = True
31
+
32
+ c = sub.add_parser(
33
+ "convergence",
34
+ help="Variant-sweep convergence: CONVERGED / ITERATE / KILL",
35
+ description="Convergence analysis for variant backtest sweeps. "
36
+ "Exit code: 0 CONVERGED, 1 ITERATE, 2 KILL, 3 error.",
37
+ )
38
+ convergence.add_arguments(c)
39
+ c.set_defaults(run=convergence.run)
40
+
41
+ f = sub.add_parser(
42
+ "fill-gap",
43
+ help="Your backtest's passive fills vs a queue proxy on market data",
44
+ description="Re-run your backtest's resting orders under a queue proxy on top-of-book and trades. "
45
+ "Exit code: 0 CONSISTENT, 1 OVERSTATED, 3 error.",
46
+ )
47
+ fillgap.add_arguments(f)
48
+ f.set_defaults(run=fillgap.run)
49
+ return p
50
+
51
+
52
+ def main(argv=None):
53
+ args = build_parser().parse_args(argv)
54
+ try:
55
+ return args.run(args)
56
+ except SystemExit as e:
57
+ # Evaluators raise SystemExit("error: ...") on bad input.
58
+ if isinstance(e.code, str):
59
+ print(e.code, file=sys.stderr)
60
+ return EXIT_ERROR
61
+ raise
62
+ except Exception as e: # a crash must not read as a verdict
63
+ print(f"error: {type(e).__name__}: {e}", file=sys.stderr)
64
+ return EXIT_ERROR
65
+
66
+
67
+ if __name__ == "__main__":
68
+ sys.exit(main())
@@ -0,0 +1,567 @@
1
+ """
2
+ backtest2live · convergence — variant-sweep convergence analyzer.
3
+
4
+ Reads a table of variant backtest results (one row per variant), measures
5
+ convergence on a chosen metric, and emits a verdict: CONVERGED / ITERATE / KILL.
6
+
7
+ Pure Python 3 standard library — no third-party dependencies, no pip install.
8
+ CSV in, Markdown report out. For Parquet/SQLite inputs, export to CSV first.
9
+
10
+ Usage:
11
+ bt2live convergence results.csv --metric sharpe_oos
12
+ bt2live convergence results.csv --metric sharpe --top-k 5 --save
13
+ bt2live convergence results.csv --json
14
+
15
+ The math is deterministic and lives here. The qualitative framing of an
16
+ ITERATE result (which next variants to try) is left to the calling agent.
17
+ """
18
+
19
+ import csv
20
+ import json
21
+ import math
22
+ import statistics
23
+ import sys
24
+ from datetime import datetime
25
+
26
+ # --- column-role heuristics ------------------------------------------------
27
+
28
+ ID_KEYWORDS = ("variant", "run", "config", "params", "param", "id", "name", "trial")
29
+ METRIC_KEYWORDS = (
30
+ "sharpe", "sortino", "calmar", "pnl", "profit", "return", "ret", "win",
31
+ "hit", "drawdown", "dd", "mdd", "loss", "score", "metric", "alpha", "ir",
32
+ "pf", "expectancy", "cagr",
33
+ )
34
+ SAMPLE_KEYWORDS = (
35
+ "n_trades", "num_trades", "trades", "n_samples", "samples", "n_obs",
36
+ "nobs", "count", "n",
37
+ )
38
+ LOWER_BETTER_KEYWORDS = ("drawdown", "dd", "mdd", "loss", "vol", "variance", "stdev", "std")
39
+
40
+ OOS_SUFFIXES = ("_oos", "_out", "_test", "_val", "_holdout", "_oss")
41
+ IS_SUFFIXES = ("_is", "_in", "_train", "_insample", "_ins")
42
+
43
+
44
+ def _to_float(s):
45
+ if s is None:
46
+ return None
47
+ s = s.strip()
48
+ if s == "":
49
+ return None
50
+ try:
51
+ v = float(s)
52
+ except ValueError:
53
+ return None
54
+ return v if math.isfinite(v) else None # inf/nan cells count as missing
55
+
56
+
57
+ def read_table(path):
58
+ """Return (headers, rows) where rows is a list of dict[str, str]."""
59
+ try:
60
+ with open(path, newline="", encoding="utf-8-sig") as fh:
61
+ reader = csv.DictReader(fh)
62
+ headers = reader.fieldnames
63
+ if not headers:
64
+ raise SystemExit(f"error: no header row found in {path!r}")
65
+ rows = [r for r in reader]
66
+ except FileNotFoundError:
67
+ raise SystemExit(f"error: file not found: {path!r}")
68
+ except OSError as e:
69
+ raise SystemExit(f"error: could not read {path!r}: {e}")
70
+ if not rows:
71
+ raise SystemExit(f"error: no data rows in {path!r}")
72
+ return headers, rows
73
+
74
+
75
+ def numeric_columns(headers, rows):
76
+ """Columns where every non-empty cell parses as a float."""
77
+ out = []
78
+ for h in headers:
79
+ vals = [_to_float(r.get(h)) for r in rows]
80
+ present = [v for v in vals if v is not None]
81
+ if present and all(_to_float(r.get(h)) is not None or (r.get(h) or "").strip() == "" for r in rows):
82
+ out.append(h)
83
+ return out
84
+
85
+
86
+ def pick_id_column(headers, numeric):
87
+ non_numeric = [h for h in headers if h not in numeric]
88
+ for h in headers:
89
+ if any(k in h.lower() for k in ID_KEYWORDS):
90
+ return h
91
+ if non_numeric:
92
+ return non_numeric[0]
93
+ return headers[0]
94
+
95
+
96
+ def pick_sample_column(numeric):
97
+ # Prefer the most specific match (longest keyword hit) to avoid grabbing a
98
+ # bare param named "n" before a real "n_trades".
99
+ best, best_len = None, 0
100
+ for h in numeric:
101
+ for k in SAMPLE_KEYWORDS:
102
+ if k in h.lower() and len(k) > best_len:
103
+ best, best_len = h, len(k)
104
+ return best
105
+
106
+
107
+ def _strip_suffix(name):
108
+ low = name.lower()
109
+ for suf in OOS_SUFFIXES + IS_SUFFIXES:
110
+ if low.endswith(suf):
111
+ return name[: -len(suf)], suf
112
+ return name, ""
113
+
114
+
115
+ def resolve_metrics(numeric, requested):
116
+ """Return (rank_col, is_col, oos_col, base)."""
117
+ if not numeric:
118
+ raise SystemExit("error: no numeric metric columns detected")
119
+
120
+ if requested:
121
+ if requested in numeric:
122
+ rank_col = requested
123
+ else:
124
+ cand = [c for c in numeric if c == requested or c.lower().startswith(requested.lower() + "_")]
125
+ oos = [c for c in cand if c.lower().endswith(OOS_SUFFIXES)]
126
+ if not cand:
127
+ raise SystemExit(
128
+ f"error: metric {requested!r} not found. Numeric columns: {', '.join(numeric)}"
129
+ )
130
+ rank_col = oos[0] if oos else cand[0]
131
+ else:
132
+ metric_like = [c for c in numeric if any(k in c.lower() for k in METRIC_KEYWORDS)]
133
+ oos = [c for c in metric_like if c.lower().endswith(OOS_SUFFIXES)]
134
+ rank_col = oos[0] if oos else (metric_like[0] if metric_like else None)
135
+ if rank_col is None:
136
+ raise SystemExit(
137
+ "error: could not auto-detect a metric. Pass --metric. "
138
+ f"Numeric columns: {', '.join(numeric)}"
139
+ )
140
+
141
+ base, _ = _strip_suffix(rank_col)
142
+ is_col = oos_col = None
143
+ for c in numeric:
144
+ b, suf = _strip_suffix(c)
145
+ if b != base:
146
+ continue
147
+ if suf in IS_SUFFIXES:
148
+ is_col = c
149
+ elif suf in OOS_SUFFIXES:
150
+ oos_col = c
151
+ # If rank_col itself carries an IS/OOS suffix, slot it in.
152
+ _, rank_suf = _strip_suffix(rank_col)
153
+ if rank_suf in OOS_SUFFIXES:
154
+ oos_col = rank_col
155
+ elif rank_suf in IS_SUFFIXES:
156
+ is_col = rank_col
157
+ return rank_col, is_col, oos_col, base
158
+
159
+
160
+ def spearman(xs, ys):
161
+ """Spearman rank correlation with average-rank tie handling. None if undefined."""
162
+ pairs = [(x, y) for x, y in zip(xs, ys) if x is not None and y is not None]
163
+ n = len(pairs)
164
+ if n < 2:
165
+ return None
166
+ rx = _avg_ranks([p[0] for p in pairs])
167
+ ry = _avg_ranks([p[1] for p in pairs])
168
+ return _pearson(rx, ry)
169
+
170
+
171
+ def _avg_ranks(vals):
172
+ order = sorted(range(len(vals)), key=lambda i: vals[i])
173
+ ranks = [0.0] * len(vals)
174
+ i = 0
175
+ while i < len(order):
176
+ j = i
177
+ while j + 1 < len(order) and vals[order[j + 1]] == vals[order[i]]:
178
+ j += 1
179
+ avg = (i + j) / 2.0 + 1.0 # ranks are 1-based
180
+ for k in range(i, j + 1):
181
+ ranks[order[k]] = avg
182
+ i = j + 1
183
+ return ranks
184
+
185
+
186
+ def _pearson(xs, ys):
187
+ n = len(xs)
188
+ mx, my = sum(xs) / n, sum(ys) / n
189
+ num = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
190
+ dx = math.sqrt(sum((x - mx) ** 2 for x in xs))
191
+ dy = math.sqrt(sum((y - my) ** 2 for y in ys))
192
+ if dx == 0 or dy == 0:
193
+ return None
194
+ return num / (dx * dy)
195
+
196
+
197
+ def relative_dispersion(vals):
198
+ """std / |mean| over the values. None if < 2 values or mean ~ 0."""
199
+ vals = [v for v in vals if v is not None]
200
+ if len(vals) < 2:
201
+ return None
202
+ mean = statistics.mean(vals)
203
+ sd = statistics.stdev(vals)
204
+ if abs(mean) < 1e-12:
205
+ return None
206
+ return sd / abs(mean)
207
+
208
+
209
+ def parameter_check(top_rows, all_rows, param_cols, getf):
210
+ """For each param, see whether the top-K occupy a contiguous slice of the
211
+ swept grid (plateau) or are scattered (isolated). Also flag edge peaks."""
212
+ findings = {}
213
+ for p in param_cols:
214
+ grid = sorted({getf(r, p) for r in all_rows if getf(r, p) is not None})
215
+ if len(grid) <= 1:
216
+ findings[p] = {"grid": grid, "contiguous": True, "edge": False, "occupied": grid}
217
+ continue
218
+ idx = {v: i for i, v in enumerate(grid)}
219
+ occ_vals = sorted({getf(r, p) for r in top_rows if getf(r, p) is not None})
220
+ occ_idx = sorted(idx[v] for v in occ_vals)
221
+ span = occ_idx[-1] - occ_idx[0]
222
+ contiguous = span <= (len(occ_idx) - 1) + 1 # allow one gap
223
+ top1_val = getf(top_rows[0], p)
224
+ edge = top1_val in (grid[0], grid[-1])
225
+ findings[p] = {
226
+ "grid": grid,
227
+ "occupied": occ_vals,
228
+ "contiguous": contiguous,
229
+ "edge": edge,
230
+ "top1": top1_val,
231
+ }
232
+ return findings
233
+
234
+
235
+ def fmt_num(x, nd=2):
236
+ if x is None:
237
+ return "n/a"
238
+ return f"{x:.{nd}f}"
239
+
240
+
241
+ def fmt_pct(x, nd=1):
242
+ if x is None:
243
+ return "n/a"
244
+ return f"{x * 100:.{nd}f}%"
245
+
246
+
247
+ def analyze(path, args):
248
+ if args.top_k < 1:
249
+ raise SystemExit("error: --top-k must be at least 1")
250
+ headers, rows = read_table(path)
251
+ numeric = numeric_columns(headers, rows)
252
+ id_col = args.id_column or pick_id_column(headers, numeric)
253
+ sample_col = args.sample_column or pick_sample_column(numeric)
254
+ rank_col, is_col, oos_col, base = resolve_metrics(numeric, args.metric)
255
+
256
+ lower_better = (
257
+ args.lower_is_better
258
+ or (not args.higher_is_better and any(k in rank_col.lower() for k in LOWER_BETTER_KEYWORDS))
259
+ )
260
+
261
+ def getf(r, col):
262
+ return _to_float(r.get(col)) if col else None
263
+
264
+ ranked = [r for r in rows if getf(r, rank_col) is not None]
265
+ if len(ranked) < 2:
266
+ raise SystemExit(f"error: need >=2 variants with a value for {rank_col!r}; got {len(ranked)}")
267
+ ranked.sort(key=lambda r: getf(r, rank_col), reverse=not lower_better)
268
+
269
+ k = min(args.top_k, len(ranked))
270
+ top = ranked[:k]
271
+
272
+ # --- metrics ---
273
+ disp_rank = relative_dispersion([getf(r, rank_col) for r in top])
274
+ disp_is = relative_dispersion([getf(r, is_col) for r in top]) if is_col else None
275
+ disp_oos = relative_dispersion([getf(r, oos_col) for r in top]) if oos_col else None
276
+
277
+ rho = None
278
+ if is_col and oos_col:
279
+ rho = spearman([getf(r, is_col) for r in ranked], [getf(r, oos_col) for r in ranked])
280
+
281
+ all_rank_vals = [getf(r, rank_col) for r in ranked]
282
+ median_all = statistics.median(all_rank_vals)
283
+ stdev_all = statistics.pstdev(all_rank_vals) if len(all_rank_vals) > 1 else 0.0
284
+ best = getf(top[0], rank_col)
285
+ if stdev_all > 0:
286
+ separation = (median_all - best) / stdev_all if lower_better else (best - median_all) / stdev_all
287
+ else:
288
+ separation = 0.0
289
+
290
+ param_cols = [
291
+ c for c in numeric
292
+ if c not in (rank_col, is_col, oos_col, sample_col, id_col)
293
+ and not any(kw in c.lower() for kw in METRIC_KEYWORDS)
294
+ ]
295
+ params = parameter_check(top, ranked, param_cols, getf)
296
+ any_isolated = any(not f["contiguous"] for f in params.values())
297
+ any_edge = any(f["edge"] for f in params.values())
298
+ plateau = bool(params) and not any_isolated
299
+
300
+ samples_ok = None
301
+ low_sample_variants = []
302
+ if sample_col:
303
+ samples_ok = True
304
+ for r in top:
305
+ n = getf(r, sample_col)
306
+ if n is not None and n < args.min_samples:
307
+ samples_ok = False
308
+ low_sample_variants.append((r.get(id_col), n))
309
+
310
+ # --- verdict ---
311
+ disp = disp_rank
312
+ kill = False
313
+ kill_reasons = []
314
+ if rho is not None and rho <= 0.0:
315
+ kill = True
316
+ kill_reasons.append(f"IS/OOS rank correlation {fmt_num(rho)} ≤ 0 — in-sample ordering does not survive out-of-sample")
317
+ if any_isolated and disp is not None and disp > 2 * args.dispersion_threshold:
318
+ kill = True
319
+ kill_reasons.append("top variants are scattered points in parameter space with high dispersion — no stable region")
320
+ if separation < 0.5:
321
+ kill = True
322
+ kill_reasons.append(f"best variant is ~indistinguishable from the median ({fmt_num(separation)}σ separation)")
323
+
324
+ converged = (
325
+ not kill
326
+ and disp is not None and disp < args.dispersion_threshold
327
+ and (rho is None or rho > args.rank_threshold)
328
+ and plateau
329
+ and (samples_ok is None or samples_ok)
330
+ and not any_edge
331
+ )
332
+
333
+ if kill:
334
+ verdict = "KILL"
335
+ elif converged:
336
+ verdict = "CONVERGED"
337
+ else:
338
+ verdict = "ITERATE"
339
+
340
+ ctx = dict(
341
+ path=path, id_col=id_col, rank_col=rank_col, is_col=is_col, oos_col=oos_col,
342
+ sample_col=sample_col, k=k, n=len(ranked), top=top, lower_better=lower_better,
343
+ disp_rank=disp_rank, disp_is=disp_is, disp_oos=disp_oos, rho=rho,
344
+ params=params, plateau=plateau, any_edge=any_edge, any_isolated=any_isolated,
345
+ samples_ok=samples_ok, low_sample_variants=low_sample_variants,
346
+ verdict=verdict, kill_reasons=kill_reasons, separation=separation,
347
+ best=best, median_all=median_all, getf=getf, args=args,
348
+ )
349
+ return ctx
350
+
351
+
352
+ def build_report(ctx):
353
+ a = ctx["args"]
354
+ run_name = a.run_name or ctx["path"].split("/")[-1]
355
+ date = a.date or datetime.now().strftime("%Y-%m-%d")
356
+ getf = ctx["getf"]
357
+ rank_col, is_col, oos_col = ctx["rank_col"], ctx["is_col"], ctx["oos_col"]
358
+ thr = a.dispersion_threshold
359
+
360
+ L = []
361
+ L.append(f"# Iteration Check | {run_name} | {date}")
362
+ L.append("")
363
+ L.append(f"## Verdict: {ctx['verdict']}")
364
+ L.append("")
365
+ L.append(f"**{_summary_line(ctx)}**")
366
+ L.append("")
367
+ L.append("---")
368
+ L.append("")
369
+ L.append("## Convergence Analysis")
370
+ L.append("")
371
+ L.append(f"Ranking metric: `{rank_col}` ({'lower is better' if ctx['lower_better'] else 'higher is better'}) · "
372
+ f"Top-{ctx['k']} of {ctx['n']} variants")
373
+ L.append("")
374
+ L.append("| Metric | Top-K Mean | Top-K Std | Dispersion | Threshold |")
375
+ L.append("|--------|-----------|-----------|------------|-----------|")
376
+ seen = set()
377
+ for col, disp in [(rank_col, ctx["disp_rank"]), (is_col, ctx["disp_is"]), (oos_col, ctx["disp_oos"])]:
378
+ if not col or col in seen:
379
+ continue
380
+ seen.add(col)
381
+ vals = [getf(r, col) for r in ctx["top"] if getf(r, col) is not None]
382
+ if not vals:
383
+ continue
384
+ mean = statistics.mean(vals)
385
+ sd = statistics.stdev(vals) if len(vals) > 1 else 0.0
386
+ flag = "—" if disp is None else ("✓" if disp < thr else "✗")
387
+ label = col + (" *(rank)*" if col == rank_col else "")
388
+ L.append(f"| {label} | {fmt_num(mean)} | {fmt_num(sd)} | {fmt_pct(disp)} | < {fmt_pct(thr,0)} {flag} |")
389
+ L.append("")
390
+ if ctx["rho"] is not None:
391
+ ok = "✓" if ctx["rho"] > a.rank_threshold else "✗"
392
+ L.append(f"**IS→OOS rank stability (Spearman):** {fmt_num(ctx['rho'])} "
393
+ f"(threshold > {fmt_num(a.rank_threshold)} {ok})")
394
+ L.append("")
395
+ L.append(f"**Read:** {_read_line(ctx)}")
396
+ L.append("")
397
+ L.append("---")
398
+ L.append("")
399
+ L.append("## Parameter-Space Check")
400
+ L.append("")
401
+ if not ctx["params"]:
402
+ L.append("No parameter columns detected — cannot assess clustering. "
403
+ "Pass parameter columns or check column inference.")
404
+ else:
405
+ clusters = []
406
+ for p, f in ctx["params"].items():
407
+ occ = ", ".join(fmt_num(v, 4).rstrip("0").rstrip(".") for v in f["occupied"])
408
+ edge = " **(edge of swept range)**" if f["edge"] else ""
409
+ clusters.append(f"- `{p}`: top-K at [{occ}]{edge} — {'contiguous' if f['contiguous'] else 'scattered'}")
410
+ L.extend(clusters)
411
+ L.append("")
412
+ L.append(f"[{'x' if not ctx['plateau'] else ' '}] Single isolated peak — fragile, retest with more samples")
413
+ L.append(f"[{'x' if ctx['plateau'] else ' '}] Plateau region — robust, safe to deploy")
414
+ L.append("")
415
+ if ctx["sample_col"]:
416
+ if ctx["samples_ok"]:
417
+ L.append(f"Sample sizes OK — all top-{ctx['k']} variants ≥ {a.min_samples} on `{ctx['sample_col']}`.")
418
+ else:
419
+ offenders = ", ".join(f"{vid} ({fmt_num(n,0)})" for vid, n in ctx["low_sample_variants"])
420
+ L.append(f"⚠ Below `--min-samples` ({a.min_samples}) on `{ctx['sample_col']}`: {offenders}")
421
+ L.append("")
422
+ L.append("---")
423
+ L.append("")
424
+ L.append("## Recommendation")
425
+ L.append("")
426
+ L.extend(_recommendation(ctx))
427
+ L.append("")
428
+ return "\n".join(L)
429
+
430
+
431
+ def _summary_line(ctx):
432
+ v = ctx["verdict"]
433
+ if v == "CONVERGED":
434
+ top_id = ctx["top"][0].get(ctx["id_col"])
435
+ # Nested same-quote f-strings are Python 3.12+ only; keep 3.9 parsing.
436
+ stable = "" if ctx["rho"] is None else f", OOS-stable (ρ={fmt_num(ctx['rho'])})"
437
+ return (f"Top-{ctx['k']} cluster tightly ({fmt_pct(ctx['disp_rank'])} dispersion) on a parameter plateau"
438
+ f"{stable} — ship `{top_id}`.")
439
+ if v == "KILL":
440
+ return ctx["kill_reasons"][0] if ctx["kill_reasons"] else "No stable edge in the swept parameter space."
441
+ return "Convergence not yet achieved — some criteria unmet (see below)."
442
+
443
+
444
+ def _read_line(ctx):
445
+ bits = []
446
+ d = ctx["disp_rank"]
447
+ if d is not None:
448
+ tight = d < ctx["args"].dispersion_threshold
449
+ bits.append(f"top-{ctx['k']} are {'clustered tightly' if tight else 'spread out'} on the ranking metric ({fmt_pct(d)})")
450
+ if ctx["rho"] is not None:
451
+ bits.append(f"IS↔OOS rank correlation is {fmt_num(ctx['rho'])}")
452
+ if not bits:
453
+ return "Limited signal — too few variants or near-zero mean for a dispersion read."
454
+ return "; ".join(bits).capitalize() + "."
455
+
456
+
457
+ def _recommendation(ctx):
458
+ v = ctx["verdict"]
459
+ if v == "CONVERGED":
460
+ top_id = ctx["top"][0].get(ctx["id_col"])
461
+ return [f"**[CONVERGED]** Top variant `{top_id}` is representative — its neighbors perform similarly "
462
+ f"and the winning region is a plateau, not a lucky point. Safe to advance this variant to the next stage."]
463
+ if v == "KILL":
464
+ out = ["**[KILL]** No region of the swept parameter space shows a stable edge. Specifically:"]
465
+ out += [f"- {r}" for r in ctx["kill_reasons"]]
466
+ out.append("Recommend abandoning this strategy line, or rethinking the metric/feature set before re-sweeping.")
467
+ return out
468
+ # ITERATE
469
+ out = ["**[ITERATE]** Convergence not yet achieved. Suggested next sweep:"]
470
+ sugg = []
471
+ for p, f in ctx["params"].items():
472
+ if f["edge"]:
473
+ sugg.append(f"- Expand `{p}` beyond its current edge (top variant sits at {fmt_num(f['top1'],4).rstrip('0').rstrip('.')}, "
474
+ f"the boundary of the swept grid)")
475
+ elif not f["contiguous"]:
476
+ sugg.append(f"- Refine the `{p}` grid around the top cluster — winners are currently scattered")
477
+ if ctx["disp_rank"] is not None and ctx["disp_rank"] >= ctx["args"].dispersion_threshold:
478
+ sugg.append(f"- Tighten dispersion: top-K spread is {fmt_pct(ctx['disp_rank'])} "
479
+ f"(> {fmt_pct(ctx['args'].dispersion_threshold,0)}) — narrow the grid or add samples")
480
+ if ctx["rho"] is not None and ctx["rho"] <= ctx["args"].rank_threshold:
481
+ sugg.append(f"- IS/OOS rank stability is weak (ρ={fmt_num(ctx['rho'])}) — likely undersampled; "
482
+ f"increase trades per variant or lengthen the OOS window")
483
+ if ctx["samples_ok"] is False:
484
+ sugg.append(f"- Rerun low-sample variants with ≥ {ctx['args'].min_samples} trades before trusting their rank")
485
+ if not sugg:
486
+ sugg.append("- Add samples and re-run; the signal is borderline on at least one criterion")
487
+ out += sugg
488
+ return out
489
+
490
+
491
+ VERDICT_EXIT = {"CONVERGED": 0, "ITERATE": 1, "KILL": 2}
492
+
493
+
494
+ def add_arguments(p):
495
+ p.add_argument("input", help="CSV file: one row per variant")
496
+ p.add_argument("--metric", help="Ranking metric column (e.g. sharpe_oos). Auto-detected if omitted.")
497
+ p.add_argument("--top-k", type=int, default=5, help="Number of top variants to assess (default 5)")
498
+ p.add_argument("--dispersion-threshold", type=float, default=0.05,
499
+ help="Max relative dispersion for CONVERGED (default 0.05 = 5%%)")
500
+ p.add_argument("--min-samples", type=int, default=200, help="Min samples/variant (default 200)")
501
+ p.add_argument("--rank-threshold", type=float, default=0.6,
502
+ help="Min IS/OOS Spearman for CONVERGED (default 0.6)")
503
+ p.add_argument("--id-column", help="Override the variant-id column")
504
+ p.add_argument("--sample-column", help="Override the sample-size column")
505
+ p.add_argument("--lower-is-better", action="store_true", help="Treat the metric as lower-is-better")
506
+ p.add_argument("--higher-is-better", action="store_true", help="Force higher-is-better (override name heuristic)")
507
+ p.add_argument("--run-name", help="Label for the report header")
508
+ p.add_argument("--date", help="Date for the report header (default: today)")
509
+ p.add_argument("--save", action="store_true",
510
+ help="Also write iteration_check_<timestamp>.md in the current directory")
511
+ p.add_argument("--json", action="store_true", help="Print a machine-readable verdict instead of Markdown")
512
+
513
+
514
+ def to_json(ctx):
515
+ """Machine-readable verdict: the numbers behind the Markdown report."""
516
+ getf = ctx["getf"]
517
+ return {
518
+ "schema": "backtest2live.convergence/1",
519
+ "input": ctx["path"],
520
+ "verdict": ctx["verdict"],
521
+ "exit_code": VERDICT_EXIT[ctx["verdict"]],
522
+ "metric": ctx["rank_col"],
523
+ "lower_is_better": ctx["lower_better"],
524
+ "is_column": ctx["is_col"],
525
+ "oos_column": ctx["oos_col"],
526
+ "sample_column": ctx["sample_col"],
527
+ "top_k": ctx["k"],
528
+ "n_variants": ctx["n"],
529
+ "top": [{"id": r.get(ctx["id_col"]), "value": getf(r, ctx["rank_col"])} for r in ctx["top"]],
530
+ "dispersion": {"rank": ctx["disp_rank"], "is": ctx["disp_is"], "oos": ctx["disp_oos"]},
531
+ "is_oos_spearman": ctx["rho"],
532
+ "separation_sigma": ctx["separation"],
533
+ "plateau": ctx["plateau"],
534
+ "edge_peak": ctx["any_edge"],
535
+ "parameters": {
536
+ p: {"occupied": f["occupied"], "contiguous": f["contiguous"], "edge": f["edge"]}
537
+ for p, f in ctx["params"].items()
538
+ },
539
+ "samples_ok": ctx["samples_ok"],
540
+ "low_sample_variants": [{"id": vid, "n": n} for vid, n in ctx["low_sample_variants"]],
541
+ "kill_reasons": ctx["kill_reasons"],
542
+ "thresholds": {
543
+ "dispersion": ctx["args"].dispersion_threshold,
544
+ "rank": ctx["args"].rank_threshold,
545
+ "min_samples": ctx["args"].min_samples,
546
+ },
547
+ }
548
+
549
+
550
+ def run(args):
551
+ """CLI entry for `bt2live convergence`. Returns the verdict exit code."""
552
+ ctx = analyze(args.input, args)
553
+ report = build_report(ctx)
554
+ if args.json:
555
+ print(json.dumps(to_json(ctx), indent=2, ensure_ascii=False, allow_nan=False))
556
+ else:
557
+ print(report)
558
+
559
+ if args.save:
560
+ stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
561
+ out_path = f"iteration_check_{stamp}.md"
562
+ with open(out_path, "w", encoding="utf-8") as fh:
563
+ fh.write(report + "\n")
564
+ print(f"\n[saved report → {out_path}]", file=sys.stderr)
565
+
566
+ # Exit code encodes the verdict for scripting: 0 CONVERGED, 1 ITERATE, 2 KILL.
567
+ return VERDICT_EXIT[ctx["verdict"]]