backtest2live 0.2.0__py3-none-any.whl

backtest2live/fillgap/__init__.py

@@ -0,0 +1,57 @@
+"""backtest2live · fill-gap — how many of your backtest's passive fills survive a queue proxy.
+
+Re-runs each resting order in your backtest's order log against public
+top-of-book and trade prints, under a queue proxy: the order fills only after
+the volume printed at its price covers the displayed queue ahead of it plus
+its own size. Reports the backtest's fill rate against the proxy's.
+
+numpy and pandas are imported inside run(), so `bt2live convergence` keeps
+working on a bare Python install.
+"""
+
+import sys
+
+SCHEMA = "backtest2live.fill-gap/1"
+VENUES = ("binance-um",)
+CTA_URL = "https://martianmobile.com/fill-autopsy?utm_source=oss&utm_medium=cli&utm_campaign=fill-gap"
+VERDICT_EXIT = {"CONSISTENT": 0, "OVERSTATED": 1}
+
+
+def add_arguments(p):
+    p.add_argument("orders", help="Your backtest's order log (CSV): time, side, price, size, and the "
+                                  "backtest's filled flag. See README for accepted column names.")
+    src = p.add_argument_group("market data (pick one)")
+    src.add_argument("--venue", choices=VENUES,
+                     help="Download public data: binance-um = Binance USDT-M futures archive "
+                          "(bookTicker ends 2024-03-30)")
+    src.add_argument("--pair", help="Symbol for --venue, e.g. BTCUSDT")
+    src.add_argument("--book", help="Your own top-of-book CSV: ts, bid_px, bid_qty, ask_px, ask_qty")
+    src.add_argument("--trades", help="Your own trades CSV: ts, price, qty, is_buyer_maker (or taker side)")
+    p.add_argument("--max-rest", type=float, default=60.0,
+                   help="Seconds an order rests when the log has no end/cancel column (default 60)")
+    p.add_argument("--latency-ms", type=float, default=0.0,
+                   help="Delay each order's arrival at the book by this much (default 0)")
+    p.add_argument("--tick", type=float, help="Price tick (inferred from the book if omitted)")
+    p.add_argument("--tolerance", type=float, default=0.10,
+                   help="Max share of claimed fills the proxy may miss before the verdict is "
+                        "OVERSTATED (default 0.10)")
+    p.add_argument("--offline", action="store_true", help="Use cached data only; never download")
+    p.add_argument("--run-name", help="Label for the report header")
+    p.add_argument("--json", action="store_true", help="Print a machine-readable result instead of Markdown")
+
+
+def run(args):
+    try:
+        import numpy  # noqa: F401
+        import pandas  # noqa: F401
+    except ImportError:
+        raise SystemExit("error: fill-gap needs numpy and pandas: pip install 'backtest2live[data]'")
+    from backtest2live.fillgap import engine, report
+
+    res = engine.evaluate(args, log=lambda m: print(m, file=sys.stderr))
+    if args.json:
+        import json
+        print(json.dumps(report.to_json(res), indent=2))
+    else:
+        print(report.markdown(res))
+    return VERDICT_EXIT[res["verdict"]]

backtest2live/fillgap/data.py

@@ -0,0 +1,289 @@
+"""Inputs for fill-gap: the user's order log, and L1 book + trade prints.
+
+Book and trades come from a loader: the Binance public archive (USDT-M
+futures, downloaded once and cached) or the user's own CSVs. Every loader
+returns the same normalized arrays: times in int64 microseconds, prices as
+floats (converted to ticks by the caller), sizes in base units.
+"""
+
+import hashlib
+import os
+import urllib.request
+import zipfile
+from datetime import date, datetime, timedelta, timezone
+
+import numpy as np
+import pandas as pd
+
+ORDER_ALIASES = {
+    "t": ("ts", "timestamp", "time", "datetime", "created_at", "submit_time", "t"),
+    "side": ("side", "direction", "dir"),
+    "px": ("price", "px", "limit_price", "limit"),
+    "size": ("size", "qty", "quantity", "amount", "sz"),
+    "filled": ("filled", "is_filled", "fill", "backtest_filled"),
+    "fill_qty": ("fill_qty", "filled_qty", "executed_qty"),
+    "t_end": ("end_ts", "cancel_ts", "expire_ts", "cancel_time", "end_time", "expiry"),
+    "pnl": ("pnl", "pnl_usd", "realized_pnl", "profit"),
+    "id": ("order_id", "id", "client_order_id"),
+}
+BOOK_ALIASES = {
+    "t": ("ts", "timestamp", "time", "transaction_time", "event_time", "t"),
+    "bid_px": ("bid_px", "bid_price", "best_bid_price", "bid"),
+    "bid_qty": ("bid_qty", "bid_size", "best_bid_qty", "bid_sz"),
+    "ask_px": ("ask_px", "ask_price", "best_ask_price", "ask"),
+    "ask_qty": ("ask_qty", "ask_size", "best_ask_qty", "ask_sz"),
+}
+TRADE_ALIASES = {
+    "t": ("ts", "timestamp", "time", "transact_time", "trade_time", "t"),
+    "px": ("price", "px"),
+    "qty": ("qty", "quantity", "size", "amount"),
+    "is_buyer_maker": ("is_buyer_maker", "buyer_maker", "m"),
+    "aggr_side": ("side", "aggressor_side", "taker_side"),
+}
+
+BUY_WORDS = {"buy", "b", "bid", "long", "1", "+1"}
+SELL_WORDS = {"sell", "s", "ask", "short", "-1"}
+TRUE_WORDS = {"true", "t", "1", "yes", "y", "1.0"}
+FALSE_WORDS = {"false", "f", "0", "no", "n", "0.0", ""}
+
+
+def _resolve(df, aliases, required, what):
+    cols = {c.lower().strip(): c for c in df.columns}
+    out = {}
+    for role, names in aliases.items():
+        for n in names:
+            if n in cols:
+                out[role] = cols[n]
+                break
+    missing = [r for r in required if r not in out]
+    if missing:
+        want = "; ".join(f"{r}: one of {', '.join(aliases[r])}" for r in missing)
+        raise SystemExit(f"error: {what} is missing columns ({want}). Found: {', '.join(df.columns)}")
+    return out
+
+
+def to_us(col):
+    """Epoch s/ms/us/ns numbers or ISO-8601 strings → int64 microseconds UTC."""
+    s = pd.Series(col)
+    if pd.api.types.is_numeric_dtype(s):
+        v = s.to_numpy(dtype=np.float64)
+        if np.isnan(v).any():
+            raise SystemExit(f"error: blank or non-numeric timestamp in column {s.name!r}")
+        mag = np.nanmedian(np.abs(v)) if len(v) else 0
+        if mag > 1e17:
+            f = 1e-3      # ns
+        elif mag > 1e14:
+            f = 1.0       # us
+        elif mag > 1e11:
+            f = 1e3       # ms
+        else:
+            f = 1e6       # s
+        return np.rint(v * f).astype(np.int64)
+    parsed = pd.to_datetime(s, utc=True, errors="coerce")
+    if parsed.isna().any():
+        bad = s[parsed.isna()].iloc[0]
+        raise SystemExit(f"error: unparseable timestamp {bad!r}")
+    # resolution-independent (pandas 2 may parse to s/ms/us/ns)
+    return ((parsed - pd.Timestamp(0, tz="UTC")) // pd.Timedelta(microseconds=1)).to_numpy(dtype=np.int64)
+
+
+def _bool(col, what):
+    s = pd.Series(col).astype(str).str.strip().str.lower()
+    bad = ~s.isin(TRUE_WORDS | FALSE_WORDS)
+    if bad.any():
+        raise SystemExit(f"error: {what} must be true/false or 1/0; got {s[bad].iloc[0]!r}")
+    return s.isin(TRUE_WORDS).to_numpy()
+
+
+def _read_csv(path, what):
+    try:
+        df = pd.read_csv(path)
+    except FileNotFoundError:
+        raise SystemExit(f"error: {what} not found: {path!r}")
+    except (OSError, UnicodeDecodeError, pd.errors.EmptyDataError, pd.errors.ParserError) as e:
+        raise SystemExit(f"error: could not read {what} {path!r}: {e}")
+    if df.empty:
+        raise SystemExit(f"error: no data rows in {what} {path!r}")
+    return df
+
+
+def _numeric(df, col, what):
+    v = pd.to_numeric(df[col], errors="coerce")
+    if v.isna().any():
+        bad = df[col][v.isna()].iloc[0]
+        raise SystemExit(f"error: non-numeric {what} in column {col!r}: {bad!r}")
+    return v.to_numpy(np.float64)
+
+
+def load_orders(path, max_rest_s):
+    df = _read_csv(path, "order log")
+    c = _resolve(df, ORDER_ALIASES, ("t", "side", "px", "size"), "order log")
+
+    side = df[c["side"]].astype(str).str.strip().str.lower()
+    unknown = ~side.isin(BUY_WORDS | SELL_WORDS)
+    if unknown.any():
+        raise SystemExit(f"error: side must be buy/sell; got {side[unknown].iloc[0]!r}")
+
+    t = to_us(df[c["t"]])
+    if "t_end" in c:
+        t_end = to_us(df[c["t_end"]])
+        end_source = c["t_end"]
+    else:
+        t_end = t + int(max_rest_s * 1e6)
+        end_source = f"--max-rest {max_rest_s:g}s"
+
+    if "filled" in c:
+        claimed = _bool(df[c["filled"]], c["filled"])
+    elif "fill_qty" in c:
+        claimed = pd.to_numeric(df[c["fill_qty"]], errors="coerce").fillna(0).to_numpy() > 0
+    else:
+        claimed = None
+
+    return {
+        "t": t,
+        "t_end": t_end,
+        "buy": side.isin(BUY_WORDS).to_numpy(),
+        "px_f": _numeric(df, c["px"], "price"),
+        "size": _numeric(df, c["size"], "size"),
+        "claimed": claimed,
+        "pnl": pd.to_numeric(df[c["pnl"]], errors="coerce").to_numpy(np.float64) if "pnl" in c else None,
+        "id": df[c["id"]].astype(str).to_numpy() if "id" in c else np.arange(len(df)).astype(str),
+        "columns": c,
+        "end_source": end_source,
+    }
+
+
+# --- user-supplied book and trades ------------------------------------------
+
+def load_user_book(path):
+    df = _read_csv(path, "book file")
+    c = _resolve(df, BOOK_ALIASES, tuple(BOOK_ALIASES), "book file")
+    out = {"t": to_us(df[c["t"]])}
+    for k in ("bid_px", "bid_qty", "ask_px", "ask_qty"):
+        out[k] = _numeric(df, c[k], k)
+    return _sort(out)
+
+
+def load_user_trades(path):
+    df = _read_csv(path, "trades file")
+    c = _resolve(df, TRADE_ALIASES, ("t", "px", "qty"), "trades file")
+    if "is_buyer_maker" in c:
+        sell_aggr = _bool(df[c["is_buyer_maker"]], c["is_buyer_maker"])
+    elif "aggr_side" in c:
+        s = df[c["aggr_side"]].astype(str).str.strip().str.lower()
+        sell_aggr = s.isin(SELL_WORDS).to_numpy()
+    else:
+        raise SystemExit("error: trades file needs the aggressor side: is_buyer_maker (true = seller hit the bid) "
+                         "or side (buy/sell = the taker's side)")
+    return _sort({"t": to_us(df[c["t"]]), "px": _numeric(df, c["px"], "price"),
+                  "qty": _numeric(df, c["qty"], "qty"), "sell_aggr": sell_aggr})
+
+
+def _sort(d):
+    o = np.argsort(d["t"], kind="stable")
+    return {k: v[o] for k, v in d.items()}
+
+
+# --- Binance public archive (USDT-M futures) ----------------------------------
+
+BINANCE_ROOT = "https://data.binance.vision/data/futures/um/daily"
+BINANCE_BOOKTICKER_LAST = date(2024, 3, 30)  # the archive stops publishing bookTicker after this day
+
+
+def cache_dir():
+    base = os.environ.get("BT2LIVE_CACHE") or os.path.join(
+        os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), "backtest2live")
+    return base
+
+
+def _fetch(kind, pair, day, offline, log):
+    name = f"{pair}-{kind}-{day.isoformat()}.zip"
+    url = f"{BINANCE_ROOT}/{kind}/{pair}/{name}"
+    path = os.path.join(cache_dir(), "data.binance.vision", "data", "futures", "um", "daily", kind, pair, name)
+    if os.path.exists(path):
+        return path
+    if offline:
+        raise SystemExit(f"error: {name} is not cached and --offline is set (cache: {cache_dir()})")
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    log(f"downloading {url}")
+    try:
+        with urllib.request.urlopen(url + ".CHECKSUM", timeout=60) as r:
+            want = r.read().decode().split()[0]
+        tmp = path + ".part"
+        h = hashlib.sha256()
+        with urllib.request.urlopen(url, timeout=600) as r, open(tmp, "wb") as fh:
+            while True:
+                chunk = r.read(1 << 20)
+                if not chunk:
+                    break
+                h.update(chunk)
+                fh.write(chunk)
+    except OSError as e:
+        raise SystemExit(f"error: could not download {url}: {e}")
+    if h.hexdigest() != want:
+        os.remove(tmp)
+        raise SystemExit(f"error: checksum mismatch for {name}")
+    os.replace(tmp, path)
+    return path
+
+
+def _read_zip_csv(path, usecols, dtype=None):
+    try:
+        with zipfile.ZipFile(path) as z:
+            with z.open(z.namelist()[0]) as fh:
+                return pd.read_csv(fh, usecols=usecols, dtype=dtype)
+    except (zipfile.BadZipFile, ValueError, IndexError) as e:
+        raise SystemExit(f"error: {os.path.basename(path)} is not a readable archive ({e}); "
+                         f"delete it from the cache and retry")
+
+
+def load_binance_um(pair, days, offline=False, log=print):
+    """bookTicker + aggTrades for the given UTC days, concatenated.
+
+    days[0] is the arrival day and must exist in the archive. Later days are
+    only the tail of the rest window: past the archive end they are dropped,
+    and those orders are evaluated on the data available.
+    """
+    if days[0] > BINANCE_BOOKTICKER_LAST:
+        raise SystemExit(
+            f"error: Binance publishes USDT-M bookTicker only through {BINANCE_BOOKTICKER_LAST}; your orders "
+            f"arrive on {days[0]}. Pass your own top-of-book and trades with --book and --trades.")
+    days = [d for d in days if d <= BINANCE_BOOKTICKER_LAST]
+    books, trades = [], []
+    for d in days:
+        b = _read_zip_csv(_fetch("bookTicker", pair, d, offline, log),
+                          ["best_bid_price", "best_bid_qty", "best_ask_price", "best_ask_qty", "transaction_time"],
+                          {"best_bid_price": "f8", "best_bid_qty": "f8", "best_ask_price": "f8",
+                           "best_ask_qty": "f8", "transaction_time": "i8"})
+        tr = _read_zip_csv(_fetch("aggTrades", pair, d, offline, log),
+                           ["price", "quantity", "transact_time", "is_buyer_maker"])
+        books.append(b)
+        trades.append(tr)
+    b = pd.concat(books, ignore_index=True)
+    tr = pd.concat(trades, ignore_index=True)
+    book = _sort({"t": b["transaction_time"].to_numpy(np.int64) * 1000,
+                  "bid_px": b["best_bid_price"].to_numpy(), "bid_qty": b["best_bid_qty"].to_numpy(),
+                  "ask_px": b["best_ask_price"].to_numpy(), "ask_qty": b["best_ask_qty"].to_numpy()})
+    ibm = tr["is_buyer_maker"]
+    if ibm.dtype != bool:
+        ibm = ibm.astype(str).str.lower().eq("true")
+    trd = _sort({"t": tr["transact_time"].to_numpy(np.int64) * 1000, "px": tr["price"].to_numpy(np.float64),
+                 "qty": tr["quantity"].to_numpy(np.float64), "sell_aggr": ibm.to_numpy()})
+    return book, trd
+
+
+def utc_days(t_from_us, t_to_us):
+    d0 = datetime.fromtimestamp(t_from_us / 1e6, tz=timezone.utc).date()
+    d1 = datetime.fromtimestamp(t_to_us / 1e6, tz=timezone.utc).date()
+    return [d0 + timedelta(days=i) for i in range((d1 - d0).days + 1)]
+
+
+def infer_tick(book):
+    """Smallest price step seen in the book, rounded to a clean decimal."""
+    px = np.unique(np.r_[book["bid_px"][:200_000], book["ask_px"][:200_000]])
+    d = np.diff(px)
+    d = d[d > 0]
+    if not len(d):
+        raise SystemExit("error: cannot infer the tick size from the book; pass --tick")
+    step = float(d.min())
+    return float(f"{step:.8g}")  # 0.09999999999 → 0.1

backtest2live/fillgap/engine.py

@@ -0,0 +1,169 @@
+"""Load inputs, run the simulation day by day, and aggregate the result."""
+
+import os
+
+import numpy as np
+
+from backtest2live.fillgap import data, sim
+
+US_PER_S = 1_000_000
+US_PER_H = 3_600 * US_PER_S
+
+
+def _ticks(px, tick):
+    return np.rint(np.asarray(px, np.float64) / tick).astype(np.int64)
+
+
+def _run_chunk(book, trades, orders, idx, tick, latency_us):
+    b = {"t": book["t"], "bid_px": _ticks(book["bid_px"], tick), "ask_px": _ticks(book["ask_px"], tick),
+         "bid_qty": book["bid_qty"], "ask_qty": book["ask_qty"]}
+    tr = {"t": trades["t"], "px": _ticks(trades["px"], tick), "qty": trades["qty"],
+          "sell_aggr": trades["sell_aggr"]}
+    px = orders["px_f"][idx] / tick
+    off = np.abs(px - np.rint(px)) > 1e-6
+    if off.any():
+        raise SystemExit(f"error: {int(off.sum())} order price(s) are not on the {tick:g} tick grid "
+                         f"(first: {orders['px_f'][idx][off][0]:g}); pass --tick or fix the log")
+    o = {"t": orders["t"][idx], "t_end": orders["t_end"][idx], "buy": orders["buy"][idx],
+         "px": np.rint(px).astype(np.int64), "size": orders["size"][idx]}
+    r = sim.simulate(b, tr, o, latency_us)
+    r["range_bps"] = sim.trailing_range_bps(
+        {"t": book["t"], "bid_px": book["bid_px"], "ask_px": book["ask_px"]}, r["t_arr"]) \
+        if len(book["t"]) else np.full(len(idx), np.nan)
+    return r
+
+
+def evaluate(args, log=print):
+    if bool(args.venue) == bool(args.book or args.trades):
+        raise SystemExit("error: pass either --venue with --pair, or --book with --trades")
+    if args.venue and not args.pair:
+        raise SystemExit("error: --venue needs --pair (e.g. --pair BTCUSDT)")
+    if (args.book or args.trades) and not (args.book and args.trades):
+        raise SystemExit("error: --book and --trades go together")
+
+    orders = data.load_orders(args.orders, args.max_rest)
+    n = len(orders["t"])
+    latency_us = int(round(args.latency_ms * 1000))
+    tick = args.tick
+
+    fields = ("status", "queue", "t_act", "touch", "t_touch", "proxy", "t_proxy", "t_arr", "range_bps")
+    out = {k: None for k in fields}
+
+    def put(idx, r):
+        for k in fields:
+            if out[k] is None:
+                out[k] = np.zeros(n, dtype=r[k].dtype)
+                if r[k].dtype.kind in "if":
+                    out[k][:] = -1 if r[k].dtype.kind == "i" else np.nan
+            out[k][idx] = r[k]
+
+    if args.book:
+        book, trades = data.load_user_book(args.book), data.load_user_trades(args.trades)
+        tick = tick or data.infer_tick(book)
+        put(np.arange(n), _run_chunk(book, trades, orders, np.arange(n), tick, latency_us))
+        source = f"your files ({os.path.basename(args.book)}, {os.path.basename(args.trades)})"
+    else:
+        pair = args.pair.upper()
+        t_arr = orders["t"] + latency_us
+        day_of = (t_arr // (86_400 * US_PER_S))
+        for d in np.unique(day_of):
+            idx = np.flatnonzero(day_of == d)
+            days = data.utc_days(int(t_arr[idx].min()), int(orders["t_end"][idx].max()))
+            book, trades = data.load_binance_um(pair, days, offline=args.offline, log=log)
+            tick = tick or data.infer_tick(book)
+            put(idx, _run_chunk(book, trades, orders, idx, tick, latency_us))
+        source = f"Binance USDT-M futures public archive, {pair}"
+
+    return aggregate(orders, out, args, tick, source)
+
+
+def _rate(mask, base):
+    k = int(base.sum())
+    return float(mask[base].mean()) if k else None
+
+
+def _median_s(t_fill, t_arr, mask):
+    m = mask & (t_fill >= 0)
+    return float(np.median((t_fill[m] - t_arr[m]) / US_PER_S)) if m.any() else None
+
+
+def aggregate(orders, r, args, tick, source):
+    st = r["status"]
+    passive = (st == sim.ACTIVE) | (st == sim.NEVER_REACHED)
+    claimed = orders["claimed"]
+    has_claim = claimed is not None
+    base = claimed if has_claim else r["touch"]
+    proxy = r["proxy"]
+
+    missed = passive & base & ~proxy
+    extra = passive & proxy & ~base
+    n_base = int((passive & base).sum())
+    missed_share = (int(missed.sum()) / n_base) if n_base else 0.0
+    verdict = "OVERSTATED" if missed_share > args.tolerance else "CONSISTENT"
+
+    hours = ((r["t_arr"] // US_PER_H) % 24).astype(int)
+    by_hour = []
+    for h in range(24):
+        m = passive & (hours == h)
+        if m.any():
+            by_hour.append({"hour": h, "orders": int(m.sum()), "baseline": _rate(base, m),
+                            "proxy": _rate(proxy, m)})
+
+    by_vol = []
+    rng = r["range_bps"]
+    ok = passive & np.isfinite(rng)
+    if ok.sum() >= 3:
+        cuts = np.quantile(rng[ok], [1 / 3, 2 / 3])
+        tercile = np.where(rng <= cuts[0], 0, np.where(rng <= cuts[1], 1, 2))
+        lo = float(np.nanmin(rng[ok]))
+        edges = [lo, float(cuts[0]), float(cuts[1]), float(np.nanmax(rng[ok]))]
+        for q, name in enumerate(("low", "mid", "high")):
+            m = ok & (tercile == q)
+            by_vol.append({"tercile": name, "range_bps": [edges[q], edges[q + 1]], "orders": int(m.sum()),
+                           "baseline": _rate(base, m), "proxy": _rate(proxy, m)})
+
+    pnl = None
+    if orders["pnl"] is not None:
+        p = np.nan_to_num(orders["pnl"])
+        pnl = {"baseline_fills": float(p[passive & base].sum()), "missed_fills": float(p[missed].sum())}
+
+    t = orders["t"]
+    return {
+        "verdict": verdict,
+        "orders_file": args.orders,
+        "run_name": args.run_name or os.path.basename(args.orders),
+        "source": source,
+        "window_utc": [int(t.min()), int(orders["t_end"].max())],
+        "tick": tick,
+        "latency_ms": args.latency_ms,
+        "order_end": orders["end_source"],
+        "tolerance": args.tolerance,
+        "baseline": "your backtest's fill flag" if has_claim else "touch rule (no fill flag in the log)",
+        "has_claim": has_claim,
+        "counts": {
+            "orders": int(len(t)),
+            "passive": int(passive.sum()),
+            "marketable": int((st == sim.MARKETABLE).sum()),
+            "no_data": int((st == sim.NO_DATA).sum()),
+            "never_reached": int((st == sim.NEVER_REACHED).sum()),
+            "baseline_fills": n_base,
+            "touch_fills": int((passive & r["touch"]).sum()),
+            "proxy_fills": int((passive & proxy).sum()),
+            "missed": int(missed.sum()),
+            "extra": int(extra.sum()),
+        },
+        "rates": {
+            "baseline": _rate(base, passive),
+            "touch": _rate(r["touch"], passive),
+            "proxy": _rate(proxy, passive),
+            "missed_share_of_baseline": missed_share,
+        },
+        "median_time_to_fill_s": {
+            "touch": _median_s(r["t_touch"], r["t_arr"], passive & r["touch"]),
+            "proxy": _median_s(r["t_proxy"], r["t_arr"], passive & proxy),
+        },
+        "by_hour_utc": by_hour,
+        "by_volatility": by_vol,
+        "pnl": pnl,
+        "missed_ids": orders["id"][missed][:1000].tolist(),
+    }

backtest2live/fillgap/report.py

@@ -0,0 +1,101 @@
+"""Markdown and JSON output for fill-gap."""
+
+from datetime import datetime, timezone
+
+from backtest2live.fillgap import CTA_URL, SCHEMA
+
+
+def _pct(x):
+    return "n/a" if x is None else f"{x * 100:.1f}%"
+
+
+def _secs(x):
+    if x is None:
+        return "n/a"
+    return f"{x * 1000:.0f} ms" if x < 1 else f"{x:.1f} s"
+
+
+def _day(us):
+    return datetime.fromtimestamp(us / 1e6, tz=timezone.utc).strftime("%Y-%m-%d")
+
+
+def _summary(res):
+    r, c = res["rates"], res["counts"]
+    who = "Your backtest fills" if res["has_claim"] else "Filling at touch gives"
+    line = (f"{who} {_pct(r['baseline'])} of its resting orders; the queue proxy fills "
+            f"{_pct(r['proxy'])}.")
+    if c["baseline_fills"]:
+        line += (f" {_pct(r['missed_share_of_baseline'])} of the counted fills "
+                 f"({c['missed']:,} of {c['baseline_fills']:,}) are not granted on this data.")
+    return line
+
+
+def markdown(res):
+    c, r = res["counts"], res["rates"]
+    w0, w1 = _day(res["window_utc"][0]), _day(res["window_utc"][1])
+    window = w0 if w0 == w1 else f"{w0} .. {w1}"
+    L = [f"# Fill Gap | {res['run_name']} | {window}", "",
+         f"## Verdict: {res['verdict']}", "", f"**{_summary(res)}**", "", "---", "",
+         "## Fill rates", "",
+         "| Rule | Fills | Fill rate |", "|------|------:|----------:|"]
+    if res["has_claim"]:
+        L.append(f"| Your backtest (`filled`) | {c['baseline_fills']:,} | {_pct(r['baseline'])} |")
+    L.append(f"| Touch rule: any print at or through your price | {c['touch_fills']:,} | {_pct(r['touch'])} |")
+    L.append(f"| Queue proxy | {c['proxy_fills']:,} | {_pct(r['proxy'])} |")
+    L += ["",
+          f"Rates are over {c['passive']:,} resting orders. Skipped: {c['marketable']:,} marketable at arrival "
+          f"(taker orders, out of scope) and {c['no_data']:,} outside the market data. "
+          f"{c['never_reached']:,} rested behind the touch and the price never came to them.", ""]
+    mt = res["median_time_to_fill_s"]
+    L.append(f"Median time to fill: {_secs(mt['touch'])} at touch, {_secs(mt['proxy'])} under the proxy.")
+    if c["extra"]:
+        L.append(f"The proxy grants {c['extra']:,} fills the baseline did not count.")
+    if res["pnl"]:
+        p = res["pnl"]
+        share = f" ({p['missed_fills'] / p['baseline_fills'] * 100:.0f}% of it)" if p["baseline_fills"] else ""
+        L.append(f"P&L your backtest booked on fills the proxy does not grant: {p['missed_fills']:,.2f} "
+                 f"of {p['baseline_fills']:,.2f}{share}.")
+    L += ["", "---", "", "## Where the fills go missing", ""]
+    if res["by_volatility"]:
+        L += ["By volatility at arrival (trailing 5-minute range of the mid):", "",
+              "| Tercile | Range (bps) | Orders | Baseline | Proxy |", "|--------|------------|-------:|------:|------:|"]
+        for v in res["by_volatility"]:
+            L.append(f"| {v['tercile']} | {v['range_bps'][0]:.1f}–{v['range_bps'][1]:.1f} | {v['orders']:,} | "
+                     f"{_pct(v['baseline'])} | {_pct(v['proxy'])} |")
+        L.append("")
+    if res["by_hour_utc"]:
+        L += ["By hour (UTC):", "", "| Hour | Orders | Baseline | Proxy |", "|-----:|-------:|------:|------:|"]
+        for h in res["by_hour_utc"]:
+            L.append(f"| {h['hour']:02d} | {h['orders']:,} | {_pct(h['baseline'])} | {_pct(h['proxy'])} |")
+        L.append("")
+    L += ["---", "", "## Inputs", "",
+          f"- Market data: {res['source']}; tick {res['tick']:g}",
+          f"- Order life: until `{res['order_end']}`" if not res["order_end"].startswith("--") else
+          f"- Order life: {res['order_end']} (no end/cancel column in the log)",
+          f"- Arrival latency: {res['latency_ms']:g} ms",
+          f"- Baseline: {res['baseline']}; verdict OVERSTATED when more than {_pct(res['tolerance'])} "
+          f"of baseline fills are not granted",
+          "", "---", "", "## What the proxy can and cannot see", "",
+          "The touch rule is an upper bound: a resting order fills only if something trades at or through "
+          "its price. The queue proxy is an estimate below that bound, built from public top-of-book and trades:",
+          "",
+          "- It grants fills too easily where it cannot see hidden or iceberg size ahead of you, your own "
+          "latency to reach the book (unless `--latency-ms` is set), or your order's effect on the flow.",
+          "- It grants fills too hard where displayed size ahead of you cancels rather than trades, and "
+          "where it measures your place in the queue from the size shown when the price reached you.",
+          "",
+          "Which way it nets out depends on the venue, the size and the regime. Your own live fills settle it: "
+          f"the calibrated check against live fills is at {CTA_URL}", ""]
+    return "\n".join(L)
+
+
+def to_json(res):
+    out = {"schema": SCHEMA}
+    out.update({k: v for k, v in res.items()})
+    out["window_utc"] = [_iso(t) for t in res["window_utc"]]
+    out["calibrated_check"] = CTA_URL
+    return out
+
+
+def _iso(us):
+    return datetime.fromtimestamp(us / 1e6, tz=timezone.utc).isoformat()