pitbacktest 0.2.0__py3-none-any.whl

pitbacktest/crypto/costs.py

@@ -0,0 +1,80 @@
+"""Trading costs and capacity for crypto perpetuals.
+
+There is no public order-book history in the archive, so costs here are *estimates*, and they are labelled as such.
+- Fees: a flat taker fee (default 5 bp, the standard retail tier). Change it to your tier.
+- Spread: a flat assumption plus a thin-contract penalty. A daily high-low estimator (Corwin-Schultz) was tried first
+  and rejected because it is invalid for crypto volatility (see `liquidity_cost_bp`).
+- Capacity: instead of pretending to model market impact without data, `participation_report` shows how large the
+  trades are relative to each contract's turnover for a given account size.
+"""
+from __future__ import annotations
+
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from ..core.costs import corwin_schultz
+
+
+def liquidity_cost_bp(panel, *, taker_fee_bp: float = 5.0, half_spread_bp: float = 2.0, window: int = 30,
+                      thin_usd: float = 1e8, thin_extra_bp: float = 5.0,
+                      spread_estimator: str = "fixed", tick_size=None) -> pd.DataFrame:
+    """(date x ticker) one-way cost in bp = taker fee + half spread (+ a penalty for thin contracts).
+
+    spread_estimator
+      "fixed"           a flat half spread (`half_spread_bp`). The default. It is an assumption, not a measurement, so
+                        use `cost_sensitivity` or a grid of values instead of trusting one number.
+      "corwin_schultz"  half the Corwin-Schultz estimate from daily highs and lows. **Do not use it for crypto.**
+                        On Binance perpetuals it gave a median spread of about 1.5% and about 38 bp one way for
+                        BTC, orders of magnitude above any real quote (the study in `studies/crypto_cross_section`
+                        documents this in Amendment 1). I did not establish why; daily crypto ranges are dominated
+                        by jumps and volatility clustering, which the estimator's assumptions do not cover. It is
+                        kept only to reproduce the preregistered run.
+
+      "tick"            half of one price tick as a share of the price: `0.5 * tick_size / close * 1e4` bp (`tick_size` is a Series by ticker,
+                        for example `ArchiveStore.symbol_rules()["tick_size"]`; a ticker without one raises). Measured against the order book
+                        (docs/crypto_spread_check.md): the quoted half spread was one tick for 9 of 12 sampled contracts (0.8 to 1.25 times the
+                        prediction). The 3 others were 10, 10 and 100 times wider: the exchange has cut their tick since, and `tick_size` is
+                        today's value, so there it is a **floor**. It is the cost of a small order at the best quote, not of a large one:
+                        impact is not in it.
+
+    Contracts whose trailing turnover is below `thin_usd` pay `thin_extra_bp` more, a blunt stand-in for impact.
+    Uses only data up to day t."""
+    adv = panel.adv(window)
+    extra = (adv < thin_usd).astype(float) * thin_extra_bp if adv is not None else 0.0
+    if spread_estimator == "corwin_schultz":
+        warnings.warn("Corwin-Schultz overstates crypto spreads by orders of magnitude; use only to reproduce old runs.",
+                      stacklevel=2)
+        cs = corwin_schultz(panel.high, panel.low)
+        half = (cs.rolling(window, min_periods=10).median() * 0.5 * 1e4).clip(lower=1.0)
+    elif spread_estimator == "tick":
+        if tick_size is None:
+            raise ValueError("spread_estimator='tick' needs tick_size, a Series by ticker")
+        tk = pd.Series(tick_size, dtype=float).reindex(panel.tickers)
+        if tk.isna().any() or (tk <= 0).any():
+            raise ValueError(f"tick_size is missing or not positive for {int((tk.isna() | (tk <= 0)).sum())} tickers "
+                             f"(first: {list(tk.index[(tk.isna() | (tk <= 0)).to_numpy()][:3])}); a delisted contract has none in the exchange rules")
+        half = 0.5 * tk / panel.close * 1e4
+    elif spread_estimator == "fixed":
+        half = half_spread_bp
+    else:
+        raise ValueError(spread_estimator)
+    return (taker_fee_bp + half + extra).astype(np.float64)
+
+
+def participation_report(panel, weights: np.ndarray, aum_usd: float, window: int = 30) -> dict:
+    """Trade size as a share of trailing daily turnover, for an account of `aum_usd`.
+
+    weights  (date x ticker) target weights, e.g. the `holdings` of a portfolio. Returns percentiles of
+    |delta weight| * aum / ADV over all trades, and the account size at which the 95th percentile would reach 1%
+    of a contract's daily turnover (a common ceiling for not moving the market)."""
+    adv = panel.adv(window).values
+    dw = np.abs(np.diff(weights, axis=0))
+    part = dw * aum_usd / np.where(adv[1:] > 0, adv[1:], np.nan)
+    part = part[(dw > 1e-9) & np.isfinite(part)]
+    if part.size == 0:
+        return {"aum_usd": aum_usd, "n_trades": 0}
+    p50, p95, p99 = (float(np.percentile(part, q)) for q in (50, 95, 99))
+    return {"aum_usd": aum_usd, "n_trades": int(part.size), "participation_p50": p50, "participation_p95": p95,
+            "participation_p99": p99, "aum_at_1pct_p95": float(aum_usd * 0.01 / p95) if p95 > 0 else np.inf}

pitbacktest/crypto/intraday.py

@@ -0,0 +1,326 @@
+"""Intraday bars for USDT perpetuals from the Binance public archive, as a point-in-time Panel.
+
+Read `docs/intraday_design.md` first. What this layer can say: how fast an edge decays with latency, and at what cost it stops
+paying. What it cannot say: whether the signal can be traded. The archive has bars, not a book, so there is no queue position, no
+partial fill and no impact below the bar.
+
+Time convention. Rows are indexed by bar **end** time (UTC, naive). A row holds only what was known at that instant, so a signal
+from row t is entered at row t + lag with lag >= 1.
+"""
+from __future__ import annotations
+
+import concurrent.futures as cf
+import io
+import zipfile
+from dataclasses import replace
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from ..core.panel import Panel
+from ..portfolio import backtest_portfolio
+from . import binance_archive as ba
+
+COLS = ["open", "high", "low", "close", "volume", "quote_volume", "trades"]
+MIN_PER_DAY = 1440
+
+
+# ------------------------------------------------------------------------------------------------------ parsing
+def parse_minute_zip(blob: bytes) -> pd.DataFrame:
+    """One archive file of 1-minute klines -> frame indexed by the bar **end** time (open time + 1 minute).
+    Millisecond and microsecond timestamps are told apart per row; header rows and duplicates are dropped."""
+    raw = ba._read_klines_zip(blob)
+    if raw.empty:
+        return pd.DataFrame(columns=COLS)
+    ts = raw["open_time"].astype("int64").to_numpy()
+    ms = np.where(ts > 10**14, ts // 1000, ts)
+    idx = (pd.to_datetime(ms, unit="ms", utc=True).tz_localize(None).astype("datetime64[ns]") + pd.Timedelta(minutes=1))   # ns in every pandas
+    df = raw[COLS].apply(pd.to_numeric, errors="coerce")
+    df.index = idx
+    df = df[~df.index.duplicated(keep="last")].sort_index()
+    return df.dropna(subset=["close"])
+
+
+def _bar_minutes(bar: str) -> int:
+    m = int(pd.Timedelta(bar) / pd.Timedelta(minutes=1))
+    if m < 1 or MIN_PER_DAY % m:
+        raise ValueError(f"bar {bar!r} must be a whole number of minutes that divides a day (1min, 5min, 15min, 1h, ...)")
+    return m
+
+
+def aggregate_bars(df1m: pd.DataFrame, bar: str) -> pd.DataFrame:
+    """1-minute frame (end-time index) -> bars labelled by their end. Open is the first minute's, close the last, high and low the
+    extremes, volumes summed; `n_min` is how many minutes the bar holds. A last bar that would end after the data ends is dropped."""
+    m = _bar_minutes(bar)
+    if df1m.empty:
+        return pd.DataFrame(columns=COLS + ["n_min"])
+    if m == 1:
+        out = df1m.copy()
+        out["n_min"] = 1
+        return out
+    opn = df1m.copy()
+    opn.index = opn.index - pd.Timedelta(minutes=1)                      # back to open time so bars align to the clock
+    g = opn.resample(f"{m}min", label="left", closed="left", origin="epoch")
+    out = pd.DataFrame({"open": g["open"].first(), "high": g["high"].max(), "low": g["low"].min(), "close": g["close"].last(),
+                        "volume": g["volume"].sum(min_count=1), "quote_volume": g["quote_volume"].sum(min_count=1),
+                        "trades": g["trades"].sum(min_count=1), "n_min": g["close"].count()})
+    out = out[out["n_min"] > 0]
+    out.index = out.index + pd.Timedelta(minutes=m)                      # label by end
+    return out[out.index <= df1m.index.max()]
+
+
+# ------------------------------------------------------------------------------------------------------ download
+def _month_keys(start: pd.Timestamp, end: pd.Timestamp) -> list[pd.Period]:
+    return list(pd.period_range(start.to_period("M"), end.to_period("M"), freq="M"))
+
+
+def _paths(store, sym: str) -> Path:
+    d = store.dir / "minutes" / sym
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def fetch_minutes(store, symbols, start, end, *, workers: int = 6, warmup_days: int = 91, getter=None, today=None) -> dict:
+    """Download 1-minute klines for `symbols` from `start - warmup_days` to `end`. Complete months come from the monthly archive,
+    the current partial month from the daily archive. Resumable: a month or day already cached, or recorded as missing (404), is
+    not requested again. `getter(key) -> bytes | None` can be injected for tests; `today` is the current UTC date."""
+    get = getter or (lambda key: ba._get(f"{ba.DL_URL}/{key}"))
+    today = pd.Timestamp(today) if today is not None else pd.Timestamp.utcnow().tz_localize(None).normalize()
+    lo = pd.Timestamp(start).normalize() - pd.Timedelta(days=warmup_days)
+    hi = min(pd.Timestamp(end).normalize(), today - pd.Timedelta(days=1))
+    cur = today.to_period("M")
+    stats = {"months": 0, "days": 0, "missing": 0, "cached": 0}
+
+    def one(sym: str) -> dict:
+        s = {"months": 0, "days": 0, "missing": 0, "cached": 0}
+        d = _paths(store, sym)
+        for per in _month_keys(lo, hi):
+            if per < cur:                                                  # a complete month
+                f, none = d / f"{per}.pkl", d / f"{per}.none"
+                if f.exists() or none.exists():
+                    s["cached"] += 1
+                    continue
+                blob = get(f"data/futures/um/monthly/klines/{sym}/1m/{sym}-1m-{per}.zip")
+                if blob is None:
+                    none.write_text("")
+                    s["missing"] += 1
+                    continue
+                parse_minute_zip(blob).to_pickle(f)                       # parsed fully first: a bad file raises and caches nothing
+                s["months"] += 1
+            else:                                                          # the partial month, one file per finished day
+                for day in pd.date_range(max(lo, per.start_time), min(hi, per.end_time.normalize())):
+                    f, none = d / f"d{day:%Y-%m-%d}.pkl", d / f"d{day:%Y-%m-%d}.none"
+                    if f.exists() or none.exists():
+                        s["cached"] += 1
+                        continue
+                    blob = get(f"data/futures/um/daily/klines/{sym}/1m/{sym}-1m-{day:%Y-%m-%d}.zip")
+                    if blob is None:
+                        none.write_text("")
+                        s["missing"] += 1
+                        continue
+                    parse_minute_zip(blob).to_pickle(f)
+                    s["days"] += 1
+        return s
+
+    with cf.ThreadPoolExecutor(workers) as ex:
+        for r in ex.map(one, list(symbols)):
+            for k in stats:
+                stats[k] += r[k]
+    return stats
+
+
+def load_minutes(store, sym: str, start, end) -> pd.DataFrame:
+    """Cached 1-minute frame for `sym` between `start` (inclusive) and `end` (inclusive day), end-time index."""
+    d = _paths(store, sym)
+    parts = [pd.read_pickle(f) for f in sorted(d.glob("*.pkl"))]
+    parts = [p for p in parts if len(p)]
+    if not parts:
+        return pd.DataFrame(columns=COLS)
+    df = pd.concat(parts)
+    df = df[~df.index.duplicated(keep="last")].sort_index()
+    return df[(df.index > pd.Timestamp(start)) & (df.index <= pd.Timestamp(end).normalize() + pd.Timedelta(days=1))]
+
+
+def parse_funding_zip(blob: bytes) -> pd.DataFrame:
+    """One monthly fundingRate archive file -> DataFrame[calc_time (ms or us), rate]."""
+    with zipfile.ZipFile(io.BytesIO(blob)) as z:
+        dd = pd.read_csv(z.open(z.namelist()[0]), dtype=str)
+    if dd.shape[1] < 3:
+        return pd.DataFrame(columns=["calc_time", "rate"])
+    dd.columns = ["calc_time", "interval", "rate"][: dd.shape[1]] + list(dd.columns[3:])
+    return dd[["calc_time", "rate"]]
+
+
+def funding_series(frames: list[pd.DataFrame]) -> pd.Series:
+    """Settlements with timestamps rounded to the nearest minute (the archive stamps them a few milliseconds late)."""
+    out = pd.concat(frames) if frames else pd.DataFrame(columns=["calc_time", "rate"])
+    t = pd.to_numeric(out["calc_time"], errors="coerce")
+    r = pd.to_numeric(out["rate"], errors="coerce")
+    ok = t.notna() & r.notna()
+    t, r = t[ok].astype("int64").to_numpy(), r[ok].to_numpy(float)
+    ms = np.where(t > 10**14, t // 1000, t)
+    idx = pd.to_datetime(ms, unit="ms", utc=True).tz_localize(None).astype("datetime64[ns]").round("min")
+    return pd.Series(r, index=idx, dtype=float).groupby(level=0).sum().sort_index()
+
+
+def fetch_funding_events(store, sym: str, *, lister=None, getter=None) -> pd.Series:
+    """Funding settlements with their instants, cached. The daily store sums them per day and loses the instant, which an
+    intraday simulation needs. `lister(sym) -> keys` and `getter(key) -> bytes | None` can be injected for tests."""
+    f = store.dir / "funding_events" / f"{sym}.pkl"
+    f.parent.mkdir(parents=True, exist_ok=True)
+    if f.exists():
+        return pd.read_pickle(f)
+    keys = lister(sym) if lister else store._months(sym, "fundingRate")
+    get = getter or (lambda key: ba._get(f"{ba.DL_URL}/{key}"))
+    frames = [parse_funding_zip(blob) for k in keys if (blob := get(k))]
+    s = funding_series(frames)
+    s.to_pickle(f)
+    return s
+
+
+# ------------------------------------------------------------------------------------------------------ the panel
+def estimate_memory(n_bars: int, n_symbols: int, n_matrices: int = 20) -> float:
+    """Rough peak gigabytes: `backtest_portfolio` peaked at about 14 (bar x contract) float64 matrices when measured on a panel
+    with close, eligible and volume (tests/memprobe in the design notes); a panel from `build_intraday_panel` also carries open,
+    high, low, funding and delisting flags, so 20 is used. Measure again if you change the engine."""
+    return n_bars * n_symbols * 8 * n_matrices / 1e9
+
+
+def build_intraday_panel(store, symbols, start, end, *, bar: str = "5min", adv_window: int = 30, min_age_days: int = 60,
+                         min_adv_usd: float = 5e6, top_n: int | None = None, entry_lag: int = 1, max_gb: float = 3.0,
+                         funding: bool = True, funding_events: dict | None = None) -> Panel:
+    """Point-in-time intraday panel. Eligibility for every bar of day D uses only days up to D - 1. Download first with
+    `fetch_minutes(store, symbols, start, end)`, which also loads the warm-up days."""
+    symbols = list(symbols)
+    if entry_lag < 1:
+        raise ValueError("entry_lag must be at least 1 bar: a bar's close is only known at its end, so it cannot be traded on")
+    m = _bar_minutes(bar)
+    start, end = pd.Timestamp(start).normalize(), pd.Timestamp(end).normalize()
+    warm = max(adv_window, min_age_days) + 1
+    lo = start - pd.Timedelta(days=warm)
+    grid_end = end + pd.Timedelta(days=1)
+    grid = pd.date_range(start + pd.Timedelta(minutes=m), grid_end, freq=f"{m}min").astype("datetime64[ns]")   # pandas 3 defaults to us
+    mem = estimate_memory(len(grid), len(symbols))
+    if mem > max_gb:
+        fit = [b for b in ("1min", "5min", "15min", "30min", "1h", "2h", "4h") if _bar_minutes(b) >= m and
+               estimate_memory(len(grid) * m // _bar_minutes(b), len(symbols)) <= max_gb]
+        raise ValueError(f"about {mem:.1f} GB needed for {len(grid):,} bars x {len(symbols)} contracts (limit {max_gb}); "
+                         f"use a longer bar ({fit[0] if fit else 'fewer contracts or a shorter period'}), fewer contracts or a shorter period")
+    bars: dict[str, pd.DataFrame] = {}
+    for s in symbols:
+        d = _paths(store, s)
+        missing = [p for p in _month_keys(lo, min(end, pd.Timestamp.utcnow().tz_localize(None).normalize() - pd.Timedelta(days=1)))
+                   if p < pd.Timestamp.utcnow().tz_localize(None).to_period("M")
+                   and not (d / f"{p}.pkl").exists() and not (d / f"{p}.none").exists()]
+        if missing:
+            raise ValueError(f"{s}: months not downloaded {[str(p) for p in missing[:4]]}; run fetch_minutes(store, symbols, start, end) "
+                             f"first (it loads {warm} warm-up days before the start)")
+        raw = load_minutes(store, s, lo, end)
+        if raw.empty:
+            continue
+        b = aggregate_bars(raw, bar)
+        bars[s] = b
+    if not bars:
+        raise ValueError("no data for any symbol")
+    cols = sorted(bars)
+
+    # ---- terminal zero-volume tail (a halted or delisted contract keeps printing flat bars): drop only that tail ------------
+    zero_interior = 0
+    for s in cols:
+        b = bars[s]
+        live = np.flatnonzero((b["quote_volume"] > 0).to_numpy())
+        if len(live) == 0:
+            bars[s] = b.iloc[0:0]
+            continue
+        zero_interior += int((b["quote_volume"].iloc[: live[-1] + 1] <= 0).sum())
+        bars[s] = b.iloc[: live[-1] + 1]
+    cols = [s for s in cols if len(bars[s])]
+
+    # ---- daily liquidity, then eligibility from the PREVIOUS day -----------------------------------------------------
+    day_of = lambda ix: (ix - pd.Timedelta(minutes=1)).floor("D")
+    dq = pd.DataFrame({s: bars[s]["quote_volume"].groupby(day_of(bars[s].index)).sum() for s in cols})
+    dq = dq.reindex(pd.date_range(dq.index.min(), max(dq.index.max(), end), freq="D"))
+    has = dq.fillna(0) > 0
+    age = has.cumsum()
+    adv = dq.where(has).rolling(adv_window, min_periods=adv_window).median()
+    ok_day = (age >= min_age_days) & (adv >= min_adv_usd)
+    if top_n:
+        rank = adv.where(ok_day).rank(axis=1, ascending=False, method="first")
+        ok_day = ok_day & (rank <= top_n)
+    ok_for_day = ok_day.shift(1, fill_value=False)                        # day D is judged by days <= D - 1
+
+    def mat(col: str) -> pd.DataFrame:
+        return pd.DataFrame({s: bars[s][col] for s in cols}).reindex(grid)
+
+    close, high, low, opn = mat("close"), mat("high"), mat("low"), mat("open")
+    qv = mat("quote_volume")
+    elig = ok_for_day.reindex(day_of(grid)).fillna(False).astype(bool).to_numpy()
+    eligible = pd.DataFrame(elig, index=grid, columns=cols) & close.notna()
+
+    # ---- delisting: last real bar of a contract that ended well before the panel end ---------------------------------------
+    last = close.apply(lambda c: c.last_valid_index())
+    da = pd.DataFrame(False, index=grid, columns=cols)
+    for s in cols:
+        if last[s] is not None and last[s] < grid[-1] - pd.Timedelta(days=3):
+            da.loc[last[s], s] = True
+
+    # ---- funding at the settlement instant ----------------------------------------------------------------------------
+    fund, n_events = None, 0
+    if funding:
+        F = np.zeros((len(grid), len(cols)))
+        for j, s in enumerate(cols):
+            ev = funding_events[s] if funding_events is not None and s in funding_events else fetch_funding_events(store, s)
+            ev = ev[(ev.index > grid[0] - pd.Timedelta(minutes=m)) & (ev.index <= grid[-1])]
+            if ev.empty:
+                continue
+            pos = grid.searchsorted(ev.index.to_numpy(), side="left")       # first bar whose end is at or after the settlement
+            ok = pos < len(grid)
+            np.add.at(F, (pos[ok], np.full(int(ok.sum()), j)), ev.to_numpy(float)[ok])
+            n_events += int(ok.sum())
+        fund = pd.DataFrame(F, index=grid, columns=cols)
+    panel = Panel(close=close, eligible=eligible, open=opn, high=high, low=low, volume=qv / close, market="CRYPTO-INTRADAY",
+                  entry_lag=entry_lag, periods_per_year=int(365 * MIN_PER_DAY / m), funding=fund, delist_after=da)
+    n = panel.eligible.sum(axis=1)
+    panel.meta = {"bar": bar, "bar_minutes": m, "symbols_used": len(cols), "bars": len(grid), "partial_bars": int(sum((bars[s]["n_min"] < m).sum() for s in cols)),
+                  "zero_volume_bars_kept": zero_interior, "delisted_in_panel": int(da.values.any(axis=0).sum()),
+                  "eligible_median": float(n[n > 0].median()) if (n > 0).any() else 0.0, "estimated_gb": round(mem, 2),
+                  "funding_events": n_events}
+    return panel
+
+
+# ------------------------------------------------------------------------------------------------------ analysis
+def backtest_intraday(panel: Panel, factor: pd.DataFrame, *, one_way_bp: float = 0.0, long_q: float = 0.2, short_q: float | None = 0.2,
+                      hold: int = 1, **kw):
+    """`backtest_portfolio` with the cost given as **one-way** basis points (the engine's scalar is a round trip), no grid and no
+    benchmark. `hold` is in bars."""
+    if panel.entry_lag < 1:
+        raise ValueError("an intraday panel needs entry_lag >= 1: a bar's close is only known when the bar ends, so it cannot be traded on")
+    kw.setdefault("benchmark", None)
+    kw.setdefault("grid", False)
+    return backtest_portfolio(panel, factor, long_q=long_q, short_q=short_q, hold=hold, spread_bp=2.0 * one_way_bp, **kw)
+
+
+def latency_sweep(panel: Panel, factor: pd.DataFrame, lags=(1, 2, 5, 15), *, one_way_bp: float = 0.0, **kw) -> pd.DataFrame:
+    """The same portfolio entered `lag` bars after the signal. A real edge decays smoothly with the delay."""
+    m = panel.meta.get("bar_minutes", 1)
+    rows = []
+    for k in lags:
+        r = backtest_intraday(replace(panel, entry_lag=int(k)), factor, one_way_bp=one_way_bp, **kw)
+        met = r.metrics
+        rows.append({"lag_bars": int(k), "delay_minutes": int(k) * m, "sharpe": met["Sharpe"], "cagr": met["CAGR"],
+                     "gross_cagr": met["gross_CAGR"], "turnover_per_bar": met["turnover_daily"]})
+    return pd.DataFrame(rows)
+
+
+def breakeven_cost(panel: Panel, factor: pd.DataFrame, **kw) -> dict:
+    """One-way cost in bp at which the net mean return is zero. Net return is linear in the cost per unit traded, so two runs
+    give it exactly: c* = mean(net at 0 bp) / mean(net at 0 bp - net at 1 bp)."""
+    kw.pop("one_way_bp", None)
+    r0 = backtest_intraday(panel, factor, one_way_bp=0.0, **kw)
+    r1 = backtest_intraday(panel, factor, one_way_bp=1.0, **kw)
+    n0, n1 = r0.net_returns, r1.net_returns
+    per_bp = float((n0 - n1).mean())
+    mean0 = float(n0.mean())
+    c = mean0 / per_bp if per_bp > 0 else float("nan")
+    return {"breakeven_one_way_bp": c, "net_mean_at_0bp": mean0, "cost_per_bp": per_bp, "gross_sharpe": r0.metrics["Sharpe"]}

pitbacktest/crypto/panel.py

@@ -0,0 +1,120 @@
+"""Build a point-in-time (PIT) Panel from the Binance USDT-M archive.
+
+Bias controls, in order of importance
+  1. Survivorship: the universe is every contract that ever traded, delisted ones included. `survivors_only=True`
+     reproduces the usual shortcut (only contracts that still trade today) so the bias can be measured, not guessed.
+  2. Universe look-ahead: a contract is eligible on day t only if, using data up to and including day t, it has
+     enough history and enough trailing dollar volume. Nothing from t+1 is used.
+  3. Stale prices: bars with zero volume (a frozen last price after a halt or a delisting) are dropped so they
+     cannot create fake zero returns or fake liquidity.
+  4. Funding: the daily funding rate is attached to the panel and charged by `backtest_portfolio`.
+  5. Delisting: the last real bar of a contract that stopped trading is marked in `delist_after`, so the engine can
+     apply an explicit delisting return instead of silently assuming the position was closed at the last price.
+
+Prices are daily UTC closes. Volumes are converted so that `close * volume` equals the USDT turnover.
+"""
+from __future__ import annotations
+
+import re
+
+import numpy as np
+import pandas as pd
+
+from ..core.panel import Panel
+from .binance_archive import ArchiveStore
+
+# Pegged coins, fiat pairs and index perpetuals are not tradable risk assets for a cross-sectional study.
+DEFAULT_EXCLUDE = re.compile(
+    r"^(USDC|BUSD|TUSD|FDUSD|USDP|DAI|USTC|EUR|GBP|AUD|TRY|BRL)USDT$|^(BTCDOM|DEFI|FOOTBALL|BLUEBIRD|ALL)USDT$|DOMUSDT$"
+)
+
+
+def build_panel(store: ArchiveStore | None = None, *, symbols: list[str] | None = None,
+                start: str = "2020-06-01", end: str | None = None,
+                adv_window: int = 30, min_adv_usd: float = 5e6, min_age_days: int = 60,
+                top_n: int | None = None, survivors_only: bool = False,
+                exclude: re.Pattern | None = DEFAULT_EXCLUDE, entry_lag: int = 1,
+                live: set[str] | None = None) -> Panel:
+    """Return a Panel with PIT eligibility, funding and delisting information.
+
+    min_adv_usd    trailing median USDT turnover needed to be eligible (median, so one spike cannot qualify a coin)
+    min_age_days   bars a contract must already have, so the first weeks after a listing are excluded
+    top_n          optionally keep only the top-N by trailing turnover among eligible contracts, each day
+    survivors_only keep only contracts still trading today (for measuring survivorship bias)
+    """
+    store = store or ArchiveStore()
+    syms = symbols or store.symbols()
+    if exclude is not None:
+        syms = [s for s in syms if not exclude.search(s)]
+    if survivors_only and live is None:
+        live = store.trading_now()
+    live = live or set()
+    bars: dict[str, pd.DataFrame] = {}
+    fund: dict[str, pd.Series] = {}
+    for s in syms:
+        f = store.dir / "daily" / f"{s}.pkl"
+        d = pd.read_pickle(f) if f.exists() else store.fetch_daily(s, False)
+        if d is None or d.empty:
+            continue
+        if survivors_only and s not in live:
+            continue
+        d = d[d["quote_volume"] > 0]                      # drop frozen, zero-volume bars (halts, delisting tail)
+        d = d[d["close"] > 0]
+        if d.empty:
+            continue
+        bars[s] = d
+        ff = store.dir / "funding" / f"{s}.pkl"
+        fund[s] = pd.read_pickle(ff) if ff.exists() else store.fetch_funding(s, False)
+    if not bars:
+        raise ValueError("no data: run pitbacktest.crypto.fetch_all() first")
+
+    idx = pd.date_range(min(d.index[0] for d in bars.values()), max(d.index[-1] for d in bars.values()), freq="D")
+    cols = sorted(bars)
+
+    def mat(col: str) -> pd.DataFrame:
+        return pd.DataFrame({s: bars[s][col] for s in cols}).reindex(idx)
+
+    close, open_, high, low = mat("close"), mat("open"), mat("high"), mat("low")
+    qv = mat("quote_volume")
+    funding = pd.DataFrame({s: fund[s] for s in cols if fund.get(s) is not None and len(fund[s])}).reindex(idx)
+    funding = funding.reindex(columns=cols)
+
+    # --- point-in-time eligibility: only information up to and including day t ---
+    age = close.notna().cumsum()
+    adv = qv.rolling(adv_window, min_periods=adv_window).median()
+    ok = close.notna() & (age >= min_age_days) & (adv >= min_adv_usd)
+    if top_n:
+        rank = adv.where(ok).rank(axis=1, ascending=False, method="first")
+        ok = ok & (rank <= top_n)
+
+    # --- delisting: last real bar of a contract that ended well before the panel end ---
+    last = close.apply(lambda c: c.last_valid_index())
+    delist_after = pd.DataFrame(False, index=idx, columns=cols)
+    for s in cols:
+        lv = last[s]
+        if lv is not None and lv < idx[-1] - pd.Timedelta(days=3):
+            delist_after.loc[lv, s] = True
+
+    # --- crypto-specific controls (replace the equity ROA / book-to-market controls) ---
+    ret = close.pct_change(fill_method=None)
+    btc = ret["BTCUSDT"] if "BTCUSDT" in ret.columns else ret.median(axis=1)
+    beta = ret.rolling(60, min_periods=40).cov(btc).div(btc.rolling(60, min_periods=40).var(), axis=0)
+    chars = {"log_dollar_vol": np.log(qv.rolling(adv_window, min_periods=adv_window).mean().clip(lower=1)),
+             "btc_beta_60": beta}
+
+    vol_base = qv / close                                   # so that close * volume == USDT turnover
+    keep = idx >= pd.Timestamp(start)
+    if end:
+        keep &= idx <= pd.Timestamp(end)
+    sl = lambda df: df.loc[keep]
+    p = Panel(close=sl(close), eligible=sl(ok), open=sl(open_), high=sl(high), low=sl(low), volume=sl(vol_base),
+              chars={k: sl(v) for k, v in chars.items()}, market="CRYPTO", entry_lag=entry_lag, periods_per_year=365,
+              funding=sl(funding), delist_after=sl(delist_after))
+    n = p.eligible.sum(axis=1)
+    p.meta = {
+        "symbols_in_archive": len(store.symbols()), "symbols_used": len(cols),
+        "survivors_only": survivors_only, "delisted_in_panel": int(delist_after.values.any(axis=0).sum()),
+        "eligible_median": float(n[n > 0].median()), "min_adv_usd": min_adv_usd, "min_age_days": min_age_days,
+        "funding_coverage": float(funding.reindex(index=p.close.index).notna().values[p.close.notna().values].mean()),
+    }
+    return p

pitbacktest/equity/__init__.py

@@ -0,0 +1,10 @@
+"""Equity survivorship tools that work without paid data.
+
+None of this removes survivorship bias from a survivors-only price source; it measures how much of the real
+universe is missing and shows, by explicit scenarios, how much that could matter. Real removal needs point-in-time data
+(see `pitbacktest.adapters.long_format` for a strict way to plug it in, and `pitbacktest.adapters.krx` for Korea).
+"""
+from .master import load_us_master, universe_coverage
+from .scenarios import inject_delistings, survivorship_scenarios, survivors_only
+
+__all__ = ["load_us_master", "universe_coverage", "inject_delistings", "survivorship_scenarios", "survivors_only"]

pitbacktest/equity/master.py

@@ -0,0 +1,60 @@
+"""A free, keyless list of US stock tickers with their listing windows, delisted ones included.
+
+Source: Tiingo's published `supported_tickers.zip` (no API key needed). It is a **security master, not a price
+source**: it tells you which tickers existed in which period. It is incomplete before about 2013 and misses some recent
+failures (for example SIVB and FRC were absent when this was written), so coverage numbers built on it are lower
+bounds on how many names a survivors-only panel is missing.
+"""
+from __future__ import annotations
+
+import io
+import urllib.request
+import zipfile
+from pathlib import Path
+
+import pandas as pd
+
+URL = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip"
+US_EXCHANGES = {"NYSE", "NASDAQ", "NYSE MKT", "NYSE ARCA", "AMEX", "NYSE American"}
+
+
+def load_us_master(cache_dir: str | Path | None = None, refresh: bool = False) -> pd.DataFrame:
+    """DataFrame [ticker, exchange, start, end, alive_today] for US-listed common stocks."""
+    d = Path(cache_dir or Path.home() / ".cache" / "quantbt").expanduser()
+    d.mkdir(parents=True, exist_ok=True)
+    f = d / "tiingo_supported_tickers.zip"
+    if refresh or not f.exists():
+        req = urllib.request.Request(URL, headers={"User-Agent": "pitbacktest-research/0.2 (public data only)"})
+        f.write_bytes(urllib.request.urlopen(req, timeout=60).read())
+    with zipfile.ZipFile(f) as z:
+        df = pd.read_csv(io.BytesIO(z.read(z.namelist()[0])))
+    return _clean(df)
+
+
+def _clean(df: pd.DataFrame) -> pd.DataFrame:
+    df = df[(df["assetType"] == "Stock") & df["exchange"].isin(US_EXCHANGES)].copy()
+    df["start"] = pd.to_datetime(df["startDate"], errors="coerce")
+    df["end"] = pd.to_datetime(df["endDate"], errors="coerce")
+    df = df.dropna(subset=["start", "end"])
+    df["alive_today"] = df["end"] >= df["end"].max() - pd.Timedelta(days=7)
+    return df[["ticker", "exchange", "start", "end", "alive_today"]].reset_index(drop=True)
+
+
+def universe_coverage(panel_tickers, master: pd.DataFrame, years=range(2010, 2025)) -> pd.DataFrame:
+    """For each year: how many US stocks were listed, how many of them are in your panel, and (the telling number)
+    how many of the names that stopped trading that year are in your panel.
+
+    A ticker only counts as present if its listing window in the master overlaps the year, so a reused ticker is not
+    credited to the company that used to have it (the master has one row per ticker and window)."""
+    have = set(panel_tickers)
+    rows = []
+    for y in years:
+        a, b = pd.Timestamp(f"{y}-01-01"), pd.Timestamp(f"{y}-12-31")
+        alive = master[(master["start"] <= b) & (master["end"] >= a)]
+        died = alive[(alive["end"] <= b) & ~alive["alive_today"]]
+        rows.append({"year": y, "listed": len(alive), "in_panel": int(alive["ticker"].isin(have).sum()),
+                     "stopped_trading": len(died), "stopped_in_panel": int(died["ticker"].isin(have).sum())})
+    out = pd.DataFrame(rows).set_index("year")
+    out["share_listed_in_panel"] = out["in_panel"] / out["listed"].replace(0, float("nan"))
+    out["share_stopped_in_panel"] = out["stopped_in_panel"] / out["stopped_trading"].replace(0, float("nan"))
+    return out