pitbacktest 0.2.0__py3-none-any.whl

pitbacktest/equity/scenarios.py

@@ -0,0 +1,104 @@
+"""Survivorship scenarios: put delistings back into a survivors-only panel and see how the result moves.
+
+This is a **what-if**, not a correction. The actual delisted companies are unknown here, so we draw them at random
+(optionally tilted toward small, illiquid or volatile names, which is where delistings concentrate in the literature) and
+give them an explicit delisting return. Run it over a grid of annual rates and delisting returns and report the range, not a
+single "corrected" number.
+
+Order-of-magnitude guide from the delisting literature (check it for your period and market): a few percent of listed
+names disappear each year, roughly half of them for performance reasons; the average return on a performance-related
+delisting is about -30% on NYSE/AMEX (Shumway 1997) and about -55% on Nasdaq (Shumway and Warther 1999).
+"""
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Callable
+
+import numpy as np
+import pandas as pd
+
+from ..core.panel import Panel
+
+
+def _weights(panel: Panel, year_start: pd.Timestamp, names: pd.Index, hazard: str) -> np.ndarray:
+    if hazard == "uniform":
+        return np.ones(len(names))
+    before = panel.dates < year_start
+    if not before.any():
+        return np.ones(len(names))                                 # first year: nothing earlier to tilt on, so uniform
+    if hazard == "volatile":
+        v = panel.close.pct_change(fill_method=None).loc[before].tail(60).std().reindex(names)
+    elif hazard == "illiquid":
+        adv = panel.adv(30)
+        v = -(adv.loc[before].tail(1).iloc[0].reindex(names)) if adv is not None else pd.Series(0.0, index=names)
+    else:
+        raise ValueError(hazard)
+    r = v.rank(pct=True).fillna(0.5).to_numpy()
+    return 0.2 + r ** 2                                            # tilted, but every name keeps a chance
+
+
+def inject_delistings(panel: Panel, *, annual_rate: float = 0.03, hazard: str = "uniform", seed: int = 0,
+                      min_age_days: int = 60) -> Panel:
+    """Return a copy of `panel` in which a fraction `annual_rate` of the names alive at the start of each calendar year stop
+    trading at a random date later that year. The last real bar is flagged in `delist_after` and prices, volume and
+    eligibility are removed afterwards."""
+    if hazard not in ("uniform", "volatile", "illiquid"):
+        raise ValueError(f"hazard must be 'uniform', 'volatile' or 'illiquid', not {hazard!r}")
+    if hazard == "illiquid" and panel.volume is None:
+        raise ValueError("hazard='illiquid' needs panel.volume")
+    if not (0 <= annual_rate <= 1):
+        raise ValueError(f"annual_rate must be between 0 and 1, got {annual_rate!r}")
+    rng = np.random.default_rng(seed)
+    close = panel.close.copy()
+    elig = panel.eligible.copy()
+    vol = None if panel.volume is None else panel.volume.copy()
+    da = (pd.DataFrame(False, index=panel.dates, columns=panel.tickers) if panel.delist_after is None
+          else panel.delist_after.copy())
+    for y in sorted(set(panel.dates.year)):
+        idx = panel.dates[panel.dates.year == y]
+        if len(idx) < 100:
+            continue                                               # skip partial years
+        alive = close.loc[idx[0]].dropna().index
+        k = int(round(annual_rate * len(alive)))
+        if k == 0:
+            continue
+        w = _weights(panel, idx[0], alive, hazard)
+        pick = rng.choice(len(alive), size=min(k, len(alive)), replace=False, p=w / w.sum())
+        for j in pick:
+            name = alive[j]
+            when = idx[int(rng.integers(min_age_days // 3, len(idx) - 1))]
+            close.loc[when + pd.Timedelta(days=1):, name] = np.nan
+            elig.loc[when + pd.Timedelta(days=1):, name] = False
+            if vol is not None:
+                vol.loc[when + pd.Timedelta(days=1):, name] = np.nan
+            da.loc[when, name] = True
+    return replace(panel, close=close, eligible=elig, volume=vol, delist_after=da)
+
+
+def survivorship_scenarios(panel: Panel, evaluate: Callable[[Panel, float], float], *,
+                           rates=(0.01, 0.03, 0.05), delist_returns=(-0.30, -0.55), hazards=("uniform", "volatile"),
+                           n: int = 10, seed: int = 0) -> pd.DataFrame:
+    """Run `evaluate(panel, delist_return)` (it should return one number, for example the net Sharpe) on the original panel
+    and on `n` randomly delisting copies for every combination. Returns the baseline and the spread of outcomes."""
+    base = float(evaluate(panel, 0.0))
+    rows = []
+    for hz in hazards:
+        for r in rates:
+            for d in delist_returns:
+                vals = [float(evaluate(inject_delistings(panel, annual_rate=r, hazard=hz, seed=seed + i), d)) for i in range(n)]
+                rows.append({"hazard": hz, "annual_rate": r, "delist_return": d, "baseline": base,
+                             "mean": float(np.mean(vals)), "p5": float(np.percentile(vals, 5)),
+                             "p95": float(np.percentile(vals, 95)), "mean_change": float(np.mean(vals) - base)})
+    return pd.DataFrame(rows)
+
+
+def survivors_only(panel: Panel, end_gap_days: int = 5) -> Panel:
+    """The usual shortcut, made explicit: keep only the securities that still have a price on the last day.
+    Compare a result on `panel` with the same result on `survivors_only(panel)` to measure survivorship bias directly."""
+    last = panel.close.apply(lambda c: c.last_valid_index())
+    keep = [c for c in panel.close.columns if last[c] is not None and last[c] >= panel.dates[-1] - pd.Timedelta(days=end_gap_days)]
+    sub = lambda d: None if d is None else d[keep]
+    return replace(panel, close=panel.close[keep], eligible=panel.eligible[keep], open=sub(panel.open), high=sub(panel.high),
+                   low=sub(panel.low), volume=sub(panel.volume), mkt_cap=sub(panel.mkt_cap),
+                   chars={k: v[keep] for k, v in panel.chars.items()}, funding=sub(panel.funding),
+                   delist_after=sub(panel.delist_after))

pitbacktest/event.py

@@ -0,0 +1,206 @@
+"""C. Event-signal test: does it hold up as individual alerts?
+
+Portfolio metrics (mean excess return) assume the mean is realised. A user receives alerts one by one, though, so the **win rate,
+median and payoff ratio** decide what it feels like.
+
+What is always decomposed
+    win rate = base rate (how the sample was picked) + lift (what the signal adds)
+        - A longer holding period raises the win rate and the base rate together. Judge by the lift only.
+        - A win rate below the base rate (lift < 0) means the signal does harm.
+    mean vs median
+        - If the signs differ, a few big winners cover many losses: it works as a portfolio and not as alerts.
+        - The typical case is a positive mean with a negative median.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass
+
+import numpy as np
+import pandas as pd
+
+from .core.controls import build_controls, xs_norm
+from .core.estimators import fama_macbeth, newey_west_t, paired_diff
+from .core.gates import GateConfig, fire_structure, run_gates
+from .core.panel import Panel
+
+
+@dataclass
+class EventResult:
+    spec: dict
+    per_horizon: dict
+    gates: dict
+    structure: dict
+    neutralized: dict | None
+    lookahead: dict
+
+
+def _trade_stats(fire: np.ndarray, cum: np.ndarray, eligible: np.ndarray,
+                 cost: float) -> dict | None:
+    """Per-trade profit distribution plus the base win rate."""
+    tr, bh, bn = [], 0.0, 0
+    for i in range(cum.shape[0]):
+        s = fire[i] & np.isfinite(cum[i])
+        k = int(s.sum())
+        if k < 1:
+            continue
+        tr.append(cum[i][s] - cost)
+        pool = eligible[i] & np.isfinite(cum[i])
+        if pool.sum() >= 20:
+            bh += float((cum[i][pool] - cost > 0).mean()) * k
+            bn += k
+    if not tr:
+        return None
+    t = np.concatenate(tr)
+    if len(t) < 100:
+        return None
+    win = float((t > 0).mean())
+    base = bh / bn if bn else np.nan
+    w, l = t[t > 0], t[t < 0]
+    return {
+        "n_trades": int(len(t)),
+        "win_rate": win * 100,
+        "base_rate": base * 100 if np.isfinite(base) else np.nan,
+        "lift_pp": (win - base) * 100 if np.isfinite(base) else np.nan,
+        "mean_bp": float(t.mean() * 1e4),
+        "median_bp": float(np.median(t) * 1e4),
+        "avg_win_bp": float(w.mean() * 1e4) if len(w) else np.nan,
+        "avg_loss_bp": float(l.mean() * 1e4) if len(l) else np.nan,
+        "payoff": float(w.mean() / abs(l.mean())) if len(l) else np.nan,
+        "p25_bp": float(np.percentile(t, 25) * 1e4),
+        "p75_bp": float(np.percentile(t, 75) * 1e4),
+        "skew_warning": bool(t.mean() > 0 > np.median(t)),
+    }
+
+
+def _daily_excess(fire: np.ndarray, cum: np.ndarray, eligible: np.ndarray,
+                  *, min_fire: int = 1, min_pool: int | None = None) -> np.ndarray:
+    """Daily excess return of the firing group (gate input).
+
+    A fixed min_pool throws away every date in a small universe (for example a 20-name test).
+    The default **adapts** to half the median universe size (at most 30).
+    """
+    if min_pool is None:
+        med = float(np.median(eligible.sum(axis=1)))
+        min_pool = int(max(5, min(30, med * 0.5)))
+    out = []
+    for i in range(cum.shape[0]):
+        s = fire[i] & np.isfinite(cum[i])
+        pool = eligible[i] & np.isfinite(cum[i])
+        if s.sum() >= min_fire and pool.sum() >= min_pool:
+            out.append(cum[i][s].mean() - cum[i][pool].mean())
+    return np.array(out)
+
+
+def backtest_event(panel: Panel, signal: pd.DataFrame, *,
+                   horizons: tuple[int, ...] = (1, 2, 5, 10, 21),
+                   cost_bp: float = 20.0,
+                   neutralize_check: bool = True,
+                   gate_config: GateConfig | None = None,
+                   null_threshold: float | None = None,
+                   delist_return: float | None = None) -> EventResult:
+    """Event-signal test.
+
+    signal   bool or 0/1 matrix (date x ticker). True = fires that day.
+    cost_bp  round-trip trading cost. If per-security measured costs exist, use the spread panel on the portfolio side.
+    delist_return  return assumed on the day after a security's last bar when it is flagged in panel.delist_after (None: carried at its last price).
+
+    With the neutralisation test on (the default) the contribution after controls is re-measured with a **dummy regression**.
+    An event signal is not a continuous factor, so it is read through a dummy coefficient and not through residuals.
+    """
+    if not horizons or any(isinstance(h, bool) or not isinstance(h, (int, np.integer)) or h < 1 for h in horizons):
+        raise ValueError(f"horizons must be whole numbers of periods, at least 1, got {horizons!r}")
+    if not (cost_bp >= 0 and np.isfinite(cost_bp)):
+        raise ValueError(f"cost_bp must be finite and not negative, got {cost_bp!r}")
+    sig = signal.reindex(index=panel.dates, columns=panel.tickers).fillna(False)
+    fire = (sig.astype(bool) & panel.eligible).values
+    ev = panel.eligible.values
+    cost = cost_bp / 1e4
+
+    lookahead = panel.assert_no_lookahead(sig.astype(float), h=max(horizons))
+    cums = {h: panel.forward(h, delist_return).values.astype(np.float64) for h in horizons}
+
+    per_h = {}
+    for h in horizons:
+        st = _trade_stats(fire, cums[h], ev, cost)
+        if st is None:
+            continue
+        de = _daily_excess(fire, cums[h], ev)
+        mu, _, t, n = newey_west_t(de, lag=max(h, 21))
+        per_h[h] = {**st, "excess_bp": float(mu * 1e4), "t": float(t), "n_days": int(n),
+                    "net_bp": float(mu * 1e4 - cost_bp)}
+
+    if not per_h:
+        raise ValueError(
+            "No horizon can be tested. Either the signal fires too rarely (fewer than 100 events) or "
+            "too few securities are eligible. Check signal, eligible and horizons.")
+    # representative horizon = where the lift is largest
+    best_h = max(per_h, key=lambda h: per_h[h].get("lift_pp", -99))
+    de = _daily_excess(fire, cums[best_h], ev)
+    if len(de) < 100:
+        warnings.warn(
+            f"Only {len(de)} days of daily excess return, so the gates cannot be trusted. "
+            f"_daily_excess counts only days with at least one firing name and a pool of at least max(5, min(30, half the "
+            f"median universe)) names, so a short sample or a rarely firing signal leaves few days.", stacklevel=2)
+
+    from .core.estimators import decile_profile
+    dec = decile_profile(sig.astype(float), panel.forward(best_h, delist_return), panel.eligible,
+                         lag=max(best_h, 21))
+    struct = fire_structure(fire, panel.dates, panel.tickers)
+    cfg = gate_config or GateConfig(null_threshold=null_threshold)
+    gates = run_gates(de, panel.dates, t_stat=per_h[best_h]["t"],
+                      net_bp=per_h[best_h]["net_bp"], rho=dec["monotonicity_rho"],
+                      fire=fire, tickers=panel.tickers, cfg=cfg, lag=max(best_h, 21))
+
+    neu = None
+    if neutralize_check:
+        neu = _dummy_neutralized(panel, fire, cums, horizons)
+
+    return EventResult(
+        spec={"horizons": list(horizons), "cost_bp": cost_bp, "entry_lag": panel.entry_lag,
+              "market": panel.market, "best_horizon": best_h,
+              "decile": dec},
+        per_horizon=per_h, gates=gates, structure=struct,
+        neutralized=neu, lookahead=lookahead)
+
+
+def _dummy_neutralized(panel: Panel, fire: np.ndarray, cums: dict,
+                       horizons: tuple[int, ...], min_obs: int = 30) -> dict:
+    """Event dummy regression: does firing still predict returns after controlling for characteristics?
+
+    Sample: every eligible name that day. Dependent: the future return. Independent: [firing dummy] + controls.
+    The dummy coefficient is 'what firing adds among names with the same characteristics'.
+    """
+    ctrl = build_controls(panel)
+    cvs = [c.values.astype(np.float64) for c in ctrl.values()]
+    ev = panel.eligible.values
+    out = {}
+    for h in horizons:
+        cum = cums[h]
+        b_raw = np.full(cum.shape[0], np.nan)
+        b_neu = np.full(cum.shape[0], np.nan)
+        for i in range(cum.shape[0]):
+            pool = ev[i] & np.isfinite(cum[i])
+            for c in cvs:
+                pool = pool & np.isfinite(c[i])
+            n = int(pool.sum())
+            if n < min_obs:
+                continue
+            d = fire[i][pool].astype(np.float64)
+            if d.sum() < 3 or d.sum() > n - 3:
+                continue
+            y = cum[i][pool]
+            try:
+                b_raw[i] = np.linalg.lstsq(np.column_stack([np.ones(n), d]), y, rcond=None)[0][1]
+                X = np.column_stack([np.ones(n), d] + [c[i][pool] for c in cvs])
+                b_neu[i] = np.linalg.lstsq(X, y, rcond=None)[0][1]
+            except np.linalg.LinAlgError:
+                continue
+        m0, _, t0, _ = newey_west_t(b_raw, lag=max(h, 21))
+        m1, _, t1, _ = newey_west_t(b_neu, lag=max(h, 21))
+        keep = (m1 / m0 * 100) if (np.isfinite(m0) and abs(m0) > 1e-9) else np.nan
+        out[h] = {"raw_bp": m0 * 1e4 if np.isfinite(m0) else np.nan, "raw_t": t0,
+                  "neutral_bp": m1 * 1e4 if np.isfinite(m1) else np.nan, "neutral_t": t1,
+                  "survival_pct": float(keep) if np.isfinite(keep) else np.nan}
+    return out

pitbacktest/execution.py

@@ -0,0 +1,310 @@
+"""Whether a trade can be done, and at what size, at the price the backtest uses.
+
+A daily backtest that trades every position change at the close assumes three things that are often false: that the security
+was open (a halted name cannot be traded), that the price was not locked at a daily limit (a name closed at the upper limit has a
+queue of buyers and no sellers, so you cannot buy it), and that you can hold any fraction of a share. This module states those
+assumptions as data the engines read.
+
+    can_buy, can_sell   `Panel` fields (bool, date x ticker). A trade that increases a position needs `can_buy` on the execution
+                        day, one that decreases it needs `can_sell`. A blocked trade does not happen and the position stays as it was.
+    tradability()       builds them from prices and volume: no price or no volume means halted; a daily move at the limit means locked.
+    at_prices()         a copy of the panel that trades at another price (the open instead of the close).
+    realize()           the engine's position path under those rules, plus whole-lot sizes and a minimum trade value.
+
+Nothing here is specific to one market. Limits and lot sizes differ by market and change by rule, so they are arguments, not a table
+inside the library.
+"""
+from __future__ import annotations
+
+from dataclasses import replace
+
+import numpy as np
+import pandas as pd
+
+from .core.costs import rate_schedule
+from .core.panel import Panel
+
+
+def tradability(panel: Panel, *, limit=None, tol: float = 0.01, halt_on_zero_volume: bool = True,
+                max_gap_days: int | None = 60) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """(can_buy, can_sell) built from the panel's prices and volume.
+
+    After a security's flagged delisting (`delist_after`) both trades are allowed, so that a position in it can be closed.
+    Halted: no close that day, or (if the panel has `volume` and `halt_on_zero_volume`) no volume. A security with a close but a
+    missing volume is also treated as halted, because an unknown is not permission. A panel without `volume` is judged by prices alone.
+    limit   the daily price limit as a fraction (0.30 for 30%), a list of (effective_date, fraction) when it changed, or None for a
+            market without one. A day whose move is at least `limit - tol` up cannot be bought (a locked-up name has no sellers) and a
+            day at least `limit - tol` down cannot be sold. The tolerance is there because tick rounding keeps the limit move slightly
+            under the limit. This is **conservative**: it also blocks days that reached the limit but traded freely before it.
+    tol     in fractions, 0 <= tol < limit.
+    max_gap_days  after this many consecutive days **without any price**, a position is treated as settled and can be closed either way
+            (None: never). An exchange settles a delisted contract and a vendor gap or a ticker that came back years later (a contract
+            that was absent for years and then listed again, in the Binance cache) is not a position you can hold through. A name with a price
+            but no volume (a Korean trading suspension) is not covered: that one really is frozen, for as long as the data says."""
+    c = panel.close
+    ok = c.notna()
+    if halt_on_zero_volume and panel.volume is not None:
+        ok = ok & (panel.volume.fillna(0.0) > 0)
+    can_buy, can_sell = ok.copy(), ok.copy()
+    if panel.delist_after is not None and bool(np.asarray(panel.delist_after.values).any()):
+        # After a flagged delisting there is no price for ever. The engines already settle such a position (`delist_return`, or the last
+        # price), so closing it is allowed in **both** directions (a long is sold, a short is bought back): refusing the exit would freeze a
+        # dead position in the book for the rest of the sample, and a frozen short is exactly what the first real-data run showed.
+        da = panel.delist_after.reindex(index=c.index, columns=c.columns).fillna(False).astype(bool)
+        gone = (da.astype(int).cumsum() - da.astype(int)) > 0
+        can_buy, can_sell = can_buy | gone, can_sell | gone
+    if max_gap_days is not None:
+        isn = c.isna().to_numpy()
+        run = np.zeros(c.shape[1], dtype=int)
+        long_gap = np.zeros(c.shape, dtype=bool)
+        for t in range(c.shape[0]):
+            run = np.where(isn[t], run + 1, 0)
+            long_gap[t] = run > max_gap_days
+        settled = pd.DataFrame(long_gap, index=c.index, columns=c.columns)
+        can_buy, can_sell = can_buy | settled, can_sell | settled
+    if limit is not None:
+        lim = (rate_schedule(panel.dates, limit, "limit").to_numpy(float) if isinstance(limit, (list, tuple))
+               else np.full(len(panel.dates), float(limit)))
+        if not (np.all(lim > 0) and np.all(lim < 1)):
+            raise ValueError(f"limit must be a fraction between 0 and 1 (0.30 for 30%), got {limit!r}")
+        if not (0 <= tol < lim.min()):
+            raise ValueError(f"tol must be at least 0 and below the limit, got {tol!r}")
+        ret = c.pct_change(fill_method=None).to_numpy(float)
+        thr = (lim - tol)[:, None]
+        with np.errstate(invalid="ignore"):
+            up, down = ret >= thr, ret <= -thr
+        can_buy = can_buy & ~pd.DataFrame(up, index=c.index, columns=c.columns)
+        can_sell = can_sell & ~pd.DataFrame(down, index=c.index, columns=c.columns)
+    return can_buy, can_sell
+
+
+def halt_runs(panel: Panel) -> np.ndarray:
+    """(date x ticker) int: how many consecutive days, counting the day itself, the security has had a price but no (or unknown) volume; 0 on a day it trades or has no price.
+    A panel without `volume` has none."""
+    c = panel.close
+    if panel.volume is None:
+        return np.zeros(c.shape, dtype=int)
+    halted = (c.notna() & ~(panel.volume.fillna(0.0) > 0)).to_numpy()
+    run = np.zeros(c.shape, dtype=int)
+    for t in range(c.shape[0]):
+        run[t] = np.where(halted[t], (run[t - 1] if t else 0) + 1, 0)
+    return run
+
+
+def freeze_hits(panel: Panel, freeze_days: int | None) -> np.ndarray | None:
+    """(date x ticker) bool indexed by the **signal date**: True where the execution day (signal date + `entry_lag`) is exactly the `freeze_days`-th day of a suspension.
+    None when `freeze_days` is None. A suspension marks a position down once and not again however long it lasts. The markdown is booked on the signal-date row, which is
+    the `freeze_days`-th suspended day minus `entry_lag`, and it enters the return of that row (close of the execution day to the close of the next), so it is the
+    `freeze_days`-th suspended day's close that takes the loss."""
+    if freeze_days is None:
+        return None
+    if isinstance(freeze_days, bool) or not isinstance(freeze_days, (int, np.integer)) or freeze_days < 1:
+        raise ValueError(f"freeze_days must be a whole number of days, at least 1, or None, got {freeze_days!r}")
+    hit = halt_runs(panel) == freeze_days
+    lag = panel.entry_lag
+    return np.vstack([hit[lag:], np.zeros((lag, hit.shape[1]), bool)]) if lag else hit
+
+
+def check_freeze_return(freeze_return: float) -> float:
+    """`freeze_return` as a float, or ValueError unless it is between -1 and 0 (a markdown is never a gain)."""
+    if isinstance(freeze_return, (bool, np.bool_)) or not isinstance(freeze_return, (int, float, np.integer, np.floating)):
+        raise ValueError(f"freeze_return must be a number between -1 and 0, got {freeze_return!r}")
+    if not (np.isfinite(freeze_return) and -1.0 <= freeze_return <= 0.0):
+        raise ValueError(f"freeze_return must be between -1 and 0 (a markdown is never a gain), got {freeze_return!r}")
+    return float(freeze_return)
+
+
+def freeze_episodes(panel: Panel, *, min_days: int = 20, relist_days: int = 20) -> pd.DataFrame:
+    """The stretches of at least `min_days` consecutive days on which a security had a price but no volume (a trading suspension), and how each ended.
+
+    One row per episode: ticker, start, end (the last suspended day), days, outcome and `ret`, the return from the last price before the suspension
+    to the price at the end of the story.
+    A day without a price (NaN) ends a suspension, so a halt with a gap of missing prices inside it is cut into two episodes. A suspension that is already running
+    on the panel's first day has no price before it: such an episode is listed, but its `ret` is NaN, and its length is truncated.
+    outcome  "resumed"               the first priced day after the suspension, and the data goes on (or the security is not flagged in `delist_after`):
+                                      ret is that day's price over the price before the suspension. That day may itself have no volume if missing prices sit between.
+             "resumed_then_delisted"  the security is flagged in `delist_after` and its last bar falls within `relist_days` of that day (a liquidation window): ret is its
+                                      last price over the price before. Without the flag the same story is labeled "resumed".
+             "ended_in_halt"          the security's last bar is a suspended day and it is flagged in `delist_after`: the price never moved, ret is 0 (NaN if there is
+                                      no price before), and the loss, if any, was never in the data
+             "ongoing"                no priced day follows: either still suspended on the last day of the panel, or the data ends in the suspension without a delisting
+                                      flag; ret is NaN
+    The point is to measure how often a position that cannot be sold is lost, instead of assuming it. Counts are small and depend on the market and
+    the period; read the distribution of `ret`, not a mean."""
+    c = panel.close
+    halted = (c.notna() & ~(panel.volume.fillna(0.0) > 0)).to_numpy() if panel.volume is not None else np.zeros(c.shape, bool)
+    px = c.to_numpy(float)
+    da = panel.delist_after.reindex(index=c.index, columns=c.columns).fillna(False).to_numpy(bool) if panel.delist_after is not None else np.zeros(c.shape, bool)
+    T, N = c.shape
+    rows = []
+    for j in range(N):
+        h = halted[:, j]
+        if not h.any():
+            continue
+        t = 0
+        valid = np.where(~np.isnan(px[:, j]))[0]
+        last_bar = int(valid[-1]) if len(valid) else -1
+        flagged = bool(da[:, j].any())
+        while t < T:
+            if not h[t]:
+                t += 1
+                continue
+            e = t
+            while e + 1 < T and h[e + 1]:
+                e += 1
+            if e - t + 1 >= min_days:
+                before = next((px[k, j] for k in range(t - 1, -1, -1) if not np.isnan(px[k, j]) and not h[k]), np.nan)
+                if e == T - 1 and not (flagged and last_bar == e):
+                    outcome, ret = "ongoing", np.nan
+                elif flagged and last_bar == e:
+                    outcome, ret = "ended_in_halt", 0.0 if np.isfinite(before) else np.nan
+                else:
+                    nxt = e + 1
+                    while nxt < T and np.isnan(px[nxt, j]):
+                        nxt += 1
+                    if nxt >= T:
+                        outcome, ret = "ongoing", np.nan
+                    elif flagged and last_bar - nxt <= relist_days:
+                        outcome, ret = "resumed_then_delisted", px[last_bar, j] / before - 1.0 if np.isfinite(before) else np.nan
+                    else:
+                        outcome, ret = "resumed", px[nxt, j] / before - 1.0 if np.isfinite(before) else np.nan
+                rows.append({"ticker": c.columns[j], "start": c.index[t], "end": c.index[e], "days": e - t + 1, "outcome": outcome, "ret": ret})
+            t = e + 1
+    return pd.DataFrame(rows, columns=["ticker", "start", "end", "days", "outcome", "ret"])
+
+
+def at_prices(panel: Panel, price="open") -> Panel:
+    """A copy of `panel` whose prices are another series, so that a position is entered and marked at it. `price` is the name of a panel
+    field (`"open"`, `"high"`, `"low"`) or a (date x ticker) frame. With the open, a signal from the close of day d is entered at the open
+    of d+lag and held to the open of d+lag+1. A price that is zero or negative is treated as missing. Only `close` changes: eligibility, volume and the rest were built from the original
+    closes, and `screen` and `backtest_event` still compute their controls from whatever `close` now holds."""
+    px = getattr(panel, price) if isinstance(price, str) else price
+    if px is None:
+        raise ValueError(f"the panel has no {price!r}")
+    if not isinstance(px, pd.DataFrame):
+        raise ValueError("price must be a field name or a (date x ticker) frame")
+    px = px.reindex(index=panel.dates, columns=panel.tickers)
+    return replace(panel, close=px.where(px > 0))                # a zero or negative price is no price: the engines treat a missing price as a return of 0, a zero would give infinity
+
+
+def exec_masks(panel: Panel):
+    """(can_buy, can_sell) as arrays indexed by the **signal date**, i.e. read on the execution day d + entry_lag. None when the panel
+    has neither field. A missing frame counts as always possible."""
+    if panel.can_buy is None and panel.can_sell is None:
+        return None, None
+    lag = panel.entry_lag
+
+    def one(f):
+        if f is None:
+            return np.ones(panel.close.shape, dtype=bool)
+        return f.shift(-lag, fill_value=True).to_numpy(bool)
+    return one(panel.can_buy), one(panel.can_sell)
+
+
+def round_to_lots(weights: np.ndarray, capital: float, price: np.ndarray, lot) -> np.ndarray:
+    """The weights you can actually hold: whole lots at the given prices. Shares = weight x capital / price, rounded to a multiple of
+    `lot`. `price` must be the **real** price level (not a back-adjusted series, whose level is arbitrary). A name with no price keeps weight NaN."""
+    with np.errstate(divide="ignore", invalid="ignore"):
+        shares = weights * capital / price
+        lots = np.round(shares / lot) * lot
+        return lots * price / capital
+
+
+def realize(target: np.ndarray, can_buy: np.ndarray | None = None, can_sell: np.ndarray | None = None, *,
+            capital: float | None = None, price: np.ndarray | None = None, lot=1.0, min_trade_value: float = 0.0, cap_gross: bool = False):
+    """The positions that result from asking for `target` row by row (row 0 is a build from cash).
+
+    Each day, per name: round the target to whole lots (when `capital` is given; `price` is read on the **same row** as the target, so pass the price of the day the
+    trade is done: `backtest_weights` shifts it by `entry_lag`); skip the trade if its value is under `min_trade_value` (a number, or one per security);
+    skip it if it increases the position where `can_buy` is False, or decreases it where `can_sell` is False. A skipped trade leaves the
+    position as it was the day before. A name with no price on a day (NaN) keeps its position.
+    The weight is what the engines keep constant, so a day without a trade keeps the previous **weight**, not the previous share count: after the price moves, the
+    shares it implies need not be a multiple of the lot (a halted name does not move, so there it is).
+
+    `cap_gross`: a position that cannot be traded ties up capital. Without this, new targets are added on top of it and the gross exposure can exceed the
+    target's. With it, the day's whole target is scaled by a k in [0, 1], the largest one found by bisection for which the resulting gross exposure does not exceed
+    the gross of the day's target (after lot rounding, so rounding itself is never cut). A position that cannot be sold stays where it is; one that cannot be
+    bought stays too unless the scaled target falls below it, in which case it is sold down, so it can shrink. A scaled target can turn a buy into a sell, which
+    changes what is blocked, so k is searched for rather than solved for, and the feasible set being an interval [0, k] is not proven (no counter-example was found
+    in 653 random small cases). On a day with nothing blocked or skipped k is exactly 1 and the result is the same as without it, with or without `capital`. If the
+    stuck positions alone exceed the target gross, nothing can meet the cap: k is 0 and the day is counted in `infeasible_days`.
+
+    Returns (positions, stats). `mean_stuck_weight` is the average over days of the weight held where the target said otherwise because a trade was
+    blocked; `longest_freeze_days` the longest run of consecutive days a wanted trade in one name was blocked; `mean_free_scale` the average k (1 without `cap_gross`), `infeasible_days` the days on which the stuck positions alone exceeded the target gross."""
+    T, N = target.shape
+    use_lots = capital is not None
+    mtv = np.asarray(min_trade_value, dtype=float)                       # a number, or one per security
+    mtv_any = bool(np.any(mtv > 0))
+    out = np.zeros_like(target, dtype=float)
+    prev = np.zeros(N)
+    asked = blocked_turn = skipped_turn = gap = stuck = k_sum = 0.0
+    n_blocked = n_skipped = infeasible = 0
+    streak = np.zeros(N, dtype=int)
+    longest = 0
+
+    def decide(raw: np.ndarray, t: int):
+        tgt = raw
+        if use_lots:
+            lt = lot[t] if isinstance(lot, np.ndarray) and lot.ndim == 2 else lot
+            tgt = round_to_lots(raw, capital, price[t], lt)
+            # A flat target needs no price to carry out (a delisted name is simply closed); a non-flat target with no price cannot be sized, so the
+            # position stays.
+            tgt = np.where(raw == 0, 0.0, np.where(np.isfinite(tgt), tgt, prev))
+        d = tgt - prev
+        stay = np.zeros(N, dtype=bool)
+        if use_lots and mtv_any:
+            stay |= (np.abs(d) * capital < mtv) & (d != 0)
+        blk = np.zeros(N, dtype=bool)
+        if can_buy is not None:
+            blk |= (d > 0) & ~can_buy[t]
+        if can_sell is not None:
+            blk |= (d < 0) & ~can_sell[t]
+        return tgt, d, stay, blk
+
+    for t in range(T):
+        raw0 = target[t]
+        k = 1.0
+        tgt, d, stay, blk = decide(raw0, t)
+        if cap_gross:
+            budget = float(np.abs(tgt).sum()) + 1e-12                       # what the day would hold with nothing in the way (after lot rounding)
+
+            def gross_at(kk: float) -> float:
+                g_tgt, _, g_stay, g_blk = decide(raw0 * kk, t)
+                return float(np.abs(np.where(g_stay | g_blk, prev, g_tgt)).sum())
+
+            if float(np.abs(np.where(stay | blk, prev, tgt)).sum()) > budget:
+                if gross_at(0.0) > budget:
+                    k = 0.0
+                    infeasible += 1
+                else:
+                    lo, hi = 0.0, 1.0                                       # lo is always feasible, hi is not
+                    while hi - lo > 1e-14:
+                        mid = 0.5 * (lo + hi)
+                        lo, hi = (mid, hi) if gross_at(mid) <= budget else (lo, mid)
+                    k = lo
+                raw = raw0 * k
+                tgt, d, stay, blk = decide(raw, t)
+            else:
+                raw = raw0
+        else:
+            raw = raw0
+        k_sum += k
+        if use_lots:
+            gap += float(np.abs(tgt - raw).sum())
+        cur = np.where(stay | blk, prev, tgt)
+        held = blk & ~stay
+        stuck += float(np.abs(cur - tgt)[held].sum())                         # weight sitting where it should not be, today
+        streak = np.where(held, streak + 1, 0)
+        longest = max(longest, int(streak.max())) if N else longest
+        if t > 0:
+            asked += float(np.abs(d).sum())
+            blocked_turn += float(np.abs(d[blk & ~stay]).sum())
+            skipped_turn += float(np.abs(d[stay]).sum())
+            n_blocked += int((blk & ~stay).sum())
+            n_skipped += int(stay.sum())
+        out[t] = cur
+        prev = cur
+    stats = {"mean_stuck_weight": stuck / T if T else 0.0, "longest_freeze_days": longest, "asked_turnover": asked, "blocked_turnover": blocked_turn, "blocked_trades": n_blocked, "blocked_turnover_share": blocked_turn / asked if asked > 0 else 0.0,
+             "min_trade_skipped": n_skipped, "min_trade_skipped_share": skipped_turn / asked if asked > 0 else 0.0,
+             "mean_abs_rounding_gap": gap / (T * N) if use_lots else 0.0, "mean_free_scale": k_sum / T if T else 1.0, "infeasible_days": infeasible}
+    return out, stats