pitbacktest 0.2.0__py3-none-any.whl

pitbacktest/adapters/yfinance.py

@@ -0,0 +1,243 @@
+"""Public-data adapter: build a Panel from yfinance.
+
+Dependency: `pip install yfinance` (not needed by the core, used only by this adapter)
+
+Usage
+    from pitbacktest.adapters.yfinance import load_panel
+    panel = load_panel(["AAPL", "MSFT", "NVDA"], "2018-01-01", "2024-12-31")
+
+Warning: the limits of free data
+    - **Survivorship bias**: a ticker list given as of today leaves out securities delisted since.
+        It is not a point-in-time universe, so performance is biased upward. Use it for exploration only.
+    - **Adjusted prices**: auto_adjust=True applies dividends and splits. Without it a split day
+        creates a fake return.
+    - Firm characteristics (ROA, book-to-market, ...) are not provided. Running without chars raises a warning,
+        and it is a legitimate one: "price controls alone cannot filter out someone else's alpha".
+"""
+
+from __future__ import annotations
+
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from ..core.panel import Panel, build_pit_eligible
+
+
+def load_panel(tickers: list[str], start: str, end: str, *,
+               min_dollar_volume: float = 1e6,
+               market: str = "US", entry_lag: int = 1,
+               with_market_cap: bool = True,
+               with_chars: bool = False,
+               shares_outstanding: dict[str, float] | None = None) -> Panel:
+    """Build a (date x ticker) panel from yfinance.
+
+    tickers            securities to look up. **Passing only securities listed today creates survivorship bias.**
+    min_dollar_volume  minimum average traded value for eligibility (20 days)
+    with_market_cap    market cap from get_shares_full (share count over time) x close.
+                       Switches on the size and turnover controls. One API call per security, so somewhat slow.
+    with_chars         builds the 5 firm characteristics from quarterly financials (log_size, log_bm, momentum, roa, asset_growth).
+                       **The key input of the neutralisation test.** Two more API calls per security.
+    shares_outstanding manual {ticker: shares}. A fallback when with_market_cap fails.
+    """
+    try:
+        import yfinance as yf
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            "yfinance is required: pip install yfinance"
+        ) from e
+
+    raw = yf.download(tickers, start=start, end=end, auto_adjust=True,
+                      progress=False, group_by="column")
+    if raw is None or raw.empty:
+        raise ValueError("yfinance returned no data (check the tickers and the period)")
+
+    def pick(field: str) -> pd.DataFrame | None:
+        if isinstance(raw.columns, pd.MultiIndex):
+            if field not in raw.columns.get_level_values(0):
+                return None
+            d = raw[field]
+        else:
+            if field not in raw.columns:
+                return None
+            d = raw[[field]].rename(columns={field: tickers[0]})
+        return d.sort_index()
+
+    close = pick("Close")
+    if close is None:
+        raise ValueError("there is no Close column")
+    close.index = pd.to_datetime(close.index).tz_localize(None)
+    volume = pick("Volume")
+    high, low, open_ = pick("High"), pick("Low"), pick("Open")
+    for d in (volume, high, low, open_):
+        if d is not None:
+            d.index = close.index
+
+    mkt_cap = None
+    if with_market_cap:
+        mkt_cap = _market_cap(yf, list(close.columns), close)
+    if mkt_cap is None and shares_outstanding:
+        so = pd.Series(shares_outstanding).reindex(close.columns)
+        if so.notna().any():
+            mkt_cap = close.mul(so, axis=1)
+    if mkt_cap is None:
+        warnings.warn("Market cap could not be built: the size and turnover controls are switched off", stacklevel=2)
+
+    chars = {}
+    if with_chars:
+        if mkt_cap is None:
+            warnings.warn("There is no market cap, so firm characteristics cannot be built", stacklevel=2)
+        else:
+            chars = _characteristics(yf, close, mkt_cap)
+
+    eligible = build_pit_eligible(close, min_adv=min_dollar_volume, volume=volume)
+    n_drop = int((~eligible).sum().sum())
+    if eligible.values.sum() == 0:
+        raise ValueError("eligible is all False: lower min_dollar_volume")
+
+    warnings.warn(
+        f"The yfinance panel is **not a point-in-time universe**. Only tickers listed today are looked up, so "
+        f"delisted securities are missing and performance is biased upward. ({n_drop:,} cells excluded)",
+        stacklevel=2)
+
+    ppy = 365 if market.upper() == "CRYPTO" else 252   # a 24/7 market has 365 trading days a year
+    n_ended = sum(1 for c in close.columns if close[c].last_valid_index() is not None
+                  and close[c].last_valid_index() < close.index[-1] - pd.Timedelta(days=30))
+    if len(close.columns) >= 30 and n_ended == 0:
+        warnings.warn("No security in this panel has a price that stops inside the period: it holds only securities alive today, "
+                      "so it has survivorship bias (yfinance hardly returns past prices of delisted or acquired securities). "
+                      "See docs/survivorship.md.", stacklevel=2)
+    return Panel(close=close, eligible=eligible, open=open_, high=high, low=low,
+                 volume=volume, mkt_cap=mkt_cap, chars=chars,
+                 market=market, entry_lag=entry_lag, periods_per_year=ppy)
+
+
+def _market_cap(yf, tickers: list[str], close: pd.DataFrame) -> pd.DataFrame | None:
+    """Share count over time x close. get_shares_full reflects changes in shares outstanding (buybacks, issuance)."""
+    cols = {}
+    for t in tickers:
+        try:
+            sh = yf.Ticker(t).get_shares_full(start=str(close.index[0].date()))
+        except Exception:  # noqa: BLE001, PERF203
+            continue
+        if sh is None or not len(sh):
+            continue
+        s = pd.Series(sh)
+        s.index = pd.to_datetime(s.index).tz_localize(None)
+        s = s[~s.index.duplicated(keep="last")].sort_index()
+        cols[t] = s.reindex(close.index, method="ffill")
+    if not cols:
+        return None
+    shares = pd.DataFrame(cols).reindex(columns=close.columns)
+    got = int(shares.notna().any().sum())
+    if got < len(tickers):
+        warnings.warn(f"Received shares for only {got}/{len(tickers)} securities: "
+                      f"for the rest the market cap is NaN, so they are left out of the controls", stacklevel=3)
+    return close * shares
+
+
+def _characteristics(yf, close: pd.DataFrame, mkt_cap: pd.DataFrame) -> dict:
+    """The five firm characteristics: standard Fama-French style definitions.
+
+    Warning: yfinance's financial coverage is short: 5 to 7 quarters and 5 years of annual data only.
+          A long backtest such as 2018 to 2024 would be empty for most of the period, so quarterly and annual data are combined and
+          **a characteristic with too little coverage is dropped altogether** (an empty column ruins the regression).
+          For long studies attach a paid financial source (Compustat, Sharadar and the like).
+
+    Disclosure lag: yfinance gives no filing date, so **period end + 45 business days** is used as an approximation.
+    """
+    LAG = 45
+    MIN_COVER = 0.30      # a characteristic is dropped if its share of valid cells is below this
+    ta, be, ni = {}, {}, {}
+    for t in close.columns:
+        try:
+            tk = yf.Ticker(t)
+            bs, inc = tk.quarterly_balance_sheet, tk.quarterly_income_stmt
+        except Exception:  # noqa: BLE001, PERF203
+            continue
+        if bs is None or bs.empty:
+            continue
+
+        def grab(df, keys):
+            if df is None or df.empty:
+                return None
+            for k in keys:
+                hit = [i for i in df.index if k in str(i)]
+                if hit:
+                    s = df.loc[hit[0]].dropna()
+                    s.index = pd.to_datetime(s.index).tz_localize(None)
+                    return s.sort_index()
+            return None
+
+        # combine quarterly (last 5 to 7) and annual (last 5 years) to widen the coverage
+        try:
+            abs_, ainc = tk.balance_sheet, tk.income_stmt
+        except Exception:  # noqa: BLE001
+            abs_ = ainc = None
+
+        def merge(q_, a_):
+            parts = [x for x in (q_, a_) if x is not None and len(x)]
+            if not parts:
+                return None
+            m = pd.concat(parts)
+            return m[~m.index.duplicated(keep="first")].sort_index()
+
+        a = merge(grab(bs, ["Total Assets"]), grab(abs_, ["Total Assets"]))
+        e = merge(grab(bs, ["Stockholders Equity", "Common Stock Equity"]),
+                  grab(abs_, ["Stockholders Equity", "Common Stock Equity"]))
+        n = merge(grab(inc, ["Net Income From Continuing Operation Net Minority Interest",
+                             "Net Income"]),
+                  grab(ainc, ["Net Income From Continuing Operation Net Minority Interest",
+                              "Net Income"]))
+        for src, dst in ((a, ta), (e, be), (n, ni)):
+            if src is not None and len(src):
+                dst[t] = src
+
+    def to_daily(d: dict) -> pd.DataFrame | None:
+        if not d:
+            return None
+        df = pd.DataFrame(d)
+        # use from the period end + LAG business days (approximating the disclosure lag)
+        df.index = df.index + pd.tseries.offsets.BDay(LAG)
+        return df.reindex(close.index.union(df.index)).ffill().reindex(close.index)\
+                 .reindex(columns=close.columns)
+
+    TA, BE, NI = to_daily(ta), to_daily(be), to_daily(ni)
+    if TA is None and BE is None:
+        warnings.warn("Quarterly financials could not be fetched, so the firm characteristics are empty", stacklevel=3)
+        return {}
+    out = {"log_size": np.log(mkt_cap.clip(lower=1)),
+           "momentum": close.pct_change(252) - close.pct_change(21)}
+    if BE is not None:
+        out["log_bm"] = np.log((BE / mkt_cap.replace(0, np.nan)).clip(lower=1e-6))
+    if NI is not None and TA is not None:
+        out["roa"] = NI / TA.replace(0, np.nan)
+    if TA is not None:
+        out["asset_growth"] = TA.pct_change(252)
+    # drop characteristics with low coverage: an empty column wipes out the regression sample
+    kept, dropped = {}, {}
+    for k, v in out.items():
+        cover = float(v.notna().values.mean())
+        (kept if cover >= MIN_COVER else dropped)[k] = cover
+        if cover >= MIN_COVER:
+            kept[k] = v
+    dropped_names = {k: f"{c*100:.0f}%" for k, c in dropped.items()}
+    kept_out = {k: out[k] for k in out if k not in dropped}
+    if dropped_names:
+        warnings.warn(
+            f"Firm characteristics {list(dropped_names)} were dropped because their coverage ({dropped_names}) is low "
+            f"(yfinance provides only 5 to 7 quarters and 5 years of annual data). "
+            f"For long backtests attach a paid financial source.", stacklevel=3)
+    got = {k: f"{float(v.notna().values.mean())*100:.0f}%" for k, v in kept_out.items()}
+    warnings.warn(f"Firm characteristic valid cells: {got} | period-end + {LAG} business-day lag applied", stacklevel=3)
+    return kept_out
+
+
+def sp500_tickers() -> list[str]:
+    """Wikipedia's current S&P 500 constituents. **A list as of today, so it has survivorship bias.**"""
+    try:
+        t = pd.read_html("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies")[0]
+        return sorted(t["Symbol"].astype(str).str.replace(".", "-", regex=False).tolist())
+    except Exception as e:  # pragma: no cover  # noqa: BLE001
+        raise RuntimeError(f"Could not fetch the S&P 500 list: {e}") from e

pitbacktest/analytics.py

@@ -0,0 +1,234 @@
+"""Reading a result: alpha and beta, information coefficients, and bootstrap intervals for the Sharpe ratio.
+
+Three questions a reviewer asks first, each with the defaults that avoid the usual mistakes.
+
+  alpha_beta                Is the return alpha, or exposure to something I could have bought? (OLS with Newey-West errors)
+  information_coefficient   Does the signal rank next-period returns? (daily rank correlation; delisted names kept)
+  sharpe_ci / sharpe_diff_ci  How precise is a Sharpe ratio, and is the difference between two of them real?
+                            (stationary block bootstrap, so autocorrelation in returns is respected)
+
+None of these proves a strategy works. They say how much the data can tell apart.
+
+References
+  Newey, West (1987, 1994): heteroskedasticity and autocorrelation consistent covariance, plug-in lag rule.
+  Politis, Romano (1994): the stationary bootstrap.
+  Lo (2002): the Sharpe ratio's sampling error is large and depends on autocorrelation.
+"""
+from __future__ import annotations
+
+import math
+import warnings
+from statistics import NormalDist
+
+import numpy as np
+import pandas as pd
+
+_N = NormalDist()
+
+
+# --------------------------------------------------------------------------------------------------- alpha and beta
+def _align(y: pd.Series, X: pd.DataFrame) -> tuple[pd.Series, pd.DataFrame]:
+    df = pd.concat([y.rename("__y__"), X], axis=1, join="inner").replace([np.inf, -np.inf], np.nan).dropna()
+    return df["__y__"], df.drop(columns="__y__")
+
+
+def alpha_beta(returns: pd.Series, factors: pd.Series | pd.DataFrame, *, periods_per_year: int = 252,
+               lag: int | None = None, min_obs: int = 60) -> dict:
+    """Regress `returns` on `factors` (for example the market) with an intercept.
+
+    Standard errors are Newey-West (Bartlett kernel, no small-sample correction), with the lag from the
+    Newey-West (1994) plug-in rule `int(4 * (T / 100) ** (2 / 9))` unless `lag` is given. The t-statistics are
+    asymptotic, compared with a normal distribution. **They are still too optimistic in finite samples**: with
+    AR(1) errors of 0.5 and 500 observations, a true alpha of zero is rejected at |t| > 1.96 in about 9% of datasets
+    instead of 5% (the uncorrected figure is 26%; `tests/test_analytics.py` A4). Read |t| below about 2.5 as no evidence. Dates are aligned on the intersection and rows with a missing
+    value are dropped; at least `min_obs` rows (and 3 per coefficient) must remain.
+
+    `alpha_annual` is the per-period intercept times `periods_per_year` (arithmetic, not compounded).
+    `appraisal` is `alpha_annual` over the annualised residual volatility.
+    Net returns include costs and funding while a benchmark does not, so a small negative alpha is expected for a
+    strategy with no edge.
+    """
+    y = pd.Series(returns).astype(float)
+    F = factors.to_frame(factors.name or "factor") if isinstance(factors, pd.Series) else factors.astype(float)
+    if F.shape[1] == 0:
+        raise ValueError("no factor columns")
+    y, F = _align(y, F)
+    T, k = len(y), F.shape[1] + 1
+    if T < max(min_obs, 3 * k):
+        raise ValueError(f"only {T} usable observations for {k} coefficients (need {max(min_obs, 3 * k)})")
+    if (F.std(ddof=0) <= 1e-12 * np.maximum(1.0, F.abs().mean())).any():      # relative: identical floats can have std ~1e-18
+        raise ValueError("a factor column is constant")
+    X = np.column_stack([np.ones(T), F.to_numpy(float)])
+    if np.linalg.matrix_rank(X) < k:
+        raise ValueError("factor columns are collinear")
+    Y = y.to_numpy(float)
+    coef, *_ = np.linalg.lstsq(X, Y, rcond=None)
+    e = Y - X @ coef
+    L = int(4 * (T / 100) ** (2 / 9)) if lag is None else int(lag)
+    L = max(0, min(L, T - 1))
+    Xe = X * e[:, None]
+    S = Xe.T @ Xe
+    for l in range(1, L + 1):
+        G = Xe[l:].T @ Xe[:-l]
+        S += (1.0 - l / (L + 1.0)) * (G + G.T)
+    bread = np.linalg.inv(X.T @ X)
+    V = bread @ S @ bread
+    se = np.sqrt(np.diag(V))
+    with np.errstate(divide="ignore", invalid="ignore"):
+        t = coef / se                                            # a perfect fit has zero error: t is inf or nan
+    sse = float(e @ e)
+    sst = float(((Y - Y.mean()) ** 2).sum())
+    resid_vol = float(e.std(ddof=k) * math.sqrt(periods_per_year))
+    ann = coef[0] * periods_per_year
+
+    def p(tv: float) -> float:
+        return float(2 * (1 - _N.cdf(abs(tv)))) if np.isfinite(tv) else float("nan")
+
+    return {"alpha": float(coef[0]), "alpha_annual": float(ann), "alpha_t": float(t[0]), "alpha_p": p(t[0]),
+            "betas": {c: {"beta": float(coef[i + 1]), "t": float(t[i + 1])} for i, c in enumerate(F.columns)},
+            "r2": float(1 - sse / sst) if sst > 0 else float("nan"), "n": T, "lag": L,
+            "resid_vol_annual": resid_vol, "appraisal": float(ann / resid_vol) if resid_vol > 0 else float("nan")}
+
+
+# ----------------------------------------------------------------------------------------- information coefficient
+def forward_returns(panel, h: int, *, delist_return: float | None = None) -> pd.DataFrame:
+    """Compounded return from close(t + lag) to close(t + lag + h), for a signal at the close of t.
+
+    A name with no price on a day earns 0 that day (it is treated as cash after its last bar), the same rule as
+    `backtest_portfolio`; with `delist_return` set, the day after a delisting flagged in `panel.delist_after` earns
+    that return instead. Dropping such names (a NaN forward return) would silently remove the losers from the test.
+    Windows that run past the end of the sample are NaN."""
+    ret = np.nan_to_num(panel.ret1().to_numpy(float), nan=0.0)
+    if delist_return is not None and panel.delist_after is not None:
+        da = panel.delist_after.reindex(index=panel.dates, columns=panel.tickers).fillna(False).to_numpy(bool)
+        nxt = np.zeros(ret.shape, dtype=bool)
+        nxt[1:] = da[:-1]
+        ret = np.where(nxt, float(delist_return), ret)
+    T = ret.shape[0]
+    g = np.ones(ret.shape)
+    for k in range(1, h + 1):
+        s = panel.entry_lag + k
+        sh = np.full(ret.shape, np.nan)
+        if s < T:
+            sh[: T - s] = 1.0 + ret[s:]
+        g = g * sh
+    return pd.DataFrame(g - 1.0, index=panel.dates, columns=panel.tickers)
+
+
+def information_coefficient(factor: pd.DataFrame, forward: pd.DataFrame, eligible: pd.DataFrame, *,
+                            min_obs: int = 20) -> pd.Series:
+    """Daily cross-sectional Spearman correlation between the factor and the forward return, among eligible names
+    with both values. Days with fewer than `min_obs` names, or no variation, are NaN."""
+    f = factor.reindex(index=forward.index, columns=forward.columns)
+    el = eligible.reindex(index=forward.index, columns=forward.columns).fillna(False).astype(bool)
+    ok = el & f.notna() & forward.notna()
+    rf = f.where(ok).rank(axis=1)
+    ry = forward.where(ok).rank(axis=1)
+    n = ok.sum(axis=1).to_numpy(float)
+    a, b = rf.to_numpy(float), ry.to_numpy(float)
+    with np.errstate(invalid="ignore", divide="ignore"), warnings.catch_warnings():
+        warnings.simplefilter("ignore", RuntimeWarning)          # all-NaN rows are expected (days with no signal)
+        a = a - np.nanmean(a, axis=1, keepdims=True)
+        b = b - np.nanmean(b, axis=1, keepdims=True)
+        num = np.nansum(a * b, axis=1)
+        den = np.sqrt(np.nansum(a * a, axis=1) * np.nansum(b * b, axis=1))
+        ic = np.where((n >= min_obs) & (den > 0), num / den, np.nan)
+    return pd.Series(ic, index=forward.index, name="ic")
+
+
+def ic_report(panel, factor: pd.DataFrame, horizons=(1, 5, 20), *, delist_return: float | None = None,
+              min_obs: int = 20) -> dict:
+    """IC summary per horizon: mean IC, its standard deviation, ICIR (mean over standard deviation), the share of
+    positive days, and a Newey-West t-statistic of the mean with lag = horizon (the forward windows overlap)."""
+    from .core.estimators import newey_west_t
+    out = {}
+    for h in horizons:
+        ic = information_coefficient(factor, forward_returns(panel, h, delist_return=delist_return),
+                                     panel.eligible, min_obs=min_obs).dropna()
+        if len(ic) < 30:
+            out[h] = {"n_days": int(len(ic)), "mean_ic": float("nan"), "std_ic": float("nan"), "icir": float("nan"),
+                      "hit_rate": float("nan"), "t_nw": float("nan")}
+            continue
+        mu, _, t, T = newey_west_t(ic.to_numpy(), lag=max(1, h))
+        sd = float(ic.std(ddof=1))
+        out[h] = {"n_days": int(T), "mean_ic": float(mu), "std_ic": sd, "icir": float(mu / sd) if sd > 0 else float("nan"),
+                  "hit_rate": float((ic > 0).mean()), "t_nw": float(t)}
+    return out
+
+
+# --------------------------------------------------------------------------------------------- bootstrap intervals
+def _stationary_indices(T: int, n: int, mean_block: float, rng: np.random.Generator) -> np.ndarray:
+    """(n, T) resampling indices for the stationary bootstrap: blocks of geometric length (mean `mean_block`),
+    wrapping around the end of the sample."""
+    p = 1.0 / max(1.0, mean_block)
+    start = rng.integers(0, T, size=(n, T))
+    new = rng.random((n, T)) < p
+    idx = np.empty((n, T), dtype=np.int64)
+    idx[:, 0] = start[:, 0]
+    for t in range(1, T):
+        idx[:, t] = np.where(new[:, t], start[:, t], (idx[:, t - 1] + 1) % T)
+    return idx
+
+
+def _sharpe_rows(R: np.ndarray, ppy: int) -> np.ndarray:
+    return R.mean(axis=1) / R.std(axis=1, ddof=1) * math.sqrt(ppy)
+
+
+def _default_block(T: int) -> float:
+    return float(max(1, round(T ** (1 / 3))))
+
+
+def sharpe_ci(returns: pd.Series, *, periods_per_year: int = 252, n: int = 2000, mean_block: float | None = None,
+              level: float = 0.95, seed: int = 0) -> dict:
+    """Percentile interval for the annualised Sharpe ratio from a stationary block bootstrap.
+
+    `mean_block` defaults to `round(T ** (1/3))`, a common rule of thumb; use a larger one if returns are strongly
+    autocorrelated. Intervals from short samples tend to be too narrow, so treat a lower bound close to zero with
+    caution. Sharpe uses risk-free 0 and the sample standard deviation (ddof=1)."""
+    x = pd.Series(returns).dropna().to_numpy(float)
+    T = len(x)
+    if T < 30:
+        raise ValueError(f"{T} observations is too few for a bootstrap")
+    b = _default_block(T) if mean_block is None else float(mean_block)
+    rng = np.random.default_rng(seed)
+    sr = []
+    for c in _chunks(n):
+        sr.append(_sharpe_rows(x[_stationary_indices(T, c, b, rng)], periods_per_year))
+    sr = np.concatenate(sr)
+    a = (1 - level) / 2
+    return {"sharpe": float(x.mean() / x.std(ddof=1) * math.sqrt(periods_per_year)),
+            "lo": float(np.quantile(sr, a)), "hi": float(np.quantile(sr, 1 - a)), "se": float(sr.std(ddof=1)),
+            "n_boot": int(n), "mean_block": b, "level": level, "T": T}
+
+
+def sharpe_diff_ci(a: pd.Series, b: pd.Series, *, periods_per_year: int = 252, n: int = 2000,
+                   mean_block: float | None = None, level: float = 0.95, seed: int = 0) -> dict:
+    """Sharpe(b) - Sharpe(a) with a **paired** stationary bootstrap: the same resampled dates are used for both
+    series, so the common movement of the two cancels, which is what makes a small difference detectable.
+    `p_boot` is twice the smaller tail of the bootstrap distribution around zero (a two-sided percentile p-value,
+    approximate). Series are aligned on their common dates."""
+    df = pd.concat([pd.Series(a).rename("a"), pd.Series(b).rename("b")], axis=1, join="inner").dropna()
+    T = len(df)
+    if T < 30:
+        raise ValueError(f"{T} common observations is too few for a bootstrap")
+    xa, xb = df["a"].to_numpy(float), df["b"].to_numpy(float)
+    bl = _default_block(T) if mean_block is None else float(mean_block)
+    rng = np.random.default_rng(seed)
+    d = []
+    for c in _chunks(n):
+        idx = _stationary_indices(T, c, bl, rng)
+        d.append(_sharpe_rows(xb[idx], periods_per_year) - _sharpe_rows(xa[idx], periods_per_year))
+    d = np.concatenate(d)
+    al = (1 - level) / 2
+    est = float((xb.mean() / xb.std(ddof=1) - xa.mean() / xa.std(ddof=1)) * math.sqrt(periods_per_year))
+    p = min(1.0, 2 * min(float((d <= 0).mean()), float((d >= 0).mean())))
+    return {"diff": est, "lo": float(np.quantile(d, al)), "hi": float(np.quantile(d, 1 - al)), "se": float(d.std(ddof=1)),
+            "p_boot": p, "n_boot": int(n), "mean_block": bl, "level": level, "T": T}
+
+
+def _chunks(n: int, size: int = 400):
+    left = n
+    while left > 0:
+        c = min(size, left)
+        yield c
+        left -= c

pitbacktest/core/controls.py

@@ -0,0 +1,126 @@
+"""Controls: 7 price-based and 5 firm characteristics.
+
+Why firm characteristics are the default
+    Testing with price controls alone makes you **mistake someone else's alpha for your own**. Without controlling for
+    profitability and value, the apparent alpha often shrinks a lot or flips sign
+    (T6 in tests/test_synthetic.py feeds a control in as the factor and reproduces this disappearance).
+    Price controls (mom20/mom120) alone cannot capture standard momentum (12-1 months), profitability or value.
+    So chars are **included automatically** when present, and a warning is raised when they are not.
+
+Normalisation
+    Everything is a per-date percentile rank -> mean 0, variance 1. Robust to outliers and lets a coefficient be read as 'bp per 1 SD'.
+"""
+
+from __future__ import annotations
+
+import warnings
+
+import numpy as np
+import pandas as pd
+
+EPS = 1e-12
+PRICE_CONTROLS = ("rev1", "rev5", "mom21", "mom63", "mom252_21", "logsize", "turnover", "vol21")
+CHAR_NAMES = ("log_size", "log_bm", "momentum", "roa", "asset_growth")
+
+
+def xs_rank(v: pd.DataFrame, eligible: pd.DataFrame) -> pd.DataFrame:
+    """Per-date percentile rank [0,1] among the eligible. A cross-sectional operation, so the market-wide component drops out by itself."""
+    return v.where(eligible).rank(axis=1, pct=True, na_option="keep")
+
+
+def xs_norm(v: pd.DataFrame, eligible: pd.DataFrame) -> pd.DataFrame:
+    """Rank -> mean 0, variance 1 scale."""
+    return ((xs_rank(v, eligible) - 0.5) * np.sqrt(12.0)).astype(np.float32)
+
+
+def build_controls(panel, *, include_chars: bool = True) -> dict[str, pd.DataFrame]:
+    """The bundle of controls. Firm characteristics are included when panel.chars exists."""
+    close, el = panel.close, panel.eligible
+    ret = close.pct_change()
+    out: dict[str, pd.DataFrame] = {
+        # rev1 is required: the order flow of mean-reversion traders is a proxy for 'the day's fall'.
+        # The 1-day reversal is the strongest short-term effect, so leaving it out makes you rediscover reversal and call it a signal.
+        "rev1": ret,
+        "rev5": close.pct_change(5),
+        "mom21": close.pct_change(21),
+        "mom63": close.pct_change(63),
+        "vol21": ret.rolling(21, min_periods=21).std(),
+    }
+    # standard momentum 12-1: mom21/63 alone cannot capture it. Its lookback is one year of bars; a panel shorter than that would give a
+    # control that is NaN everywhere, and every date would silently drop out of the regressions, so it is left out (with a warning).
+    lookback = int(getattr(panel, "periods_per_year", 252))
+    if lookback < len(close) - 21:
+        out["mom252_21"] = close.pct_change(lookback) - close.pct_change(21)
+    else:
+        warnings.warn(f"The 12-1 momentum control needs {lookback} bars of history and the panel has {len(close)}: it is left out of the controls.",
+                      stacklevel=2)
+    if panel.mkt_cap is not None:
+        out["logsize"] = np.log(panel.mkt_cap.replace(0, np.nan))
+        adv = panel.adv(20)
+        if adv is not None:
+            out["turnover"] = adv / (panel.mkt_cap + EPS)
+
+    if include_chars:
+        if not panel.chars:
+            warnings.warn(
+                "Firm characteristics (chars) are missing. With price controls alone the ROA and book-to-market exposures are not captured, so "
+                "'someone else's alpha' can be mistaken for the factor's own. "
+                "Fill chars through an adapter, or state include_chars=False explicitly.",
+                stacklevel=2,
+            )
+        for k, v in panel.chars.items():
+            out[f"char_{k}"] = v
+
+    return {k: xs_norm(v, el).astype(np.float32) for k, v in out.items() if v is not None}
+
+
+def neutralize(factor: pd.DataFrame, controls: dict[str, pd.DataFrame],
+               eligible: pd.DataFrame, *, min_obs: int = 50) -> pd.DataFrame:
+    """Per-date cross-sectional regression residual: orthogonalised against the controls.
+
+    For continuous factors. Event (0/1) signals use the dummy regression in event.py.
+    A day whose residual is below 1e-5 of the factor's size (the controls explain the factor almost completely) stays NaN.
+    """
+    fv = factor.values.astype(np.float64)
+    ev = eligible.values
+    cvs = [c.values.astype(np.float64) for c in controls.values()]
+    out = np.full(fv.shape, np.nan)
+    for i in range(fv.shape[0]):
+        ok = ev[i] & np.isfinite(fv[i])
+        for c in cvs:
+            ok &= np.isfinite(c[i])
+        if ok.sum() < min_obs:
+            continue
+        y = fv[i][ok]
+        X = np.column_stack([np.ones(ok.sum())] + [c[i][ok] for c in cvs])
+        try:
+            Q, _ = np.linalg.qr(X)
+        except np.linalg.LinAlgError:
+            continue
+        res = y - Q @ (Q.T @ y)
+        # A factor inside the span of the controls leaves only rounding noise (about 1e-8 of its size in float32).
+        # Left in, a later rank-normalisation would blow that noise up to unit scale and the result would depend on the
+        # BLAS build. No information is left, so the day stays NaN.
+        if not (res.std() > 1e-5 * y.std()):
+            continue
+        out[i][ok] = res
+    return pd.DataFrame(out, index=factor.index, columns=factor.columns)
+
+
+def build_chars_from_financials(close: pd.DataFrame, mkt_cap: pd.DataFrame,
+                                book_equity: pd.DataFrame | None = None,
+                                net_income: pd.DataFrame | None = None,
+                                total_assets: pd.DataFrame | None = None) -> dict:
+    """Five firm characteristics: standard Fama-French style definitions.
+
+    Quarterly financials are passed already forward-filled to daily (the adapter's job).
+    """
+    chars = {"log_size": np.log(mkt_cap.clip(lower=1)),
+             "momentum": close.pct_change(252) - close.pct_change(21)}
+    if book_equity is not None:
+        chars["log_bm"] = np.log((book_equity / mkt_cap.replace(0, np.nan)).clip(lower=1e-6))
+    if net_income is not None and total_assets is not None:
+        chars["roa"] = net_income / total_assets.replace(0, np.nan)
+    if total_assets is not None:
+        chars["asset_growth"] = total_assets.pct_change(252)
+    return chars