pitbacktest 0.2.0__py3-none-any.whl

pitbacktest/__init__.py

@@ -0,0 +1,27 @@
+"""pitbacktest: a backtest toolkit that measures its own biases (US, Korea, crypto).
+
+Three entry points
+    screen()             A. Factor screening with neutralisation          (research stage)
+    backtest_event()     C. Event signals                                  (alert products)
+    backtest_portfolio() B. Portfolio alpha (CAGR, MDD, Sharpe)
+
+Design principles
+    - The core is plain pandas and numpy; data sources live in `adapters`.
+    - There is one timing convention and it is enforced with an assert.
+    - Controls for firm characteristics are the default.
+    - The statistics that decide a verdict are computed with controls; uncontrolled figures are returned next to them, not in their place.
+"""
+
+from .core.panel import Panel, build_pit_eligible
+from .core.gates import GateConfig
+from .event import backtest_event
+from .portfolio import backtest_portfolio, assert_timing
+from .screen import screen
+from . import validation, analytics, execution
+from .ledger import Ledger
+from .weights import backtest_weights, capacity_curve, ImpactModel
+from .shorting import shortable_from_bans
+
+__version__ = "0.2.0"
+__all__ = ["shortable_from_bans", "execution", "validation", "analytics", "Ledger", "backtest_weights", "capacity_curve", "ImpactModel", "Panel", "build_pit_eligible", "GateConfig", "screen",
+           "backtest_event", "backtest_portfolio", "assert_timing"]

pitbacktest/adapters/krx.py

@@ -0,0 +1,249 @@
+"""Korean equities from the official KRX OpenAPI, point in time and survivorship-free.
+
+How it removes survivorship bias: the API returns **every stock that was listed on a given date**, including the ones
+delisted or merged later (on 2015-01-02, 125 of the 899 KOSPI names are gone by 2024). Fetching day by day therefore
+builds the universe the way it really was.
+
+How it handles corporate actions without a price-adjustment table: each row carries the change versus the *reference
+price* (`CMPPREVDD_PRC`), and the reference price already reflects splits and rights issues. So
+`close / (close - change) - 1` is the adjusted return. It matched the published return to within rounding for all 953
+KOSPI stocks on 2024-01-02, and gives -2.08% (not -98%) for Samsung Electronics on its 50:1 split day (2018-05-04).
+The adjusted close used by the panel is the cumulative product of those returns. Dividends are not included (price
+return only).
+
+You need a KRX OpenAPI key (free, the services have to be approved on openapi.krx.co.kr). Put it in the environment as
+KRX_OPENAPI_KEY or pass `key=`; `load_key(env_path)` reads only that one variable from a .env file.
+
+    fetch_days("2013-01-01", "2026-10-06", "~/.cache/quantbt/krx")      # resumable, about 2 calls per trading day
+    panel = build_krx_panel("~/.cache/quantbt/krx", start="2014-01-01")
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from ..core.panel import Panel
+
+BASE = "https://data-dbg.krx.co.kr/svc/apis"
+ENDPOINTS = {"KOSPI": "sto/stk_bydd_trd", "KOSDAQ": "sto/ksq_bydd_trd"}
+RENAME = {"ISU_CD": "code", "ISU_NM": "name", "MKT_NM": "market", "TDD_CLSPRC": "close", "CMPPREVDD_PRC": "change",
+          "TDD_OPNPRC": "open", "TDD_HGPRC": "high", "TDD_LWPRC": "low", "ACC_TRDVOL": "volume", "ACC_TRDVAL": "value",
+          "MKTCAP": "mktcap", "LIST_SHRS": "shares"}
+# Not tradable common stock for a cross-sectional study: SPACs, REITs and funds listed as stocks.
+# Korean name patterns that mark SPACs (스팩), REITs (리츠), numbered funds (N호), and infrastructure funds (인프라, 맥쿼리). These must stay Korean:
+# they are matched against the names KRX returns.
+DEFAULT_EXCLUDE = re.compile(r"스팩|리츠|\d+호|인프라|맥쿼리")
+
+
+class QuotaExceeded(RuntimeError):
+    """The API refused further calls (daily limit). Run again later; the cache keeps what was fetched."""
+
+
+def load_key(env_path: str | Path | None = None) -> str:
+    """KRX_OPENAPI_KEY from the environment, else from a .env file. Only that one variable is read."""
+    if os.environ.get("KRX_OPENAPI_KEY"):
+        return os.environ["KRX_OPENAPI_KEY"]
+    if env_path:
+        for line in Path(env_path).expanduser().read_text().splitlines():
+            k, _, v = line.partition("=")
+            if k.strip() == "KRX_OPENAPI_KEY":
+                return v.strip().strip('"').strip("'")
+    raise KeyError("KRX_OPENAPI_KEY is not set")
+
+
+def _call(path: str, date: str, key: str, retries: int = 4) -> list[dict]:
+    req = urllib.request.Request(f"{BASE}/{path}?basDd={date}", headers={"AUTH_KEY": key, "User-Agent": "pitbacktest-research/0.2"})
+    delay = 1.0
+    for i in range(retries + 1):
+        try:
+            with urllib.request.urlopen(req, timeout=40) as r:
+                return json.loads(r.read()).get("OutBlock_1", [])
+        except urllib.error.HTTPError as e:
+            body = e.read()[:200].decode("utf-8", "replace")
+            if e.code in (429, 403) or "limit" in body.lower() or "초과" in body:   # "초과" = "exceeded", the wording of the KRX quota message
+                raise QuotaExceeded(f"HTTP {e.code}: {body}") from None
+            if e.code >= 500 and i < retries:
+                time.sleep(delay); delay *= 2
+                continue
+            raise
+        except (urllib.error.URLError, TimeoutError, ConnectionError, ValueError):   # ValueError: an empty or non-JSON body
+            if i < retries:
+                time.sleep(delay); delay *= 2
+                continue
+            raise
+    return []
+
+
+def _frame(rows: list[dict]) -> pd.DataFrame:
+    df = pd.DataFrame(rows).rename(columns=RENAME)[list(RENAME.values())]
+    for c in ("close", "change", "open", "high", "low", "volume", "value", "mktcap", "shares"):
+        df[c] = pd.to_numeric(df[c].replace({"-": np.nan, "": np.nan}), errors="coerce")
+    return df
+
+
+def fetch_days(start, end, store_dir, *, key: str | None = None, markets=("KOSPI", "KOSDAQ"), sleep: float = 0.1,
+               max_calls: int | None = None, caller=None, progress: bool = True) -> dict:
+    """Download one file per weekday into `store_dir`. Resumable: finished days (and holidays) are skipped.
+    `caller(path, date)` can be injected for tests. Stops cleanly if the API reports a quota limit."""
+    store = Path(store_dir).expanduser()
+    store.mkdir(parents=True, exist_ok=True)
+    key = key or (None if caller else load_key())
+    call = caller or (lambda p, d: _call(p, d, key))
+    done = {"days": 0, "holidays": 0, "skipped": 0, "calls": 0, "stopped": None}
+    for ts in pd.bdate_range(start, end):
+        d = ts.strftime("%Y%m%d")
+        if (store / f"{d}.pkl").exists() or (store / f"{d}.hol").exists():
+            done["skipped"] += 1
+            continue
+        if max_calls is not None and done["calls"] + len(markets) > max_calls:
+            done["stopped"] = "max_calls"
+            break
+        frames = []
+        try:
+            for m in markets:
+                rows = call(ENDPOINTS[m], d)
+                done["calls"] += 1
+                if rows:
+                    frames.append(_frame(rows))
+                time.sleep(sleep)
+        except QuotaExceeded as e:
+            done["stopped"] = f"quota: {e}"
+            break
+        if frames and len(frames) < len(markets) and caller is None:
+            done.setdefault("partial", []).append(d)                   # one market answered empty: do not save, retry next run
+            continue
+        if frames:
+            pd.concat(frames).to_pickle(store / f"{d}.pkl")
+            done["days"] += 1
+        else:
+            (store / f"{d}.hol").write_text("")                      # holiday: both markets empty
+            done["holidays"] += 1
+        if progress and (done["days"] + done["holidays"]) % 100 == 0:
+            print(f"  {d}  {done}", flush=True)
+    return done
+
+
+def build_krx_panel(store_dir, *, start=None, end=None, min_age_days: int = 60, adv_window: int = 30,
+                    min_value_krw: float = 1e9, common_only: bool = True, exclude=DEFAULT_EXCLUDE,
+                    reuse_gap_days: int = 120, entry_lag: int = 1, drop_suspect_above: float | None = None) -> Panel:
+    """Point-in-time Panel of Korean stocks from the cached daily files.
+
+    eligible on day t uses data up to t only: at least `min_age_days` bars of history, trailing median traded value of
+    at least `min_value_krw`, positive volume that day, common shares only (code ends in 0) and no SPAC or REIT names.
+    A code that disappears for more than `reuse_gap_days` and comes back is treated as a different security (suffix #2).
+
+    Known defect of the return rule. The adjusted return is `close / (close - change) - 1`, which is right when the exchange's reference price already
+    reflects a split or rights issue. When a security is **suspended and a consolidation or capital reduction happens meanwhile**, the first day of trading
+    again carries a `change` against the old, unadjusted close, and the formula returns a move of thousands of percent that never happened (one case in the
+    cache: 2,080 won to 625,000 won, +29,948%). Every daily return beyond +-100% is listed in `meta["suspect_returns"]` (ticker, date, return, and whether the
+    previous day had zero or missing volume): 38 of 7.8 million in the whole cache (from 2013; the largest is +6,699,900%), 14 of them on such a day, and 29 and 10 since June 2015.
+    All are upward: a return cannot go below -100%. `drop_suspect_above=1.0` treats
+    such a day as a return of 0 (the price chain is cut there, the true move is unknown); the default None leaves the series as it was, so earlier results
+    reproduce. The strategies in this repository do not hold suspended names (eligibility needs volume), so none of them was affected; a position that is
+    **stuck** in a suspended name (`Panel.can_buy`/`can_sell`, `freeze_*`) can be."""
+    store = Path(store_dir).expanduser()
+    files = sorted(store.glob("*.pkl"))
+    if start:
+        files = [f for f in files if f.stem >= pd.Timestamp(start).strftime("%Y%m%d")]
+    if end:
+        files = [f for f in files if f.stem <= pd.Timestamp(end).strftime("%Y%m%d")]
+    if not files:
+        raise ValueError("no cached days: run fetch_days() first")
+    parts = []
+    for f in files:
+        d = pd.read_pickle(f)
+        d["date"] = pd.Timestamp(f.stem)
+        parts.append(d)
+    df = pd.concat(parts, ignore_index=True).sort_values(["code", "date"])
+    gap = df.groupby("code")["date"].diff().dt.days
+    life = (gap > reuse_gap_days).groupby(df["code"]).cumsum()
+    df["sid"] = np.where(life == 0, df["code"], df["code"] + "#" + (life + 1).astype(str))
+    names = df.drop_duplicates("sid", keep="last").set_index("sid")["name"].to_dict()
+    market_names = sorted(df["market"].dropna().unique())
+    df["_mk"] = df["market"].map({m: i for i, m in enumerate(market_names)}).astype("float64")
+
+    def mat(c: str) -> pd.DataFrame:
+        return df.pivot(index="date", columns="sid", values=c).sort_index()
+
+    close, chg = mat("close"), mat("change")
+    ref = close - chg
+    ret = (close / ref - 1.0).where(ref > 0)
+    first = close.notna() & ~close.notna().cumsum().shift(1, fill_value=0).astype(bool)   # a security's first bar
+    ret = ret.mask(first)                                                    # the listing-day move is not tradable
+    vol_prev0 = (mat("volume").fillna(0) <= 0).shift(1, fill_value=False)
+    sus = ret.stack()
+    sus = sus[sus.abs() > 1.0]
+    suspect = pd.DataFrame({"ticker": sus.index.get_level_values(1), "date": sus.index.get_level_values(0), "ret": sus.to_numpy(),
+                            "after_suspension": [bool(vol_prev0.loc[d, t]) for d, t in sus.index]}).reset_index(drop=True)
+    if drop_suspect_above is not None:
+        if not (drop_suspect_above > 0):
+            raise ValueError(f"drop_suspect_above must be positive, got {drop_suspect_above!r}")
+        ret = ret.mask(ret.abs() > drop_suspect_above)                      # unknown, so the chain treats it as 0
+    adj = 100.0 * np.exp(np.log1p(ret.fillna(0.0)).cumsum().where(close.notna()))
+    k = adj / close                                                          # adjust open/high/low to the same level
+    value, vol_raw = mat("value"), mat("volume")
+    age = close.notna().cumsum()
+    adv = value.rolling(adv_window, min_periods=adv_window).median()
+    ok = close.notna() & (vol_raw > 0) & (age >= min_age_days) & (adv >= min_value_krw)
+    if common_only:
+        ok = ok & pd.Series({c: c.split("#")[0].endswith("0") for c in ok.columns}).reindex(ok.columns).values
+    if exclude is not None:
+        bad = [s for s in ok.columns if exclude.search(names.get(s, ""))]
+        ok[bad] = False
+    last = close.apply(lambda c: c.last_valid_index())
+    da = pd.DataFrame(False, index=close.index, columns=close.columns)
+    for s in close.columns:
+        if last[s] is not None and last[s] < close.index[-1] - pd.Timedelta(days=5):
+            da.loc[last[s], s] = True
+    def intraday(c: str) -> pd.DataFrame:
+        """Open, high or low adjusted to the close's level. The raw files write 0 on a day without trades (volume 0), which is no price: it becomes NaN."""
+        m = mat(c)
+        return (m * k).where(m > 0)
+
+    p = Panel(close=adj, eligible=ok, open=intraday("open"), high=intraday("high"), low=intraday("low"),
+              volume=value / adj, mkt_cap=mat("mktcap"), market="KR", periods_per_year=245, entry_lag=entry_lag,
+              delist_after=da)
+    n = ok.sum(axis=1)
+    mk = mat("_mk").fillna(-1).astype("int8")                                # the market of each security on each day (-1: not listed)
+    p.meta = {"suspect_returns": suspect, "market_codes": {i: m for i, m in enumerate(market_names)}, "market_by_date": mk,
+              "raw_close": close.astype("float32"),                  # the real price level (won), for whole-share sizes; `close` is back-adjusted
+              "securities": int(close.shape[1]), "delisted_in_panel": int(da.values.any(axis=0).sum()),
+              "eligible_median": float(n[n > 0].median()), "first_day": str(close.index[0].date()),
+              "last_day": str(close.index[-1].date()), "names": names}
+    return p
+
+
+def sell_tax_panel(panel: Panel, schedule: dict) -> pd.DataFrame:
+    """(date x ticker) sell-side transaction tax in bp, for `backtest_portfolio(sell_bp=...)` and `backtest_weights(sell_bp=...)`.
+
+    schedule  {market name: [(effective_date, bp), ...]}, for example {"KOSPI": [("2025-01-01", 15.0), ...], "KOSDAQ": [...]}. Each
+              market's rate holds from its effective date until the next entry. The panel must come from `build_krx_panel`, which
+              records the market of every security **on every day** (a stock that moved from KOSDAQ to KOSPI changes rate the day it
+              moves). A date before the first entry of a market, or a market in the data that is missing from `schedule`, raises.
+
+    The library ships no rate table: the rates and their effective dates are law and change, so you pass the ones you have checked
+    against the National Tax Service or the statute. Include every levy that the seller pays (the securities transaction tax and, for
+    KOSPI, the rural development special tax). Days on which a security is not listed get the highest rate of that day, the cautious
+    choice, because a position held through a gap is still charged when it is sold."""
+    from ..core.costs import rate_schedule
+    mk, codes = panel.meta.get("market_by_date"), panel.meta.get("market_codes")
+    if mk is None or codes is None:
+        raise ValueError("this panel has no market information: build it with build_krx_panel")
+    unknown = sorted(set(codes.values()) - set(schedule))
+    if unknown:
+        raise ValueError(f"no tax schedule for market(s) {unknown}; give one for every market in the data ({sorted(codes.values())})")
+    rates = {m: rate_schedule(panel.dates, schedule[m], f"schedule[{m!r}]").to_numpy(float) for m in codes.values()}
+    arr = mk.reindex(index=panel.dates, columns=panel.tickers).to_numpy()
+    worst = np.max(np.vstack([rates[m] for m in codes.values()]), axis=0)
+    out = np.repeat(worst[:, None], arr.shape[1], axis=1)
+    for i, m in codes.items():
+        out = np.where(arr == i, rates[m][:, None], out)
+    return pd.DataFrame(out, index=panel.dates, columns=panel.tickers)

pitbacktest/adapters/long_format.py

@@ -0,0 +1,87 @@
+"""Bring your own point-in-time data: a long table (one row per security and day) into a Panel, with strict checks.
+
+This is the door for CRSP (WRDS), Sharadar, Norgate, Polygon or any other source that keeps delisted securities. The
+framework cannot create survivorship-free data; it can refuse to accept data that is ambiguous.
+
+Required columns (names are configurable): a **permanent security id** (not the ticker), a date and a close.
+Optional: volume, open/high/low, market cap, a point-in-time `in_universe` flag (for example S&P 500 membership on that
+date), a `delist_return` on the last row of a security, and a ticker column (only used to detect ticker reuse).
+
+Checks: no duplicate (id, date), positive closes, sorted dates. Delisting is detected as the last bar of a security that
+ended well before the panel end. If a delisting return is given it is compounded into the last close (the CRSP
+convention: the return on the last day is (1 + r_last) * (1 + r_delist) - 1), so the engine sees a real loss.
+"""
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from ..core.panel import Panel
+
+
+def panel_from_long(df: pd.DataFrame, *, id_col: str = "id", date_col: str = "date", close_col: str = "close",
+                    volume_col: str | None = None, open_col: str | None = None, high_col: str | None = None,
+                    low_col: str | None = None, mktcap_col: str | None = None, in_universe_col: str | None = None,
+                    delist_return_col: str | None = None, ticker_col: str | None = None,
+                    min_age_days: int = 60, adv_window: int = 30, min_adv: float | None = None,
+                    ended_gap_days: int = 30, entry_lag: int = 1, market: str = "US",
+                    periods_per_year: int = 252) -> Panel:
+    """Build a Panel from a long table, one row per security and day (columns and checks: see the module docstring).
+
+    Raises KeyError for a missing required column and ValueError for duplicate (id, date) rows or non-positive closes.
+    Eligibility is `in_universe_col` when given; otherwise at least `min_age_days` of history and, when `min_adv` and a volume column are
+    given, a trailing median traded value (close x volume over `adv_window` rows) of at least `min_adv`. `volume_col` is a quantity.
+    A security whose last row is more than `ended_gap_days` before the last date is flagged in `delist_after`, and only for those a
+    `delist_return_col` value is compounded into the last close. `panel.meta` records the row and security counts, how many ended early,
+    how many delisting returns were applied, and the tickers used by more than one id."""
+    need = [id_col, date_col, close_col]
+    miss = [c for c in need if c not in df.columns]
+    if miss:
+        raise KeyError(f"missing columns: {miss}")
+    d = df.copy()
+    d[date_col] = pd.to_datetime(d[date_col])
+    if d.duplicated([id_col, date_col]).any():
+        raise ValueError("duplicate (id, date) rows: the id must identify one security on one date")
+    if (d[close_col] <= 0).any():
+        raise ValueError("non-positive closes: clean them or drop the rows")
+    d = d.sort_values([id_col, date_col])
+    meta: dict = {"rows": int(len(d)), "securities": int(d[id_col].nunique())}
+
+    last_row = d.groupby(id_col).tail(1).set_index(id_col)
+    end = d[date_col].max()
+    ended = last_row[date_col] < end - pd.Timedelta(days=ended_gap_days)
+    meta["ended_before_end"] = int(ended.sum())
+    meta["ended_share"] = float(ended.mean())
+
+    n_dl = 0
+    if delist_return_col and delist_return_col in d.columns:
+        idx = d.groupby(id_col).tail(1).index
+        dr = d.loc[idx, delist_return_col]
+        hit = idx[dr.notna().values & ended.reindex(d.loc[idx, id_col]).values]
+        d.loc[hit, close_col] = d.loc[hit, close_col] * (1.0 + d.loc[hit, delist_return_col])
+        n_dl = len(hit)
+    meta["delist_returns_applied"] = n_dl
+
+    def mat(c: str) -> pd.DataFrame:
+        return d.pivot(index=date_col, columns=id_col, values=c).sort_index()
+
+    close = mat(close_col)
+    vol = mat(volume_col) if volume_col else None
+    age = close.notna().cumsum()
+    if in_universe_col:
+        ok = mat(in_universe_col).fillna(False).astype(bool) & close.notna()
+    else:
+        ok = close.notna() & (age >= min_age_days)
+        if min_adv is not None and vol is not None:
+            ok &= (close * vol).rolling(adv_window, min_periods=adv_window).median() >= min_adv
+    da = pd.DataFrame(False, index=close.index, columns=close.columns)
+    for s in ended[ended].index:
+        da.loc[last_row.loc[s, date_col], s] = True
+    if ticker_col and ticker_col in d.columns:
+        t = d.groupby(ticker_col)[id_col].nunique()
+        meta["reused_tickers"] = sorted(t[t > 1].index.astype(str).tolist())      # one ticker, several securities
+    p = Panel(close=close, eligible=ok, open=mat(open_col) if open_col else None, high=mat(high_col) if high_col else None,
+              low=mat(low_col) if low_col else None, volume=vol, mkt_cap=mat(mktcap_col) if mktcap_col else None,
+              market=market, periods_per_year=periods_per_year, entry_lag=entry_lag, delist_after=da)
+    p.meta = meta
+    return p

pitbacktest/adapters/tiingo.py

@@ -0,0 +1,236 @@
+"""Tiingo end-of-day prices (free account) as a point-in-time panel that keeps delisted securities.
+
+What it is for
+  Tiingo's end-of-day API returns the full history of many US stocks that were later acquired or delisted, which yfinance
+  does not. The free tier is small, so this adapter is built for a **random sample** of tickers, not the whole market:
+  `draw_order` gives a seeded random order and `fetch_symbols` downloads in that order, resumably, and stops cleanly when
+  the account runs out of requests. Any prefix of the order is a random sample.
+
+What it does not fix
+  A probe (`docs/tiingo_probe.py`) found the history of 23 of 29 takeover or rename cases but of **none** of 12 bankruptcies
+  and rescue sales. A panel from this adapter is less survivor-biased than one from yfinance, not unbiased. Treat any
+  survivors-versus-this comparison as a lower bound.
+
+Choices worth knowing
+  - Returns use `adjClose` (split and dividend adjusted, so total return). Eligibility uses the raw `close` and `volume`.
+  - A ticker with several listing windows in the ticker list (reuse) becomes several securities, each cut to its own window.
+  - A security whose last bar is more than `end_gap_days` before the last calendar date is flagged in `delist_after`.
+  - A ticker for which the API has nothing is recorded (`.none`) and counted, never silently replaced.
+  - Only the `TIINGO_API_KEY` variable is read.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from ..core.panel import Panel
+
+BASE = "https://api.tiingo.com/tiingo/daily"
+KEEP = ("date", "close", "volume", "adjClose")
+# With `full=True` the open, high, low (adjusted for splits and dividends like adjClose), the dividend paid and the split factor are kept too.
+FULL = KEEP + ("open", "high", "low", "adjOpen", "adjHigh", "adjLow", "divCash", "splitFactor")
+_NOT_PLAIN = re.compile(r"[-.]")
+
+
+class QuotaExceeded(RuntimeError):
+    """The API refused further requests (rate or monthly limit). Run again later; the cache keeps what was fetched."""
+
+
+def load_key(env_path: str | Path | None = None) -> str:
+    """TIINGO_API_KEY from the environment, else from a .env file. Only that one variable is read."""
+    if os.environ.get("TIINGO_API_KEY"):
+        return os.environ["TIINGO_API_KEY"]
+    if env_path:
+        for line in Path(env_path).expanduser().read_text().splitlines():
+            k, _, v = line.partition("=")
+            if k.strip() == "TIINGO_API_KEY":
+                return v.strip().strip('"').strip("'")
+    raise KeyError("TIINGO_API_KEY is not set")
+
+
+# ---------------------------------------------------------------------------------------------------- the sample
+def study_frame(master: pd.DataFrame, since: str = "2013-01-01") -> pd.DataFrame:
+    """Rows of the ticker list that can enter a study starting at `since`: plain tickers (no '-' or '.', and no
+    five-letter ticker ending in W, U, R or P: warrants, units, rights, preferreds) whose window ends on or after `since`."""
+    t = master["ticker"].astype(str)
+    plain = ~t.str.contains(_NOT_PLAIN) & ~((t.str.len() == 5) & t.str[-1].isin(list("WURP")))
+    return master[plain & (master["end"] >= pd.Timestamp(since))].reset_index(drop=True)
+
+
+def draw_order(tickers, seed: int = 0) -> list[str]:
+    """A seeded random permutation of the unique tickers (sorted first, so it does not depend on the input order).
+    Download in this order and every prefix is a simple random sample."""
+    u = sorted(set(map(str, tickers)))
+    rng = np.random.default_rng(seed)
+    return [u[i] for i in rng.permutation(len(u))]
+
+
+# ---------------------------------------------------------------------------------------------------- downloading
+def _call(ticker: str, start: str, key: str, retries: int = 3):
+    """Bars for one ticker, or None if the API does not know it. Raises QuotaExceeded on HTTP 429."""
+    url = f"{BASE}/{ticker.lower()}/prices?startDate={start}&format=json"
+    req = urllib.request.Request(url, headers={"Authorization": f"Token {key}", "Content-Type": "application/json",
+                                               "User-Agent": "pitbacktest-research/0.2"})
+    delay = 1.0
+    for i in range(retries + 1):
+        try:
+            with urllib.request.urlopen(req, timeout=60) as r:
+                data = json.loads(r.read())
+            # The monthly limit on distinct symbols (500 on a free account) comes back as HTTP 200 with a message, not as a 429.
+            if isinstance(data, dict) and re.search(r"run over|allocation|look ?up|upgrade", str(data.get("detail", "")), re.I):
+                raise QuotaExceeded(f"HTTP 200: {data.get('detail')}")
+            return data
+        except urllib.error.HTTPError as e:
+            body = e.read()[:300].decode("utf-8", "replace")
+            if e.code == 429:
+                raise QuotaExceeded(f"HTTP 429: {body}") from None
+            if e.code == 404:
+                return None
+            if e.code in (401, 403):
+                raise PermissionError(f"HTTP {e.code}: {body}") from None
+            if e.code >= 500 and i < retries:
+                time.sleep(delay); delay *= 2
+                continue
+            raise
+        except (urllib.error.URLError, TimeoutError, ConnectionError, ValueError):
+            if i < retries:
+                time.sleep(delay); delay *= 2
+                continue
+            raise
+    return None
+
+
+def _check(ticker: str, rows, fields=KEEP) -> list[dict]:
+    if not isinstance(rows, list):
+        raise RuntimeError(f"Tiingo {ticker}: unexpected response type {type(rows).__name__}")
+    out = []
+    for r in rows:
+        if not all(k in r for k in fields):
+            raise RuntimeError(f"Tiingo {ticker}: bar without {[k for k in fields if k not in r]}; the format changed")
+        out.append({k: (str(r[k])[:10] if k == "date" else r[k]) for k in fields})
+    return out
+
+
+def _safe(ticker: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_]", "_", ticker.upper())
+
+
+def fetch_symbols(tickers, store_dir, *, start: str = "2012-01-01", key: str | None = None, sleep: float = 0.3,
+                  max_new: int | None = None, caller=None, progress: bool = True, full: bool = False) -> dict:
+    """Download one file per ticker into `store_dir` (`T.json` with the needed fields, or `T.none` if the API has nothing).
+    Resumable: tickers already stored are skipped. `max_new` limits requests in this run. Stops cleanly on a quota error.
+    `full=True` keeps the open, high, low, dividend and split factor as well (use a different `store_dir` from a store made without it).
+    `caller(ticker, start)` can be injected for tests (return None for an unknown ticker)."""
+    store = Path(store_dir).expanduser()
+    store.mkdir(parents=True, exist_ok=True)
+    key = key or (None if caller else load_key())
+    call = caller or (lambda t, s: _call(t, s, key))
+    done = {"fetched": 0, "none": 0, "skipped": 0, "calls": 0, "stopped": None}
+    for t in tickers:
+        f = store / f"{_safe(t)}.json"
+        n = store / f"{_safe(t)}.none"
+        if f.exists() or n.exists():
+            done["skipped"] += 1
+            continue
+        if max_new is not None and done["calls"] >= max_new:
+            done["stopped"] = "max_new"
+            break
+        try:
+            rows = call(t, start)
+        except QuotaExceeded as e:
+            done["stopped"] = f"quota: {e}"
+            break
+        done["calls"] += 1
+        if rows is None or (isinstance(rows, list) and len(rows) == 0):
+            n.write_text("")
+            done["none"] += 1
+        else:
+            f.write_text(json.dumps(_check(t, rows, FULL if full else KEEP)))        # validated first: a malformed answer is never saved
+            done["fetched"] += 1
+        if progress and done["calls"] % 25 == 0:
+            print(f"  {done['calls']} requests, {done['fetched']} with data, {done['none']} none", flush=True)
+        time.sleep(sleep)
+    return done
+
+
+# ---------------------------------------------------------------------------------------------------- the panel
+def build_tiingo_panel(store_dir, master: pd.DataFrame, tickers, *, start: str = "2012-01-01", end: str | None = None,
+                       min_age_bars: int = 60, adv_window: int = 30, min_dollar_volume: float = 2e6, min_price: float = 1.0,
+                       min_names_per_date: int = 100, end_gap_days: int = 5, entry_lag: int = 1) -> Panel:
+    """Point-in-time panel from the stored files. Eligibility on day t uses only data up to t.
+
+    Each listing window of a ticker in `master` (columns ticker, start, end) is one security, cut to that window.
+    The trading calendar is the set of dates on which at least `min_names_per_date` securities have a bar.
+    `start` is the first date kept: pass the start of the warm-up year and cut the study period afterwards. The age of a
+    security counts bars from the start of the stored data (`fetch_symbols(start=...)`), so a warm-up year of 60+ bars
+    makes that irrelevant for the study period."""
+    store = Path(store_dir).expanduser()
+    adj, raw, vol, oh, div, meta = {}, {}, {}, {"adjOpen": {}, "adjHigh": {}, "adjLow": {}}, {}, {"tickers_with_data": 0, "tickers_none": 0, "tickers_missing": 0, "windows_cut": 0}
+    for t in map(str, tickers):
+        f, n = store / f"{_safe(t)}.json", store / f"{_safe(t)}.none"
+        if n.exists():
+            meta["tickers_none"] += 1
+            continue
+        if not f.exists():
+            meta["tickers_missing"] += 1
+            continue
+        bars = pd.DataFrame(json.loads(f.read_text()))
+        if bars.empty:
+            meta["tickers_none"] += 1
+            continue
+        meta["tickers_with_data"] += 1
+        bars["date"] = pd.to_datetime(bars["date"])
+        bars = bars.drop_duplicates("date", keep="last").set_index("date").sort_index()
+        wins = master[master["ticker"].astype(str) == t].sort_values("start")
+        if wins.empty:
+            wins = pd.DataFrame({"start": [bars.index[0]], "end": [bars.index[-1]]})
+        multi = len(wins) > 1
+        meta["windows_cut"] += int(multi)
+        for _, w in wins.iterrows():
+            b = bars.loc[(bars.index >= w["start"]) & (bars.index <= w["end"])]
+            b = b[(b["adjClose"] > 0) & (b["close"] > 0)]
+            if b.empty:
+                continue
+            sid = f"{t}@{w['start']:%Y%m%d}" if multi else t
+            adj[sid], raw[sid], vol[sid] = b["adjClose"], b["close"], b["volume"]
+            if "adjOpen" in b.columns:                                  # a store made with full=True
+                for k in oh:
+                    oh[k][sid] = b[k]
+                div[sid] = b["divCash"]
+    if not adj:
+        raise ValueError("no usable securities in the store")
+    A, R, V = pd.DataFrame(adj), pd.DataFrame(raw), pd.DataFrame(vol)
+    cal = A.index[(A.notna().sum(axis=1) >= min_names_per_date) & (A.index >= pd.Timestamp(start))]
+    if end:
+        cal = cal[cal <= pd.Timestamp(end)]
+    A, R, V = A.reindex(cal), R.reindex(cal), V.reindex(cal)
+    dvol = R * V
+    # eligibility, all trailing
+    age = A.notna().cumsum()
+    med = dvol.rolling(adv_window, min_periods=adv_window).median()
+    elig = (age >= min_age_bars) & (med >= min_dollar_volume) & (R >= min_price) & (V > 0) & A.notna()
+    last = A.apply(lambda c: c.last_valid_index())
+    last_date = cal[-1]
+    da = pd.DataFrame(False, index=cal, columns=A.columns)
+    for c in A.columns:
+        if last[c] is not None and last[c] < last_date - pd.Timedelta(days=end_gap_days):
+            da.loc[last[c], c] = True
+    r = A.pct_change(fill_method=None)
+    extra = {}
+    if oh["adjOpen"]:
+        O, Hh, Ll = (pd.DataFrame(oh[k]).reindex(index=cal, columns=A.columns) for k in ("adjOpen", "adjHigh", "adjLow"))
+        extra = {"open": O, "high": Hh, "low": Ll}
+        meta["div_cash"] = pd.DataFrame(div).reindex(index=cal, columns=A.columns).astype("float32")   # dollars per share paid that day (raw prices)
+    meta["suspect_returns_gt_10x"] = int((r.abs() > 10).sum().sum())
+    meta["securities"] = int(A.shape[1])
+    meta["raw_close"] = R.astype("float32")                      # the real price level (dollars), for whole-share sizes; `close` is adjusted
+    return Panel(close=A, eligible=elig, volume=dvol / A, market="US", entry_lag=entry_lag, periods_per_year=252,
+                 delist_after=da, meta=meta, **extra)