pitbacktest 0.2.0__py3-none-any.whl

pitbacktest/core/panel.py

@@ -0,0 +1,306 @@
+"""Panel convention: the input contract of every test.
+
+Design principles
+    - The core is **plain pandas and numpy** and does not depend on any external data source (adapters do that).
+    - Every matrix is (date x ticker) and the index and columns must be identical.
+    - `eligible` is the **point-in-time universe**: True only for securities that could actually be traded that day.
+        It is the only device against survivorship bias, so it is a required input.
+
+Timing convention (the only one used in the whole project)
+    A signal is **fixed** at close(t). Entry is at close(t+entry_lag) and exit at close(t+entry_lag+h).
+    entry_lag defaults to 1: "see the signal, buy the next day". 0 assumes a fill at the same close (aggressive).
+    Breaking this convention is caught by Panel.assert_no_lookahead().
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+from typing import Iterable
+
+import numpy as np
+import pandas as pd
+
+REQUIRED = ("close", "eligible")
+OPTIONAL = ("open", "high", "low", "volume", "mkt_cap", "funding", "delist_after")
+
+
+@dataclass
+class Panel:
+    """A bundle of (date x ticker) matrices.
+
+    close    adjusted closing price: the only source of returns
+    eligible point-in-time universe bool: listed, tradable and passing the liquidity condition that day
+    The rest is optional. With volume and mkt_cap the liquidity and size controls are switched on.
+    """
+
+    close: pd.DataFrame
+    eligible: pd.DataFrame
+    open: pd.DataFrame | None = None
+    high: pd.DataFrame | None = None
+    low: pd.DataFrame | None = None
+    volume: pd.DataFrame | None = None
+    mkt_cap: pd.DataFrame | None = None
+    chars: dict[str, pd.DataFrame] = field(default_factory=dict)
+    market: str = "KR"
+    entry_lag: int = 1
+    # Trading days per year used for annualising: 252 for stocks, 365 for 24/7 markets (crypto). A wrong value makes CAGR, Sharpe and volatility wrong.
+    periods_per_year: int = 252
+    # Optional: futures funding (date x ticker, daily sum, positive means longs pay). Delisting flag (True on the last real bar). Notes for adapters.
+    funding: pd.DataFrame | None = None
+    delist_after: pd.DataFrame | None = None
+    # Optional: (date x ticker) bool, True where a security can be sold short on the execution day (borrowable, and no short-selling ban). None means every
+    # security can. A date or security missing from the frame counts as **not** shortable: an unknown is not permission.
+    shortable: pd.DataFrame | None = None
+    # Optional: (date x ticker) bool, False where a trade that increases (can_buy) or decreases (can_sell) a position cannot be done on that day
+    # (halted, or locked at a daily price limit). None means always possible; a date or security missing from the frame counts as not possible.
+    # `execution.tradability` builds them from prices and volume.
+    can_buy: pd.DataFrame | None = None
+    can_sell: pd.DataFrame | None = None
+    meta: dict = field(default_factory=dict)
+
+    # ---------------------------------------------------------------- construction and validation
+    def __post_init__(self) -> None:
+        self.close = self.close.sort_index()
+        if self.close.index.has_duplicates:                    # before any reindex, which would fail with a less helpful pandas message
+            raise ValueError("close.index has duplicate dates")
+        self.eligible = self.eligible.reindex(
+            index=self.close.index, columns=self.close.columns
+        ).fillna(False).astype(bool)
+        for k in OPTIONAL:
+            v = getattr(self, k)
+            if v is not None:
+                setattr(self, k, v.reindex(index=self.close.index, columns=self.close.columns))
+        self.chars = {
+            k: v.reindex(index=self.close.index, columns=self.close.columns)
+            for k, v in self.chars.items()
+        }
+        if self.delist_after is not None:
+            self.delist_after = self.delist_after.fillna(False).astype(bool)
+        for k in ("shortable", "can_buy", "can_sell"):
+            v = getattr(self, k)
+            if v is not None:
+                setattr(self, k, v.reindex(index=self.close.index, columns=self.close.columns).fillna(False).astype(bool))
+        self.validate()
+
+    def validate(self) -> None:
+        """Raise ValueError if the panel breaks its invariants: the date index ascending and without duplicates, at least one eligible
+        security, `entry_lag` not negative, and at least 10 eligible securities on half of the dates or more. Runs on construction."""
+        if not self.close.index.is_monotonic_increasing:
+            raise ValueError("close.index is not ascending")
+        if self.close.index.has_duplicates:
+            raise ValueError("close.index has duplicate dates")
+        if self.eligible.values.sum() == 0:
+            raise ValueError("eligible is all False")
+        if self.entry_lag < 0:
+            raise ValueError("entry_lag must be 0 or more")
+        n = self.eligible.sum(axis=1)
+        if (n[n > 0] < 10).mean() > 0.5:
+            raise ValueError(
+                "eligible has fewer than 10 securities on more than half of the dates: cross-sectional tests are impossible"
+            )
+
+    # ---------------------------------------------------------------- derived
+    @property
+    def dates(self) -> pd.DatetimeIndex:
+        return self.close.index
+
+    @property
+    def tickers(self) -> pd.Index:
+        return self.close.columns
+
+    def ret1(self) -> pd.DataFrame:
+        """Daily return close(t)/close(t-1) - 1."""
+        return self.close.pct_change(fill_method=None)
+
+    def forward(self, h: int, delist_return: float | None = None) -> pd.DataFrame:
+        """Return from close(t + lag) to close(t + lag + h) for a signal at t: a plain price ratio, NaN when either price is missing.
+
+        A security flagged in `delist_after` is carried at its last price after its last bar (as cash), or at `last price x (1 + delist_return)`
+        when `delist_return` is given, so a holding period that runs into a delisting keeps its result instead of dropping out. Without that
+        the statistics of `screen` and `backtest_event` would leave out exactly the trades that ended in a delisting. A security that is only
+        missing for some days (a halt) still gives NaN when an end price is missing. Compare `analytics.forward_returns`, which compounds daily
+        returns and treats a missing price as a return of 0."""
+        lag = self.entry_lag
+        px = self.close
+        if self.delist_after is not None and bool(np.asarray(self.delist_after.values).any()):
+            da = self.delist_after.reindex(index=px.index, columns=px.columns).fillna(False).astype(bool).astype(int)
+            gone = (da.cumsum() - da) > 0                                   # strictly after the security's last bar
+            carried = px.ffill() * (1.0 + (0.0 if delist_return is None else float(delist_return)))
+            px = px.where(~gone, carried)
+        entry = px.shift(-lag)
+        exit_ = px.shift(-(lag + h))
+        return (exit_ / entry - 1.0).astype(np.float32)
+
+    def fingerprint(self) -> str:
+        """Short hash of the numbers this panel holds (every matrix that is present, including volume and market cap, which
+        the impact model and the size controls read), used by the trial ledger to tell whether two runs saw the same data.
+        The date index and the ticker names are not hashed: the same values in the same shape give the same fingerprint."""
+        import hashlib
+        h = hashlib.sha256()
+        mats = [(n, getattr(self, n)) for n in ("close", "eligible", "open", "high", "low", "volume", "mkt_cap", "funding", "delist_after", "shortable", "can_buy", "can_sell")]
+        mats += [(f"chars.{k}", self.chars[k]) for k in sorted(self.chars)]
+        for name, v in mats:
+            if v is not None:
+                a = np.ascontiguousarray(v.to_numpy(dtype=np.float32))
+                h.update(name.encode() + repr(a.shape).encode() + a.tobytes())
+        h.update(f"{self.entry_lag}|{self.periods_per_year}|{self.market}".encode())
+        return h.hexdigest()[:16]
+
+    def adv(self, window: int = 20) -> pd.DataFrame:
+        """Average traded value. None if there is no volume."""
+        if self.volume is None:
+            return None
+        return (self.close * self.volume).rolling(window, min_periods=window).mean()
+
+    # ---------------------------------------------------------------- safeguards
+    def truncate(self, last: pd.Timestamp) -> "Panel":
+        """A copy that holds only the rows up to and including `last`: what a user standing on that date would have had."""
+        cut = lambda f: None if f is None else f.loc[:last]
+        return replace(self, close=self.close.loc[:last], eligible=self.eligible.loc[:last], open=cut(self.open), high=cut(self.high), low=cut(self.low),
+                       volume=cut(self.volume), mkt_cap=cut(self.mkt_cap), funding=cut(self.funding), delist_after=cut(self.delist_after),
+                       shortable=cut(self.shortable), can_buy=cut(self.can_buy), can_sell=cut(self.can_sell),
+                       chars={k: v.loc[:last] for k, v in self.chars.items()})
+
+    def assert_causal(self, make_signal, n_cuts: int = 6, seed: int = 0, min_history: int = 300, rtol: float = 1e-9) -> dict:
+        """The exact look-ahead check: a signal that is a function of the panel must not change when the future is removed.
+
+        `make_signal(panel)` returns the (date x ticker) signal. For `n_cuts` random dates t (at least `min_history` rows in, so that rolling windows are full) the signal is
+        rebuilt from the panel truncated at t, and its last row must equal the row of the full-data signal at t, name by name (NaN equal to NaN). Any use of a later row
+        (a negative shift, a centred window, a mean or standard deviation over the whole history, a rank across time) makes the rows differ.
+        Returns {"pass", "cuts", "mismatched_cuts", "max_abs_diff"}. It checks that the function is causal; it does not say the signal is good, and it cannot see look-ahead in
+        the *data* itself (a price already revised), only in how the signal is computed. It needs the function, not a frame: that is what makes it exact."""
+        full = make_signal(self)
+        if not isinstance(full, pd.DataFrame):
+            raise ValueError("make_signal must return a (date x ticker) DataFrame")
+        n = len(self.dates)
+        if n <= min_history + 1:
+            raise ValueError(f"the panel has {n} rows; at least {min_history + 2} are needed to cut it")
+        rng = np.random.default_rng(seed)
+        cuts = sorted(int(i) for i in rng.choice(np.arange(min_history, n - 1), size=min(n_cuts, n - 1 - min_history), replace=False))
+        bad, worst = [], 0.0
+        for i in cuts:
+            t = self.dates[i]
+            part = make_signal(self.truncate(t))
+            a = part.iloc[-1].reindex(self.tickers).to_numpy(float)
+            b = full.loc[t].reindex(self.tickers).to_numpy(float)
+            same_nan = np.isnan(a) == np.isnan(b)
+            both = ~np.isnan(a) & ~np.isnan(b)
+            diff = float(np.max(np.abs(a[both] - b[both]) / (np.abs(b[both]) + 1e-12))) if both.any() else 0.0
+            worst = max(worst, diff)
+            if (not same_nan.all()) or diff > rtol:
+                bad.append(str(t.date()))
+        return {"pass": not bad, "cuts": [str(self.dates[i].date()) for i in cuts], "mismatched_cuts": bad, "max_abs_diff": worst}
+
+    def assert_no_lookahead(self, signal: pd.DataFrame, h: int = 5,
+                            n_null: int = 8, seed: int = 0) -> dict:
+        """A weak heuristic for look-ahead; prefer `assert_causal`, which is exact. If the result improves when the signal is pushed one day **later**, it is looking at the future.
+
+        **Known blind spots, measured on real data (see docs/market_validation.md):** a signal that is the future return itself is **not** flagged (delaying it by a day
+        makes it worse, not better), and a legitimate signal whose own edge is negative **is** flagged (delaying shrinks the loss, which reads as an improvement).
+        Treat a pass as "no evidence", never as proof, and a fail on a signal with a negative edge as possibly false.
+
+        A normal signal gets worse when delayed (the information decays).
+        If it improves after the delay, the signal already contains future information.
+
+        Warning: judging by a bare `lagged > base` makes a **random factor wrong 50% of the time**:
+              both are near zero so the sign flips by chance. So the noise scale (sd) of the spread is measured with random shuffles
+              and twice that sd is used as the tolerance.
+        """
+        fwd = self.forward(h)
+        base = _spread(signal, fwd, self.eligible)
+        lagged = _spread(signal.shift(1), fwd, self.eligible)
+        rng = np.random.default_rng(seed)
+        nulls = []
+        for _ in range(n_null):
+            sh = signal.values.copy()
+            ev = self.eligible.values
+            for i in range(sh.shape[0]):
+                j = np.where(ev[i])[0]
+                if len(j) > 1:
+                    sh[i, j] = sh[i, rng.permutation(j)]
+            v = _spread(pd.DataFrame(sh, index=signal.index, columns=signal.columns),
+                        fwd, self.eligible)
+            if np.isfinite(v):
+                nulls.append(v)
+        tol = 2 * float(np.std(nulls)) if len(nulls) >= 3 else 0.0
+        ok = not (np.isfinite(base) and np.isfinite(lagged) and lagged > base + tol)
+        return {"base_bp": base * 1e4, "lagged_bp": lagged * 1e4,
+                "tol_bp": tol * 1e4, "pass": bool(ok)}
+
+    def audit(self) -> dict:
+        """Data integrity audit: run it once before any test.
+
+        It catches common data defects:
+            - silent truncation   (data vanishes wholesale after some date)
+            - adjusted-price mismatch (more extreme gaps than daily moves: the adj_open defect type)
+            - universe jumps      (the population triples over time, for example)
+        """
+        out = {}
+        n = self.eligible.sum(axis=1)
+        out["dates"] = len(self.dates)
+        out["tickers"] = int(self.eligible.any(axis=0).sum())
+        out["eligible_mean"] = float(n.mean())
+        yr = n.groupby(self.dates.year).mean()
+        out["eligible_by_year"] = {int(k): float(v) for k, v in yr.items()}
+        out["universe_growth"] = float(yr.iloc[-1] / yr.iloc[0]) if len(yr) > 1 and yr.iloc[0] else np.nan
+
+        r = self.ret1().where(self.eligible)
+        out["ret_extreme_pct"] = float((r.abs() > 0.30).mean().mean() * 100)
+        gap = None
+        if self.open is not None:
+            gap = (self.open / self.close.shift(1) - 1.0).where(self.eligible)
+            out["gap_extreme_pct"] = float((gap.abs() > 0.30).mean().mean() * 100)
+            ratio = out["gap_extreme_pct"] / max(out["ret_extreme_pct"], 1e-9)
+            out["gap_vs_daily_ratio"] = float(ratio)
+            out["adj_price_consistent"] = bool(ratio < 3.0)
+
+        # silent truncation: security coverage drops sharply after some date
+        cov = self.close.notna().sum(axis=1)
+        if len(cov) > 60:
+            tail = cov.iloc[-20:].mean(); body = cov.iloc[:-20].median()
+            out["tail_coverage_ratio"] = float(tail / body) if body else np.nan
+            out["truncation_suspected"] = bool(tail < body * 0.5)
+
+        # suspected survivorship bias: in a real market a steady share of securities disappears every year. If almost no security's price stops before the panel's end,
+        # the panel holds only 'securities alive today'. Measured: of 41 well-known delisted or acquired stocks looked up in yfinance,
+        # none came back with a correct history (docs/survivorship.md).
+        last = self.close.apply(lambda c: c.last_valid_index())
+        ended = last.dropna() < (self.dates[-1] - pd.Timedelta(days=30))
+        span_years = (self.dates[-1] - self.dates[0]).days / 365.25
+        out["ended_before_end_share"] = float(ended.mean()) if len(ended) else np.nan
+        out["survivorship_suspected"] = bool(len(ended) >= 30 and span_years >= 3 and ended.mean() < 0.01)
+        return out
+
+
+def _spread(signal: pd.DataFrame, fwd: pd.DataFrame, eligible: pd.DataFrame,
+            q: float = 0.10) -> float:
+    """Mean top-q minus bottom-q spread (internal helper for the look-ahead check)."""
+    rk = signal.where(eligible).rank(axis=1, pct=True, na_option="keep")
+    hi, lo = ((rk >= 1 - q) & eligible).values, ((rk <= q) & eligible).values
+    c = fwd.values.astype(np.float64)
+    out = []
+    for i in range(c.shape[0]):
+        a, b = hi[i] & np.isfinite(c[i]), lo[i] & np.isfinite(c[i])
+        if a.sum() >= 3 and b.sum() >= 3:
+            out.append(c[i][a].mean() - c[i][b].mean())
+    return float(np.mean(out)) if out else np.nan
+
+
+def build_pit_eligible(close: pd.DataFrame, *, listed: pd.DataFrame | None = None,
+                       min_adv: float = 0.0, volume: pd.DataFrame | None = None,
+                       exclude: pd.DataFrame | None = None) -> pd.DataFrame:
+    """Helper that builds a point-in-time universe mask.
+
+    listed   listing flag bool (falls back to whether close exists)
+    min_adv  minimum average traded value (needs volume)
+    exclude  mask of securities to exclude, such as administrative issues and trading halts (True = exclude)
+    """
+    ok = close.notna() & (close > 0)
+    if listed is not None:
+        ok &= listed.reindex_like(close).fillna(False).astype(bool)
+    if min_adv > 0 and volume is not None:
+        adv = (close * volume).rolling(20, min_periods=20).mean()
+        ok &= adv >= min_adv
+    if exclude is not None:
+        ok &= ~exclude.reindex_like(close).fillna(False).astype(bool)
+    return ok.fillna(False)

pitbacktest/crypto/__init__.py

@@ -0,0 +1,12 @@
+"""Crypto perpetual-futures data and panel construction (Binance USDT-M).
+
+The point of this subpackage is to remove, as far as public data allows, three biases that crypto backtests
+usually carry: survivorship (delisted contracts are included), universe look-ahead (eligibility uses only data up
+to the signal date) and ignored funding (the daily funding rate is charged to the position).
+"""
+from .binance_archive import ArchiveStore, fetch_all
+from .panel import build_panel
+from .costs import liquidity_cost_bp, participation_report
+from . import intraday
+
+__all__ = ["ArchiveStore", "fetch_all", "build_panel", "liquidity_cost_bp", "participation_report", "intraday"]

pitbacktest/crypto/binance_archive.py

@@ -0,0 +1,287 @@
+"""Binance public data archive (data.binance.vision) for USDT-margined perpetuals.
+
+Why the archive and not the live API: the archive keeps **delisted** contracts (LUNA, FTT and hundreds of
+others), so a universe built from it is not limited to today's survivors. The live API is used only to fill the
+last partial month for contracts that still trade.
+
+Daily bars are UTC days. Funding is settled every 8 hours (older contracts) or per `funding_interval_hours`;
+`daily_funding` sums every settlement that falls inside the UTC day.
+
+Everything is cached under `cache_dir` (default ~/.cache/quantbt/binance_um) so a download happens once.
+"""
+from __future__ import annotations
+
+import concurrent.futures as cf
+import io
+import json
+import re
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import zipfile
+from pathlib import Path
+
+import pandas as pd
+
+LIST_URL = "https://s3-ap-northeast-1.amazonaws.com/data.binance.vision"
+DL_URL = "https://data.binance.vision"
+API = "https://fapi.binance.com"
+KL_COLS = ["open_time", "open", "high", "low", "close", "volume", "close_time", "quote_volume", "trades",
+           "taker_buy_base", "taker_buy_quote", "ignore"]
+UA = {"User-Agent": "pitbacktest-research/0.2 (public data only)"}
+
+
+def _get(url: str, retries: int = 4, timeout: float = 30) -> bytes | None:
+    """GET with backoff. Returns None on 404 (a missing archive file is normal)."""
+    url = urllib.parse.quote(url, safe=":/?&=%")                  # non-ASCII symbols (e.g. Chinese tickers) need encoding
+    delay = 1.0
+    for i in range(retries + 1):
+        try:
+            with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout) as r:
+                return r.read()
+        except urllib.error.HTTPError as e:
+            if e.code == 404:
+                return None
+            if e.code in (418, 429, 500, 502, 503, 504) and i < retries:
+                time.sleep(delay * (3 if e.code in (418, 429) else 1))
+                delay *= 2
+                continue
+            raise
+        except (urllib.error.URLError, TimeoutError, ConnectionError):
+            if i < retries:
+                time.sleep(delay)
+                delay *= 2
+                continue
+            raise
+    return None
+
+
+def _list(prefix: str) -> list[str]:
+    """All keys under a prefix (S3 listing, paginated)."""
+    keys, marker = [], ""
+    while True:
+        q = {"prefix": prefix, "max-keys": "1000"}
+        if marker:
+            q["marker"] = marker
+        x = (_get(f"{LIST_URL}?{urllib.parse.urlencode(q)}") or b"").decode("utf-8")
+        got = re.findall(r"<Key>([^<]+)</Key>", x)
+        keys += got
+        if "<IsTruncated>true</IsTruncated>" in x and got:
+            marker = got[-1]
+        else:
+            return keys
+
+
+def _list_dirs(prefix: str) -> list[str]:
+    out, marker = [], ""
+    while True:
+        q = {"prefix": prefix, "delimiter": "/", "max-keys": "1000"}
+        if marker:
+            q["marker"] = marker
+        x = (_get(f"{LIST_URL}?{urllib.parse.urlencode(q)}") or b"").decode("utf-8")
+        got = re.findall(r"<Prefix>([^<]+)</Prefix>", x)
+        got = [g for g in got if g != prefix]
+        out += got
+        nm = re.search(r"<NextMarker>([^<]+)</NextMarker>", x)
+        if "<IsTruncated>true</IsTruncated>" in x and nm:
+            marker = nm.group(1)
+        else:
+            return out
+
+
+def _read_klines_zip(blob: bytes) -> pd.DataFrame:
+    with zipfile.ZipFile(io.BytesIO(blob)) as z:
+        raw = pd.read_csv(z.open(z.namelist()[0]), header=None, names=KL_COLS, dtype=str)
+    t = pd.to_numeric(raw["open_time"], errors="coerce")        # some files carry a header row
+    raw = raw[t.notna()].copy()
+    raw["open_time"] = t[t.notna()].astype("int64")
+    return raw
+
+
+def _frame_from_klines(raw: pd.DataFrame) -> pd.DataFrame:
+    if raw.empty:
+        return pd.DataFrame()
+    ts = raw["open_time"].astype("int64")
+    unit = "us" if ts.iloc[0] > 10**14 else "ms"                # newer archives use microseconds
+    idx = pd.to_datetime(ts, unit=unit, utc=True).dt.tz_localize(None).dt.normalize()
+    df = raw[["open", "high", "low", "close", "volume", "quote_volume", "trades"]].apply(pd.to_numeric, errors="coerce")
+    df.index = idx
+    df = df[~df.index.duplicated(keep="last")].sort_index()
+    return df
+
+
+class ArchiveStore:
+    """Local cache plus downloader for one market (USDT-M perpetuals)."""
+
+    def __init__(self, cache_dir: str | Path | None = None):
+        self.dir = Path(cache_dir or Path.home() / ".cache" / "quantbt" / "binance_um").expanduser()
+        (self.dir / "daily").mkdir(parents=True, exist_ok=True)
+        (self.dir / "funding").mkdir(parents=True, exist_ok=True)
+
+    # ------------------------------------------------------------ symbols
+    def symbols(self, refresh: bool = False, quote: str = "USDT") -> list[str]:
+        """Every USDT perpetual that ever existed in the archive (delisted ones included).
+        Delivery contracts such as BTCUSDT_250627 are excluded."""
+        f = self.dir / "symbols.json"
+        if f.exists() and not refresh:
+            return json.loads(f.read_text())
+        pre = "data/futures/um/monthly/klines/"
+        names = [d[len(pre):].strip("/") for d in _list_dirs(pre)]
+        syms = sorted(n for n in names if n.endswith(quote) and "_" not in n)
+        f.write_text(json.dumps(syms))
+        return syms
+
+    def trading_now(self) -> set[str]:
+        """Symbols the live exchange reports as TRADING (used to know which ones need the live top-up)."""
+        raw = _get(f"{API}/fapi/v1/exchangeInfo")
+        info = json.loads(raw)
+        return {s["symbol"] for s in info["symbols"] if s.get("contractType") == "PERPETUAL" and s.get("status") == "TRADING"}
+
+    def symbol_rules(self, refresh: bool = False) -> pd.DataFrame:
+        """Trading rules per contract from the live exchange-info endpoint (no key): quantity step, minimum quantity, minimum order value,
+        price tick, status and the onboard and delivery dates, indexed by symbol. **Today's values only**: the endpoint has no history, so a rule
+        that changed (BTCUSDT's minimum order value is not what it was in 2020) is applied to the past as it is now, and a contract that is no longer
+        listed (delisted ones) is absent. The frame carries the day it was fetched in `.attrs["fetched"]`. Cached as `symbol_rules.json`."""
+        f = self.dir / "symbol_rules.json"
+        if f.exists() and not refresh:
+            blob = json.loads(f.read_text())
+        else:
+            raw = _get(f"{API}/fapi/v1/exchangeInfo")
+            if raw is None:
+                raise RuntimeError("exchange info is not available")
+            rows = {}
+            for s in json.loads(raw)["symbols"]:
+                flt = {x["filterType"]: x for x in s.get("filters", [])}
+                lot, mn, pf = flt.get("LOT_SIZE"), flt.get("MIN_NOTIONAL"), flt.get("PRICE_FILTER")
+                if lot is None or mn is None:
+                    continue
+                rows[s["symbol"]] = {"step_size": float(lot["stepSize"]), "min_qty": float(lot["minQty"]), "min_notional": float(mn["notional"]),
+                                     "tick_size": float(pf["tickSize"]) if pf else float("nan"), "status": s.get("status"),
+                                     "contract_type": s.get("contractType"), "onboard_ms": s.get("onboardDate"), "delivery_ms": s.get("deliveryDate")}
+            blob = {"fetched": pd.Timestamp.now("UTC").strftime("%Y-%m-%d"), "rules": rows}
+            f.write_text(json.dumps(blob))
+        df = pd.DataFrame.from_dict(blob["rules"], orient="index")
+        df.attrs["fetched"] = blob["fetched"]
+        return df
+
+    # ------------------------------------------------------------ one symbol
+    def _months(self, sym: str, kind: str) -> list[str]:
+        keys = _list(f"data/futures/um/monthly/{kind}/{sym}/" + ("1d/" if kind == "klines" else ""))
+        return sorted(k for k in keys if k.endswith(".zip"))
+
+    def fetch_daily(self, sym: str, live: bool) -> pd.DataFrame | None:
+        """Daily bars (open, high, low, close, volume, quote_volume, trades) indexed by UTC date: the monthly archive files plus, when `live`
+        is True, the live API for the days after the last archived one. The unfinished current day is dropped. Cached as a pickle."""
+        f = self.dir / "daily" / f"{sym}.pkl"
+        if f.exists():
+            return pd.read_pickle(f)
+        parts = []
+        for k in self._months(sym, "klines"):
+            blob = _get(f"{DL_URL}/{k}")
+            if blob:
+                parts.append(_read_klines_zip(blob))
+        raw = pd.concat(parts) if parts else pd.DataFrame(columns=KL_COLS)
+        df = _frame_from_klines(raw)
+        if live:                                                   # top-up from the live API
+            start = (df.index[-1] + pd.Timedelta(days=1)) if len(df) else pd.Timestamp("2019-09-01")
+            ms = int(start.tz_localize("UTC").timestamp() * 1000)
+            blob = _get(f"{API}/fapi/v1/klines?symbol={sym}&interval=1d&startTime={ms}&limit=1500")
+            if blob:
+                rows = json.loads(blob)
+                if rows:
+                    live_df = _frame_from_klines(pd.DataFrame(rows, columns=KL_COLS).assign(
+                        open_time=lambda d: d["open_time"].astype("int64")))
+                    df = pd.concat([df, live_df]).pipe(lambda d: d[~d.index.duplicated(keep="last")]).sort_index()
+        if len(df):
+            today = pd.Timestamp.utcnow().tz_localize(None).normalize()
+            df = df[df.index < today]                              # never keep an unfinished bar
+        df.to_pickle(f)
+        return df
+
+    def fetch_funding(self, sym: str, live: bool) -> pd.Series | None:
+        """Funding rate summed per UTC day (every settlement inside the day), from the monthly fundingRate files plus, when `live` is True,
+        the live API. The current day is dropped. Cached as a pickle. For settlement instants see `crypto.intraday.fetch_funding_events`."""
+        f = self.dir / "funding" / f"{sym}.pkl"
+        if f.exists():
+            return pd.read_pickle(f)
+        frames = []
+        for k in self._months(sym, "fundingRate"):
+            blob = _get(f"{DL_URL}/{k}")
+            if not blob:
+                continue
+            with zipfile.ZipFile(io.BytesIO(blob)) as z:
+                d = pd.read_csv(z.open(z.namelist()[0]), dtype=str)
+            if d.shape[1] >= 3:
+                d.columns = ["calc_time", "interval", "rate"][: d.shape[1]] + list(d.columns[3:])
+                frames.append(d[["calc_time", "rate"]])
+        out = pd.concat(frames) if frames else pd.DataFrame(columns=["calc_time", "rate"])
+        out["calc_time"] = pd.to_numeric(out["calc_time"], errors="coerce")
+        out["rate"] = pd.to_numeric(out["rate"], errors="coerce")
+        out = out.dropna()
+        if live:
+            last = out["calc_time"].max() if len(out) else 1567296000000
+            blob = _get(f"{API}/fapi/v1/fundingRate?symbol={sym}&startTime={int(last) + 1}&limit=1000")
+            if blob:
+                rows = json.loads(blob)
+                if rows:
+                    add = pd.DataFrame({"calc_time": [r["fundingTime"] for r in rows], "rate": [float(r["fundingRate"]) for r in rows]})
+                    out = pd.concat([out, add])
+        if out.empty:
+            s = pd.Series(dtype=float)
+        else:
+            ts = out["calc_time"].astype("int64")
+            unit = "us" if ts.iloc[0] > 10**14 else "ms"
+            day = pd.to_datetime(ts, unit=unit, utc=True).dt.tz_localize(None).dt.normalize()
+            s = out.set_index(day)["rate"].groupby(level=0).sum().sort_index()   # sum of settlements in the UTC day
+        today = pd.Timestamp.utcnow().tz_localize(None).normalize()
+        s = s[s.index < today]
+        s.to_pickle(f)
+        return s
+
+
+def fetch_all(store: ArchiveStore | None = None, symbols: list[str] | None = None, workers: int = 12,
+              progress: bool = True) -> dict:
+    """Download (or load from cache) daily bars and funding for every symbol. Returns counts."""
+    store = store or ArchiveStore()
+    syms = symbols or store.symbols()
+    live = store.trading_now()
+    done = {"ok": 0, "empty": 0, "error": 0}
+    errs: list[str] = []
+
+    def one(s: str):
+        d = store.fetch_daily(s, s in live)
+        f = store.fetch_funding(s, s in live)
+        return s, (0 if d is None else len(d)), (0 if f is None else len(f))
+
+    with cf.ThreadPoolExecutor(workers) as ex:
+        futs = {ex.submit(one, s): s for s in syms}
+        for n, fu in enumerate(cf.as_completed(futs), 1):
+            try:
+                _, nd, nf = fu.result()
+                done["ok" if nd else "empty"] += 1
+            except Exception as e:                                  # one bad symbol must not stop the run
+                done["error"] += 1
+                errs.append(f"{futs[fu]}: {type(e).__name__}: {e}")
+            if progress and n % 25 == 0:
+                print(f"  {n}/{len(syms)}  {done}", flush=True)
+    done["errors"] = errs[:20]
+    return done
+
+
+def execution_rules(rules: pd.DataFrame, tickers, *, missing: str = "raise") -> tuple[pd.Series, pd.Series]:
+    """(lot, min_trade_value) per ticker for `backtest_weights(lot=..., min_trade_value=...)`, from `ArchiveStore.symbol_rules()`.
+
+    lot is the quantity step (at least the minimum quantity), in units of the contract; min_trade_value the minimum order value in the quote
+    currency. `rules` holds today's values only and has no delisted contracts. For a ticker it does not know, `missing="raise"` (default) raises,
+    and `missing="nan"` returns NaN for it so that you can leave it out of a small-account run. No substitute rule is invented: a step is in units of
+    the contract, which are worth anything from a thousandth of a dollar to tens of thousands, so there is no safe value to fill in."""
+    if missing not in ("raise", "nan"):
+        raise ValueError(f"missing must be 'raise' or 'nan', not {missing!r}")
+    tickers = pd.Index(tickers)
+    unknown = tickers[~tickers.isin(rules.index)]
+    if len(unknown) and missing == "raise":
+        raise ValueError(f"no trading rules for {len(unknown)} tickers (first: {list(unknown[:3])}); they are probably delisted. "
+                         f"Pass missing='nan' to get NaN for them and leave them out")
+    r = rules.reindex(tickers)
+    return r[["step_size", "min_qty"]].max(axis=1).rename("lot"), r["min_notional"].rename("min_trade_value")