factorflow 0.1.0__py3-none-any.whl

factorflow/sim/pipelines.py

@@ -0,0 +1,147 @@
+"""Example pipelines used by the demo, the validation and the benchmarks."""
+
+from __future__ import annotations
+
+from factorflow.graph import Pipeline
+from factorflow.sim.market import STYLES
+from factorflow.terms import BarData, Classifier, Column, Factor, Filter, where
+
+close, volume, dollar = BarData.close, BarData.volume, BarData.dollar
+high, low = BarData.high, BarData.low
+industry: Classifier = Column("industry", Classifier)
+signal: Factor = Column("signal")
+
+
+def beta_window_for(history_days: int) -> int:
+    """200 days, or the history less a month when that is shorter (at least 20)."""
+    return max(20, min(200, history_days - 30))
+
+
+def daily_pipeline(n_assets: int, beta_window: int = 200) -> Pipeline:
+    """Return the daily research pipeline.
+
+    Outputs are returns, beta to the equal-weight market, styles, the planted
+    signal, and decile screens.
+
+    Parameters
+    ----------
+    n_assets : int
+        Names in the book. Sets the decile width to ``max(1, n_assets // 10)``.
+    beta_window : int, optional
+        Rolling beta length in daily bars. Default is 200.
+
+    Returns
+    -------
+    Pipeline
+    """
+    r = close.pct_change(1)
+    logret = (close / close.shift(1)).log()
+    mkt = r - r.demean()  # equal-weight market return, broadcast to every asset
+    universe: Filter = (dollar.rolling_mean(20) > 0) & close.notnull()
+    decile = max(1, n_assets // 10)
+    cols = {
+        "ret": r,
+        "logret": logret,
+        "mkt": mkt,
+        "beta": r.rolling_beta(mkt, beta_window),
+        "vol20": r.rolling_std(20),
+        "mom20": close.pct_change(20),
+        "ind_rel": r.demean(groupby=industry),
+        "signal_z": signal.zscore(mask=universe),
+        "long": signal.top(decile, mask=universe),
+        "short": signal.bottom(decile, mask=universe),
+        **{f"{k}_z": Column(k).zscore(mask=universe) for k in STYLES},
+    }
+    return Pipeline(cols, screen=universe)
+
+
+def minute_pipeline() -> Pipeline:
+    """Intraday pipeline shown live by the demo (about 40 graph nodes)."""
+    r = close.pct_change(1)
+    liquid: Filter = volume.rolling_mean(30) > 0
+    vwap30 = dollar.rolling_sum(30) / volume.rolling_sum(30)
+    mom30 = close.pct_change(30)
+    vol30 = r.rolling_std(30)
+    score = (
+        mom30.zscore(groupby=industry, mask=liquid) + signal.zscore(mask=liquid) - vol30.zscore(mask=liquid)
+    )
+    cols = {
+        "close": close,
+        "ret1": r,
+        "mom30": mom30,
+        "vol30": vol30,
+        "vwap_dev": close / vwap30 - 1.0,
+        "rel_vol": volume / volume.rolling_mean(60),
+        "range30": (high.rolling_max(30) - low.rolling_min(30)) / close,
+        "ewm_ret": r.ewm_mean(10.0),
+        "score": score,
+        "rank": score.rank(mask=liquid, ascending=False),
+        "long": where(score.top(10, mask=liquid), 1.0, 0.0),
+    }
+    return Pipeline(cols, screen=liquid)
+
+
+def crypto_pipeline() -> Pipeline:
+    """24/7 minute pipeline for the crypto majors (beta to BTC via the cross-section)."""
+    r = close.pct_change(1)
+    btc = r - r.demean()  # with 3 names this is the equal-weight crypto market
+    return Pipeline(
+        {
+            "close": close,
+            "ret1": r,
+            "mom60": close.pct_change(60),
+            "vol60": r.rolling_std(60),
+            "beta240": r.rolling_beta(btc, 240),
+            "dollar60": dollar.rolling_sum(60),
+            "mom_rank": close.pct_change(60).rank(ascending=False),
+        }
+    )
+
+
+def wide_pipeline(n_factors: int = 100) -> Pipeline:
+    """Build the wide benchmark pipeline.
+
+    This is the 100-factor pipeline used by ``benchmarks/bench_minute.py``,
+    including industry groups. ``n_factors`` sets how many style-like terms
+    are added.
+    """
+    c, v, h, lo = close, volume, high, low
+    r = c.pct_change(1)
+    liquid = v.rolling_mean(30) > 0
+    windows = [5, 10, 15, 20, 30, 60, 90, 120]
+    gens = [
+        lambda w: c.pct_change(w),
+        lambda w: r.rolling_std(w),
+        lambda w: c / c.rolling_mean(w) - 1.0,
+        lambda w: v.rolling_mean(w),
+        lambda w: (c - lo.rolling_min(w)) / (h.rolling_max(w) - lo.rolling_min(w)),
+        lambda w: c.ts_rank(w),
+        lambda w: r.rolling_corr(v.pct_change(1), w),
+        lambda w: r.ewm_mean(w / 2),
+        lambda w: c.pct_change(w).zscore(mask=liquid, groupby=industry),
+        lambda w: r.rolling_std(w).rank(mask=liquid),
+        lambda w: (r.rolling_mean(w) / r.rolling_std(w)).demean(groupby=industry),
+        lambda w: r.rolling_beta(r.demean(), w),
+    ]
+    cols: dict = {}
+    i = 0
+    while len(cols) < n_factors:
+        g = gens[i % len(gens)]
+        w = windows[(i // len(gens)) % len(windows)]
+        cols[f"f{len(cols):03d}"] = g(w)
+        i += 1
+    return Pipeline(cols, screen=liquid)
+
+
+def event_pipeline() -> Pipeline:
+    """Per-asset time-series pipeline for event (tick/volume/dollar) bars."""
+    r = close.pct_change(1)
+    return Pipeline(
+        {
+            "ret": r,
+            "vol20": r.rolling_std(20),
+            "mom10": close.pct_change(10),
+            "ewm": r.ewm_mean(5.0),
+            "logc": close.log(),
+        }
+    )

factorflow/sim/replay.py

@@ -0,0 +1,385 @@
+"""Replay a trade stream through :class:`~factorflow.LiveEngine` and check it against batch.
+
+``replay`` streams trades in timestamp order. Before each trade it calls ``advance(ts)``
+(sealing every bar whose ``end + grace`` has passed, forward-filling empty bars), then
+``tick``. The open bar is revised on a market-time cadence (``revise_every_ns``, like a
+dashboard refresh) and/or every ``revise_every_ticks`` trades. With ``speed`` set, the
+driver sleeps so that market time runs ``speed`` times faster than the wall clock (1.0 =
+real time); without it, it runs as fast as possible.
+
+Non-bar inputs (signal, classifiers, style exposures) come from ``extras``: a
+``(times, {column: (T, N) array})`` pair aligned to bar ends; whenever a new bar opens, its
+row is written with ``set_open``, exactly as a feed handler would.
+
+``verify_replay`` runs the batch engine on the same bars and asserts that every sealed live
+row is bit-for-bit equal to the batch row (and every sampled open revision equals a batch run
+on sealed history + the open bar treated as final).
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from factorflow.bars import FIELDS, Trades
+from factorflow.batch import BatchEngine
+from factorflow.events import EventBarBuilder, EventBars, EventPipeline, build_event_bars
+from factorflow.graph import Graph, Pipeline, compile_pipeline
+from factorflow.grid import NS, BarGrid
+from factorflow.live import LiveEngine, Snapshot
+from factorflow.panel import Panel
+
+
+@dataclass
+class ReplayStats:
+    """Counters from one :func:`replay` call.
+
+    Notes
+    -----
+    ``tick_s`` is seconds spent inside ``LiveEngine.tick``. ``revise_ms`` and
+    ``seal_ms`` are per-call latencies in milliseconds. ``sealed`` keeps every
+    sealed snapshot, in order. ``market_s`` is the span of trade timestamps, in seconds.
+    """
+
+    n_ticks: int = 0
+    wall_s: float = 0.0
+    market_s: float = 0.0
+    tick_s: float = 0.0  # time inside tick() (bar accumulation + routing)
+    revise_ms: list[float] = field(default_factory=list)
+    seal_ms: list[float] = field(default_factory=list)
+    sealed: list[Snapshot] = field(default_factory=list)
+    open_checked: int = 0
+
+    @property
+    def ticks_per_s(self) -> float:
+        """Return trades processed per wall-clock second.
+
+        Returns
+        -------
+        float
+            NaN if no wall time was recorded.
+        """
+        return self.n_ticks / self.wall_s if self.wall_s > 0 else float("nan")
+
+    def _q(self, xs: list[float], q: float) -> float:
+        return float(np.quantile(xs, q)) if xs else float("nan")
+
+    def summary(self) -> dict[str, float]:
+        """Return the counters as a flat dict of floats.
+
+        Returns
+        -------
+        dict of str to float
+            Includes tick throughput, speedup against real time, tick
+            microseconds, and revise and seal percentiles.
+        """
+        return {
+            "ticks": self.n_ticks,
+            "wall_s": self.wall_s,
+            "ticks_per_s": self.ticks_per_s,
+            "speedup_vs_realtime": self.market_s / self.wall_s if self.wall_s else float("nan"),
+            "tick_us": 1e6 * self.tick_s / self.n_ticks if self.n_ticks else float("nan"),
+            "revises": len(self.revise_ms),
+            "revise_ms_p50": self._q(self.revise_ms, 0.5),
+            "revise_ms_p95": self._q(self.revise_ms, 0.95),
+            "revise_ms_p99": self._q(self.revise_ms, 0.99),
+            "seals": len(self.seal_ms),
+            "seal_ms_p50": self._q(self.seal_ms, 0.5),
+            "seal_ms_p95": self._q(self.seal_ms, 0.95),
+        }
+
+
+class Extras:
+    """Row lookup for non-bar input columns: the row of the first bar end >= ``open_end``."""
+
+    def __init__(self, times: np.ndarray, columns: Mapping[str, np.ndarray]):
+        """Store non-bar columns aligned to bar ends.
+
+        Parameters
+        ----------
+        times : array of int64
+            Bar ends, nanoseconds UTC, non-decreasing.
+        columns : mapping of str to ndarray
+            Each array has shape ``(T, N)``.
+        """
+        self.times = np.asarray(times, np.int64)
+        self.columns = dict(columns)
+
+    def row(self, open_end: int) -> dict[str, np.ndarray]:
+        """Return the column row for the bar that contains ``open_end``.
+
+        Parameters
+        ----------
+        open_end : int
+            Open bar end, nanoseconds UTC.
+
+        Returns
+        -------
+        dict of str to ndarray
+            One ``(N,)`` row per column. If ``open_end`` is past the last time, the last row is used.
+        """
+        r = min(int(np.searchsorted(self.times, open_end, side="left")), len(self.times) - 1)
+        return {k: v[r] for k, v in self.columns.items()}
+
+
+def replay(
+    engine: LiveEngine,
+    trades: Trades,
+    *,
+    start: int = 0,
+    stop: int | None = None,
+    revise_every_ns: int | None = NS,
+    revise_every_ticks: int | None = None,
+    extras: Extras | None = None,
+    speed: float | None = None,
+    until: int | None = None,
+    on_revise: Callable[[Snapshot], None] | None = None,
+    on_seal: Callable[[Snapshot], None] | None = None,
+    check_open: Callable[[LiveEngine, Snapshot], None] | None = None,
+    check_open_every: int = 0,
+    stats: ReplayStats | None = None,
+) -> ReplayStats:
+    """Stream ``trades[start:stop]`` through ``engine`` (which must have a grid).
+
+    ``until`` (ns) finally advances the clock to seal every bar ending by then.
+    ``check_open(engine, snap)`` is called on every ``check_open_every``-th revision.
+    """
+    if engine.grid is None:
+        raise ValueError("replay needs a LiveEngine with a BarGrid")
+    st = stats or ReplayStats()
+    stop = len(trades) if stop is None else stop
+    a_, t_, p_, q_ = trades.asset, trades.ts, trades.price, trades.size
+    pc = time.perf_counter
+    wall0 = pc()
+    t_first = int(t_[start]) if stop > start else 0
+    last_open = None
+
+    def sealed_now(snaps: list[Snapshot], dt: float) -> None:
+        if not snaps:
+            return
+        per = 1e3 * dt / len(snaps)
+        for s in snaps:
+            st.seal_ms.append(per)
+            st.sealed.append(s)
+            if on_seal is not None:
+                on_seal(s)
+
+    def open_bar_inputs() -> None:
+        nonlocal last_open
+        if extras is not None and engine.open_end != last_open:
+            for k, v in extras.row(int(engine.open_end)).items():  # type: ignore[arg-type]
+                if k in engine.extra:  # columns the pipeline does not read are skipped
+                    engine.set_open(k, v)
+        last_open = engine.open_end
+
+    open_bar_inputs()
+    next_revise = (t_first // revise_every_ns + 1) * revise_every_ns if revise_every_ns else None
+    since = 0
+    n_rev = 0
+    for i in range(start, stop):
+        ts = int(t_[i])
+        if speed:
+            # pace market time; keep sealing bars on the clock while no trades arrive
+            while (lag := (ts - t_first) / NS / speed - (pc() - wall0)) > 0:
+                time.sleep(min(lag, 0.05))
+                now = t_first + int((pc() - wall0) * speed * NS)
+                t0 = pc()
+                snaps = engine.advance(min(now, ts))
+                sealed_now(snaps, pc() - t0)
+                if snaps:
+                    open_bar_inputs()
+        # market-time revisions due before this trade
+        if next_revise is not None and ts >= next_revise:
+            if since:
+                n_rev += 1
+                _timed_revise(engine, st, on_revise, check_open, check_open_every, n_rev)
+                since = 0
+            next_revise = (ts // revise_every_ns + 1) * revise_every_ns  # type: ignore[operator]
+        t0 = pc()
+        snaps = engine.advance(ts)
+        sealed_now(snaps, pc() - t0)
+        if snaps:
+            open_bar_inputs()
+        t0 = pc()
+        engine.tick(int(a_[i]), ts, float(p_[i]), float(q_[i]))
+        st.tick_s += pc() - t0
+        since += 1
+        if revise_every_ticks and since >= revise_every_ticks:
+            n_rev += 1
+            _timed_revise(engine, st, on_revise, check_open, check_open_every, n_rev)
+            since = 0
+    if since:
+        n_rev += 1
+        _timed_revise(engine, st, on_revise, check_open, check_open_every, n_rev)
+    if until is not None:
+        t0 = pc()
+        snaps = engine.advance(until)
+        sealed_now(snaps, pc() - t0)
+    st.n_ticks += stop - start
+    st.wall_s += pc() - wall0
+    if stop > start:
+        st.market_s += (int(t_[stop - 1]) - t_first) / NS
+    return st
+
+
+def _timed_revise(engine, st, on_revise, check_open, every, n) -> None:
+    t0 = time.perf_counter()
+    snap = engine.revise()
+    st.revise_ms.append(1e3 * (time.perf_counter() - t0))
+    if on_revise is not None:
+        on_revise(snap)
+    if check_open is not None and every and n % every == 0:
+        check_open(engine, snap)
+        st.open_checked += 1
+
+
+# ====================================================================== verification
+def _equal(a: np.ndarray, b: np.ndarray) -> bool:
+    return bool(np.array_equal(a, b, equal_nan=a.dtype.kind == "f"))
+
+
+@dataclass
+class VerifyResult:
+    """The outcome of comparing a live replay with batch.
+
+    Notes
+    -----
+    ``mismatches`` is empty when every checked row matched.
+    ``sealed_rows`` is how many bars the live engine sealed.
+    """
+
+    grid: str
+    sealed_rows: int
+    open_checked: int
+    mismatches: list[str]
+    stats: ReplayStats
+
+    @property
+    def ok(self) -> bool:
+        """Return whether the replay matched and sealed at least one row.
+
+        Returns
+        -------
+        bool
+            ``True`` when ``mismatches`` is empty and ``sealed_rows > 0``.
+        """
+        return not self.mismatches and self.sealed_rows > 0
+
+
+def verify_replay(
+    pipeline: Pipeline | Graph,
+    full: Panel,
+    trades: Trades,
+    grid: BarGrid,
+    history_rows: int,
+    *,
+    extra_columns: tuple[str, ...] = (),
+    revise_every_ns: int | None = NS,
+    revise_every_ticks: int | None = None,
+    check_open_every: int = 0,
+    partial_threshold: float = 0.0,
+    label: str = "",
+) -> VerifyResult:
+    """Live replay of the trades after ``full.head(history_rows)`` vs batch on ``full``.
+
+    ``full`` must be the batch bars of the same trades on ``grid`` (``build_time_bars``),
+    optionally preceded by history rows, plus any ``extra_columns``.
+    """
+    g = pipeline if isinstance(pipeline, Graph) else compile_pipeline(pipeline)
+    batch = BatchEngine().run(g, full)
+    hist = full.head(history_rows)
+    live = LiveEngine(g, hist, grid, partial_threshold=partial_threshold)
+    extras = Extras(full.times, {k: full.columns[k] for k in extra_columns}) if extra_columns else None
+    start = int(np.searchsorted(trades.ts, int(full.times[history_rows - 1]), side="left"))
+    mism: list[str] = []
+
+    def check_open(eng: LiveEngine, snap: Snapshot) -> None:
+        n_sealed = history_rows + len(stats.sealed)
+        bar_cols = (*FIELDS, "has_trade")
+        row = {}
+        for k in full.columns:
+            if k in bar_cols:
+                row[k] = eng.bar.row(k)
+            elif k in eng.extra:
+                row[k] = eng.extra[k]
+            else:  # not a pipeline input; any value will do
+                row[k] = full.columns[k][min(n_sealed, len(full.times) - 1)]
+        ref = BatchEngine().run(g, full.head(n_sealed).with_row(row, int(eng.open_end)))  # type: ignore[arg-type]
+        for k, v in snap.values.items():
+            if not _equal(v, ref.values[k][n_sealed]):
+                mism.append(f"open row {n_sealed} column {k}")
+
+    stats = ReplayStats()
+    replay(
+        live,
+        trades,
+        start=start,
+        revise_every_ns=revise_every_ns,
+        revise_every_ticks=revise_every_ticks,
+        extras=extras,
+        until=int(full.times[-1]) + grid.grace_ns,
+        check_open=check_open if check_open_every else None,
+        check_open_every=check_open_every,
+        stats=stats,
+    )
+    sealed = stats.sealed
+    expect = len(full.times) - history_rows
+    if len(sealed) != expect:
+        mism.append(f"sealed {len(sealed)} bars, expected {expect}")
+    for j, snap in enumerate(sealed[:expect]):
+        r = history_rows + j
+        if snap.ts != int(full.times[r]):
+            mism.append(f"sealed bar {j} stamped {snap.ts}, batch row {r} is {int(full.times[r])}")
+            break
+        for k, v in snap.values.items():
+            if not _equal(v, batch.values[k][r]):
+                mism.append(f"sealed row {r} column {k}")
+        if batch.screen is not None and snap.screen is not None and not _equal(snap.screen, batch.screen[r]):
+            mism.append(f"sealed row {r} screen")
+    return VerifyResult(label or repr(grid), len(sealed), stats.open_checked, mism, stats)
+
+
+def verify_event_pipeline(
+    pipeline: Pipeline | Graph,
+    trades: Trades,
+    asset: int,
+    builder_factory: Callable[[], EventBarBuilder],
+    history_frac: float = 0.3,
+    label: str = "",
+) -> VerifyResult:
+    """One asset's event bars (tick/volume/dollar): live EventPipeline vs batch."""
+    g = pipeline if isinstance(pipeline, Graph) else compile_pipeline(pipeline)
+    m = trades.asset == asset
+    ts, px, sz = trades.ts[m], trades.price[m], trades.size[m]
+    b = builder_factory()
+    bars, _ = build_event_bars(ts, px, sz, b.kind, b.rule, b.overshoot, b.day_key)
+    batch = BatchEngine().run(g, bars.panel())
+    cut = int(len(ts) * history_frac)
+    hb = builder_factory()
+    hist_bars = EventBars.from_bars(hb.update_many(ts[:cut].tolist(), px[:cut].tolist(), sz[:cut].tolist()))
+    # resume the same builder state: replay the history trades into it, then go live
+    ep = EventPipeline(g, hb, history=hist_bars)
+    stats = ReplayStats()
+    t0 = time.perf_counter()
+    sealed: list[Snapshot] = []
+    for t, p, q in zip(ts[cut:].tolist(), px[cut:].tolist(), sz[cut:].tolist(), strict=True):
+        for s in ep.update(t, p, q):
+            if not s.is_open:
+                sealed.append(s)
+    stats.wall_s = time.perf_counter() - t0
+    stats.n_ticks = len(ts) - cut
+    mism: list[str] = []
+    h = len(hist_bars)
+    if h + len(sealed) != len(bars):
+        mism.append(f"live sealed {h + len(sealed)} bars, batch has {len(bars)}")
+    for j, s in enumerate(sealed):
+        r = h + j
+        if r >= len(bars):
+            break
+        for k, v in s.values.items():
+            if not _equal(v, batch.values[k][r]):
+                mism.append(f"event row {r} column {k}")
+    return VerifyResult(label or f"event[{asset}]", len(sealed), 0, mism, stats)