factorflow 0.1.0__py3-none-any.whl

factorflow/panel.py

@@ -0,0 +1,181 @@
+"""Panel: dense ``T × N`` arrays on a bar grid.
+
+Rows are bars, labeled by end time. Columns are named arrays of shape ``(T, N)``
+with one column per asset. Missing factor values are NaN. Classifier codes use
+the column's own dtype (``-1`` is the engine's missing classifier code when the
+column is read as a :class:`~factorflow.terms.Classifier`).
+
+Examples
+--------
+>>> import numpy as np
+>>> from factorflow import Panel
+>>> p = Panel(np.arange(3), ("AAA", "BBB"), {"close": np.ones((3, 2))})
+>>> p.shape
+(3, 2)
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+
+import numpy as np
+import polars as pl
+
+
+@dataclass
+class Panel:
+    """One history: bar-end times, asset names, and dense columns.
+
+    Parameters
+    ----------
+    times : array of int64
+        Bar end, nanoseconds since the Unix epoch, UTC. Must be non-decreasing.
+        Event bars may share a timestamp. Length ``T``.
+    assets : sequence of str
+        Asset names, length ``N``. Order is the column order of every array.
+    columns : dict of str to ndarray, optional
+        Each array has shape ``(T, N)``. Omitted columns are simply absent;
+        a pipeline that reads one will fail at runtime with ``KeyError``.
+    starts : array of int64 or None, optional
+        Bar start, same clock and length as ``times``. ``None`` if the caller
+        does not have starts.
+
+    Raises
+    ------
+    ValueError
+        If ``times`` decreases, or a column's shape is not ``(T, N)``.
+    """
+
+    times: np.ndarray  # int64 ns since epoch (UTC), bar end; non-decreasing (event bars may share a stamp)
+    assets: tuple[str, ...]
+    columns: dict[str, np.ndarray] = field(default_factory=dict)
+    starts: np.ndarray | None = None  # optional bar start times
+
+    def __post_init__(self) -> None:
+        """Check shapes and that bar times do not go backwards.
+
+        Raises
+        ------
+        ValueError
+            If ``times`` is not non-decreasing, or a column shape is not ``(T, N)``.
+        """
+        self.times = np.asarray(self.times, dtype=np.int64)
+        self.assets = tuple(self.assets)
+        t, n = len(self.times), len(self.assets)
+        if t > 1 and not np.all(np.diff(self.times) >= 0):
+            raise ValueError("panel times must be non-decreasing")
+        for k, v in self.columns.items():
+            if v.shape != (t, n):
+                raise ValueError(f"column {k!r} has shape {v.shape}, expected {(t, n)}")
+
+    @property
+    def shape(self) -> tuple[int, int]:
+        """Number of bars and number of assets.
+
+        Returns
+        -------
+        tuple of int
+            ``(T, N)``.
+        """
+        return len(self.times), len(self.assets)
+
+    def head(self, rows: int) -> Panel:
+        """Return the first ``rows`` bars.
+
+        Parameters
+        ----------
+        rows : int
+            How many leading bars to keep. Not a time span.
+
+        Returns
+        -------
+        Panel
+            A new panel. Arrays are views of the originals.
+        """
+        return Panel(
+            self.times[:rows],
+            self.assets,
+            {k: v[:rows] for k, v in self.columns.items()},
+            None if self.starts is None else self.starts[:rows],
+        )
+
+    def with_row(self, row: Mapping[str, np.ndarray], time: int) -> Panel:
+        """Return a copy with one bar appended.
+
+        Used to treat an open bar as if it were already sealed, which is how a
+        live revision is checked against :class:`~factorflow.batch.BatchEngine`.
+
+        Parameters
+        ----------
+        row : mapping of str to ndarray
+            One value per asset for every column this panel already has.
+            Dtypes follow the existing columns.
+        time : int
+            End time of the new bar, nanoseconds UTC.
+
+        Returns
+        -------
+        Panel
+            ``T + 1`` bars. ``starts`` is dropped.
+        """
+        cols = {
+            k: np.vstack([v, np.asarray(row[k], dtype=v.dtype)[None, :]]) for k, v in self.columns.items()
+        }
+        return Panel(np.r_[self.times, np.int64(time)], self.assets, cols)
+
+    @classmethod
+    def from_frame(
+        cls,
+        df: pl.DataFrame,
+        assets: Sequence[str] | None = None,
+        time_col: str = "ts",
+        asset_col: str = "asset",
+    ) -> Panel:
+        """Build a panel from a long polars frame.
+
+        Parameters
+        ----------
+        df : polars.DataFrame
+            One row per ``(ts, asset)``. Other columns become panel columns.
+            ``ts`` may be int64 nanoseconds or a UTC datetime. Missing pairs
+            become NaN. Non-float columns are cast to float64.
+        assets : sequence of str or None, optional
+            Column order. ``None`` sorts the names that appear in ``df``.
+        time_col : str, optional
+            Timestamp column. Default is ``"ts"``.
+        asset_col : str, optional
+            Asset-name column. Default is ``"asset"``.
+
+        Returns
+        -------
+        Panel
+            Dense ``(T, N)`` columns. Assets in ``df`` that are not in
+            ``assets`` are dropped.
+
+        Examples
+        --------
+        >>> import polars as pl
+        >>> from factorflow import Panel
+        >>> df = pl.DataFrame({"ts": [0, 0], "asset": ["A", "B"], "close": [1.0, 2.0]})
+        >>> Panel.from_frame(df).assets
+        ('A', 'B')
+        """
+        ts = df[time_col]
+        if ts.dtype != pl.Int64:
+            ts = ts.cast(pl.Datetime("ns", "UTC")).cast(pl.Int64)
+        df = df.with_columns(ts.alias("__t"))
+        times = np.unique(df["__t"].to_numpy())
+        names = tuple(assets) if assets is not None else tuple(sorted(df[asset_col].unique().to_list()))
+        ti = np.searchsorted(times, df["__t"].to_numpy())
+        amap = {a: i for i, a in enumerate(names)}
+        ai = np.array([amap.get(a, -1) for a in df[asset_col].to_list()])
+        keep = ai >= 0
+        cols = {}
+        for c in df.columns:
+            if c in (time_col, asset_col, "__t"):
+                continue
+            arr = np.full((len(times), len(names)), np.nan)
+            arr[ti[keep], ai[keep]] = df[c].to_numpy()[keep].astype(np.float64)
+            cols[c] = arr
+        return cls(times, names, cols)

factorflow/sim/__init__.py

@@ -0,0 +1,22 @@
+"""Synthetic market data with known ground truth, a replay driver and validation tools.
+
+    from factorflow.sim import SimConfig, simulate
+    mkt = simulate(SimConfig(seed=7))          # equities + BTC/ETH/SOL, deterministic
+    mkt.equities.daily                         # daily Panel (signal, industry, styles columns)
+    mkt.equities.trades                        # ticks of the last ``intraday_days`` sessions
+
+CLI: ``python -m factorflow.sim demo | validate | bench``.
+"""
+
+from factorflow.sim.market import (
+    STYLES,
+    AssetSim,
+    CryptoSpec,
+    EquitySpec,
+    SimConfig,
+    SimMarket,
+    Truth,
+    simulate,
+)
+
+__all__ = ["STYLES", "AssetSim", "CryptoSpec", "EquitySpec", "SimConfig", "SimMarket", "Truth", "simulate"]

factorflow/sim/__main__.py

@@ -0,0 +1,210 @@
+"""``python -m factorflow.sim {demo,validate,bench}``."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+
+import numpy as np
+
+from factorflow.grid import NS, BarGrid
+from factorflow.live import LiveEngine, Snapshot
+from factorflow.sim.market import EXTRA_COLUMNS, SimConfig, simulate
+from factorflow.sim.pipelines import beta_window_for, crypto_pipeline, daily_pipeline, minute_pipeline
+from factorflow.sim.replay import Extras, replay, verify_replay
+
+
+def _fmt_ts(ns: int) -> str:
+    return str(np.datetime64(int(ns), "ns").astype("datetime64[s]")).replace("T", " ") + " UTC"
+
+
+def _verify(mkt, check_every: int = 0) -> bool:
+    """Replay every intraday tick through LiveEngine and compare sealed rows with batch.
+
+    Revision cadence (market time): daily bars every 60 s, equity minute bars every 1 s,
+    crypto minute bars every 5 s.
+    """
+    eq, cr = mkt.equities, mkt.crypto
+    p, _ = eq.time_bars("1min")
+    first = int(np.searchsorted(p.times, eq.closes[eq.intraday[0]], side="right"))
+    runs = [
+        (
+            "equities, daily bars (ticks revise the open daily bar)",
+            eq,
+            daily_pipeline(len(eq.assets), beta_window_for(eq.n_hist)),
+            eq.daily,
+            eq.daily_grid(),
+            eq.n_hist,
+            EXTRA_COLUMNS,
+            60,
+        ),
+        (
+            "equities, 1-minute bars",
+            eq,
+            minute_pipeline(),
+            p,
+            BarGrid("XNYS", "1min"),
+            first,
+            ("signal", "industry"),
+            1,
+        ),
+    ]
+    if cr is not None:
+        cp, _ = cr.time_bars("1min")
+        first_c = int(np.searchsorted(cp.times, cr.closes[cr.intraday[0]], side="right"))
+        runs.append(
+            ("crypto 24/7, 1-minute bars", cr, crypto_pipeline(), cp, BarGrid("24/7", "1min"), first_c, (), 5)
+        )
+    ok = True
+    for label, sim, pipe, panel, grid, h, extra, every_s in runs:
+        t0 = time.perf_counter()
+        v = verify_replay(
+            pipe,
+            panel,
+            sim.trades,
+            grid,
+            h,
+            extra_columns=extra,
+            revise_every_ns=every_s * NS,
+            check_open_every=check_every,
+        )
+        s = v.stats.summary()
+        status = "OK " if v.ok else "FAIL"
+        print(
+            f"  [{status}] {label}: {v.sealed_rows} sealed rows == batch (bit for bit), "
+            f"{v.open_checked} open revisions checked; {int(s['ticks']):,} ticks, {s['revises']} revisions "
+            f"every {every_s} s, revise p50 {s['revise_ms_p50']:.2f} ms ({time.perf_counter() - t0:.1f} s)"
+        )
+        for m in v.mismatches[:5]:
+            print("        ", m)
+        ok &= v.ok
+    return ok
+
+
+def _live_view(mkt, speed: float, seconds: float, top: int, out=sys.stdout) -> None:
+    """Replay the last session's ticks through a minute-grid engine, printing live values."""
+    eq = mkt.equities
+    p, _ = eq.time_bars("1min")
+    last = eq.intraday[-1]
+    h = int(np.searchsorted(p.times, eq.opens[last], side="right"))
+    live = LiveEngine(minute_pipeline(), p.head(h), BarGrid("XNYS", "1min"))
+    extras = Extras(p.times, {k: p.columns[k] for k in ("signal", "industry")})
+    t_open = int(eq.opens[last])
+    start = int(np.searchsorted(eq.trades.ts, t_open, side="left"))
+    stop = int(np.searchsorted(eq.trades.ts, t_open + int(seconds * speed * NS), side="left"))
+    tty = out.isatty()
+    last_print = [0.0]
+    n_sealed = [0]
+    assets = eq.assets
+
+    def show(snap: Snapshot, force: bool = False) -> None:
+        now = time.perf_counter()
+        if not force and now - last_print[0] < 1.0:
+            return
+        last_print[0] = now
+        v = snap.values
+        order = [int(i) for i in np.argsort(v["rank"]) if np.isfinite(v["rank"][i])][:top]
+        flag = "LIVE (open bar, provisional)" if snap.is_open else "SEALED (final)"
+        lines = [
+            f"bar ending {_fmt_ts(snap.ts)}  [{flag}]  sealed bars this run: {n_sealed[0]}  "
+            f"changed: {', '.join(sorted(snap.changed)) or '-'}",
+            f"  {'asset':<6} {'is_open':<7} {'close':>9} {'mom30':>8} {'vol30':>8} {'vwap_dev':>9} "
+            f"{'score':>7} {'rank':>4}",
+        ]
+        for i in order:
+            lines.append(
+                f"  {assets[i]:<6} {snap.is_open!s:<7} {v['close'][i]:9.2f} {v['mom30'][i]:8.4f} "
+                f"{v['vol30'][i]:8.5f} {v['vwap_dev'][i]:9.5f} {v['score'][i]:7.3f} {v['rank'][i]:4.0f}"
+            )
+        if tty:
+            out.write("\x1b[2J\x1b[H")
+        out.write("\n".join(lines) + "\n\n")
+        out.flush()
+
+    def on_seal(snap: Snapshot) -> None:
+        n_sealed[0] += 1
+        show(snap)
+
+    st = replay(
+        live, eq.trades, start=start, stop=stop, extras=extras, speed=speed, on_revise=show, on_seal=on_seal
+    )
+    snap = live.revise()
+    show(snap, force=True)
+    s = st.summary()
+    print(
+        f"replayed {int(s['ticks']):,} ticks ({st.market_s / 60:.1f} market minutes at {speed:g}x), "
+        f"{len(st.seal_ms)} bars sealed; revise p50 {s['revise_ms_p50']:.2f} ms, "
+        f"seal p50 {s['seal_ms_p50']:.2f} ms"
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run ``demo``, ``validate``, or ``bench`` from the command line.
+
+    Parameters
+    ----------
+    argv : list of str or None, optional
+        Arguments after the program name. ``None`` reads ``sys.argv``.
+
+    Returns
+    -------
+    int
+        0 when the live replay matched batch, 1 otherwise. ``bench`` returns 0.
+    """
+    ap = argparse.ArgumentParser(prog="python -m factorflow.sim", description=__doc__)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    for name in ("demo", "validate", "bench"):
+        sp = sub.add_parser(name)
+        sp.add_argument("--seed", type=int, default=7)
+        if name != "bench":
+            sp.add_argument("--names", type=int, default=100)
+            sp.add_argument("--history-days", type=int, default=252)
+            sp.add_argument("--intraday-days", type=int, default=3)
+        if name == "demo":
+            sp.add_argument("--speed", type=float, default=60.0, help="market seconds per wall second")
+            sp.add_argument("--seconds", type=float, default=15.0, help="wall seconds of live replay")
+            sp.add_argument("--top", type=int, default=10)
+            sp.add_argument("--skip-validation", action="store_true")
+        if name == "validate":
+            sp.add_argument(
+                "--check-open",
+                type=int,
+                default=0,
+                help="also check every Nth open revision against batch (slow)",
+            )
+        if name == "bench":
+            sp.add_argument("--names", type=int, nargs="+", default=[100, 500])
+            sp.add_argument("--minutes", type=int, default=30)
+    a = ap.parse_args(argv)
+
+    if a.cmd == "bench":
+        from factorflow.sim.bench import main as bench_main
+
+        bench_main(tuple(a.names), a.minutes, a.seed)
+        return 0
+
+    from factorflow.sim.validate import validate
+
+    cfg = SimConfig(seed=a.seed, history_days=a.history_days, intraday_days=a.intraday_days).scaled(n=a.names)
+    t0 = time.perf_counter()
+    mkt = simulate(cfg)
+    print(f"== simulated market (seed {a.seed}, {time.perf_counter() - t0:.1f} s)")
+    print(mkt.summary())
+    ok = True
+    if a.cmd == "validate" or not a.skip_validation:
+        print("\n== live replay vs batch")
+        ok = _verify(mkt, getattr(a, "check_open", 0))
+        print("\n== validation against the planted truth")
+        print(validate(mkt).report())
+    if a.cmd == "demo":
+        print(f"\n== live: last session replayed at {a.speed:g}x for {a.seconds:g} s (Ctrl-C to stop)")
+        try:
+            _live_view(mkt, a.speed, a.seconds, a.top)
+        except KeyboardInterrupt:
+            print("stopped")
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

factorflow/sim/bench.py

@@ -0,0 +1,130 @@
+"""Replay throughput and latency on simulated minute data (``python -m factorflow.sim bench``)."""
+
+from __future__ import annotations
+
+import os
+import platform
+import time
+from dataclasses import dataclass
+
+import numpy as np
+
+from factorflow.graph import compile_pipeline
+from factorflow.grid import NS, BarGrid
+from factorflow.live import LiveEngine
+from factorflow.sim.market import SimConfig, simulate
+from factorflow.sim.pipelines import minute_pipeline, wide_pipeline
+from factorflow.sim.replay import Extras, ReplayStats, replay
+
+
+@dataclass
+class BenchRow:
+    """One benchmark measurement.
+
+    Notes
+    -----
+    ``names`` is the equity count. ``nodes`` is the compiled graph size.
+    ``stats`` comes from :meth:`ReplayStats.summary`.
+    """
+
+    names: int
+    pipeline: str
+    nodes: int
+    mode: str
+    stats: dict[str, float]
+
+
+def run_bench(
+    names: tuple[int, ...] = (100, 500),
+    minutes: int = 30,
+    seed: int = 7,
+    pipelines: tuple[str, ...] = ("demo", "wide100"),
+) -> list[BenchRow]:
+    """Replay the first ``minutes`` of a session for each universe size.
+
+    The window is the busiest part of the intraday U-shape. The engine is a
+    minute-grid :class:`~factorflow.live.LiveEngine` warmed on the previous
+    session.
+
+    Modes are ``ingest`` (ticks and seals, no revisions), ``revise@1s`` (a
+    revision every second of market time), and ``revise@tick`` (a revision
+    after every trade, first 5 minutes only).
+    """
+    rows: list[BenchRow] = []
+    grid = BarGrid("XNYS", "1min")
+    for n in names:
+        cfg = SimConfig(seed=seed, history_days=30, intraday_days=2, crypto=None).scaled(n=n)
+        eq = simulate(cfg).equities
+        panel, _ = eq.time_bars("1min")
+        first_day_bars = int(np.searchsorted(panel.times, eq.closes[eq.intraday[0]], side="right"))
+        hist = panel.head(first_day_bars)
+        extras = Extras(panel.times, {k: panel.columns[k] for k in ("signal", "industry")})
+        t_open = int(eq.opens[eq.intraday[1]])
+        start = int(np.searchsorted(eq.trades.ts, t_open, side="left"))
+        stop = int(np.searchsorted(eq.trades.ts, t_open + minutes * 60 * NS, side="left"))
+        stop5 = int(np.searchsorted(eq.trades.ts, t_open + 5 * 60 * NS, side="left"))
+        for pname in pipelines:
+            pipe = minute_pipeline() if pname == "demo" else wide_pipeline(100)
+            g = compile_pipeline(pipe)
+            for mode, kw, hi in (
+                ("ingest", {"revise_every_ns": None}, stop),
+                ("revise@1s", {"revise_every_ns": NS}, stop),
+                ("revise@tick", {"revise_every_ns": None, "revise_every_ticks": 1}, stop5),
+            ):
+                live = LiveEngine(g, hist, grid)
+                st = ReplayStats()
+                replay(live, eq.trades, start=start, stop=hi, extras=extras, stats=st, **kw)  # type: ignore[arg-type]
+                rows.append(BenchRow(n, pname, len(g.order), mode, st.summary()))
+    return rows
+
+
+def format_bench(rows: list[BenchRow], minutes: int) -> str:
+    """Format benchmark rows as a text table.
+
+    Parameters
+    ----------
+    rows : list of BenchRow
+        Measurements from :func:`run_bench`.
+    minutes : int
+        How many minutes of the session were replayed, except the per-tick mode, which uses 5.
+
+    Returns
+    -------
+    str
+        A header plus one line per row. Latencies are milliseconds.
+    """
+    L = [
+        f"factorflow replay benchmark: XNYS 1-minute grid, first {minutes} min of a session "
+        f"(revise@tick: first 5 min)",
+        f"python {platform.python_version()}  numpy {np.__version__}  "
+        f"{platform.processor() or platform.machine()}  cpus={os.cpu_count()}",
+        f"  {'names':>5} {'pipeline':<8} {'nodes':>5} {'mode':<11} {'ticks':>8} {'ticks/s':>9} "
+        f"{'x realtime':>10} {'tick us':>7} {'revise p50/p95 ms':>18} {'seal p50/p95 ms':>16}",
+    ]
+    for r in rows:
+        s = r.stats
+        rev = f"{s['revise_ms_p50']:.2f}/{s['revise_ms_p95']:.2f}" if s["revises"] > 1 else "-"
+        L.append(
+            f"  {r.names:>5} {r.pipeline:<8} {r.nodes:>5} {r.mode:<11} {int(s['ticks']):>8} "
+            f"{s['ticks_per_s']:>9,.0f} {s['speedup_vs_realtime']:>10,.0f} {s['tick_us']:>7.2f} "
+            f"{rev:>18} {s['seal_ms_p50']:.2f}/{s['seal_ms_p95']:.2f}"
+        )
+    return "\n".join(L)
+
+
+def main(names: tuple[int, ...], minutes: int, seed: int) -> None:
+    """Run the benchmark and print the table.
+
+    Parameters
+    ----------
+    names : tuple of int
+        Equity universe sizes.
+    minutes : int
+        Minutes of the session to replay.
+    seed : int
+        SimConfig seed.
+    """
+    t0 = time.perf_counter()
+    rows = run_bench(names, minutes, seed)
+    print(format_bench(rows, minutes))
+    print(f"(total {time.perf_counter() - t0:.0f} s)")