quantred 2.4.0__py3-none-any.whl

quantred/data/edges.py

@@ -0,0 +1,81 @@
+"""The pandas edge — the *only* sanctioned crossings between the lazy columnar core
+and pandas.
+
+The lazy compute model keeps the panel in Arrow/Polars and never
+materializes the whole thing into pandas. But pandas is still the right currency at
+two narrow places: a **per-symbol leaf** where the data is provably small (the
+indicator math the strategy already speaks), and the **final report** a human reads.
+
+Routing every crossing through this module makes the edge *greppable*: a
+``to_pandas`` on a full panel is a bug, not a convenience, and a reviewer can find
+every place the column store collapses to row-major pandas by searching for these
+two functions. Nothing else in the codebase should call ``.to_pandas()`` /
+``pl.from_pandas`` on panel-wide data directly.
+
+Requires the ``store`` extra (``polars``).
+"""
+
+from typing import TYPE_CHECKING, Optional, Union
+
+import pandas as pd
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    import polars as pl
+
+
+def to_pandas(
+    frame: Union["pl.DataFrame", "pl.LazyFrame"],
+    *,
+    index: Optional[str] = None,
+    streaming: bool = True,
+) -> pd.DataFrame:
+    """Collapse a Polars frame to pandas — the sanctioned core→edge crossing.
+
+    Accepts either an eager :class:`polars.DataFrame` or a :class:`polars.LazyFrame`;
+    a lazy frame is collected first (with the streaming engine by default, so a large
+    plan materializes in bounded-memory chunks rather than all at once). Pass
+    ``index`` to set a column as the resulting pandas index (e.g. ``"ts"`` for a
+    per-symbol leaf, ``"symbol"`` for a one-timestamp cross-section).
+
+    This is a deliberate narrowing point: call it at a small per-symbol leaf or the
+    final report, never on the full multi-symbol panel.
+    """
+    import polars as pl
+
+    if isinstance(frame, pl.LazyFrame):
+        frame = collect_streaming(frame) if streaming else frame.collect()
+    pdf = frame.to_pandas()
+    if index is not None:
+        if index not in pdf.columns:
+            raise KeyError(f"index column {index!r} not in frame; columns are {list(pdf.columns)}")
+        pdf = pdf.set_index(index)
+    return pdf
+
+
+def from_pandas(df: pd.DataFrame, *, include_index: bool = True) -> "pl.DataFrame":
+    """Lift a pandas frame into Polars — the sanctioned edge→core crossing.
+
+    Used to bring a per-symbol leaf (or a provider frame) back into the columnar
+    core. By default the pandas index is materialized as a column; pass
+    ``include_index=False`` to drop it. A provider/OHLCV frame is typically indexed by
+    an *unnamed* ``DatetimeIndex`` — Polars would materialize that as a column literally
+    named ``"None"``, so we name it first: a ``DatetimeIndex`` becomes ``ts`` (the
+    promise that a time index survives as a real ``ts`` column), any other unnamed
+    index becomes ``index``.
+    """
+    import polars as pl
+
+    if include_index and df.index.name is None:
+        df = df.rename_axis("ts" if isinstance(df.index, pd.DatetimeIndex) else "index")
+    return pl.from_pandas(df, include_index=include_index)
+
+
+def collect_streaming(lazy: "pl.LazyFrame") -> "pl.DataFrame":
+    """Materialize a :class:`polars.LazyFrame` via the **streaming** engine.
+
+    The streaming engine processes the query plan in bounded-memory chunks, so a plan
+    over a larger-than-RAM panel collects without ever holding the whole result. This
+    is the one terminal the lazy helpers in :mod:`quantred.data.compute` are meant to end
+    on; it lives here (next to ``to_pandas``) because materializing *is* the edge.
+    """
+    return lazy.collect(engine="streaming")

quantred/data/features.py

@@ -0,0 +1,120 @@
+"""Feature producers - populate a :class:`FeaturePanel`'s columns from scanned bars.
+
+Each producer is the cross-sectional analog of an indicator: it reads the
+point-in-time bars for the universe and writes one or more feature columns. Three
+today - a risk producer (beta + residual volatility), a factor-exposure producer
+(the risk model's standardized exposures, for factor-neutral alphas), and a
+generic score producer (apply any per-name scorer). New producers (liquidity,
+transaction-cost params) slot in the same way without the consumers changing.
+"""
+
+from typing import Callable, Dict, Optional, Sequence
+
+import pandas as pd
+
+from quantred.analytics import metrics as m
+from quantred.data.panel import FeaturePanel
+from quantred.indicators import indicators
+
+#: A scorer reads one symbol's (as-of-sliced) bars and returns a continuous score.
+Scorer = Callable[[pd.DataFrame], float]
+
+#: Panel column prefix for factor-exposure features (``exp_market``, ``exp_size``, ...).
+EXPOSURE_PREFIX = "exp_"
+
+
+def add_risk_features(
+    panel: FeaturePanel,
+    bars: Dict[str, pd.DataFrame],
+    benchmark_bars: Optional[pd.DataFrame],
+    periods_per_year: float,
+    default_residual_vol: float = 0.20,
+) -> FeaturePanel:
+    """Write ``beta`` and ``residual_vol`` columns (annualized) for each name.
+
+    Residual volatility strips the benchmark-explained part of each return; it is
+    the ``sigma`` the alpha-scaling identity needs. With no benchmark series
+    available it falls back to total volatility (beta unknown, set to 1.0) and
+    records ``benchmark_available=False`` in the panel meta.
+
+    **The alpha/risk vol seam.** This ``residual_vol`` is
+    computed **unconditionally** (a flat trailing-window figure) and deliberately
+    stays that way — it scales alpha magnitude (``α = ω·IC·z``, Case 1) and the
+    refinement's Case logic, and an alpha shouldn't double in size just because last week was
+    wild. Only the *risk-model* Σ used for portfolio construction / tracking-error
+    (:mod:`quantred.risk.conditional`) conditions on recent volatility. Do not "unify"
+    the two into one vol-squared concept — that reintroduces exactly the position-
+    doubling-in-a-vol-spike bug this separation exists to prevent.
+    """
+    available = benchmark_bars is not None and not benchmark_bars.empty
+    betas: Dict[str, float] = {}
+    vols: Dict[str, float] = {}
+    for symbol in panel.symbols:
+        frame = bars.get(symbol)
+        if frame is None or frame.empty:
+            continue
+        close = frame["close"]
+        if available:
+            beta = indicators.calculate_beta(close, benchmark_bars["close"])
+            vol = indicators.calculate_residual_volatility(
+                close, benchmark_bars["close"], beta, periods_per_year
+            )
+        else:
+            beta = 1.0
+            vol = m.annualized_volatility(close.pct_change().dropna(), int(round(periods_per_year)))
+        betas[symbol] = beta
+        vols[symbol] = vol or default_residual_vol
+
+    panel.set("beta", betas)
+    panel.set("residual_vol", vols)
+    panel.meta["benchmark_available"] = available
+    return panel
+
+
+def add_factor_exposure_features(
+    panel: FeaturePanel,
+    bars: Dict[str, pd.DataFrame],
+    benchmark_bars: Optional[pd.DataFrame],
+    factors: Sequence[str],
+) -> FeaturePanel:
+    """Write standardized risk-model factor exposures as ``exp_<factor>`` columns.
+
+    The columns come from the same exposure builder the factor risk model uses
+    (:func:`quantred.risk.exposures.build_factor_exposures`), so "factor-neutral" in the
+    alpha pipeline means neutral to the *risk model's* factors — one definition,
+    both places. The market factor reuses the panel's ``beta`` column when present
+    (same regression, already run by :func:`add_risk_features`). If the build
+    qualifies fewer than two names, **no columns are written** — the refinement then
+    falls back to plain-beta neutralization rather than silently doing nothing.
+    Names missing a single factor get a NaN exposure (mean-imputed downstream).
+    """
+    from quantred.risk.exposures import build_factor_exposures
+
+    if not list(factors):
+        return panel
+    betas = panel.get("beta") if panel.has("beta") else None
+    frame = build_factor_exposures(bars, benchmark_bars, factors=list(factors), betas=betas)
+    if frame.empty:
+        return panel
+    for name in frame.columns:
+        panel.set(f"{EXPOSURE_PREFIX}{name}", frame[name])
+    return panel
+
+
+def add_score_feature(
+    panel: FeaturePanel,
+    scorer: Scorer,
+    bars: Dict[str, pd.DataFrame],
+    column: str = "score",
+) -> FeaturePanel:
+    """Write a continuous ``score`` column by applying ``scorer`` to each name's bars."""
+    scores: Dict[str, float] = {}
+    for symbol in panel.symbols:
+        frame = bars.get(symbol)
+        if frame is None or frame.empty:
+            continue
+        value = scorer(frame)
+        if value is not None and not pd.isna(value):
+            scores[symbol] = float(value)
+    panel.set(column, scores)
+    return panel

quantred/data/panel.py

@@ -0,0 +1,61 @@
+"""The cross-sectional feature panel - all the data in one place, as of one moment.
+
+A :class:`FeaturePanel` is the single table every cross-sectional module reads from
+and writes to: rows are symbols, columns are features (a signal's score, residual
+volatility, beta, factor exposures, and - once refined - z and alpha). It is the
+unit that makes the active-management stack cohesive: a strategy/scanner *produces*
+a score column, the risk layer produces beta/residual_vol, the alpha layer consumes
+those and produces z/alpha, and portfolio construction consumes the whole row.
+Stacked over time these panels are the data a factor-importance or signal-combination
+search runs over - "which factors matter right now" is a query against the panel.
+
+Point-in-time by construction: a panel is built from a :mod:`quantred.data.scan` at one
+``as_of``, so every column is leakage-safe. Today it wraps a pandas frame (small
+universes); the same shape is what an Arrow/Polars columnar store would back at
+scale, so the consumers above never learn where the data physically lives.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Mapping, Union
+
+import pandas as pd
+
+Values = Union[pd.Series, Mapping[str, float]]
+
+
+@dataclass
+class FeaturePanel:
+    """A symbol-indexed table of features for one universe at one ``as_of``."""
+
+    as_of: datetime
+    features: pd.DataFrame
+    #: Cross-sectional flags set by producers/refiners (e.g. low_confidence).
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def for_universe(cls, as_of: datetime, symbols: List[str]) -> "FeaturePanel":
+        """An empty panel whose rows are ``symbols`` (column-less until populated)."""
+        index = pd.Index(list(dict.fromkeys(symbols)), name="symbol")
+        return cls(as_of=as_of, features=pd.DataFrame(index=index))
+
+    def set(self, name: str, values: Values) -> "FeaturePanel":
+        """Add/replace a feature column, aligned to the panel's symbols."""
+        series = values if isinstance(values, pd.Series) else pd.Series(values, dtype=float)
+        self.features[name] = series.reindex(self.features.index)
+        return self
+
+    def get(self, name: str) -> pd.Series:
+        """Return a feature column (raises ``KeyError`` if absent)."""
+        return self.features[name]
+
+    def has(self, name: str) -> bool:
+        return name in self.features.columns
+
+    @property
+    def symbols(self) -> List[str]:
+        return list(self.features.index)
+
+    @property
+    def columns(self) -> List[str]:
+        return list(self.features.columns)

quantred/data/scan.py

@@ -0,0 +1,85 @@
+"""The data-access seam - one place to scan bars point-in-time.
+
+Every cross-sectional research flow (alphas, and soon risk / portfolio /
+information analysis) needs the same thing: the bars for a universe *as of* a
+rebalance timestamp, with nothing after it. This module is that single seam, and
+the single home of the leakage guard (``<= as_of``) - so no call site re-implements
+slicing and accidentally lets a future bar through.
+
+Today it is pandas-backed over :class:`MarketDataClient`. The ``scan(universe,
+timeframe, as_of, lookback)`` signature is deliberately the same contract an
+out-of-core Arrow/Polars/DuckDB source would implement, so growing the storage
+tier is a new adapter behind this interface, not a rewrite of the layers above.
+"""
+
+from datetime import datetime
+from typing import Dict, List, Protocol, runtime_checkable
+
+import pandas as pd
+
+from quantred.marketdata.client import MarketDataClient, TimeframeLike
+from quantred.utils.timeutils import shift_days
+
+
+@runtime_checkable
+class BarSource(Protocol):
+    """A point-in-time bar source: bars for a universe, never past ``as_of``."""
+
+    def scan(
+        self, universe: List[str], timeframe: TimeframeLike, as_of: datetime, lookback_days: int
+    ) -> Dict[str, pd.DataFrame]:
+        """Return ``{symbol: OHLCV}`` over ``(as_of - lookback, as_of]`` per symbol."""
+        ...
+
+
+class ClientBarSource:
+    """A :class:`BarSource` backed by the existing :class:`MarketDataClient`."""
+
+    def __init__(self, data_client: MarketDataClient):
+        self._client = data_client
+
+    def scan(
+        self,
+        universe: List[str],
+        timeframe: TimeframeLike,
+        as_of: datetime,
+        lookback_days: int = 365,
+    ) -> Dict[str, pd.DataFrame]:
+        # Fetch with end == as_of (defends leakage for real feeds), then slice
+        # defensively (synthetic/fake feeds ignore the window, so the slice is the
+        # real guard). De-dup the universe so a benchmark listed twice is fetched once.
+        start = shift_days(as_of, -lookback_days)
+        raw = self._client.get_bars(list(dict.fromkeys(universe)), timeframe, start, as_of)
+        out: Dict[str, pd.DataFrame] = {}
+        for symbol, frame in raw.items():
+            sliced = slice_to_as_of(frame, as_of)
+            if sliced is not None and not sliced.empty:
+                out[symbol] = sliced
+        return out
+
+
+def slice_to_as_of(frame: pd.DataFrame, as_of: datetime) -> pd.DataFrame:
+    """Return only bars at or before ``as_of`` - the leakage guard.
+
+    Aligns the cutoff to the frame in **both** directions. Handling only the aware
+    index / naive cutoff pair was enough while callers passed naive cutoffs, and
+    stopped being enough the moment the scanner began localizing its window
+    unconditionally: an aware cutoff against a naive index raised ``Invalid
+    comparison between dtype=datetime64[ns] and Timestamp`` on any provider whose
+    bar timestamps carry no zone - a frame this guard had always accepted.
+
+    Being the single seam means owning the whole coercion, not half of it. A naive
+    bar index is taken as UTC, the assumption
+    :func:`~quantred.utils.timeutils.localize_index_to_new_york` already makes about
+    a provider's timestamps.
+    """
+    if frame is None or frame.empty:
+        return frame
+    ts = pd.Timestamp(as_of)
+    idx = frame.index
+    if isinstance(idx, pd.DatetimeIndex):
+        if idx.tz is not None:
+            ts = ts.tz_localize(idx.tz) if ts.tzinfo is None else ts.tz_convert(idx.tz)
+        elif ts.tzinfo is not None:
+            ts = ts.tz_convert("UTC").tz_localize(None)
+    return frame.loc[idx <= ts]

quantred/data/store.py

@@ -0,0 +1,269 @@
+"""Out-of-core bar storage: a Parquet/Arrow store behind the same ``scan()`` seam.
+
+The load-bearing assumption everywhere else is that an in-memory pandas frame is the
+currency between layers — fine for a few hundred symbols of daily bars, fatal for the
+volumes a real system sees. This module is the first storage tier that scales past
+RAM: bars live in **partitioned Parquet**, and a scan reads them through Arrow with
+**predicate/projection pushdown** — the ``as_of`` filter is applied at the storage
+layer, so a point-in-time scan physically never reads rows after ``as_of``.
+
+Crucially it implements the *same* :class:`~quantred.data.scan.BarSource` contract as the
+in-memory :class:`~quantred.data.scan.ClientBarSource`, so it is a **drop-in**: the alpha,
+risk, and portfolio layers above never learn where the data physically lives. pandas
+appears only at the edge (the returned per-symbol frames), never as the cross-layer
+currency. Requires the ``store`` extra (``pyarrow``).
+"""
+
+import shutil
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional, Sequence
+
+import pandas as pd
+
+from quantred.marketdata.client import TimeframeLike
+from quantred.marketdata.timeframe import Timeframe
+from quantred.utils.timeutils import shift_days
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    import polars as pl
+
+#: The enforced bar schema columns (the Arrow boundary contract).
+BAR_COLUMNS = ["ts", "symbol", "open", "high", "low", "close", "volume"]
+
+
+class ParquetBarStore:
+    """A point-in-time :class:`BarSource` backed by symbol/date-partitioned Parquet.
+
+    Layout: ``<root>/<timeframe>/symbol=<SYM>/year=<YYYY>/part.parquet``. One timeframe
+    per subtree, then a Hive ``symbol=…/year=…`` partitioning that serves both access
+    patterns (a symbol's full history, and a date window across symbols). A scan prunes
+    to the **partition files that overlap the date window** (file-level pruning) and
+    then pushes the exact ``as_of`` window into the Parquet reader (row-group pruning),
+    so a one-year scan of a decade-deep store physically opens ~one year of files.
+
+    Year is the partition granularity: coarse enough that daily/most-intraday data
+    keeps a sane file count, fine enough that a typical lookback window touches one or
+    two partitions. Writing a symbol replaces its whole subtree (all years).
+    """
+
+    def __init__(self, root: str):
+        self.root = Path(root)
+
+    # ------------------------------------------------------------------ #
+    # Write
+    # ------------------------------------------------------------------ #
+    def write(self, bars: Dict[str, pd.DataFrame], timeframe: TimeframeLike = "1Day") -> None:
+        """Persist ``{symbol: OHLCV}`` to the store (replacing each symbol's subtree).
+
+        Each symbol's bars are split into ``year=<YYYY>`` partition files (by the bar's
+        UTC year, the same basis the scan window prunes on). The symbol's existing
+        subtree is removed first so a rewrite never leaves a stale year partition behind.
+        """
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+
+        tf = str(Timeframe.parse(timeframe) if isinstance(timeframe, str) else timeframe)
+        for symbol, frame in bars.items():
+            if frame is None or frame.empty:
+                continue
+            records = self._to_records(symbol, frame)
+            sdir = self._symbol_dir(tf, symbol)
+            if sdir.exists():
+                shutil.rmtree(sdir)  # full replace: no stale year partitions survive
+            for year, group in records.groupby(records["ts"].dt.year):
+                part = sdir / f"year={int(year)}"
+                part.mkdir(parents=True, exist_ok=True)
+                table = pa.Table.from_pandas(group, preserve_index=False)
+                pq.write_table(table, part / "part.parquet")
+
+    def delete_symbol(self, symbol: str, timeframe: TimeframeLike = "1Day") -> None:
+        """Remove a symbol's entire cached subtree (all years) for one timeframe.
+
+        The corporate-action lever: a split/dividend backfill invalidates
+        everything cached for a symbol, not just a sub-range, so the next fetch
+        starts clean rather than merging fresh data with stale pre-adjustment bars.
+        """
+        tf = str(Timeframe.parse(timeframe) if isinstance(timeframe, str) else timeframe)
+        sdir = self._symbol_dir(tf, symbol)
+        if sdir.exists():
+            shutil.rmtree(sdir)
+
+    # ------------------------------------------------------------------ #
+    # Scan (the BarSource contract)
+    # ------------------------------------------------------------------ #
+    def scan(
+        self, universe: List[str], timeframe: TimeframeLike, as_of: datetime, lookback_days: int = 365
+    ) -> Dict[str, pd.DataFrame]:
+        """Return ``{symbol: OHLCV}`` over ``(as_of - lookback, as_of]`` — pushed down.
+
+        Only the ``year=`` partition files overlapping the window are opened (file-level
+        pruning), and the timestamp window is then a Parquet filter, so rows outside it
+        (in particular any bar *after* ``as_of``) are never read into memory.
+        """
+        import pyarrow.compute as pc
+        import pyarrow.dataset as ds
+
+        tf = str(Timeframe.parse(timeframe) if isinstance(timeframe, str) else timeframe)
+        if not (self.root / tf).exists():
+            return {}
+
+        start, end = self._window(as_of, lookback_days)
+        out: Dict[str, pd.DataFrame] = {}
+        for symbol in dict.fromkeys(universe):
+            files = self._window_files(tf, symbol, start, end)
+            if not files:
+                continue
+            dataset = ds.dataset(files, format="parquet")
+            # Predicate pushdown: only ts in (start, end] is read off disk.
+            table = dataset.to_table(filter=(pc.field("ts") > start) & (pc.field("ts") <= end))
+            if table.num_rows:
+                out[symbol] = self._from_records(table.to_pandas())
+        return out
+
+    # ------------------------------------------------------------------ #
+    # Partition geometry (symbol=…/year=… Hive layout)
+    # ------------------------------------------------------------------ #
+    def _tf_root(self, timeframe: TimeframeLike) -> Path:
+        tf = str(Timeframe.parse(timeframe) if isinstance(timeframe, str) else timeframe)
+        return self.root / tf
+
+    def _symbol_dir(self, tf: str, symbol: str) -> Path:
+        return self.root / tf / f"symbol={symbol}"
+
+    @staticmethod
+    def _window(as_of: datetime, lookback_days: int) -> tuple:
+        """Normalize ``as_of`` to UTC and return the ``(start, end]`` window."""
+        moment = pd.Timestamp(as_of)
+        # Shift in the value's own zone first: in UTC, a New York midnight minus N days
+        # crosses DST and lands an hour off the boundary.
+        start = shift_days(moment, -lookback_days)
+
+        def utc(t):
+            return t.tz_localize("UTC") if t.tzinfo is None else t.tz_convert("UTC")
+
+        return utc(start), utc(moment)
+
+    def _year_files(self, sdir: Path, lo_year: Optional[int], hi_year: Optional[int]) -> List[str]:
+        """``part.parquet`` paths in ``sdir``'s ``year=`` partitions within ``[lo, hi]``."""
+        if not sdir.exists():
+            return []
+        files = []
+        for ydir in sorted(sdir.glob("year=*")):
+            try:
+                year = int(ydir.name.split("=", 1)[1])
+            except (IndexError, ValueError):
+                continue
+            if (lo_year is None or year >= lo_year) and (hi_year is None or year <= hi_year):
+                part = ydir / "part.parquet"
+                if part.exists():
+                    files.append(str(part))
+        return files
+
+    def _window_files(self, tf: str, symbol: str, start: pd.Timestamp, end: pd.Timestamp) -> List[str]:
+        """The partition files for ``symbol`` overlapping ``[start.year, end.year]``."""
+        return self._year_files(self._symbol_dir(tf, symbol), start.year, end.year)
+
+    def partition_paths(
+        self,
+        universe: Sequence[str],
+        timeframe: TimeframeLike = "1Day",
+        as_of: Optional[datetime] = None,
+        lookback_days: int = 365,
+    ) -> List[str]:
+        """The Parquet partition files backing ``universe`` (existing partitions only).
+
+        The file list a DuckDB/Polars set-based query (:func:`quantred.data.compute.sql_query`)
+        or :meth:`scan_lazy` reads over. With ``as_of`` given the list is pruned to the
+        ``year=`` partitions overlapping ``(as_of - lookback, as_of]`` (file-level
+        date pruning); without it, every year partition for the symbols is returned.
+        """
+        tf = str(Timeframe.parse(timeframe) if isinstance(timeframe, str) else timeframe)
+        if as_of is None:
+            lo_year = hi_year = None
+        else:
+            start, end = self._window(as_of, lookback_days)
+            lo_year, hi_year = start.year, end.year
+        paths: List[str] = []
+        for symbol in dict.fromkeys(universe):
+            paths.extend(self._year_files(self._symbol_dir(tf, symbol), lo_year, hi_year))
+        return paths
+
+    def scan_lazy(
+        self,
+        universe: List[str],
+        timeframe: TimeframeLike,
+        as_of: datetime,
+        lookback_days: int = 365,
+        columns: Optional[Sequence[str]] = None,
+    ) -> "pl.LazyFrame":
+        """Return a **lazy** long-format panel for ``universe`` over ``(as_of - lookback, as_of]``.
+
+        The compute-tier counterpart to :meth:`scan`: instead of materializing
+        per-symbol pandas frames, it returns a Polars :class:`~polars.LazyFrame` over
+        the Parquet partitions with the ``as_of`` window pushed into the reader as a
+        predicate (rows after ``as_of`` are physically never read) and an optional
+        column projection pushed down too. ``as_of`` may be naive (treated as UTC) or
+        tz-aware; the window matches the eager :meth:`scan` exactly (strict ``>`` on the
+        lower bound, ``<=`` on ``as_of``). Nothing executes until a terminal collect
+        (:func:`quantred.data.edges.collect_streaming`). The leaf scan + pushdown stream; a
+        downstream ``over(...)``/sort buffers (see :mod:`quantred.data.compute` on what is
+        genuinely bounded-memory). Requires the ``store`` extra (``polars``).
+
+        An empty universe yields an empty but **schema-carrying** LazyFrame, so a
+        window helper composed onto it resolves its source column and returns an empty
+        frame rather than raising (the lazy analogue of :meth:`scan` returning ``{}``).
+        """
+        import polars as pl
+
+        schema = {
+            "ts": pl.Datetime("us", "UTC"),
+            "symbol": pl.Utf8,
+            "open": pl.Float64,
+            "high": pl.Float64,
+            "low": pl.Float64,
+            "close": pl.Float64,
+            "volume": pl.Float64,
+        }
+
+        # File-level date pruning: only year partitions overlapping the window.
+        paths = self.partition_paths(universe, timeframe, as_of=as_of, lookback_days=lookback_days)
+        if not paths:
+            empty = pl.DataFrame(schema=schema).lazy()
+            if columns is not None:
+                empty = empty.select(list(dict.fromkeys(["ts", "symbol", *columns])))
+            return empty
+
+        start, end = self._window(as_of, lookback_days)
+        # hive_partitioning=false: `symbol` is a real column in each file; don't also
+        # synthesize it from the `symbol=…` directory (that collides).
+        lf = pl.scan_parquet(paths, hive_partitioning=False)
+        # Predicate pushdown: the window (in particular the as_of upper bound) is
+        # pushed into the Parquet SCAN, so post-as_of rows are never read.
+        lf = lf.filter((pl.col("ts") > pl.lit(start)) & (pl.col("ts") <= pl.lit(end)))
+        if columns is not None:
+            keep = list(dict.fromkeys(["ts", "symbol", *columns]))
+            lf = lf.select(keep)
+        return lf
+
+    # ------------------------------------------------------------------ #
+    # The pandas edge (the only sanctioned crossings)
+    # ------------------------------------------------------------------ #
+    @staticmethod
+    def _to_records(symbol: str, frame: pd.DataFrame) -> pd.DataFrame:
+        """OHLCV frame (ts index) → a flat, schema-typed records frame for Parquet."""
+        records = frame.reset_index()
+        records = records.rename(columns={records.columns[0]: "ts"})
+        ts = pd.to_datetime(records["ts"], utc=True)
+        records["ts"] = ts
+        records["symbol"] = symbol
+        for col in ("open", "high", "low", "close", "volume"):
+            records[col] = records[col].astype("float64")
+        return records[BAR_COLUMNS]
+
+    @staticmethod
+    def _from_records(records: pd.DataFrame) -> pd.DataFrame:
+        """Records frame → an OHLCV frame indexed by a sorted UTC timestamp."""
+        frame = records.sort_values("ts").set_index("ts")
+        frame.index = pd.DatetimeIndex(frame.index, name="timestamp")
+        return frame[["open", "high", "low", "close", "volume"]]

quantred/demo/__init__.py

@@ -0,0 +1,22 @@
+"""What `quantred demo` runs on, and what a bare install falls back to.
+
+Not examples to copy — that is ``example``, which
+``quantred init --example-pack`` hands you as a repository you own. These exist for
+three narrower jobs:
+
+* ``quantred demo`` needs a strategy to put through the pipeline, offline and with no
+  keys, and it cannot depend on anything the user has not installed yet.
+* ``--strategy`` and ``--scanner`` need a default that resolves on a bare install, or
+  every command fails at lookup rather than saying something useful.
+* There should be one smallest complete strategy in the tree to read.
+
+They are registered through the same two entry-point groups a private pack uses, and
+declared in this project's own ``pyproject.toml``. That is deliberate: the discovery
+path the whole private-strategy feature rests on is then exercised by every install
+rather than only by tests. The registry names them directly as well, because
+enumeration order across distributions is undefined and these two names are reserved -
+a pack cannot take them, and which class answers to them is not left to chance.
+
+``demo_trend`` and ``demo_volume`` are named for what they are for. Neither is an edge,
+and neither is where your work goes.
+"""