quantred 2.4.0__py3-none-any.whl

quantred/costs/__init__.py

@@ -0,0 +1,11 @@
+"""Transaction costs: price the cost of trading so research metrics are net, not gross.
+
+A parametric model (commission + half-spread + square-root market impact) charged on
+every simulated fill, and an alpha haircut + linear turnover term for the optimizer.
+Research-clock only - the live path gets real fills from the broker.
+"""
+
+from quantred.costs.base import CostModel, Trade, TradeCost
+from quantred.costs.parametric import ParametricCostModel
+
+__all__ = ["CostModel", "Trade", "TradeCost", "ParametricCostModel"]

quantred/costs/base.py

@@ -0,0 +1,126 @@
+"""Transaction-cost model core - what it costs to change a position.
+
+Every Sharpe and equity curve the backtester produces is *gross* until a cost model
+charges trading. Gross results are the most reliable way to fool yourself: the
+strategies that look best in-sample are very often the highest-turnover ones, which
+are exactly the ones costs destroy. Active management treats cost as first-class -
+it caps the transfer coefficient in ``IR ≈ TC·IC·√BR``.
+
+This module owns the shared types: :class:`Trade` (what is being traded, plus the
+liquidity context), :class:`TradeCost` (the commission / spread / impact breakdown),
+and the :class:`CostModel` interface. The concrete model lives in
+:mod:`quantred.costs.parametric`.
+
+**A pure pricing helper, and both clocks may use it.** It said "research-clock only:
+the live path gets real fills from the broker" long after that stopped being true -
+``execution.live_trader._cost_estimate`` imports it and calls ``cost()`` on the order
+path, deliberately, so the modelled number in a live report is comparable with the one
+the backtest was judged on. Nothing is broken by that: ``costs/`` is not one of the
+packages the trade-clock rule names, and what lives here is arithmetic over its
+arguments - no model, no optimizer, no database, no network. But a stale boundary
+comment is how the next person justifies a real violation, so the claim is corrected
+rather than left to be discovered.
+
+What must stay true is the reason the boundary was drawn: what this computes is a
+*prediction*, and a broker fee is an *observation*. The live path records them
+separately and must never merge them.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class Trade:
+    """A trade to be priced, with the trailing liquidity context it's priced against."""
+
+    symbol: str
+    shares: float  # signed; cost is charged on the absolute size
+    price: float
+    adv: float  # trailing average daily volume (shares), as-of the trade
+    daily_vol: float  # trailing daily return volatility (fraction), as-of the trade
+    spread: float = 0.0005  # quoted spread (fraction, e.g. 0.0005 = 5 bps)
+
+    @property
+    def notional(self) -> float:
+        return abs(self.shares) * self.price
+
+    @property
+    def participation(self) -> float:
+        """Fraction of a day's volume this trade demands (`|q| / ADV`)."""
+        return abs(self.shares) / self.adv if self.adv > 0 else float("inf")
+
+
+@dataclass
+class TradeCost:
+    """The decomposed dollar cost of a trade (commission + spread + impact)."""
+
+    commission: float
+    spread_cost: float
+    impact_cost: float
+    participation: float
+    capped: bool  # participation exceeded the realistic cap → fill is optimistic
+
+    @property
+    def total(self) -> float:
+        return self.commission + self.spread_cost + self.impact_cost
+
+
+class CostModel(ABC):
+    """Prices the cost of changing a position."""
+
+    @abstractmethod
+    def cost(self, trade: Trade) -> TradeCost:
+        """Return the decomposed dollar cost of ``trade``."""
+
+    def cost_rate(self, trade: Trade) -> float:
+        """One-way cost as a fraction of traded notional."""
+        notional = trade.notional
+        return self.cost(trade).total / notional if notional > 0 else 0.0
+
+    def turnover_cost_rate(self, spread: float = None) -> float:
+        """Linear (size-independent) one-way cost per unit of traded notional.
+
+        The size-independent part of a trade's cost — the optimizer's L1 turnover
+        coefficient (the cost-aware objective). A model with no linear cost returns 0
+        (the default);
+        :class:`~quantred.costs.parametric.ParametricCostModel` returns commission + s/2.
+        """
+        return 0.0
+
+    def impact_coefficient(self, daily_vol: float, adv_dollar: float, capital: float) -> float:
+        """√-impact coefficient ``k`` for the optimizer's ``Σ kᵢ·|Δwᵢ|^{3/2}`` term.
+
+        The realistic square-root impact is convex in size but not quadratic, so it
+        enters the portfolio optimizer's objective as a conic term rather than the
+        risk quadratic.
+        ``k`` is *per unit of capital, per rebalance*: the impact cost as a fraction of
+        capital of trading ``|Δwᵢ|`` (a fraction of the ``capital`` book) is
+        ``k·|Δwᵢ|^{3/2}``. A model with no impact returns 0 (the default).
+        """
+        return 0.0
+
+    def carry_cost(self, notional: float, is_short: bool, holding_years: float) -> float:
+        """Financing cost of *holding* a position (borrow on shorts). Default: none."""
+        return 0.0
+
+    def borrow_rate(self, override: Optional[float] = None) -> float:
+        """Annualized borrow rate (fraction of notional per year) charged on a short
+        holding - the portfolio optimizer's per-name short-side cost tilt,
+        threaded the same way :meth:`turnover_cost_rate` threads the per-name spread.
+        ``override`` is a per-name rate from :class:`~quantred.portfolio.optimizer.CostInputs`
+        (a locate-desk quote or a manual hard-to-borrow override); a model with no
+        borrow cost returns 0 regardless of ``override``.
+        """
+        return 0.0
+
+    def annual_cost_rate(self, trade: Trade, holding_period_years: float) -> float:
+        """Round-trip cost amortized over the holding period - the alpha haircut rate.
+
+        A name held for a month pays its round-trip cost ~12x a year; a +4%/yr alpha
+        with a 2% round-trip held one month is deeply negative, not a +4% opportunity.
+        """
+        if holding_period_years <= 0:
+            return float("inf")
+        return 2.0 * self.cost_rate(trade) / holding_period_years

quantred/costs/parametric.py

@@ -0,0 +1,131 @@
+"""Parametric cost model: commission + half-spread + square-root market impact.
+
+For a trade of ``q`` shares at price ``p`` in a name with average daily volume
+``ADV`` and quoted spread ``s``:
+
+    cost($) = commission + (s/2)·|q|·p + impact_rate·|q|·p
+
+with the empirically robust **square-root law** for impact (Almgren et al.):
+
+    impact_rate = η · σ_daily · √(|q| / ADV)
+
+This is concave in size *per share* (√) but convex in *total* cost
+(``|q|·impact_rate ∝ |q|^{3/2}``) - the property that makes an optimizer prefer to
+spread a target across names. A linear impact fallback is offered for a fully
+convex-quadratic optimizer formulation. Defaults are calibrated to liquid US
+equities and are all overridable.
+"""
+
+import math
+from typing import Optional
+
+from quantred.costs.base import CostModel, Trade, TradeCost
+
+
+class ParametricCostModel(CostModel):
+    """Commission + half-spread + (square-root or linear) market impact."""
+
+    def __init__(
+        self,
+        commission_bps: float = 1.0,
+        default_spread_bps: float = 5.0,
+        impact_eta: float = 0.3,
+        participation_cap: float = 0.10,
+        annual_borrow_bps: float = 50.0,
+        linear_impact: bool = False,
+    ):
+        self.commission_rate = commission_bps / 1e4
+        self.default_spread = default_spread_bps / 1e4
+        self.impact_eta = impact_eta
+        self.participation_cap = participation_cap
+        self.annual_borrow_rate = annual_borrow_bps / 1e4  # financing cost to hold a short
+        self.linear_impact = linear_impact  # linear participation (convex-quadratic) vs √-law
+
+    def cost(self, trade: Trade) -> TradeCost:
+        notional = trade.notional
+        if notional <= 0:
+            return TradeCost(0.0, 0.0, 0.0, 0.0, capped=False)
+
+        spread = trade.spread if trade.spread is not None else self.default_spread
+        participation = trade.participation
+
+        commission = self.commission_rate * notional
+        spread_cost = (spread / 2.0) * notional
+        if math.isinf(participation):
+            impact_rate = 0.0  # no ADV info → can't size impact; commission+spread only
+        elif self.linear_impact:
+            impact_rate = self.impact_eta * trade.daily_vol * participation
+        else:
+            impact_rate = self.impact_eta * trade.daily_vol * math.sqrt(participation)
+        impact_cost = impact_rate * notional
+
+        return TradeCost(
+            commission=commission,
+            spread_cost=spread_cost,
+            impact_cost=impact_cost,
+            participation=participation,
+            capped=participation > self.participation_cap,
+        )
+
+    def carry_cost(self, notional: float, is_short: bool, holding_years: float) -> float:
+        """Financing cost of *holding* a position - borrow on shorts, accrued over time.
+
+        Long positions have no borrow cost here (margin financing is a separate, later
+        concern); a short pays ``borrow_rate · notional · holding_years``. This is what
+        stops a short-heavy strategy from being silently flattered.
+        """
+        if not is_short or holding_years <= 0:
+            return 0.0
+        return self.annual_borrow_rate * abs(notional) * holding_years
+
+    def borrow_rate(self, override: float = None) -> float:
+        """Annualized borrow rate - the flat ``annual_borrow_bps`` default, or a
+        per-name override (a locate-desk quote) when the caller has one.
+        """
+        return self.annual_borrow_rate if override is None else override
+
+    def turnover_cost_rate(self, spread: float = None) -> float:
+        """Linear cost per unit of turnover notional - the optimizer's L1 cost term.
+
+        The size-independent part (commission + half-spread); impact is omitted here
+        because it's non-linear in trade size (it belongs in the √-law accounting).
+        """
+        s = self.default_spread if spread is None else spread
+        return self.commission_rate + s / 2.0
+
+    def impact_coefficient(self, daily_vol: float, adv_dollar: float, capital: float) -> float:
+        """√-impact coefficient ``k`` (per unit capital) for the optimizer's conic term.
+
+        The √-law impact rate is ``η·σ·√(participation)`` with ``participation =
+        |q|/ADV``. Trading ``|Δw|`` of a ``capital``-dollar book is ``q = |Δw|·capital/p``
+        shares, so ``participation = |Δw|·capital / (p·ADV) = |Δw|·capital / ADV$``. The
+        impact cost as a *fraction of capital* is then ``η·σ·√(capital/ADV$)·|Δw|^{3/2}``,
+        so ``k = η·σ·√(capital/ADV$)``. Zero when ADV is unavailable (mirrors
+        :meth:`cost`, which charges no impact without volume) or in linear-impact mode
+        (that fallback is quadratic-in-size, not conic, and isn't fed to the optimizer).
+        """
+        # `not (x > 0)` (rather than `x <= 0`) also rejects NaN, so a bad ADV/vol input
+        # drops the impact term instead of poisoning the solve with a NaN coefficient.
+        if self.linear_impact or not (adv_dollar > 0) or not (capital > 0) or not (daily_vol > 0):
+            return 0.0
+        return self.impact_eta * daily_vol * math.sqrt(capital / adv_dollar)
+
+
+def cost_curvature(k_imp: float, typical_trade: float) -> Optional[float]:
+    """Local quadratic curvature ``c2 = d2cost/dDeltaw2`` of the √-impact term at a
+    representative trade size: ``cost(Δw) = k·|Δw|^{3/2}`` (impact
+    only - the linear term is a kink, not curvature, at any Δw != 0), so
+    ``d2/dΔw2 = (3/4)·k·|Δw|^{-1/2}``. This is the curvature a quadratic-cost
+    closed form needs to derive the trading rate κ
+    (:func:`quantred.portfolio.policy.derive_kappa`); our cost is linear+3/2-power, so
+    this is a *calibrated-at-realistic-size* approximation, not exact -
+    validated empirically by the net-of-cost A/B, not asserted.
+
+    Returns ``None`` (curvature undefined) when there is no impact term
+    (``k_imp <= 0``, e.g. no capital/ADV so the solve is linear-only) or the trade
+    size is non-positive - the signal for :func:`~quantred.portfolio.policy.derive_kappa`
+    to fall back to band-only behavior rather than divide by (near) zero.
+    """
+    if not (k_imp > 0) or not (typical_trade > 0):
+        return None
+    return 0.75 * k_imp / math.sqrt(typical_trade)

quantred/data/__init__.py

@@ -0,0 +1,29 @@
+"""Cross-sectional data substrate: scan bars point-in-time, assemble a feature panel.
+
+The single place data is gathered for cross-sectional research (alphas, risk,
+portfolio construction, information analysis): :func:`~quantred.data.scan.BarSource`
+scans a universe as of a timestamp (leakage-safe), and a :class:`~quantred.data.panel.FeaturePanel`
+holds every name's features in one table that producers fill and consumers read.
+"""
+
+from quantred.data.features import Scorer, add_factor_exposure_features, add_risk_features, add_score_feature
+from quantred.data.panel import FeaturePanel
+from quantred.data.scan import BarSource, ClientBarSource, slice_to_as_of
+from quantred.data.store import BAR_COLUMNS, ParquetBarStore
+
+# Lazy out-of-core compute lives in quantred.data.compute / quantred.data.edges and
+# is imported on demand — both need the optional ``store`` extra (polars/duckdb), so
+# they are intentionally not pulled into the base-install import path here.
+
+__all__ = [
+    "BarSource",
+    "ClientBarSource",
+    "slice_to_as_of",
+    "FeaturePanel",
+    "Scorer",
+    "add_factor_exposure_features",
+    "add_risk_features",
+    "add_score_feature",
+    "ParquetBarStore",
+    "BAR_COLUMNS",
+]

quantred/data/compute.py

@@ -0,0 +1,332 @@
+"""Lazy, out-of-core compute over the bar panel (Polars · DuckDB).
+
+The *storage* half of out-of-core (a Parquet/Arrow ``BarSource`` behind ``scan()``)
+lives in the store module. This module is the *compute* half: the hot, panel-wide
+operations expressed as a **lazy** plan, rather than eagerly materializing a ``T×N``
+pandas panel into RAM.
+
+Two engines, sharing Arrow so it isn't either/or:
+
+- **Polars ``LazyFrame``** for dataframe-style transforms — rolling-window indicators
+  become window expressions ``.over("symbol", order_by="ts")``, and the
+  cross-sectional z-score / winsorize / rank become ``.over("ts")`` expressions.
+  Nothing materializes until a terminal :func:`~quantred.data.edges.collect_streaming`.
+- **DuckDB SQL** (:func:`sql_query`) for set-based work over the Parquet store
+  (aggregations, joins) — its out-of-core engine streams the scan too.
+
+Three disciplines this module keeps:
+
+1. **Pushdown survives the plan.** A lazy scan reads only the columns/row-ranges it
+   needs and the ``as_of`` predicate is pushed into the Parquet reader — never a
+   post-materialize filter (see :meth:`ParquetBarStore.scan_lazy`).
+2. **Determinism under multithreading.** Polars/DuckDB parallelize; float reduction
+   order can vary. The cross-sectional reductions here are order-invariant
+   (mean/std/quantile), and :data:`CANONICAL_SORT` pins a stable output order so
+   repeated runs are byte-identical.
+3. **The pandas edge stays narrow.** Every crossing back to pandas routes through
+   :mod:`quantred.data.edges`; this module returns Polars frames / numpy, never pandas.
+
+**A note on what is genuinely bounded-memory.** The leaf Parquet scan (with its
+pushed-down predicate/projection) streams, and :func:`streaming_covariance` is bounded
+by the universe size, not by history length. But in the current Polars engine a
+``.over(...)`` window groupby and a top-level :func:`sort_canonical` are *buffering*
+operators — they materialize their input — so a pipeline that ends on a cross-sectional
+``over("ts")`` plus a canonical sort has peak memory ~ the panel it sorts, not O(chunk).
+The honest out-of-core wins today are the streaming scan, the as-of pushdown, and the
+streaming covariance accumulator; the set-based DuckDB path (which can spill) is the
+route to a genuinely bounded cross-sectional reduction. The window *math* is correct
+either way; only the memory profile of the buffering stages is not O(1) in history.
+
+The legacy pandas implementations (:mod:`quantred.indicators.indicators`,
+:mod:`quantred.alphas.refine`, :mod:`quantred.risk.sample`) are kept as the equivalence
+oracle — the tests prove these lazy ports match them within tolerance.
+
+Requires the ``store`` extra (``polars``; ``duckdb`` for :func:`sql_query`).
+"""
+
+from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Tuple
+
+import numpy as np
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    import polars as pl
+
+#: Column names of the long-format bar panel (matches the Parquet store schema).
+TS = "ts"
+SYMBOL = "symbol"
+
+#: Canonical output ordering. Pinning an explicit sort makes a multithreaded collect
+#: byte-identical run to run (determinism).
+CANONICAL_SORT = [TS, SYMBOL]
+
+
+def sort_canonical(lf: "pl.LazyFrame") -> "pl.LazyFrame":
+    """Pin the deterministic ``(ts, symbol)`` output order on a lazy plan."""
+    return lf.sort(CANONICAL_SORT)
+
+
+# --------------------------------------------------------------------------- #
+# Time-series indicators — Polars window expressions, partitioned per symbol.
+# Pandas (quantred/indicators) stays the per-symbol leaf and the equivalence oracle.
+# --------------------------------------------------------------------------- #
+def _ordered_over(expr: "pl.Expr", by: str, order: str = TS) -> "pl.Expr":
+    """A window expression evaluated per ``by`` group, **sorted by ``order`` within it**.
+
+    A rolling/lagged indicator is meaningless unless the window sees rows in time
+    order, and the on-disk/scan order is not guaranteed to be ts-ascending (multiple
+    row groups, concatenated files, an unsorted write). Passing ``order_by`` makes the
+    window sort *inside* the group, so the result is independent of physical row order
+    — matching the eager :meth:`ParquetBarStore.scan` path, which sorts each symbol by
+    ts. (A post-hoc :func:`sort_canonical` cannot fix this: it reorders the finished
+    output, not the sequence the window was computed over.)
+    """
+    return expr.over(by, order_by=order)
+
+
+def with_sma(
+    lf: "pl.LazyFrame", period: int, *, src: str = "close", out: str = "sma", by: str = SYMBOL
+) -> "pl.LazyFrame":
+    """Add a simple moving average column, per symbol (matches ``calculate_sma``).
+
+    Polars ``rolling_mean`` defaults ``min_periods == window_size``, so the head is
+    null until the window fills — exactly pandas' ``rolling(period).mean()``. The
+    window is ts-ordered within each symbol (see :func:`_ordered_over`).
+    """
+    import polars as pl
+
+    return lf.with_columns(_ordered_over(pl.col(src).rolling_mean(window_size=period), by).alias(out))
+
+
+def with_ema(
+    lf: "pl.LazyFrame", period: int, *, src: str = "close", out: str = "ema", by: str = SYMBOL
+) -> "pl.LazyFrame":
+    """Add an exponential moving average column, per symbol (matches ``calculate_ema``).
+
+    Uses ``adjust=False`` to match pandas' ``ewm(span=period, adjust=False).mean()``
+    (recursive form seeded on the first observation), ts-ordered within each symbol.
+
+    Equivalence holds for a gap-free source. An *interior* null/NaN between two valid
+    bars diverges from pandas: pandas (``ignore_na=False``) decays the weight across the
+    gap, Polars re-weights as if the missing bar were absent. A clean bar series has no
+    interior gaps (the only nulls are the leading warm-up, which agree); feed
+    gap-filled input if a source can drop interior bars.
+    """
+    import polars as pl
+
+    return lf.with_columns(_ordered_over(pl.col(src).ewm_mean(span=period, adjust=False), by).alias(out))
+
+
+def with_returns(
+    lf: "pl.LazyFrame", *, src: str = "close", out: str = "ret", by: str = SYMBOL
+) -> "pl.LazyFrame":
+    """Add a one-bar simple return column, per symbol (matches ``close.pct_change()``).
+
+    ts-ordered within each symbol. Note ``pct_change`` off a zero/again-zero price is a
+    genuine NaN/inf (``0/0``, ``x/0``) just as in pandas; the cross-sectional helpers
+    below treat such non-finite values as missing.
+    """
+    import polars as pl
+
+    return lf.with_columns(_ordered_over(pl.col(src).pct_change(), by).alias(out))
+
+
+# --------------------------------------------------------------------------- #
+# Cross-sectional refinement — Polars ``.over("ts")`` expressions on the panel.
+# quantred/alphas/refine.py stays the equivalence oracle.
+#
+# Missing-data contract: the pandas oracle (refine.py) runs on ``.dropna()`` — it
+# skips non-finite names and standardizes/ranks the rest. Polars reductions skip
+# *null* but PROPAGATE NaN/inf, which would poison the whole cross-section (one NaN
+# score → every name's z collapses to 0; a NaN gets handed the top rank). So each
+# helper first maps non-finite to null via ``_finite`` — then a single contaminated
+# name no longer destroys the rest of the cross-section, matching the oracle.
+# --------------------------------------------------------------------------- #
+def _finite(src: str) -> "pl.Expr":
+    """``src`` with non-finite values (NaN/±inf) mapped to null, so reductions skip them."""
+    import polars as pl
+
+    col = pl.col(src)
+    return pl.when(col.is_finite()).then(col).otherwise(None)
+
+
+def cross_sectional_zscore(
+    lf: "pl.LazyFrame", *, src: str = "score", out: str = "z", by: str = TS
+) -> "pl.LazyFrame":
+    """Cross-sectional z-score ``(x - mean) / std`` per timestamp (matches ``refine.zscore``).
+
+    Uses the population std (``ddof=0``) so the cross-section has unit dispersion.
+    Non-finite names are skipped (null out) exactly as the oracle's ``dropna``; a
+    degenerate cross-section (zero std across the finite names) maps to all-zeros — a
+    universe with no spread expresses no view, the same contract as the pandas oracle.
+    """
+    import polars as pl
+
+    x = _finite(src)
+    mean = x.mean().over(by)
+    std = x.std(ddof=0).over(by)
+    z = (
+        pl.when(x.is_null())  # a missing/non-finite name stays missing (oracle: NaN)
+        .then(None)
+        .when((std == 0) | std.is_null())  # no spread among finite names → no view
+        .then(pl.lit(0.0))
+        .otherwise((x - mean) / std)
+    )
+    return lf.with_columns(z.alias(out))
+
+
+def cross_sectional_winsorize(
+    lf: "pl.LazyFrame",
+    *,
+    src: str = "score",
+    out: Optional[str] = None,
+    lower: float = 0.025,
+    upper: float = 0.975,
+    by: str = TS,
+) -> "pl.LazyFrame":
+    """Clip ``src`` to its ``[lower, upper]`` cross-sectional quantiles (matches ``refine.winsorize``).
+
+    Quantiles use linear interpolation to match pandas' ``Series.quantile`` default and
+    are taken over the finite names only (non-finite stay missing). Writes back into
+    ``src`` unless ``out`` is given.
+    """
+
+    out = out or src
+    x = _finite(src)
+    lo = x.quantile(lower, interpolation="linear").over(by)
+    hi = x.quantile(upper, interpolation="linear").over(by)
+    return lf.with_columns(x.clip(lower_bound=lo, upper_bound=hi).alias(out))
+
+
+def cross_sectional_rank(
+    lf: "pl.LazyFrame", *, src: str = "score", out: str = "rank", by: str = TS
+) -> "pl.LazyFrame":
+    """Add a cross-sectional rank per timestamp (1 = smallest), average ties.
+
+    ``method="average"`` matches pandas' ``Series.rank()`` default. Non-finite names are
+    null (Polars ranks null as null, like pandas), so a missing score is never handed a
+    tradable rank.
+    """
+
+    return lf.with_columns(_finite(src).rank(method="average").over(by).alias(out))
+
+
+def cross_sectional_demean(
+    lf: "pl.LazyFrame", *, src: str = "score", out: str = "z", by: str = TS
+) -> "pl.LazyFrame":
+    """Subtract the cross-sectional mean per timestamp (matches ``refine.demean``).
+
+    Mean is taken over the finite names; non-finite names stay missing.
+    """
+
+    x = _finite(src)
+    return lf.with_columns((x - x.mean().over(by)).alias(out))
+
+
+# --------------------------------------------------------------------------- #
+# Streaming covariance — accumulate cross-products by streaming the return panel
+# instead of materializing the full T×N matrix. quantred/risk/sample.SampleCovariance
+# is the equivalence oracle (this reproduces its population MLE estimate).
+# --------------------------------------------------------------------------- #
+def streaming_covariance(
+    return_chunks: Iterable[np.ndarray],
+) -> Tuple[np.ndarray, int]:
+    """Per-bar covariance accumulated over chunks of an aligned return panel.
+
+    Each chunk is a ``rows × N`` block of an already-aligned (complete-case) return
+    panel — the same panel :func:`quantred.risk.base.build_return_panel` produces, but fed
+    in row-chunks so no more than one chunk plus the ``N×N`` accumulator is ever in
+    memory. Memory is bounded by the **universe size**, not the length of history —
+    the point of an out-of-core covariance.
+
+    Accumulates the uncentered sum ``s = Σ xₜ`` and second-moment ``S₂ = Σ xₜxₜᵀ``,
+    then returns the population covariance ``Σ = S₂/n − (s/n)(s/n)ᵀ``. The result
+    matches the single-pass estimate to machine epsilon regardless of how the rows are
+    split (a tested invariant; not byte-identical, since float summation order varies
+    with chunking). The uncentered form is *algebraically* identical to centering first,
+    and *numerically* matches :class:`~quantred.risk.sample.SampleCovariance` to machine
+    epsilon for the inputs it is contracted on — **return-scale, near-zero-mean** rows
+    (what :func:`quantred.risk.base.build_return_panel` produces from ``pct_change``). It is
+    NOT for price levels: when the mean dwarfs the spread (|mean| ≫ std), ``S₂/n`` and
+    ``(s/n)(s/n)ᵀ`` are two large near-equal numbers and their difference loses
+    precision (catastrophic cancellation). Feed returns, not levels.
+
+    Returns ``(Σ_per_bar, n_observations)``; with fewer than two observations Σ is the
+    zero matrix (mirroring the oracle's degenerate-case contract).
+    """
+    s: Optional[np.ndarray] = None
+    s2: Optional[np.ndarray] = None
+    count = 0
+    n_cols = 0
+
+    for chunk in return_chunks:
+        x = np.asarray(chunk, dtype=float)
+        if x.ndim != 2 or x.shape[0] == 0:
+            continue
+        if s is None:
+            n_cols = x.shape[1]
+            s = np.zeros(n_cols)
+            s2 = np.zeros((n_cols, n_cols))
+        elif x.shape[1] != n_cols:
+            raise ValueError(f"chunk has {x.shape[1]} columns, expected {n_cols}")
+        s += x.sum(axis=0)
+        s2 += x.T @ x
+        count += x.shape[0]
+
+    if count < 2 or s is None:
+        return np.zeros((n_cols, n_cols)), count
+    mean = s / count
+    cov = s2 / count - np.outer(mean, mean)
+    return cov, count
+
+
+def iter_row_chunks(matrix: np.ndarray, chunk_rows: int) -> Iterable[np.ndarray]:
+    """Yield row-blocks of ``matrix`` of at most ``chunk_rows`` rows.
+
+    The bridge from a materialized return panel to :func:`streaming_covariance`: lets
+    a caller feed an in-memory panel in bounded slices, or a streaming reader hand
+    chunks straight through.
+    """
+    if chunk_rows < 1:
+        raise ValueError("chunk_rows must be >= 1")
+    for start in range(0, len(matrix), chunk_rows):
+        yield matrix[start : start + chunk_rows]
+
+
+# --------------------------------------------------------------------------- #
+# DuckDB SQL path — set-based work over the Parquet store, out-of-core.
+# --------------------------------------------------------------------------- #
+def sql_query(
+    parquet_glob: str | Sequence[str],
+    sql: str,
+    *,
+    params: Optional[Sequence] = None,
+    view: str = "bars",
+) -> "pl.DataFrame":
+    """Run a DuckDB SQL query over Parquet, returned as a Polars frame (zero-copy Arrow).
+
+    Registers ``parquet_glob`` (a path/glob or a list of paths) as a view named
+    ``view`` (default ``bars``) so the query reads ``FROM bars``. DuckDB's reader does
+    its own projection/predicate pushdown and streams the scan, so an aggregation over
+    a multi-file store never materializes the whole thing. The result returns as
+    Polars (shared Arrow) — collapse it to pandas only at the edge via
+    :func:`quantred.data.edges.to_pandas`.
+
+    ``params`` binds DuckDB ``?`` placeholders (e.g. an ``as_of`` cutoff), keeping the
+    point-in-time filter inside the pushed-down scan rather than a post-filter.
+    """
+    import duckdb
+
+    paths: List[str] = [parquet_glob] if isinstance(parquet_glob, str) else list(parquet_glob)
+    con = duckdb.connect()
+    try:
+        # hive_partitioning=false: the store embeds `symbol` as a real column, so we
+        # must not also synthesize it from the `symbol=…` path (that would collide).
+        # Register the scan as a relation (CREATE VIEW can't take a prepared param).
+        con.register(view, con.read_parquet(paths, hive_partitioning=False))
+        arrow = con.execute(sql, list(params) if params is not None else None).to_arrow_table()
+    finally:
+        con.close()
+
+    import polars as pl
+
+    return pl.from_arrow(arrow)