rangebar 11.6.1__cp313-cp313-macosx_11_0_arm64.whl

rangebar/streaming.py

@@ -0,0 +1,300 @@
+# ADR: docs/adr/2026-01-31-realtime-streaming-api.md
+"""Real-time streaming API for range bar construction.
+
+This module provides async Python APIs for constructing range bars from live
+data sources (Binance WebSocket, Exness tick feeds).
+
+Architecture:
+- Low-level: Callback-based Rust bindings (PyBinanceLiveStream)
+- High-level: Python async generators built on top
+
+Examples
+--------
+Async generator (recommended for most use cases):
+
+>>> import asyncio
+>>> from rangebar.streaming import stream_binance_live
+>>>
+>>> async def main():
+...     async for bar in stream_binance_live("BTCUSDT", threshold_bps=250):
+...         print(f"New bar: {bar['close']}")
+...
+>>> asyncio.run(main())
+
+Low-level callback interface:
+
+>>> from rangebar.streaming import BinanceLiveStream
+>>>
+>>> stream = BinanceLiveStream("BTCUSDT", threshold_decimal_bps=250)
+>>> stream.connect()
+>>> while stream.is_connected:
+...     bar = stream.next_bar(timeout_ms=5000)
+...     if bar:
+...         print(f"New bar: {bar['close']}")
+
+Custom data source with StreamingRangeBarProcessor:
+
+>>> from rangebar.streaming import StreamingRangeBarProcessor
+>>>
+>>> processor = StreamingRangeBarProcessor(250)
+>>> for trade in my_trade_source():
+...     bars = processor.process_trade(trade)
+...     for bar in bars:
+...         print(f"Completed bar: {bar['close']}")
+"""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import time
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ._core import (
+    BinanceLiveStream,
+    StreamingConfig,
+    StreamingMetrics,
+    StreamingRangeBarProcessor,
+)
+
+if TYPE_CHECKING:
+    from typing import Any
+
+__all__ = [
+    "BinanceLiveStream",
+    "ReconnectionConfig",
+    "StreamingConfig",
+    "StreamingError",
+    "StreamingMetrics",
+    "StreamingRangeBarProcessor",
+    "stream_binance_live",
+]
+
+
+class StreamingError(Exception):
+    """Error during streaming operation."""
+
+
+@dataclass
+class ReconnectionConfig:
+    """Configuration for automatic reconnection with jitter.
+
+    Attributes:
+        max_retries: Maximum reconnection attempts (0 = infinite)
+        initial_delay_s: Initial delay before first retry
+        max_delay_s: Maximum delay between retries
+        backoff_factor: Multiplier for exponential backoff
+        jitter_factor: Random jitter range (0.5 = ±50% of delay)
+        max_total_duration_s: Maximum total time spent reconnecting (0 = infinite)
+
+    Notes:
+        Jitter is applied to prevent thundering herd when multiple clients
+        reconnect simultaneously. The actual delay is:
+        `delay * (1 - jitter_factor + random() * 2 * jitter_factor)`
+
+        For jitter_factor=0.5, delay varies from 50% to 150% of base delay.
+    """
+
+    max_retries: int = 0  # 0 = infinite
+    initial_delay_s: float = 1.0
+    max_delay_s: float = 60.0
+    backoff_factor: float = 2.0
+    jitter_factor: float = 0.5  # ±50% jitter to prevent thundering herd
+    max_total_duration_s: float = 0.0  # 0 = infinite
+
+
+async def stream_binance_live(
+    symbol: str,
+    threshold_bps: int = 250,
+    *,
+    reconnect: bool = True,
+    reconnect_config: ReconnectionConfig | None = None,
+) -> AsyncIterator[dict[str, Any]]:
+    """Stream range bars from Binance WebSocket in real-time.
+
+    This is an async generator that yields completed range bars as they
+    are constructed from live trade data.
+
+    Args:
+        symbol: Trading pair (e.g., "BTCUSDT")
+        threshold_bps: Range bar threshold in decimal basis points (250 = 0.25%)
+        reconnect: Whether to automatically reconnect on disconnect
+        reconnect_config: Custom reconnection settings
+
+    Yields:
+        Range bar dicts with OHLCV + microstructure features
+
+    Raises:
+        StreamingError: If connection fails and reconnection is disabled
+
+    Example:
+        >>> async for bar in stream_binance_live("BTCUSDT", threshold_bps=250):
+        ...     print(f"New bar: {bar['close']}, OFI: {bar['ofi']}")
+    """
+    if reconnect_config is None:
+        reconnect_config = ReconnectionConfig()
+
+    retry_count = 0
+    current_delay = reconnect_config.initial_delay_s
+    reconnect_start_time: float | None = None
+
+    while True:
+        try:
+            # Create stream and connect
+            stream = BinanceLiveStream(symbol, threshold_bps)
+
+            # Run connect in thread pool to avoid blocking event loop
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(None, stream.connect)
+
+            # Reset retry state on successful connection
+            retry_count = 0
+            current_delay = reconnect_config.initial_delay_s
+            reconnect_start_time = None
+
+            # Yield bars as they arrive
+            while stream.is_connected:
+                # Poll for bar with timeout (non-blocking via thread pool)
+                # Bind stream as default arg to avoid B023 closure issue
+                bar = await loop.run_in_executor(
+                    None, lambda s=stream: s.next_bar(timeout_ms=1000)
+                )
+
+                if bar is not None:
+                    yield bar
+
+                # Allow other coroutines to run
+                await asyncio.sleep(0)
+
+            # Stream disconnected
+            if not reconnect:
+                break
+
+        except Exception as e:
+            if not reconnect:
+                msg = f"Stream connection failed: {e}"
+                raise StreamingError(msg) from e
+
+            # Track reconnection start time
+            if reconnect_start_time is None:
+                reconnect_start_time = time.monotonic()
+
+            retry_count += 1
+
+            # Check max retries
+            if (
+                reconnect_config.max_retries > 0
+                and retry_count > reconnect_config.max_retries
+            ):
+                msg = f"Max retries ({reconnect_config.max_retries}) exceeded"
+                raise StreamingError(msg) from e
+
+            # Check max total duration
+            if reconnect_config.max_total_duration_s > 0:
+                elapsed = time.monotonic() - reconnect_start_time
+                if elapsed > reconnect_config.max_total_duration_s:
+                    msg = (
+                        f"Max reconnection duration "
+                        f"({reconnect_config.max_total_duration_s:.0f}s) exceeded"
+                    )
+                    raise StreamingError(msg) from e
+
+            # Apply jitter to prevent thundering herd
+            # Jitter range: [1 - jitter_factor, 1 + jitter_factor]
+            jitter_multiplier = 1.0 - reconnect_config.jitter_factor + (
+                random.random() * 2 * reconnect_config.jitter_factor
+            )
+            jittered_delay = current_delay * jitter_multiplier
+
+            # Log and wait before retry
+            print(
+                f"Stream disconnected, retrying in {jittered_delay:.1f}s "
+                f"(attempt {retry_count})"
+            )
+            await asyncio.sleep(jittered_delay)
+
+            # Exponential backoff (applied to base delay, not jittered)
+            current_delay = min(
+                current_delay * reconnect_config.backoff_factor,
+                reconnect_config.max_delay_s,
+            )
+
+
+class AsyncStreamingProcessor:
+    """Async wrapper for StreamingRangeBarProcessor.
+
+    This class provides an async interface for processing trades from
+    any data source into range bars.
+
+    Example:
+        >>> processor = AsyncStreamingProcessor(250)
+        >>> async for trade in my_async_trade_source():
+        ...     bars = await processor.process_trade(trade)
+        ...     for bar in bars:
+        ...         await handle_new_bar(bar)
+    """
+
+    def __init__(self, threshold_decimal_bps: int) -> None:
+        """Create async streaming processor.
+
+        Args:
+            threshold_decimal_bps: Range bar threshold (250 = 0.25%)
+        """
+        self._processor = StreamingRangeBarProcessor(threshold_decimal_bps)
+        self._lock = asyncio.Lock()
+
+    async def process_trade(self, trade: dict[str, Any]) -> list[dict[str, Any]]:
+        """Process a single trade asynchronously.
+
+        Args:
+            trade: Trade dict with timestamp, price, quantity/volume
+
+        Returns:
+            List of completed bar dicts (usually 0 or 1)
+        """
+        async with self._lock:
+            loop = asyncio.get_event_loop()
+            return await loop.run_in_executor(
+                None, self._processor.process_trade, trade
+            )
+
+    async def process_trades(
+        self, trades: list[dict[str, Any]]
+    ) -> list[dict[str, Any]]:
+        """Process multiple trades asynchronously.
+
+        Args:
+            trades: List of trade dicts
+
+        Returns:
+            List of completed bar dicts
+        """
+        async with self._lock:
+            loop = asyncio.get_event_loop()
+            return await loop.run_in_executor(
+                None, self._processor.process_trades, trades
+            )
+
+    async def get_incomplete_bar(self) -> dict[str, Any] | None:
+        """Get current incomplete bar asynchronously."""
+        async with self._lock:
+            loop = asyncio.get_event_loop()
+            return await loop.run_in_executor(
+                None, self._processor.get_incomplete_bar
+            )
+
+    @property
+    def trades_processed(self) -> int:
+        """Number of trades processed."""
+        return self._processor.trades_processed
+
+    @property
+    def bars_generated(self) -> int:
+        """Number of bars generated."""
+        return self._processor.bars_generated
+
+    def get_metrics(self) -> StreamingMetrics:
+        """Get streaming metrics."""
+        return self._processor.get_metrics()

rangebar/validation/__init__.py

@@ -0,0 +1,69 @@
+"""Validation framework for microstructure features (Issue #25) and cache integrity (Issue #39).
+
+Provides tiered validation for market microstructure features and cache operations:
+- Tier 0: Cache staleness detection (<5ms) - schema evolution support
+- Tier 0: Post-storage validation after cache operations (<1 sec) - Issue #39
+- Tier 1: Auto-validation on every precompute (<30 sec)
+- Tier 2: Statistical validation before production ML (~10 min)
+- Tier 3: Feature importance and drift analysis (30+ min, on-demand)
+- Continuity: Tiered gap classification for range bar data (Issue #19)
+"""
+
+from .cache_staleness import (
+    StalenessResult,
+    detect_staleness,
+    validate_schema_version,
+)
+from .continuity import (
+    ASSET_CLASS_MULTIPLIERS,
+    VALIDATION_PRESETS,
+    AssetClass,
+    ContinuityError,
+    ContinuityWarning,
+    GapInfo,
+    GapTier,
+    TieredValidationResult,
+    TierSummary,
+    TierThresholds,
+    ValidationPreset,
+    detect_asset_class,
+    validate_continuity,
+    validate_continuity_tiered,
+    validate_junction_continuity,
+)
+from .post_storage import (
+    ValidationResult,
+    compute_dataframe_checksum,
+    validate_ohlc_invariants,
+    validate_post_storage,
+)
+from .tier1 import FEATURE_COLS, validate_tier1
+from .tier2 import validate_tier2
+
+__all__ = [
+    "ASSET_CLASS_MULTIPLIERS",
+    "FEATURE_COLS",
+    "VALIDATION_PRESETS",
+    "AssetClass",
+    "ContinuityError",
+    "ContinuityWarning",
+    "GapInfo",
+    "GapTier",
+    "StalenessResult",
+    "TierSummary",
+    "TierThresholds",
+    "TieredValidationResult",
+    "ValidationPreset",
+    "ValidationResult",
+    "compute_dataframe_checksum",
+    "detect_asset_class",
+    "detect_staleness",
+    "validate_continuity",
+    "validate_continuity_tiered",
+    "validate_junction_continuity",
+    "validate_ohlc_invariants",
+    "validate_post_storage",
+    "validate_schema_version",
+    "validate_tier1",
+    "validate_tier2",
+]

rangebar/validation/cache_staleness.py

@@ -0,0 +1,277 @@
+# polars-exception: ClickHouse cache returns Pandas for backtesting.py
+"""Cache staleness detection for schema evolution.
+
+This module provides content-based validation to detect stale cached data
+that was computed with older versions lacking microstructure features.
+
+Tier 0 validation: Fast staleness detection (<5ms for 100K bars).
+Run on every cache read when microstructure features are requested.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Literal
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+from rangebar.constants import MICROSTRUCTURE_COLUMNS
+
+logger = logging.getLogger(__name__)
+
+# Semantic version has 3 parts
+_VERSION_PARTS = 3
+
+
+@dataclass
+class StalenessResult:
+    """Result of cache staleness detection.
+
+    Attributes
+    ----------
+    is_stale : bool
+        True if cached data is detected as stale and should be invalidated.
+    reason : str | None
+        Human-readable description of why data is stale (None if not stale).
+    confidence : Literal["high", "medium", "low"]
+        Confidence level of staleness detection.
+    checks_passed : dict[str, bool]
+        Individual validation checks and their results.
+    recommendations : list[str]
+        Suggested actions to resolve staleness.
+    """
+
+    is_stale: bool
+    reason: str | None = None
+    confidence: Literal["high", "medium", "low"] = "high"
+    checks_passed: dict[str, bool] = field(default_factory=dict)
+    recommendations: list[str] = field(default_factory=list)
+
+
+def _check_vwap(
+    df: pd.DataFrame,
+    checks: dict[str, bool],
+    reasons: list[str],
+) -> None:
+    """Validate VWAP is within [Low, High] and not all zeros."""
+    if "vwap" not in df.columns:
+        return
+
+    vwap_all_zero = (df["vwap"] == 0).all()
+    checks["vwap_not_all_zero"] = not vwap_all_zero
+
+    if vwap_all_zero:
+        reasons.append("All VWAP values are zero (pre-v7.0 cache data)")
+    else:
+        vwap_valid = (df["vwap"] >= df["Low"]) & (df["vwap"] <= df["High"])
+        checks["vwap_bounded"] = vwap_valid.all()
+        if not vwap_valid.all():
+            invalid_count = (~vwap_valid).sum()
+            reasons.append(f"VWAP outside [Low, High] for {invalid_count} bars")
+
+
+def _check_bounded_columns(
+    df: pd.DataFrame,
+    checks: dict[str, bool],
+    reasons: list[str],
+) -> None:
+    """Validate bounded microstructure columns are within expected ranges."""
+    # OFI in [-1, 1]
+    if "ofi" in df.columns:
+        ofi_bounded = df["ofi"].between(-1, 1).all()
+        checks["ofi_bounded"] = ofi_bounded
+        if not ofi_bounded:
+            reasons.append("OFI values outside [-1, 1] range")
+
+    # Turnover imbalance in [-1, 1]
+    if "turnover_imbalance" in df.columns:
+        ti_bounded = df["turnover_imbalance"].between(-1, 1).all()
+        checks["turnover_imbalance_bounded"] = ti_bounded
+        if not ti_bounded:
+            reasons.append("Turnover imbalance outside [-1, 1] range")
+
+    # Duration non-negative
+    if "duration_us" in df.columns:
+        duration_valid = (df["duration_us"] >= 0).all()
+        checks["duration_non_negative"] = duration_valid
+        if not duration_valid:
+            reasons.append("Negative duration values detected")
+
+    # Aggregation density >= 1
+    if "aggregation_density" in df.columns:
+        agg_valid = (df["aggregation_density"] >= 1).all()
+        checks["aggregation_density_valid"] = agg_valid
+        if not agg_valid:
+            reasons.append("Aggregation density < 1 detected")
+
+
+def _check_volume_consistency(
+    df: pd.DataFrame,
+    checks: dict[str, bool],
+    reasons: list[str],
+) -> None:
+    """Validate buy_volume + sell_volume == Volume."""
+    required_cols = {"buy_volume", "sell_volume", "Volume"}
+    if not required_cols.issubset(df.columns):
+        return
+
+    vol_sum = df["buy_volume"] + df["sell_volume"]
+    vol_diff = (vol_sum - df["Volume"]).abs()
+    vol_match = (vol_diff < 1e-6 * df["Volume"].abs().clip(lower=1e-10)).all()
+    checks["volume_consistency"] = vol_match
+    if not vol_match:
+        reasons.append("buy_volume + sell_volume != Volume")
+
+
+def _check_trade_counts(
+    df: pd.DataFrame,
+    checks: dict[str, bool],
+    reasons: list[str],
+) -> None:
+    """Validate trade count columns have valid values."""
+    if "individual_trade_count" in df.columns:
+        counts_valid = (df["individual_trade_count"] >= 1).all()
+        checks["trade_counts_valid"] = counts_valid
+        if not counts_valid:
+            reasons.append("Invalid trade count values (< 1)")
+
+
+def _check_all_microstructure_zero(
+    df: pd.DataFrame,
+    checks: dict[str, bool],
+    reasons: list[str],
+) -> None:
+    """Check if all microstructure columns are zero (indicates stale data)."""
+    micro_cols_present = [c for c in MICROSTRUCTURE_COLUMNS if c in df.columns]
+    if not micro_cols_present:
+        return
+
+    all_micro_zero = all((df[col] == 0).all() for col in micro_cols_present)
+    checks["microstructure_not_all_zero"] = not all_micro_zero
+    if all_micro_zero:
+        reasons.append("All microstructure columns are zero (pre-v7.0 cache data)")
+
+
+def _determine_staleness(
+    checks: dict[str, bool],
+    reasons: list[str],
+) -> StalenessResult:
+    """Determine final staleness result from individual checks."""
+    high_confidence_checks = [
+        "vwap_bounded",
+        "vwap_not_all_zero",
+        "ofi_bounded",
+        "turnover_imbalance_bounded",
+        "duration_non_negative",
+        "aggregation_density_valid",
+        "trade_counts_valid",
+        "microstructure_not_all_zero",
+    ]
+
+    high_conf_failures = [
+        k for k in high_confidence_checks if k in checks and not checks[k]
+    ]
+
+    is_stale = len(high_conf_failures) > 0
+
+    # Determine confidence level
+    confidence: Literal["high", "medium", "low"] = "high"
+    if is_stale and not high_conf_failures:
+        confidence = "medium"
+
+    # Build recommendations
+    recommendations: list[str] = []
+    if is_stale:
+        recommendations.append(
+            "Invalidate cache entry and recompute with current version"
+        )
+        if "vwap_not_all_zero" in high_conf_failures:
+            recommendations.append("Data appears to be from pre-v7.0")
+        if "microstructure_not_all_zero" in high_conf_failures:
+            recommendations.append("Data appears to be from pre-v7.0")
+        recommendations.append("Run: get_range_bars(..., use_cache=False)")
+
+    return StalenessResult(
+        is_stale=is_stale,
+        reason="; ".join(reasons) if reasons else None,
+        confidence=confidence,
+        checks_passed=checks,
+        recommendations=recommendations,
+    )
+
+
+def detect_staleness(
+    df: pd.DataFrame,
+    require_microstructure: bool = True,
+) -> StalenessResult:
+    """Detect stale cached data using content-based validation.
+
+    This is Tier 0 validation: fast staleness detection (<5ms for 100K bars).
+    Run on every cache read before returning data to caller.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Cached range bar DataFrame, possibly with microstructure columns.
+    require_microstructure : bool, default=True
+        If True, check for valid microstructure columns.
+
+    Returns
+    -------
+    StalenessResult
+        Detection result with confidence level and specific failures.
+    """
+    checks: dict[str, bool] = {}
+    reasons: list[str] = []
+
+    if require_microstructure:
+        _check_vwap(df, checks, reasons)
+        _check_bounded_columns(df, checks, reasons)
+        _check_volume_consistency(df, checks, reasons)
+        _check_trade_counts(df, checks, reasons)
+        _check_all_microstructure_zero(df, checks, reasons)
+
+    return _determine_staleness(checks, reasons)
+
+
+def validate_schema_version(
+    cached_version: str | None,
+    min_version: str,
+) -> bool:
+    """Check if cached data meets minimum schema version requirement.
+
+    Parameters
+    ----------
+    cached_version : str | None
+        Version string from cached data (e.g., "7.0.0").
+    min_version : str
+        Minimum required version (e.g., "7.0.0").
+
+    Returns
+    -------
+    bool
+        True if cached_version >= min_version.
+    """
+    if not cached_version:
+        return False
+
+    try:
+        cached_parts = [int(x) for x in cached_version.split(".")[:_VERSION_PARTS]]
+        min_parts = [int(x) for x in min_version.split(".")[:_VERSION_PARTS]]
+
+        # Pad to 3 parts
+        while len(cached_parts) < _VERSION_PARTS:
+            cached_parts.append(0)
+        while len(min_parts) < _VERSION_PARTS:
+            min_parts.append(0)
+
+        return tuple(cached_parts) >= tuple(min_parts)
+    except (ValueError, AttributeError):
+        logger.warning(
+            "Invalid version format: cached=%r, min=%r",
+            cached_version,
+            min_version,
+        )
+        return False