rangebar 11.6.1__cp313-cp313-macosx_11_0_arm64.whl

rangebar/clickhouse/query_operations.py

@@ -0,0 +1,345 @@
+# polars-exception: backtesting.py requires Pandas DataFrames with DatetimeIndex
+# Issue #46: Modularization M6 - Extract query operations from cache.py
+"""Query operations for ClickHouse range bar cache.
+
+Provides mixin methods for retrieving range bars by count (get_n_bars)
+and by timestamp range (get_bars_by_timestamp_range). Used by RangeBarCache
+via mixin inheritance.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pandas as pd
+
+from ..constants import (
+    MICROSTRUCTURE_COLUMNS,
+    MIN_VERSION_FOR_MICROSTRUCTURE,
+)
+from ..conversion import normalize_arrow_dtypes
+from ..exceptions import CacheReadError
+
+logger = logging.getLogger(__name__)
+
+
+class QueryOperationsMixin:
+    """Mixin providing query operations for RangeBarCache.
+
+    Requires `self.client` and `self.count_bars()` from parent class.
+    """
+
+    def get_n_bars(
+        self,
+        symbol: str,
+        threshold_decimal_bps: int,
+        n_bars: int,
+        before_ts: int | None = None,
+        include_microstructure: bool = False,
+        min_schema_version: str | None = None,
+    ) -> tuple[pd.DataFrame | None, int]:
+        """Get N bars from cache, ordered chronologically (oldest first).
+
+        Uses ORDER BY timestamp_ms DESC LIMIT N for efficient retrieval,
+        then reverses in Python for chronological order.
+
+        Parameters
+        ----------
+        symbol : str
+            Trading symbol (e.g., "BTCUSDT")
+        threshold_decimal_bps : int
+            Threshold in decimal basis points
+        n_bars : int
+            Maximum number of bars to retrieve
+        before_ts : int | None
+            Only get bars with timestamp_ms < before_ts.
+            If None, gets most recent bars.
+        include_microstructure : bool
+            If True, includes vwap, buy_volume, sell_volume columns
+        min_schema_version : str | None
+            Minimum schema version required for cache hit. If specified,
+            only returns data with rangebar_version >= min_schema_version.
+            When include_microstructure=True and min_schema_version=None,
+            automatically requires version >= 7.0.0.
+
+        Returns
+        -------
+        tuple[pd.DataFrame | None, int]
+            (bars_df, available_count) where:
+            - bars_df is OHLCV DataFrame (or None if no bars)
+            - available_count is total bars available (may be > len(bars_df))
+        """
+        # First get the count (for reporting)
+        available_count = self.count_bars(symbol, threshold_decimal_bps, before_ts)
+
+        if available_count == 0:
+            return None, 0
+
+        # Select columns
+        base_cols = """
+            timestamp_ms,
+            open as Open,
+            high as High,
+            low as Low,
+            close as Close,
+            volume as Volume
+        """
+        if include_microstructure:
+            base_cols += """,
+            vwap,
+            buy_volume,
+            sell_volume,
+            duration_us,
+            ofi,
+            vwap_close_deviation,
+            price_impact,
+            kyle_lambda_proxy,
+            trade_intensity,
+            volume_per_trade,
+            aggression_ratio,
+            aggregation_density,
+            turnover_imbalance
+        """
+
+        # Determine effective min version for schema evolution filtering
+        effective_min_version = min_schema_version
+        if include_microstructure and effective_min_version is None:
+            effective_min_version = MIN_VERSION_FOR_MICROSTRUCTURE
+
+        # Build version filter if specified
+        version_filter = ""
+        if effective_min_version:
+            version_filter = """
+              AND rangebar_version != ''
+              AND rangebar_version >= {min_version:String}"""
+
+        if before_ts is not None:
+            # Split path: with end_ts filter
+            query = f"""
+                SELECT {base_cols}
+                FROM rangebar_cache.range_bars FINAL
+                WHERE symbol = {{symbol:String}}
+                  AND threshold_decimal_bps = {{threshold:UInt32}}
+                  AND timestamp_ms < {{end_ts:Int64}}
+                  {version_filter}
+                ORDER BY timestamp_ms DESC
+                LIMIT {{n_bars:UInt64}}
+            """
+            params: dict[str, str | int] = {
+                "symbol": symbol,
+                "threshold": threshold_decimal_bps,
+                "end_ts": before_ts,
+                "n_bars": n_bars,
+            }
+            if effective_min_version:
+                params["min_version"] = effective_min_version
+            df = self.client.query_df_arrow(query, parameters=params)
+        else:
+            # Split path: no end_ts filter (most recent)
+            query = f"""
+                SELECT {base_cols}
+                FROM rangebar_cache.range_bars FINAL
+                WHERE symbol = {{symbol:String}}
+                  AND threshold_decimal_bps = {{threshold:UInt32}}
+                  {version_filter}
+                ORDER BY timestamp_ms DESC
+                LIMIT {{n_bars:UInt64}}
+            """
+            params = {
+                "symbol": symbol,
+                "threshold": threshold_decimal_bps,
+                "n_bars": n_bars,
+            }
+            if effective_min_version:
+                params["min_version"] = effective_min_version
+            df = self.client.query_df_arrow(query, parameters=params)
+
+        if df.empty:
+            return None, available_count
+
+        # Reverse to chronological order (oldest first)
+        df = df.iloc[::-1].reset_index(drop=True)
+
+        # Convert to TZ-aware UTC DatetimeIndex (Issue #20: match get_range_bars output)
+        df["timestamp"] = pd.to_datetime(df["timestamp_ms"], unit="ms", utc=True)
+        df = df.set_index("timestamp")
+        df = df.drop(columns=["timestamp_ms"])
+
+        # Convert PyArrow dtypes to numpy for compatibility
+        df = normalize_arrow_dtypes(df)
+
+        # Convert microstructure columns if present
+        if include_microstructure:
+            df = normalize_arrow_dtypes(df, columns=list(MICROSTRUCTURE_COLUMNS))
+
+        return df, available_count
+
+    def get_bars_by_timestamp_range(
+        self,
+        symbol: str,
+        threshold_decimal_bps: int,
+        start_ts: int,
+        end_ts: int,
+        include_microstructure: bool = False,
+        include_exchange_sessions: bool = False,
+        ouroboros_mode: str = "year",
+        min_schema_version: str | None = None,
+    ) -> pd.DataFrame | None:
+        """Get bars within a timestamp range (for get_range_bars cache lookup).
+
+        Unlike get_range_bars() which requires exact CacheKey match,
+        this method queries by timestamp range, returning any cached bars
+        that fall within [start_ts, end_ts].
+
+        Parameters
+        ----------
+        symbol : str
+            Trading symbol (e.g., "BTCUSDT")
+        threshold_decimal_bps : int
+            Threshold in decimal basis points
+        start_ts : int
+            Start timestamp in milliseconds (inclusive)
+        end_ts : int
+            End timestamp in milliseconds (inclusive)
+        include_microstructure : bool
+            If True, includes vwap, buy_volume, sell_volume columns
+        include_exchange_sessions : bool
+            If True, includes exchange_session_* columns (Issue #8)
+        ouroboros_mode : str
+            Ouroboros reset mode: "year", "month", or "week" (default: "year")
+            Plan: sparkling-coalescing-dijkstra.md
+        min_schema_version : str | None
+            Minimum schema version required for cache hit. If specified,
+            only returns data with rangebar_version >= min_schema_version.
+            When include_microstructure=True and min_schema_version=None,
+            automatically requires version >= 7.0.0.
+
+        Returns
+        -------
+        pd.DataFrame | None
+            OHLCV DataFrame with TZ-aware UTC timestamps if found, None otherwise.
+            Returns None if no bars exist in the range or version mismatch.
+
+        Raises
+        ------
+        CacheReadError
+            If the query fails due to database errors.
+        """
+        # Build column list
+        base_cols = """
+            timestamp_ms,
+            open as Open,
+            high as High,
+            low as Low,
+            close as Close,
+            volume as Volume
+        """
+        if include_microstructure:
+            base_cols += """,
+            vwap,
+            buy_volume,
+            sell_volume,
+            duration_us,
+            ofi,
+            vwap_close_deviation,
+            price_impact,
+            kyle_lambda_proxy,
+            trade_intensity,
+            volume_per_trade,
+            aggression_ratio,
+            aggregation_density,
+            turnover_imbalance
+        """
+
+        # Issue #8: Exchange session flags
+        if include_exchange_sessions:
+            base_cols += """,
+            exchange_session_sydney,
+            exchange_session_tokyo,
+            exchange_session_london,
+            exchange_session_newyork
+        """
+
+        # Ouroboros mode filter ensures cache isolation between modes
+        # Plan: sparkling-coalescing-dijkstra.md
+
+        # Determine effective min version for schema evolution filtering
+        effective_min_version = min_schema_version
+        if include_microstructure and effective_min_version is None:
+            # Auto-require v7.0.0+ for microstructure features
+            effective_min_version = MIN_VERSION_FOR_MICROSTRUCTURE
+
+        # Build version filter if specified
+        version_filter = ""
+        if effective_min_version:
+            version_filter = """
+              AND rangebar_version != ''
+              AND rangebar_version >= {min_version:String}"""
+
+        query = f"""
+            SELECT {base_cols}
+            FROM rangebar_cache.range_bars FINAL
+            WHERE symbol = {{symbol:String}}
+              AND threshold_decimal_bps = {{threshold:UInt32}}
+              AND ouroboros_mode = {{ouroboros_mode:String}}
+              AND timestamp_ms >= {{start_ts:Int64}}
+              AND timestamp_ms <= {{end_ts:Int64}}
+              {version_filter}
+            ORDER BY timestamp_ms
+        """
+
+        # Build parameters
+        params: dict[str, str | int] = {
+            "symbol": symbol,
+            "threshold": threshold_decimal_bps,
+            "ouroboros_mode": ouroboros_mode,
+            "start_ts": start_ts,
+            "end_ts": end_ts,
+        }
+        if effective_min_version:
+            params["min_version"] = effective_min_version
+
+        try:
+            df = self.client.query_df_arrow(query, parameters=params)
+        except (OSError, RuntimeError) as e:
+            logger.exception(
+                "Cache read failed for %s @ %d dbps (range query)",
+                symbol,
+                threshold_decimal_bps,
+            )
+            msg = f"Failed to read bars for {symbol}: {e}"
+            raise CacheReadError(
+                msg,
+                symbol=symbol,
+                operation="read_range",
+            ) from e
+
+        if df.empty:
+            logger.debug(
+                "Cache miss for %s @ %d dbps (range: %d-%d)",
+                symbol,
+                threshold_decimal_bps,
+                start_ts,
+                end_ts,
+            )
+            return None
+
+        logger.debug(
+            "Cache hit: %d bars for %s @ %d dbps (range query)",
+            len(df),
+            symbol,
+            threshold_decimal_bps,
+        )
+
+        # Convert to TZ-aware UTC DatetimeIndex (matches get_range_bars output)
+        df["timestamp"] = pd.to_datetime(df["timestamp_ms"], unit="ms", utc=True)
+        df = df.set_index("timestamp")
+        df = df.drop(columns=["timestamp_ms"])
+
+        # Convert PyArrow dtypes to numpy float64 for compatibility
+        df = normalize_arrow_dtypes(df)
+
+        if include_microstructure:
+            df = normalize_arrow_dtypes(df, columns=list(MICROSTRUCTURE_COLUMNS))
+
+        return df

rangebar/clickhouse/schema.sql

@@ -0,0 +1,187 @@
+-- ClickHouse schema for rangebar cache
+-- Stores computed range bars (Tier 2)
+--
+-- Note: Raw tick data (Tier 1) is stored locally using Parquet files.
+-- See rangebar.storage.TickStorage for tick data caching.
+--
+-- Usage:
+--   CREATE DATABASE IF NOT EXISTS rangebar_cache;
+--   Then run this file or use RangeBarCache._ensure_schema()
+
+-- ============================================================================
+-- Migration for v5.0.0 (from v4.x)
+-- ============================================================================
+-- Run this ONCE if upgrading from rangebar-py v4.x with existing cache:
+--
+-- ALTER TABLE rangebar_cache.range_bars
+--     RENAME COLUMN threshold_bps TO threshold_decimal_bps;
+--
+-- Note: New installations do not need this migration.
+
+-- ============================================================================
+-- Migration for v7.2.0 (Issue #32: rename aggregation_efficiency)
+-- ============================================================================
+-- Run this ONCE if upgrading from rangebar-py v7.1.x with existing cache:
+--
+-- ALTER TABLE rangebar_cache.range_bars
+--     RENAME COLUMN aggregation_efficiency TO aggregation_density;
+--
+-- Note: New installations do not need this migration.
+
+-- ============================================================================
+-- Migration for v10.x (Ouroboros: cyclical reset boundaries)
+-- ============================================================================
+-- Run this ONCE if upgrading from rangebar-py v9.x with existing cache:
+--
+-- ALTER TABLE rangebar_cache.range_bars
+--     ADD COLUMN ouroboros_mode LowCardinality(String) DEFAULT 'week';
+--
+-- Note: New installations do not need this migration.
+-- Plan: /Users/terryli/.claude/plans/sparkling-coalescing-dijkstra.md
+
+-- ============================================================================
+-- Migration for v12.x (Issue #8: Exchange sessions integration)
+-- ============================================================================
+-- Run this ONCE if upgrading from rangebar-py v11.x with existing cache:
+--
+-- ALTER TABLE rangebar_cache.range_bars
+--     ADD COLUMN exchange_session_sydney UInt8 DEFAULT 0,
+--     ADD COLUMN exchange_session_tokyo UInt8 DEFAULT 0,
+--     ADD COLUMN exchange_session_london UInt8 DEFAULT 0,
+--     ADD COLUMN exchange_session_newyork UInt8 DEFAULT 0;
+--
+-- Note: New installations do not need this migration.
+
+-- ============================================================================
+-- Migration for v13.x (Issue #59: Inter-bar features from lookback window)
+-- ============================================================================
+-- Run this ONCE if upgrading from rangebar-py v12.x with existing cache:
+--
+-- ALTER TABLE rangebar_cache.range_bars
+--     -- Tier 1: Core features (7 features)
+--     ADD COLUMN lookback_trade_count Nullable(UInt32) DEFAULT NULL,
+--     ADD COLUMN lookback_ofi Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_duration_us Nullable(Int64) DEFAULT NULL,
+--     ADD COLUMN lookback_intensity Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_vwap_raw Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_vwap_position Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_count_imbalance Nullable(Float64) DEFAULT NULL,
+--     -- Tier 2: Statistical features (5 features)
+--     ADD COLUMN lookback_kyle_lambda Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_burstiness Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_volume_skew Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_volume_kurt Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_price_range Nullable(Float64) DEFAULT NULL,
+--     -- Tier 3: Advanced features (4 features)
+--     ADD COLUMN lookback_kaufman_er Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_garman_klass_vol Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_hurst Nullable(Float64) DEFAULT NULL,
+--     ADD COLUMN lookback_permutation_entropy Nullable(Float64) DEFAULT NULL;
+--
+-- Note: New installations do not need this migration.
+
+-- ============================================================================
+-- Computed Range Bars Cache (Tier 2)
+-- ============================================================================
+-- Stores computed range bars with all parameters as cache key
+-- Cache hit requires exact match on: symbol, threshold, time range
+
+CREATE TABLE IF NOT EXISTS rangebar_cache.range_bars (
+    -- Cache key components
+    symbol LowCardinality(String),
+    threshold_decimal_bps UInt32,
+
+    -- OHLCV data
+    timestamp_ms Int64,
+    open Float64,
+    high Float64,
+    low Float64,
+    close Float64,
+    volume Float64,
+
+    -- Market microstructure (from rangebar-core)
+    vwap Float64 DEFAULT 0,
+    buy_volume Float64 DEFAULT 0,
+    sell_volume Float64 DEFAULT 0,
+    individual_trade_count UInt32 DEFAULT 0,
+    agg_record_count UInt32 DEFAULT 0,
+
+    -- Microstructure features (Issue #25)
+    duration_us Int64 DEFAULT 0,
+    ofi Float64 DEFAULT 0,
+    vwap_close_deviation Float64 DEFAULT 0,
+    price_impact Float64 DEFAULT 0,
+    kyle_lambda_proxy Float64 DEFAULT 0,
+    trade_intensity Float64 DEFAULT 0,
+    volume_per_trade Float64 DEFAULT 0,
+    aggression_ratio Float64 DEFAULT 0,
+    aggregation_density Float64 DEFAULT 1,
+    turnover_imbalance Float64 DEFAULT 0,
+
+    -- Ouroboros (cyclical reset boundaries, v10.x)
+    ouroboros_mode LowCardinality(String) DEFAULT 'week',
+
+    -- Exchange session flags (Issue #8: indicates active traditional market sessions)
+    exchange_session_sydney UInt8 DEFAULT 0,
+    exchange_session_tokyo UInt8 DEFAULT 0,
+    exchange_session_london UInt8 DEFAULT 0,
+    exchange_session_newyork UInt8 DEFAULT 0,
+
+    -- Inter-bar features (Issue #59: computed from lookback window BEFORE bar opens)
+    -- Tier 1: Core features (7 features)
+    lookback_trade_count Nullable(UInt32) DEFAULT NULL,
+    lookback_ofi Nullable(Float64) DEFAULT NULL,
+    lookback_duration_us Nullable(Int64) DEFAULT NULL,
+    lookback_intensity Nullable(Float64) DEFAULT NULL,
+    lookback_vwap_raw Nullable(Float64) DEFAULT NULL,
+    lookback_vwap_position Nullable(Float64) DEFAULT NULL,
+    lookback_count_imbalance Nullable(Float64) DEFAULT NULL,
+    -- Tier 2: Statistical features (5 features)
+    lookback_kyle_lambda Nullable(Float64) DEFAULT NULL,
+    lookback_burstiness Nullable(Float64) DEFAULT NULL,
+    lookback_volume_skew Nullable(Float64) DEFAULT NULL,
+    lookback_volume_kurt Nullable(Float64) DEFAULT NULL,
+    lookback_price_range Nullable(Float64) DEFAULT NULL,
+    -- Tier 3: Advanced features (4 features)
+    lookback_kaufman_er Nullable(Float64) DEFAULT NULL,
+    lookback_garman_klass_vol Nullable(Float64) DEFAULT NULL,
+    lookback_hurst Nullable(Float64) DEFAULT NULL,
+    lookback_permutation_entropy Nullable(Float64) DEFAULT NULL,
+
+    -- Cache metadata
+    cache_key String,                        -- Hash of full parameters
+    rangebar_version String DEFAULT '',      -- Version for invalidation
+    source_start_ts Int64 DEFAULT 0,         -- Input data time range
+    source_end_ts Int64 DEFAULT 0,
+    computed_at DateTime64(3) DEFAULT now64(3)
+)
+ENGINE = ReplacingMergeTree(computed_at)
+-- Partition by symbol, threshold, and month
+PARTITION BY (symbol, threshold_decimal_bps, toYYYYMM(toDateTime(timestamp_ms / 1000)))
+-- Order for efficient lookups
+ORDER BY (symbol, threshold_decimal_bps, timestamp_ms);
+
+-- ============================================================================
+-- Materialized Views (Optional - for analytics)
+-- ============================================================================
+
+-- View: Daily trade volume by symbol
+-- CREATE MATERIALIZED VIEW IF NOT EXISTS rangebar_cache.daily_trade_volume
+-- ENGINE = SummingMergeTree()
+-- PARTITION BY toYYYYMM(date)
+-- ORDER BY (symbol, date)
+-- AS SELECT
+--     symbol,
+--     toDate(toDateTime(timestamp_ms / 1000)) AS date,
+--     count() AS trade_count,
+--     sum(quantity) AS total_volume
+-- FROM rangebar_cache.raw_trades
+-- GROUP BY symbol, date;
+
+-- ============================================================================
+-- Indexes (ClickHouse creates automatically based on ORDER BY)
+-- ============================================================================
+-- No additional indexes needed - ORDER BY creates primary key index
+-- ClickHouse uses sparse indexing which is efficient for our access patterns:
+-- - Symbol + time range queries
+-- - Symbol + threshold lookups