rangebar 11.6.1__cp313-cp313-macosx_11_0_arm64.whl

rangebar/clickhouse/tunnel.py

@@ -0,0 +1,222 @@
+"""SSH tunnel manager for remote ClickHouse hosts.
+
+This module provides SSH tunnel management for connecting to ClickHouse
+on remote GPU workstations when direct network access is not available.
+"""
+
+from __future__ import annotations
+
+import socket
+import subprocess
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+
+def _find_free_port() -> int:
+    """Find a free local port.
+
+    Returns
+    -------
+    int
+        Available port number
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("localhost", 0))
+        return s.getsockname()[1]
+
+
+def _is_port_open(host: str, port: int, timeout: float = 0.5) -> bool:
+    """Check if a port is open.
+
+    Parameters
+    ----------
+    host : str
+        Host to check
+    port : int
+        Port to check
+    timeout : float
+        Connection timeout
+
+    Returns
+    -------
+    bool
+        True if port is open
+    """
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.settimeout(timeout)
+            return s.connect_ex((host, port)) == 0
+    except OSError:
+        return False
+
+
+class SSHTunnel:
+    """Manages SSH tunnel to remote ClickHouse host.
+
+    Creates an SSH tunnel from a local port to the remote host's
+    ClickHouse port (default 8123). Use as a context manager.
+
+    Parameters
+    ----------
+    ssh_alias : str
+        SSH alias from ~/.ssh/config
+    remote_port : int
+        Remote ClickHouse port (default: 8123)
+    local_port : int | None
+        Local port to use (default: auto-assign)
+
+    Examples
+    --------
+    >>> with SSHTunnel("my-gpu-host") as local_port:
+    ...     client = get_client("localhost", local_port)
+    ...     # Use client...
+    """
+
+    def __init__(
+        self,
+        ssh_alias: str,
+        remote_port: int = 8123,
+        local_port: int | None = None,
+    ) -> None:
+        """Initialize tunnel configuration."""
+        self.ssh_alias = ssh_alias
+        self.remote_port = remote_port
+        self._local_port = local_port
+        self._process: subprocess.Popen[bytes] | None = None
+
+    @property
+    def local_port(self) -> int | None:
+        """Get the local port (assigned when tunnel starts)."""
+        return self._local_port
+
+    @property
+    def is_active(self) -> bool:
+        """Check if tunnel is active."""
+        return (
+            self._process is not None
+            and self._process.poll() is None
+            and self._local_port is not None
+            and _is_port_open("localhost", self._local_port)
+        )
+
+    def start(self, timeout: float = 5.0) -> int:
+        """Start the SSH tunnel.
+
+        Parameters
+        ----------
+        timeout : float
+            Maximum time to wait for tunnel to be ready
+
+        Returns
+        -------
+        int
+            Local port the tunnel is listening on
+
+        Raises
+        ------
+        RuntimeError
+            If tunnel fails to start
+        """
+        if self._process is not None:
+            msg = "Tunnel already started"
+            raise RuntimeError(msg)
+
+        # Assign local port if not specified
+        if self._local_port is None:
+            self._local_port = _find_free_port()
+
+        # Start SSH tunnel process
+        self._process = subprocess.Popen(
+            [
+                "ssh",
+                "-N",  # Don't execute remote command
+                "-o",
+                "ExitOnForwardFailure=yes",
+                "-o",
+                "ServerAliveInterval=30",
+                "-o",
+                "ServerAliveCountMax=3",
+                "-L",
+                f"{self._local_port}:localhost:{self.remote_port}",
+                self.ssh_alias,
+            ],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+
+        # Wait for tunnel to be ready
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            # Check if port is open first - handles SSH ControlMaster case
+            # where the ssh process exits immediately after handing off to master
+            if _is_port_open("localhost", self._local_port):
+                return self._local_port
+
+            # Check if process died with error (non-zero exit)
+            exit_code = self._process.poll()
+            if exit_code is not None:
+                # Exit code 0 with ControlMaster means forwarding was handed off
+                # to master connection - give it a moment to activate
+                if exit_code == 0:
+                    time.sleep(0.2)
+                    if _is_port_open("localhost", self._local_port):
+                        # Forwarding via ControlMaster succeeded, but we don't
+                        # own the process anymore - set to None so stop() is a no-op
+                        self._process = None
+                        return self._local_port
+                # Non-zero exit or port still not open after ControlMaster handoff
+                stderr = ""
+                if self._process.stderr:
+                    stderr = self._process.stderr.read().decode()
+                msg = (
+                    f"SSH tunnel to {self.ssh_alias} failed "
+                    f"(exit={exit_code}): {stderr}"
+                )
+                self._process = None
+                raise RuntimeError(msg)
+
+            time.sleep(0.1)
+
+        # Timeout - kill process
+        self._cleanup()
+        msg = f"SSH tunnel to {self.ssh_alias} timed out after {timeout}s"
+        raise RuntimeError(msg)
+
+    def stop(self) -> None:
+        """Stop the SSH tunnel."""
+        self._cleanup()
+
+    def _cleanup(self) -> None:
+        """Clean up tunnel resources."""
+        if self._process is not None:
+            try:
+                self._process.terminate()
+                self._process.wait(timeout=5)
+            except (subprocess.TimeoutExpired, OSError):
+                try:
+                    self._process.kill()
+                    self._process.wait(timeout=1)
+                except (subprocess.TimeoutExpired, OSError):
+                    pass
+            finally:
+                self._process = None
+
+    def __enter__(self) -> int:
+        """Start tunnel and return local port."""
+        return self.start()
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        """Stop tunnel on exit."""
+        self.stop()
+
+    def __del__(self) -> None:
+        """Clean up on garbage collection."""
+        self._cleanup()

rangebar/constants.py

@@ -0,0 +1,288 @@
+"""Constants and presets for rangebar-py.
+
+This module centralizes all constants to eliminate duplication across the codebase.
+Import from here instead of defining locally.
+
+SSoT (Single Source of Truth) for:
+- MICROSTRUCTURE_COLUMNS: Optional microstructure feature columns
+- TIER1_SYMBOLS: High-liquidity crypto symbols
+- THRESHOLD_PRESETS: Named threshold values in decimal basis points
+- THRESHOLD_DECIMAL_MIN/MAX: Valid threshold range
+- _CRYPTO_BASES: Known crypto base symbols for asset class detection
+- _FOREX_CURRENCIES: Known forex currencies for asset class detection
+- MEM_GUARDS: Memory guard registry (Issue #49)
+"""
+
+from __future__ import annotations
+
+# =============================================================================
+# Schema Version Constants (Cache Evolution)
+# =============================================================================
+# Used for cache validation and schema evolution tracking.
+# Increment when schema changes require cache invalidation.
+#
+# Version history:
+# - 6.0.0: OHLCV only (legacy, pre-microstructure)
+# - 7.0.0: Added 15 microstructure columns (Issue #25)
+# - 10.0.0: Added ouroboros_mode column
+# - 11.0.0: Current version with modular architecture
+
+SCHEMA_VERSION_OHLCV_ONLY: str = "6.0.0"  # Pre-microstructure (legacy)
+SCHEMA_VERSION_MICROSTRUCTURE: str = "7.0.0"  # Added 15 microstructure columns
+SCHEMA_VERSION_OUROBOROS: str = "10.0.0"  # Added ouroboros_mode column
+
+# Minimum versions required for features
+MIN_VERSION_FOR_MICROSTRUCTURE: str = SCHEMA_VERSION_MICROSTRUCTURE
+MIN_VERSION_FOR_OUROBOROS: str = SCHEMA_VERSION_OUROBOROS
+
+# =============================================================================
+# Microstructure Columns (Issue #25, v7.0+)
+# =============================================================================
+# These columns are optional and only present when include_microstructure=True
+# or when bars are generated with microstructure features enabled.
+#
+# IMPORTANT: Keep this list in sync with:
+# - crates/rangebar-core/src/bar.rs (Rust struct fields)
+# - python/rangebar/clickhouse/schema.sql (ClickHouse columns)
+
+MICROSTRUCTURE_COLUMNS: tuple[str, ...] = (
+    # Basic extended columns
+    "vwap",
+    "buy_volume",
+    "sell_volume",
+    "individual_trade_count",
+    "agg_record_count",
+    # Microstructure features (Issue #25)
+    "duration_us",
+    "ofi",
+    "vwap_close_deviation",
+    "price_impact",
+    "kyle_lambda_proxy",
+    "trade_intensity",
+    "volume_per_trade",
+    "aggression_ratio",
+    "aggregation_density",
+    "turnover_imbalance",
+)
+
+# =============================================================================
+# Inter-Bar Feature Columns (Issue #59, v12.0+)
+# =============================================================================
+# Computed from a lookback window of trades BEFORE each bar opens.
+# All features are Optional - None when no lookback data available.
+#
+# IMPORTANT: Keep this list in sync with:
+# - crates/rangebar-core/src/interbar.rs (Rust feature computation)
+# - crates/rangebar-core/src/types.rs (RangeBar struct fields)
+# - python/rangebar/clickhouse/schema.sql (ClickHouse columns)
+
+INTER_BAR_FEATURE_COLUMNS: tuple[str, ...] = (
+    # Tier 1: Core features (7 features, min 1 trade)
+    "lookback_trade_count",
+    "lookback_ofi",
+    "lookback_duration_us",
+    "lookback_intensity",
+    "lookback_vwap_raw",
+    "lookback_vwap_position",
+    "lookback_count_imbalance",
+    # Tier 2: Statistical features (5 features)
+    "lookback_kyle_lambda",
+    "lookback_burstiness",
+    "lookback_volume_skew",
+    "lookback_volume_kurt",
+    "lookback_price_range",
+    # Tier 3: Advanced features (4 features, min 60+ trades)
+    "lookback_kaufman_er",
+    "lookback_garman_klass_vol",
+    "lookback_hurst",
+    "lookback_permutation_entropy",
+)
+
+# =============================================================================
+# Tier-1 Symbols (high-liquidity, available on all Binance markets)
+# =============================================================================
+
+TIER1_SYMBOLS: tuple[str, ...] = (
+    "AAVE",
+    "ADA",
+    "AVAX",
+    "BCH",
+    "BNB",
+    "BTC",
+    "DOGE",
+    "ETH",
+    "FIL",
+    "LINK",
+    "LTC",
+    "NEAR",
+    "SOL",
+    "SUI",
+    "UNI",
+    "WIF",
+    "WLD",
+    "XRP",
+)
+
+# =============================================================================
+# Threshold Range (from rangebar-core)
+# =============================================================================
+
+THRESHOLD_DECIMAL_MIN: int = 1  # 1 dbps = 0.001%
+THRESHOLD_DECIMAL_MAX: int = 100_000  # 100,000 dbps = 100%
+
+# =============================================================================
+# Threshold Presets (in decimal basis points)
+# =============================================================================
+# 1 dbps = 0.001% = 0.00001 (one-tenth of a basis point)
+# Example: 250 dbps = 0.25%
+
+THRESHOLD_PRESETS: dict[str, int] = {
+    "micro": 10,  # 10 dbps = 0.01% (scalping)
+    "tight": 50,  # 50 dbps = 0.05% (day trading)
+    "standard": 100,  # 100 dbps = 0.1% (swing trading)
+    "medium": 250,  # 250 dbps = 0.25% (default)
+    "wide": 500,  # 500 dbps = 0.5% (position trading)
+    "macro": 1000,  # 100bps = 1% (long-term)
+}
+
+# =============================================================================
+# Asset Class Detection Helpers
+# =============================================================================
+
+# Common crypto base symbols for detection
+_CRYPTO_BASES: frozenset[str] = frozenset(
+    {
+        "BTC",
+        "ETH",
+        "BNB",
+        "SOL",
+        "XRP",
+        "ADA",
+        "DOGE",
+        "DOT",
+        "MATIC",
+        "AVAX",
+        "LINK",
+        "UNI",
+        "ATOM",
+        "LTC",
+        "ETC",
+        "XLM",
+        "ALGO",
+        "NEAR",
+        "FIL",
+        "APT",
+    }
+)
+
+# Common forex base/quote currencies
+_FOREX_CURRENCIES: frozenset[str] = frozenset(
+    {
+        "EUR",
+        "USD",
+        "GBP",
+        "JPY",
+        "CHF",
+        "AUD",
+        "NZD",
+        "CAD",
+        "SEK",
+        "NOK",
+    }
+)
+
+# =============================================================================
+# Continuity Validation Constants
+# =============================================================================
+
+# Default tolerance for junction continuity validation (0.01% = 0.0001)
+CONTINUITY_TOLERANCE_PCT: float = 0.0001
+
+# =============================================================================
+# Exchange Session Column Names (Ouroboros feature)
+# =============================================================================
+
+EXCHANGE_SESSION_COLUMNS: tuple[str, ...] = (
+    "exchange_session_sydney",
+    "exchange_session_tokyo",
+    "exchange_session_london",
+    "exchange_session_newyork",
+)
+
+# =============================================================================
+# All Optional Columns (for cache operations)
+# =============================================================================
+# Union of microstructure + exchange session columns
+
+ALL_OPTIONAL_COLUMNS: tuple[str, ...] = (
+    *MICROSTRUCTURE_COLUMNS,
+    *EXCHANGE_SESSION_COLUMNS,
+)
+
+# =============================================================================
+# Memory Guard Registry (Issue #49)
+# =============================================================================
+# Each guard prevents a specific memory exhaustion pattern.
+# Code references use "# MEM-XXX:" comments for traceability.
+#
+# Guards are organized by pipeline stage:
+#   Loading  → MEM-001, MEM-004, MEM-007, MEM-010
+#   Process  → MEM-002, MEM-003
+#   Concat   → MEM-006, MEM-008
+#   Test     → MEM-005
+#   Process  → MEM-009
+#
+# When adding a new guard, assign the next number and add an entry here.
+
+MEM_GUARDS: dict[str, dict[str, str]] = {
+    "MEM-001": {
+        "description": "Avoid map_elements() in Parquet parsing (native Polars ops)",
+        "location": "storage/parquet.py:185",
+        "stage": "loading",
+    },
+    "MEM-002": {
+        "description": "Process trades in 100K chunks (~15 MB each)",
+        "location": "orchestration/helpers.py:274, processors/api.py:371",
+        "stage": "processing",
+    },
+    "MEM-003": {
+        "description": "Select columns BEFORE .collect() on LazyFrame",
+        "location": "orchestration/helpers.py:236, processors/api.py:341",
+        "stage": "processing",
+    },
+    "MEM-004": {
+        "description": "Guard read_ticks() with size estimation before .collect()",
+        "location": "storage/parquet.py",
+        "stage": "loading",
+    },
+    "MEM-005": {
+        "description": "gc.collect() after each test to prevent accumulation",
+        "location": "tests/conftest.py:26",
+        "stage": "testing",
+    },
+    "MEM-006": {
+        "description": "Use Polars concat instead of pandas for memory efficiency",
+        "location": "conversion.py:107, orchestration/precompute.py:404",
+        "stage": "concatenation",
+    },
+    "MEM-007": {
+        "description": "Guard deprecated _fetch_binance() with date range limit",
+        "location": "orchestration/helpers.py:136",
+        "stage": "loading",
+    },
+    "MEM-008": {
+        "description": "Streaming bar accumulation (avoid holding all in memory)",
+        "location": "orchestration/range_bars.py",
+        "stage": "concatenation",
+    },
+    "MEM-009": {
+        "description": "Process-level RLIMIT_AS cap (MemoryError instead of OOM kill)",
+        "location": "resource_guard.py",
+        "stage": "process",
+    },
+    "MEM-010": {
+        "description": "Pre-flight memory estimation before tick loading",
+        "location": "resource_guard.py",
+        "stage": "loading",
+    },
+}

rangebar/conversion.py

@@ -0,0 +1,177 @@
+"""Conversion utilities for rangebar-py.
+
+This module provides dtype conversion, normalization, and DataFrame manipulation
+utilities used throughout the codebase. These functions handle:
+- Converting bar dictionaries to Polars/pandas DataFrames
+- Concatenating DataFrames with consistent dtypes
+- Normalizing datetime precision (Issue #44 fix)
+- Converting PyArrow dtypes to numpy for compatibility
+
+SSoT (Single Source of Truth) for:
+- _bars_list_to_polars: Convert bar dicts to Polars DataFrame
+- _concat_pandas_via_polars: Memory-efficient DataFrame concatenation
+- normalize_temporal_precision: Fix mixed datetime precision
+- normalize_arrow_dtypes: Convert PyArrow to numpy dtypes
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pandas as pd
+
+if TYPE_CHECKING:
+    import polars as pl
+
+
+def _bars_list_to_polars(
+    bars: list[dict],
+    include_microstructure: bool = False,
+) -> pl.DataFrame:
+    """Convert list of bar dicts to Polars DataFrame in backtesting.py format.
+
+    Parameters
+    ----------
+    bars : list[dict]
+        List of bar dictionaries from processor
+    include_microstructure : bool
+        Include microstructure columns
+
+    Returns
+    -------
+    pl.DataFrame
+        DataFrame with OHLCV columns (capitalized), DatetimeIndex
+    """
+    import polars as pl
+
+    if not bars:
+        return pl.DataFrame()
+
+    bars_df = pl.DataFrame(bars)
+
+    # Convert timestamp to datetime
+    if "timestamp" in bars_df.columns:
+        bars_df = bars_df.with_columns(
+            pl.col("timestamp")
+            .str.to_datetime(format="%Y-%m-%dT%H:%M:%S%.f%:z")
+            .alias("timestamp")
+        )
+
+    # Rename to backtesting.py format
+    rename_map = {
+        "open": "Open",
+        "high": "High",
+        "low": "Low",
+        "close": "Close",
+        "volume": "Volume",
+    }
+    bars_df = bars_df.rename(
+        {k: v for k, v in rename_map.items() if k in bars_df.columns}
+    )
+
+    # Select columns
+    base_cols = ["timestamp", "Open", "High", "Low", "Close", "Volume"]
+    if include_microstructure:
+        # Include all columns
+        return bars_df
+    # Only OHLCV columns
+    available = [c for c in base_cols if c in bars_df.columns]
+    return bars_df.select(available)
+
+
+def normalize_temporal_precision(pldf: pl.DataFrame) -> pl.DataFrame:
+    """Normalize datetime columns to microsecond precision.
+
+    This prevents SchemaError when concatenating DataFrames with mixed
+    datetime precision (e.g., μs vs ns). See Issue #44.
+
+    Parameters
+    ----------
+    pldf : pl.DataFrame
+        Polars DataFrame to normalize
+
+    Returns
+    -------
+    pl.DataFrame
+        DataFrame with all datetime columns cast to microsecond precision
+    """
+    import polars as pl
+
+    for col in pldf.columns:
+        if pldf[col].dtype.is_temporal():
+            pldf = pldf.with_columns(pl.col(col).dt.cast_time_unit("us"))
+    return pldf
+
+
+def _concat_pandas_via_polars(dfs: list[pd.DataFrame]) -> pd.DataFrame:
+    """Concatenate pandas DataFrames using Polars for memory efficiency (MEM-006).
+
+    This function uses Polars' more efficient concatenation instead of pd.concat,
+    reducing memory fragmentation and improving performance for large datasets.
+
+    Parameters
+    ----------
+    dfs : list[pd.DataFrame]
+        List of pandas DataFrames to concatenate
+
+    Returns
+    -------
+    pd.DataFrame
+        Concatenated DataFrame with sorted DatetimeIndex
+    """
+    import polars as pl
+
+    if not dfs:
+        return pd.DataFrame()
+
+    if len(dfs) == 1:
+        return dfs[0]
+
+    # Convert to Polars
+    pl_dfs = [pl.from_pandas(df.reset_index()) for df in dfs]
+
+    # Normalize datetime columns to consistent precision (μs) before concat
+    # This prevents SchemaError when months have mixed precision (Issue #44)
+    normalized = [normalize_temporal_precision(pldf) for pldf in pl_dfs]
+    combined = pl.concat(normalized)
+
+    # Sort by timestamp/index column
+    index_col = "timestamp" if "timestamp" in combined.columns else combined.columns[0]
+    combined = combined.sort(index_col)
+
+    # Convert back to pandas with proper index
+    result = combined.to_pandas()
+    if index_col in result.columns:
+        result = result.set_index(index_col)
+
+    return result
+
+
+def normalize_arrow_dtypes(
+    df: pd.DataFrame, columns: list[str] | None = None
+) -> pd.DataFrame:
+    """Convert PyArrow dtypes to numpy for compatibility.
+
+    ClickHouse query_df_arrow returns double[pyarrow], but process_trades
+    returns float64. This function normalizes the dtypes.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame potentially containing PyArrow dtypes
+    columns : list[str] | None
+        Columns to normalize. If None, normalizes OHLCV columns.
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with numpy dtypes
+    """
+    if columns is None:
+        columns = ["Open", "High", "Low", "Close", "Volume"]
+
+    for col in columns:
+        if col in df.columns:
+            df[col] = df[col].astype("float64")
+
+    return df