rangebar 11.6.1__cp313-cp313-macosx_11_0_arm64.whl

rangebar/cli.py

@@ -0,0 +1,298 @@
+"""Command-line interface for rangebar-py cache management.
+
+This module provides CLI commands for managing the ClickHouse cache,
+including status checks, population, and clearing operations.
+
+Usage
+-----
+After installation, the CLI is available as `rangebar`:
+
+    $ rangebar status BTCUSDT
+    $ rangebar populate BTCUSDT --start 2024-01-01 --end 2024-06-30
+    $ rangebar clear BTCUSDT --confirm
+
+Or run as a module:
+
+    $ python -m rangebar.cli status BTCUSDT
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from datetime import UTC, datetime
+
+import click
+
+# Configure logging for CLI
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+
+@click.group()
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output")
+@click.version_option(package_name="rangebar")
+def cli(verbose: bool) -> None:
+    """rangebar-py cache management CLI.
+
+    Manage ClickHouse cache for computed range bars.
+    """
+    if verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+
+@cli.command()
+@click.argument("symbol")
+@click.option(
+    "--threshold",
+    "-t",
+    type=int,
+    default=250,
+    help="Threshold in dbps (default: 250)",
+)
+def status(symbol: str, threshold: int) -> None:
+    """Show cache status for a symbol.
+
+    Example: rangebar status BTCUSDT
+    """
+    try:
+        from .clickhouse import RangeBarCache
+    except ImportError as e:
+        click.echo(f"Error: ClickHouse support not available: {e}", err=True)
+        sys.exit(1)
+
+    try:
+        with RangeBarCache() as cache:
+            # Get bar count
+            count = cache.count_bars(symbol, threshold)
+
+            # Get timestamp range
+            oldest = cache.get_oldest_bar_timestamp(symbol, threshold)
+            newest = cache.get_newest_bar_timestamp(symbol, threshold)
+
+            click.echo(f"Symbol: {symbol}")
+            click.echo(f"Threshold: {threshold} dbps")
+            click.echo(f"Cached bars: {count:,}")
+
+            if oldest and newest:
+                oldest_dt = datetime.fromtimestamp(oldest / 1000)  # noqa: DTZ006
+                newest_dt = datetime.fromtimestamp(newest / 1000)  # noqa: DTZ006
+                click.echo(f"Date range: {oldest_dt.date()} to {newest_dt.date()}")
+            else:
+                click.echo("Date range: N/A (no cached bars)")
+
+    except ConnectionError as e:
+        click.echo(f"Error: Cannot connect to ClickHouse: {e}", err=True)
+        sys.exit(1)
+
+
+@cli.command()
+@click.argument("symbol")
+@click.option("--start", "-s", required=True, help="Start date (YYYY-MM-DD)")
+@click.option("--end", "-e", required=True, help="End date (YYYY-MM-DD)")
+@click.option(
+    "--threshold",
+    "-t",
+    type=int,
+    default=250,
+    help="Threshold in dbps (default: 250)",
+)
+@click.option(
+    "--notify/--no-notify",
+    default=True,
+    help="Send Telegram notifications (default: on)",
+)
+def populate(
+    symbol: str,
+    start: str,
+    end: str,
+    threshold: int,
+    notify: bool,
+) -> None:
+    """Populate cache for a date range.
+
+    Fetches tick data and computes range bars, storing results in ClickHouse.
+
+    Example: rangebar populate BTCUSDT --start 2024-01-01 --end 2024-06-30
+    """
+    # Validate dates
+    try:
+        datetime.strptime(start, "%Y-%m-%d")  # noqa: DTZ007
+        datetime.strptime(end, "%Y-%m-%d")  # noqa: DTZ007
+    except ValueError:
+        click.echo("Error: Invalid date format. Use YYYY-MM-DD", err=True)
+        sys.exit(1)
+
+    # Enable Telegram notifications if requested
+    if notify:
+        try:
+            from .notify.telegram import enable_telegram_notifications
+
+            enable_telegram_notifications()
+        except ImportError:
+            click.echo("Warning: Telegram notifications not available", err=True)
+
+    click.echo(f"Populating cache for {symbol} from {start} to {end}...")
+    click.echo(f"Threshold: {threshold} dbps")
+
+    try:
+        from . import get_range_bars
+
+        df = get_range_bars(
+            symbol,
+            start,
+            end,
+            threshold_decimal_bps=threshold,
+            use_cache=True,
+            fetch_if_missing=True,
+        )
+
+        click.echo(f"Computed {len(df):,} bars")
+        click.echo("Bars should now be cached in ClickHouse")
+
+    except (ValueError, RuntimeError) as e:
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(1)
+
+
+@cli.command()
+@click.argument("symbol")
+@click.option(
+    "--threshold",
+    "-t",
+    type=int,
+    default=None,
+    help="Threshold in dbps (default: all thresholds)",
+)
+@click.option("--confirm", is_flag=True, help="Confirm deletion (required)")
+def clear(symbol: str, threshold: int | None, confirm: bool) -> None:
+    """Clear cache for a symbol.
+
+    Requires --confirm flag to prevent accidental deletion.
+
+    Example: rangebar clear BTCUSDT --confirm
+    """
+    if not confirm:
+        click.echo("Error: Add --confirm flag to delete cache data", err=True)
+        click.echo(f"  rangebar clear {symbol} --confirm")
+        sys.exit(1)
+
+    try:
+        from .clickhouse import RangeBarCache
+    except ImportError as e:
+        click.echo(f"Error: ClickHouse support not available: {e}", err=True)
+        sys.exit(1)
+
+    try:
+        with RangeBarCache() as cache:
+            if threshold:
+                # Clear specific threshold
+                click.echo(f"Clearing cache for {symbol} @ {threshold} dbps...")
+
+                # Get count before clearing
+                count = cache.count_bars(symbol, threshold)
+                if count == 0:
+                    click.echo("No cached bars found")
+                    return
+
+                # Delete using timestamp range (all time)
+                cache.invalidate_range_bars_by_range(
+                    symbol=symbol,
+                    threshold_decimal_bps=threshold,
+                    start_timestamp_ms=0,
+                    end_timestamp_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
+                )
+                click.echo(f"Cleared {count:,} bars (deletion is async)")
+            else:
+                click.echo(f"Clearing all cache for {symbol}...")
+                # Would need to query all thresholds and clear each
+                # For now, just show a message
+                click.echo(
+                    "Note: Clearing all thresholds not yet implemented. "
+                    "Specify --threshold to clear a specific threshold."
+                )
+
+    except ConnectionError as e:
+        click.echo(f"Error: Cannot connect to ClickHouse: {e}", err=True)
+        sys.exit(1)
+
+
+@cli.command()
+def list_symbols() -> None:
+    """List all cached symbols and their thresholds."""
+    try:
+        from .clickhouse import RangeBarCache
+    except ImportError as e:
+        click.echo(f"Error: ClickHouse support not available: {e}", err=True)
+        sys.exit(1)
+
+    try:
+        with RangeBarCache() as cache:
+            # Query distinct symbol/threshold combinations
+            query = """
+                SELECT symbol, threshold_decimal_bps, count(*) as bar_count
+                FROM rangebar_cache.range_bars FINAL
+                GROUP BY symbol, threshold_decimal_bps
+                ORDER BY symbol, threshold_decimal_bps
+            """
+            result = cache.client.query(query)
+
+            if not result.result_rows:
+                click.echo("No cached data found")
+                return
+
+            click.echo("Cached symbols:")
+            click.echo("-" * 50)
+            click.echo(f"{'Symbol':<12} {'Threshold':<12} {'Bars':>12}")
+            click.echo("-" * 50)
+
+            for row in result.result_rows:
+                symbol, threshold, count = row
+                click.echo(f"{symbol:<12} {threshold:<12} {count:>12,}")
+
+    except ConnectionError as e:
+        click.echo(f"Error: Cannot connect to ClickHouse: {e}", err=True)
+        sys.exit(1)
+
+
+@cli.command()
+def test_telegram() -> None:
+    """Send a test Telegram notification."""
+    try:
+        from .notify.telegram import is_configured, send_telegram
+    except ImportError:
+        click.echo("Error: Telegram module not available", err=True)
+        sys.exit(1)
+
+    if not is_configured():
+        click.echo(
+            "Error: Telegram not configured. "
+            "Set RANGEBAR_TELEGRAM_TOKEN environment variable.",
+            err=True,
+        )
+        sys.exit(1)
+
+    success = send_telegram(
+        "<b>rangebar-py CLI Test</b>\n\n"
+        "This is a test notification from the rangebar CLI.\n"
+        "If you see this, Telegram notifications are working correctly."
+    )
+
+    if success:
+        click.echo("Test notification sent successfully")
+    else:
+        click.echo("Failed to send test notification", err=True)
+        sys.exit(1)
+
+
+def main() -> None:
+    """Entry point for the CLI."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

rangebar/clickhouse/CLAUDE.md

@@ -0,0 +1,139 @@
+# ClickHouse Cache Layer
+
+**Parent**: [/python/rangebar/CLAUDE.md](/python/rangebar/CLAUDE.md) | **Schema**: [schema.sql](./schema.sql)
+
+---
+
+## Cache Population
+
+### Native Method (Recommended)
+
+Use `get_range_bars()` with `fetch_if_missing=True` to populate the cache:
+
+```python
+from rangebar import get_range_bars
+
+# Populate threshold 100 data for BTCUSDT
+df = get_range_bars(
+    "BTCUSDT",
+    start_date="2023-06-01",
+    end_date="2025-12-01",
+    threshold_decimal_bps=100,
+    use_cache=True,
+    fetch_if_missing=True,
+    include_microstructure=False,
+)
+print(f"Fetched {len(df):,} bars")
+```
+
+**How it works**:
+
+1. Checks ClickHouse cache for existing data
+2. If missing, fetches raw tick data from Binance (day-by-day to prevent OOM)
+3. Computes range bars via Rust backend
+4. Stores results in `rangebar_cache.range_bars` table
+5. Returns pandas DataFrame
+
+### Populating Remote Hosts
+
+Run on the target machine:
+
+```bash
+# SSH to host and run in tmux (long-running)
+ssh <host> "tmux new-session -d -s rangebar-fetch 'cd ~/alpha-forge-research && source .venv/bin/activate && python3 << \"PYEOF\"
+from rangebar import get_range_bars
+import time
+
+print(\"Fetching BTCUSDT threshold 100 data...\")
+start = time.time()
+df = get_range_bars(
+    \"BTCUSDT\",
+    start_date=\"2023-06-01\",
+    end_date=\"2025-12-01\",
+    threshold_decimal_bps=100,
+    use_cache=True,
+    fetch_if_missing=True,
+)
+elapsed = time.time() - start
+print(f\"Fetched {len(df):,} bars in {elapsed:.1f}s\")
+PYEOF
+ 2>&1 | tee ~/fetch_rangebar.log'"
+
+# Monitor progress
+ssh <host> "tail -f ~/fetch_rangebar.log"
+```
+
+---
+
+## Checking Cache Status
+
+### Using cache_status.py script
+
+```bash
+python scripts/cache_status.py
+```
+
+### Direct ClickHouse query
+
+```python
+import clickhouse_connect
+client = clickhouse_connect.get_client(host='localhost')
+result = client.query('''
+    SELECT symbol, threshold_decimal_bps, count(*) as bars,
+           min(timestamp_ms) as earliest, max(timestamp_ms) as latest
+    FROM rangebar_cache.range_bars
+    GROUP BY symbol, threshold_decimal_bps
+''')
+for row in result.result_rows:
+    print(f"{row[0]} @ {row[1]} dbps: {row[2]:,} bars")
+```
+
+---
+
+## Threshold Presets
+
+| Preset       | Value | Use Case         |
+| ------------ | ----- | ---------------- |
+| `"micro"`    | 10    | Scalping         |
+| `"tight"`    | 50    | Day trading      |
+| `"standard"` | 100   | Swing trading    |
+| `"medium"`   | 250   | Default          |
+| `"wide"`     | 500   | Position trading |
+| `"macro"`    | 1000  | Long-term        |
+
+---
+
+## Host-Specific Cache Status
+
+| Host        | Symbols                                     | Thresholds Cached     | Notes             |
+| ----------- | ------------------------------------------- | --------------------- | ----------------- |
+| bigblack    | BTCUSDT, ETHUSDT, SOLUSDT, BNBUSDT (crypto) | 25, 50, 100, 200 dbps | Primary GPU host  |
+| bigblack    | EURUSD (forex)                              | 50, 100, 200 dbps     | Exness Raw_Spread |
+| littleblack | 700                                         | 700 dbps              | Secondary host    |
+| local       | varies                                      | varies                | Development       |
+
+**Total cached**: 260M+ bars (crypto) + 130K bars (forex)
+
+To add a threshold to a host, run the population script above on that host.
+
+---
+
+## Files
+
+| File                  | Purpose                                                        |
+| --------------------- | -------------------------------------------------------------- |
+| `cache.py`            | RangeBarCache class, core cache operations                     |
+| `bulk_operations.py`  | BulkStoreMixin (store_bars_bulk, store_bars_batch)             |
+| `query_operations.py` | QueryOperationsMixin (get_n_bars, get_bars_by_timestamp_range) |
+| `schema.sql`          | ClickHouse table schema (v7.0: 10 microstructure cols)         |
+| `config.py`           | Connection configuration                                       |
+| `preflight.py`        | Installation checks                                            |
+| `tunnel.py`           | SSH tunnel support                                             |
+
+---
+
+## Related
+
+- [/CLAUDE.md](/CLAUDE.md) - Project hub
+- [/python/rangebar/CLAUDE.md](/python/rangebar/CLAUDE.md) - Python API
+- [/scripts/cache_status.py](/scripts/cache_status.py) - Status script

rangebar/clickhouse/__init__.py

@@ -0,0 +1,100 @@
+"""ClickHouse cache layer for computed range bars.
+
+This module provides caching for computed range bars (Tier 2) using ClickHouse.
+Raw tick data (Tier 1) is stored locally via `rangebar.storage.TickStorage`.
+
+Configuration
+-------------
+Set environment variables via mise (recommended) or directly:
+
+    # Connection mode (RANGEBAR_MODE)
+    export RANGEBAR_MODE=local  # Force localhost:8123 only
+    export RANGEBAR_MODE=cloud  # Require CLICKHOUSE_HOST env var
+    export RANGEBAR_MODE=auto   # Auto-detect (default)
+
+    # Host configuration (for AUTO/CLOUD modes)
+    export RANGEBAR_CH_HOSTS="host1,host2"  # SSH aliases from ~/.ssh/config
+    export RANGEBAR_CH_PRIMARY="host1"       # Default host
+    export CLICKHOUSE_HOST="localhost"       # Direct host (CLOUD mode)
+
+If no env vars set, falls back to localhost:8123 in AUTO mode.
+
+Example
+-------
+>>> from rangebar.clickhouse import RangeBarCache, get_available_clickhouse_host
+>>> from rangebar import process_trades_to_dataframe_cached
+>>>
+>>> # Check ClickHouse availability
+>>> host = get_available_clickhouse_host()
+>>> print(f"Using ClickHouse at {host.host} via {host.method}")
+>>>
+>>> # Use cached processing
+>>> df = process_trades_to_dataframe_cached(trades, symbol="BTCUSDT")
+
+See Also
+--------
+rangebar.storage.TickStorage : Local Parquet storage for raw tick data
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+
+from .cache import CacheKey, RangeBarCache
+from .client import (
+    ClickHouseQueryError,
+    ClickHouseUnavailableError,
+    get_client,
+)
+from .config import ClickHouseConfig, ConnectionMode, get_connection_mode
+from .mixin import ClickHouseClientMixin
+from .preflight import (
+    ClickHouseNotConfiguredError,
+    HostConnection,
+    InstallationLevel,
+    PreflightResult,
+    detect_clickhouse_state,
+    get_available_clickhouse_host,
+)
+from .tunnel import SSHTunnel
+
+__all__ = [
+    # Sorted for ruff RUF022
+    "CacheKey",
+    "ClickHouseClientMixin",
+    "ClickHouseConfig",
+    "ClickHouseNotConfiguredError",
+    "ClickHouseQueryError",
+    "ClickHouseUnavailableError",
+    "ConnectionMode",
+    "HostConnection",
+    "InstallationLevel",
+    "PreflightResult",
+    "RangeBarCache",
+    "SSHTunnel",
+    "detect_clickhouse_state",
+    "get_available_clickhouse_host",
+    "get_client",
+    "get_connection_mode",
+]
+
+
+def _emit_import_warning() -> None:
+    """Emit warning at import time if ClickHouse not ready."""
+    try:
+        state = detect_clickhouse_state()
+        if state.level < InstallationLevel.RUNNING_NO_SCHEMA:
+            warnings.warn(
+                f"ClickHouse cache not available: {state.message}. "
+                f"Cached functions will fail. {state.action_required or ''}",
+                UserWarning,
+                stacklevel=3,
+            )
+    except Exception:
+        pass  # Don't fail import on preflight errors
+
+
+# Optional: emit warning at import time (can be disabled via env var)
+if not os.getenv("RANGEBAR_SKIP_IMPORT_CHECK"):
+    _emit_import_warning()