gapless-crypto-clickhouse 7.1.0__py3-none-any.whl

gapless_crypto_clickhouse/clickhouse/__init__.py

@@ -0,0 +1,17 @@
+"""
+ClickHouse connection and schema management for gapless-crypto-data v4.0.0.
+
+Provides ClickHouseConnection class for database operations using clickhouse-driver.
+Replaces QuestDB implementation (ADR-0003) for future-proofing and ecosystem maturity.
+
+Usage:
+    from gapless_crypto_clickhouse.clickhouse import ClickHouseConnection
+
+    with ClickHouseConnection() as conn:
+        conn.execute("SELECT 1")
+"""
+
+from .config import ClickHouseConfig
+from .connection import ClickHouseConnection
+
+__all__ = ["ClickHouseConnection", "ClickHouseConfig"]

gapless_crypto_clickhouse/clickhouse/config.py

@@ -0,0 +1,119 @@
+"""
+ClickHouse configuration for gapless-crypto-data v4.0.0.
+
+Environment variable support for connection parameters.
+Follows same pattern as QuestDBConfig (ADR-0003).
+
+Error Handling: Raise and propagate (no fallback, no defaults for required params)
+"""
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass
+class ClickHouseConfig:
+    """
+    ClickHouse connection configuration.
+
+    Attributes:
+        host: ClickHouse server hostname (default: localhost)
+        port: Native protocol port (default: 9000)
+        http_port: HTTP interface port (default: 8123)
+        database: Database name (default: default)
+        user: Username (default: default)
+        password: Password (default: empty)
+        secure: Enable TLS/SSL for secure connections (default: False, required for ClickHouse Cloud)
+
+    Environment Variables:
+        CLICKHOUSE_HOST: Override host
+        CLICKHOUSE_PORT: Override native protocol port
+        CLICKHOUSE_HTTP_PORT: Override HTTP port
+        CLICKHOUSE_DATABASE: Override database name
+        CLICKHOUSE_USER: Override username
+        CLICKHOUSE_PASSWORD: Override password
+        CLICKHOUSE_SECURE: Enable TLS/SSL (set to 'true' for ClickHouse Cloud)
+
+    Example:
+        # Default configuration (localhost)
+        config = ClickHouseConfig.from_env()
+
+        # Custom configuration
+        config = ClickHouseConfig(host="clickhouse.example.com", port=9000)
+    """
+
+    host: str = "localhost"
+    port: int = 9000
+    http_port: int = 8123
+    database: str = "default"
+    user: str = "default"
+    password: str = ""
+    secure: bool = False
+
+    @classmethod
+    def from_env(cls) -> "ClickHouseConfig":
+        """
+        Create configuration from environment variables.
+
+        Returns:
+            ClickHouseConfig with values from environment or defaults
+
+        Raises:
+            ValueError: If CLICKHOUSE_PORT is not a valid integer
+
+        Example:
+            export CLICKHOUSE_HOST=clickhouse.example.com
+            export CLICKHOUSE_PORT=9000
+            config = ClickHouseConfig.from_env()
+        """
+        try:
+            port = int(os.getenv("CLICKHOUSE_PORT", "9000"))
+            http_port = int(os.getenv("CLICKHOUSE_HTTP_PORT", "8123"))
+        except ValueError as e:
+            raise ValueError(
+                f"Invalid CLICKHOUSE_PORT or CLICKHOUSE_HTTP_PORT (must be integer): {e}"
+            ) from e
+
+        return cls(
+            host=os.getenv("CLICKHOUSE_HOST", "localhost"),
+            port=port,
+            http_port=http_port,
+            database=os.getenv("CLICKHOUSE_DATABASE", "default"),
+            user=os.getenv("CLICKHOUSE_USER", "default"),
+            password=os.getenv("CLICKHOUSE_PASSWORD", ""),
+            secure=os.getenv("CLICKHOUSE_SECURE", "false").lower() in ("true", "1", "yes"),
+        )
+
+    def validate(self) -> None:
+        """
+        Validate configuration parameters.
+
+        Raises:
+            ValueError: If any parameter is invalid
+
+        Example:
+            config = ClickHouseConfig(port=-1)
+            config.validate()  # Raises ValueError
+        """
+        if not self.host:
+            raise ValueError("host cannot be empty")
+
+        if not (1 <= self.port <= 65535):
+            raise ValueError(f"port must be between 1 and 65535, got {self.port}")
+
+        if not (1 <= self.http_port <= 65535):
+            raise ValueError(f"http_port must be between 1 and 65535, got {self.http_port}")
+
+        if not self.database:
+            raise ValueError("database cannot be empty")
+
+        if not self.user:
+            raise ValueError("user cannot be empty")
+
+    def __repr__(self) -> str:
+        """String representation (hide password)."""
+        return (
+            f"ClickHouseConfig(host='{self.host}', port={self.port}, "
+            f"http_port={self.http_port}, database='{self.database}', "
+            f"user='{self.user}', password='***', secure={self.secure})"
+        )

gapless_crypto_clickhouse/clickhouse/connection.py

@@ -0,0 +1,269 @@
+"""
+ClickHouse connection management for gapless-crypto-clickhouse v6.0.0.
+
+Provides context-managed connection to ClickHouse using clickhouse-connect with Apache Arrow support.
+Replaces clickhouse-driver (ADR-0023) for 3x faster queries and 4x less memory.
+
+Error Handling: Raise and propagate (no fallback, no retry, no silent failures)
+SLOs: Availability (connection health checks), Correctness (query validation),
+      Observability (connection logging), Maintainability (standard HTTP client)
+
+Usage:
+    from gapless_crypto_clickhouse.clickhouse import ClickHouseConnection
+
+    with ClickHouseConnection() as conn:
+        df = conn.query_dataframe("SELECT * FROM ohlcv FINAL LIMIT 10")
+        print(df)  # pandas DataFrame with Arrow-optimized internals
+"""
+
+import logging
+from typing import Any, Dict, List, Optional, Tuple
+
+import clickhouse_connect
+import pandas as pd
+
+from .config import ClickHouseConfig
+
+logger = logging.getLogger(__name__)
+
+
+class ClickHouseConnection:
+    """
+    Context-managed ClickHouse connection with Apache Arrow support.
+
+    Provides execute() for queries and insert_dataframe() for bulk inserts.
+    Uses HTTP protocol (port 8123) with Apache Arrow for zero-copy DataFrame creation.
+
+    Attributes:
+        config: ClickHouse configuration
+        client: clickhouse-connect Client instance
+
+    Error Handling:
+        - Connection failures raise Exception
+        - Query failures raise Exception
+        - No retries, no fallbacks (raise and propagate policy)
+
+    Performance:
+        - Arrow-optimized queries: 3x faster DataFrame creation
+        - Zero-copy when possible: 4x less memory
+        - HTTP protocol: nginx/reverse proxy compatible
+
+    Example:
+        with ClickHouseConnection() as conn:
+            # Execute query (returns tuples)
+            result = conn.execute("SELECT COUNT(*) FROM ohlcv")
+
+            # Query DataFrame (Arrow-optimized internally)
+            df = conn.query_dataframe("SELECT * FROM ohlcv FINAL LIMIT 10")
+
+            # Insert DataFrame
+            df = pd.DataFrame({"col": [1, 2, 3]})
+            conn.insert_dataframe(df, "test_table")
+    """
+
+    def __init__(self, config: Optional[ClickHouseConfig] = None) -> None:
+        """
+        Initialize ClickHouse connection.
+
+        Args:
+            config: ClickHouse configuration (default: from environment)
+
+        Raises:
+            ValueError: If configuration is invalid
+            Exception: If connection fails
+
+        Example:
+            # Default configuration (localhost)
+            conn = ClickHouseConnection()
+
+            # Custom configuration
+            config = ClickHouseConfig(host="clickhouse.example.com")
+            conn = ClickHouseConnection(config)
+        """
+        self.config = config or ClickHouseConfig.from_env()
+        self.config.validate()
+
+        logger.info(
+            f"Initializing ClickHouse connection: {self.config.host}:{self.config.http_port} "
+            f"(HTTP protocol with Arrow support)"
+        )
+
+        try:
+            # clickhouse-connect uses HTTP protocol (port 8123 local, 8443 Cloud)
+            self.client = clickhouse_connect.get_client(
+                host=self.config.host,
+                port=self.config.http_port,
+                database=self.config.database,
+                username=self.config.user,
+                password=self.config.password,
+                secure=self.config.secure,  # Enable TLS/SSL for ClickHouse Cloud
+                # Performance settings
+                settings={
+                    "max_block_size": 100000,  # Batch size for queries
+                },
+            )
+        except Exception as e:
+            raise Exception(
+                f"Failed to connect to ClickHouse at {self.config.host}:{self.config.http_port}: {e}"
+            ) from e
+
+    def __enter__(self) -> "ClickHouseConnection":
+        """Context manager entry with schema validation."""
+        if not self.health_check():
+            raise Exception("ClickHouse health check failed during context manager entry")
+
+        # Schema validation (ADR-0024)
+        from .schema_validator import SchemaValidationError, SchemaValidator
+
+        try:
+            validator = SchemaValidator(self)
+            validator.validate_schema()
+            logger.info("Schema validation passed")
+        except SchemaValidationError as e:
+            logger.error(f"Schema validation failed: {e}")
+            raise
+
+        logger.debug("ClickHouse connection opened")
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        """Context manager exit (cleanup)."""
+        if self.client:
+            self.client.close()
+            logger.debug("ClickHouse connection closed")
+
+    def health_check(self) -> bool:
+        """
+        Verify ClickHouse connection is alive.
+
+        Returns:
+            True if connection is healthy, False otherwise
+
+        Example:
+            conn = ClickHouseConnection()
+            if conn.health_check():
+                print("Connection healthy")
+        """
+        try:
+            result = self.client.command("SELECT 1")
+            if result != 1:
+                logger.error(f"Health check failed: unexpected result {result}")
+                return False
+            logger.debug("ClickHouse health check passed")
+            return True
+        except Exception as e:
+            logger.error(f"ClickHouse health check failed: {e}")
+            return False
+
+    def execute(self, query: str, params: Optional[Dict[str, Any]] = None) -> List[Tuple[Any, ...]]:
+        """
+        Execute SQL query with parameter substitution.
+
+        Args:
+            query: SQL query string (use {name:Type} placeholders for clickhouse-connect)
+            params: Query parameters (dict mapping placeholder names to values)
+
+        Returns:
+            List of result tuples
+
+        Raises:
+            Exception: If query execution fails
+
+        Example:
+            # Simple query
+            result = conn.execute("SELECT 1")  # [(1,)]
+
+            # Parameterized query (clickhouse-connect format)
+            result = conn.execute(
+                "SELECT * FROM ohlcv WHERE symbol = {symbol:String}",
+                params={'symbol': 'BTCUSDT'}
+            )
+        """
+        try:
+            logger.debug(f"Executing query: {query[:100]}...")
+            result = self.client.query(query, parameters=params or {})
+            rows = result.result_rows
+            logger.debug(f"Query returned {len(rows)} rows")
+            return rows
+        except Exception as e:
+            raise Exception(f"Query execution failed: {query[:100]}...\nError: {e}") from e
+
+    def query_dataframe(self, query: str, params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        """
+        Execute SQL query and return results as pandas DataFrame with Arrow optimization.
+
+        Args:
+            query: SQL query string (use {name:Type} placeholders for clickhouse-connect)
+            params: Query parameters (dict mapping placeholder names to values)
+
+        Returns:
+            pandas DataFrame with query results (Arrow-optimized internally)
+
+        Raises:
+            Exception: If query execution fails
+
+        Performance:
+            - Arrow format enabled: 3x faster DataFrame creation
+            - Zero-copy when compatible: 4x less memory
+            - Automatic fallback if Arrow not available
+
+        Example:
+            # Simple query
+            df = conn.query_dataframe("SELECT * FROM ohlcv FINAL LIMIT 10")
+
+            # Parameterized query
+            df = conn.query_dataframe(
+                "SELECT * FROM ohlcv FINAL WHERE symbol = {symbol:String}",
+                params={'symbol': 'BTCUSDT'}
+            )
+        """
+        try:
+            logger.debug(f"Executing query (DataFrame, Arrow-optimized): {query[:100]}...")
+            # Use Arrow-optimized query method for 3x faster DataFrame creation
+            df = self.client.query_df_arrow(query, parameters=params or {})
+            logger.debug(f"Query returned {len(df)} rows (Arrow-optimized)")
+            return df
+        except Exception as e:
+            raise Exception(f"Query execution failed: {query[:100]}...\nError: {e}") from e
+
+    def insert_dataframe(self, df: pd.DataFrame, table: str) -> int:
+        """
+        Bulk insert DataFrame to ClickHouse table.
+
+        Args:
+            df: pandas DataFrame with data to insert
+            table: Target table name
+
+        Returns:
+            Number of rows inserted
+
+        Raises:
+            Exception: If insert fails
+            ValueError: If DataFrame is empty or has invalid schema
+
+        Example:
+            df = pd.DataFrame({
+                'timestamp': pd.to_datetime(['2024-01-01']),
+                'symbol': ['BTCUSDT'],
+                'open': [50000.0]
+            })
+            rows = conn.insert_dataframe(df, 'ohlcv')
+            print(f"Inserted {rows} rows")
+        """
+        if df.empty:
+            logger.warning(f"Empty DataFrame, skipping insert to {table}")
+            return 0
+
+        try:
+            logger.info(f"Inserting {len(df)} rows to {table}")
+
+            # Use standard insert (Arrow benefits are mainly on query side)
+            self.client.insert_df(table, df)
+
+            logger.info(f"Successfully inserted {len(df)} rows to {table}")
+            return len(df)
+
+        except Exception as e:
+            raise Exception(f"Bulk insert failed for table {table} ({len(df)} rows): {e}") from e
+        except ValueError as e:
+            raise ValueError(f"Invalid DataFrame schema for table {table}: {e}") from e

gapless_crypto_clickhouse/clickhouse/schema.sql

@@ -0,0 +1,98 @@
+-- ClickHouse Schema for gapless-crypto-data v4.0.0
+-- ADR-0005: ClickHouse Migration for Future-Proofing
+--
+-- ReplacingMergeTree engine with deterministic versioning for zero-gap guarantee.
+-- Preserves ADR-0004 futures support (instrument_type column for spot/futures).
+--
+-- Error Handling: Raise and propagate (no silent failures)
+-- SLOs: Availability, Correctness (zero-gap via _version), Observability, Maintainability
+
+CREATE TABLE IF NOT EXISTS ohlcv (
+    -- Primary timestamp (microsecond precision - ADR-0021)
+    -- Upgraded from DateTime64(3) to support Binance's 2025-01-01 format transition:
+    --   Spot data: microseconds (16 digits) after 2025-01-01
+    --   Futures data: milliseconds (13 digits), converted to microseconds during ingestion
+    timestamp DateTime64(6) CODEC(DoubleDelta, LZ4),
+
+    -- Metadata columns (low-cardinality, optimized for indexing)
+    symbol LowCardinality(String) CODEC(ZSTD(3)),           -- Trading pair (e.g., "BTCUSDT")
+    timeframe LowCardinality(String) CODEC(ZSTD(3)),       -- Timeframe (e.g., "1h", "1mo")
+    instrument_type LowCardinality(String) CODEC(ZSTD(3)), -- 'spot' or 'futures-um' (ADR-0004, ADR-0021)
+    data_source LowCardinality(String) CODEC(ZSTD(3)),     -- 'cloudfront'
+
+    -- OHLCV data (core price/volume metrics)
+    open Float64 CODEC(Gorilla, LZ4),
+    high Float64 CODEC(Gorilla, LZ4),
+    low Float64 CODEC(Gorilla, LZ4),
+    close Float64 CODEC(Gorilla, LZ4),
+    volume Float64 CODEC(Gorilla, LZ4),
+
+    -- Additional microstructure metrics (Binance 11-column format)
+    close_time DateTime64(6) CODEC(DoubleDelta, LZ4),  -- Upgraded to microsecond precision
+    quote_asset_volume Float64 CODEC(Gorilla, LZ4),
+    number_of_trades Int64 CODEC(Delta, LZ4),
+    taker_buy_base_asset_volume Float64 CODEC(Gorilla, LZ4),
+    taker_buy_quote_asset_volume Float64 CODEC(Gorilla, LZ4),
+
+    -- Futures-specific data (ADR-0021, v3.2.0+)
+    funding_rate Nullable(Float64) CODEC(Gorilla, LZ4),  -- NULL for spot, initially NULL for futures
+
+    -- Deduplication support (application-level, preserves zero-gap guarantee)
+    _version UInt64 CODEC(Delta, LZ4),  -- Deterministic hash of row content
+    _sign Int8 DEFAULT 1                -- ReplacingMergeTree sign (1 for active rows)
+
+) ENGINE = ReplacingMergeTree(_version)
+ORDER BY (timestamp, symbol, timeframe, instrument_type)
+PARTITION BY toYYYYMMDD(timestamp)
+SETTINGS
+    index_granularity = 8192,           -- Default granularity (good for time-series)
+    allow_nullable_key = 0,             -- Disallow NULL in ORDER BY keys (data quality)
+    merge_with_ttl_timeout = 86400;     -- Merge within 24 hours (background deduplication)
+
+-- Rationale:
+-- 1. ReplacingMergeTree(_version): Handles duplicates via background merges
+--    - _version is deterministic hash of (timestamp, OHLCV, symbol, timeframe, instrument_type)
+--    - Identical writes → identical _version → consistent merge outcome
+--    - Preserves zero-gap guarantee via deterministic deduplication
+--
+-- 2. ORDER BY composite key: (timestamp, symbol, timeframe, instrument_type)
+--    - Optimizes queries filtering by these columns
+--    - ClickHouse uses ORDER BY as primary key (unlike PostgreSQL)
+--
+-- 3. PARTITION BY toYYYYMMDD(timestamp): Daily partitions
+--    - Matches ADR-0003 QuestDB partition strategy (PARTITION BY DAY)
+--    - Enables efficient partition pruning for date-range queries
+--
+-- 4. LowCardinality(String): ClickHouse equivalent to QuestDB SYMBOL
+--    - Optimizes storage for low-cardinality columns (symbol, timeframe, etc.)
+--    - Automatic dictionary encoding (similar to SYMBOL capacity)
+--
+-- 5. CODEC compression:
+--    - DoubleDelta: Optimized for timestamps (sequential values)
+--    - Gorilla: Optimized for float values (OHLCV data)
+--    - Delta: Optimized for integer sequences (number_of_trades)
+--    - ZSTD: General-purpose compression for string columns
+--
+-- 6. DateTime64(6): Microsecond precision (ADR-0021)
+--    - Upgraded from DateTime64(3) to support Binance's 2025-01-01 format transition
+--    - Spot data: microseconds (16 digits) after 2025-01-01
+--    - Futures data: milliseconds (13 digits), converted to microseconds during ingestion
+--    - Universal microsecond precision prevents timestamp errors
+--    - ClickHouse equivalent to QuestDB TIMESTAMP type
+
+-- Zero-Gap Guarantee:
+-- Unlike QuestDB DEDUP ENABLE UPSERT KEYS (immediate consistency),
+-- ClickHouse uses eventual consistency (duplicates visible until merge).
+-- Application-level deterministic versioning ensures consistent merge outcomes.
+--
+-- Query pattern for deduplicated results:
+--   SELECT * FROM ohlcv FINAL WHERE symbol = 'BTCUSDT' AND timeframe = '1h';
+--
+-- FINAL keyword forces deduplication at query time (10-30% performance overhead).
+-- This is an acceptable trade-off for zero-gap guarantee preservation.
+
+-- Migration from QuestDB (ADR-0003):
+-- QuestDB SYMBOL → ClickHouse LowCardinality(String)
+-- QuestDB DEDUP ENABLE UPSERT KEYS → ClickHouse ReplacingMergeTree(_version)
+-- QuestDB PARTITION BY DAY → ClickHouse PARTITION BY toYYYYMMDD(timestamp)
+-- QuestDB PostgreSQL wire protocol → ClickHouse native protocol (clickhouse-driver)