gapless-crypto-clickhouse 7.1.0__py3-none-any.whl

gapless_crypto_clickhouse/utils/etag_cache.py

@@ -0,0 +1,194 @@
+"""ETag-based HTTP caching for immutable Binance Vision data.
+
+CloudFront CDN provides ETags for all monthly ZIP files. Since historical data
+is immutable, ETags enable bandwidth-efficient re-runs through 304 Not Modified
+responses (90%+ bandwidth reduction).
+
+SLO Targets:
+    Availability: 100% - handles cache corruption gracefully
+    Correctness: 100% - cache mismatches trigger full download
+    Observability: All cache hits/misses logged
+    Maintainability: Follows XDG Base Directory Specification
+
+Architecture:
+    - Cache location: $HOME/.cache/gapless-crypto-data/etags.json
+    - Standard library only (pathlib + json)
+    - Exception-only failure (no silent fallbacks)
+"""
+
+import json
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ETagCache:
+    """HTTP ETag cache manager for Binance Vision immutable data.
+
+    Manages ETag-based caching to avoid re-downloading immutable historical data.
+    Uses XDG Base Directory Specification for cache location.
+
+    Cache Structure:
+        {
+            "https://data.binance.vision/.../BTCUSDT-1h-2024-01.zip": {
+                "etag": "efcd0b4716abb9d950262a26fcb6ba43",
+                "last_checked": "2025-10-16T16:30:00Z",
+                "file_size": 12845632
+            }
+        }
+
+    Examples:
+        >>> cache = ETagCache()
+        >>> cache.update_etag(url, "abc123", 1024000)
+        >>> etag = cache.get_etag(url)
+        >>> print(f"Cache hit: {etag}")
+        Cache hit: abc123
+
+    Note:
+        Cache is persistent across runs. Corrupted cache files are automatically
+        deleted and recreated. All errors propagate (exception-only failure).
+    """
+
+    def __init__(self, cache_dir: Optional[Path] = None):
+        """Initialize ETag cache manager.
+
+        Args:
+            cache_dir: Override default cache directory location.
+                      Default: $HOME/.cache/gapless-crypto-data/
+
+        Raises:
+            OSError: If cache directory creation fails
+        """
+        if cache_dir is None:
+            # Follow XDG Base Directory Specification
+            home = Path.home()
+            self.cache_dir = home / ".cache" / "gapless-crypto-data"
+        else:
+            self.cache_dir = cache_dir
+
+        self.cache_file = self.cache_dir / "etags.json"
+
+        # Create cache directory if not exists
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load cache (or create empty)
+        self._cache: Dict[str, Dict] = self._load_cache()
+
+    def _load_cache(self) -> Dict[str, Dict]:
+        """Load cache from disk.
+
+        Returns:
+            Cache dictionary mapping URLs to ETag metadata
+
+        Raises:
+            json.JSONDecodeError: If cache file is corrupted (auto-deleted)
+            OSError: If file read fails (propagated)
+        """
+        if not self.cache_file.exists():
+            logger.debug(f"Cache file not found, creating new cache: {self.cache_file}")
+            return {}
+
+        try:
+            with open(self.cache_file, "r") as f:
+                cache_data = json.load(f)
+                logger.debug(f"Loaded ETag cache with {len(cache_data)} entries")
+                return cache_data
+        except json.JSONDecodeError as e:
+            logger.error(f"Corrupted ETag cache file, deleting: {e}")
+            self.cache_file.unlink()  # Delete corrupted cache
+            raise ValueError(
+                f"ETag cache corrupted at {self.cache_file}. "
+                f"Deleted corrupted file. Original error: {e}"
+            ) from e
+
+    def _save_cache(self) -> None:
+        """Save cache to disk.
+
+        Raises:
+            OSError: If file write fails (propagated)
+            json.JSONEncodeError: If cache data not JSON-serializable (propagated)
+        """
+        with open(self.cache_file, "w") as f:
+            json.dump(self._cache, f, indent=2)
+        logger.debug(f"Saved ETag cache with {len(self._cache)} entries")
+
+    def get_etag(self, url: str) -> Optional[str]:
+        """Get ETag for URL from cache.
+
+        Args:
+            url: Full URL to Binance Vision file
+
+        Returns:
+            ETag string if cached, None if not found
+
+        Examples:
+            >>> cache = ETagCache()
+            >>> etag = cache.get_etag("https://data.binance.vision/.../BTCUSDT-1h-2024-01.zip")
+            >>> if etag:
+            ...     print("Cache hit")
+            ... else:
+            ...     print("Cache miss")
+        """
+        entry = self._cache.get(url)
+        if entry:
+            logger.debug(f"Cache hit for {url}: {entry['etag']}")
+            return entry["etag"]
+        else:
+            logger.debug(f"Cache miss for {url}")
+            return None
+
+    def update_etag(self, url: str, etag: str, file_size: int) -> None:
+        """Update cache with new ETag metadata.
+
+        Args:
+            url: Full URL to Binance Vision file
+            etag: ETag from HTTP response header
+            file_size: Content-Length from HTTP response
+
+        Raises:
+            OSError: If cache save fails (propagated)
+        """
+        self._cache[url] = {
+            "etag": etag,
+            "last_checked": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+            "file_size": file_size,
+        }
+        self._save_cache()
+        logger.debug(f"Updated cache for {url}: {etag}")
+
+    def invalidate(self, url: str) -> None:
+        """Remove URL from cache (ETag mismatch scenario).
+
+        Args:
+            url: Full URL to invalidate
+
+        Raises:
+            OSError: If cache save fails (propagated)
+        """
+        if url in self._cache:
+            del self._cache[url]
+            self._save_cache()
+            logger.warning(f"Invalidated cache entry for {url}")
+
+    def get_cache_stats(self) -> Dict[str, int]:
+        """Get cache statistics for observability.
+
+        Returns:
+            Dictionary with cache entry count and total cached file size
+        """
+        total_size = sum(entry.get("file_size", 0) for entry in self._cache.values())
+        return {"total_entries": len(self._cache), "total_cached_size": total_size}
+
+    def clear_cache(self) -> None:
+        """Clear all cache entries.
+
+        Raises:
+            OSError: If cache file deletion fails (propagated)
+        """
+        self._cache = {}
+        if self.cache_file.exists():
+            self.cache_file.unlink()
+        logger.info("Cleared ETag cache")

gapless_crypto_clickhouse/utils/timeframe_constants.py

@@ -0,0 +1,90 @@
+"""Centralized timeframe constants for data collection and gap detection.
+
+This module provides single source of truth for timeframe-to-interval mappings
+used across collectors and gap fillers, eliminating code duplication and
+preventing calculation bugs.
+
+SLO Targets:
+    Maintainability: Single source of truth eliminates 3+ code duplications
+    Correctness: All 13 timeframes map to accurate minute values
+    Availability: Supports full spectrum from 1s to 1d timeframes
+"""
+
+from datetime import timedelta
+from typing import Dict
+
+import pandas as pd
+
+# Timeframe to minutes mapping (single source of truth)
+TIMEFRAME_TO_MINUTES: Dict[str, float] = {
+    "1s": 1 / 60,  # 1 second = 1/60 minute
+    "1m": 1,
+    "3m": 3,
+    "5m": 5,
+    "15m": 15,
+    "30m": 30,
+    "1h": 60,
+    "2h": 120,
+    "4h": 240,
+    "6h": 360,
+    "8h": 480,
+    "12h": 720,
+    "1d": 1440,  # 24 hours = 1440 minutes
+}
+
+# Pandas Timedelta mapping (derived from minutes)
+TIMEFRAME_TO_TIMEDELTA: Dict[str, pd.Timedelta] = {
+    timeframe: pd.Timedelta(minutes=minutes) for timeframe, minutes in TIMEFRAME_TO_MINUTES.items()
+}
+
+# Python timedelta mapping (for non-pandas contexts)
+TIMEFRAME_TO_PYTHON_TIMEDELTA: Dict[str, timedelta] = {
+    timeframe: timedelta(minutes=minutes) for timeframe, minutes in TIMEFRAME_TO_MINUTES.items()
+}
+
+# Binance API interval mapping (for API parameter compatibility)
+TIMEFRAME_TO_BINANCE_INTERVAL: Dict[str, str] = {
+    "1s": "1s",
+    "1m": "1m",
+    "3m": "3m",
+    "5m": "5m",
+    "15m": "15m",
+    "30m": "30m",
+    "1h": "1h",
+    "2h": "2h",
+    "4h": "4h",
+    "6h": "6h",
+    "8h": "8h",
+    "12h": "12h",
+    "1d": "1d",
+}
+
+# Validation: All timeframes must be present in all mappings
+_EXPECTED_TIMEFRAMES = {
+    "1s",
+    "1m",
+    "3m",
+    "5m",
+    "15m",
+    "30m",
+    "1h",
+    "2h",
+    "4h",
+    "6h",
+    "8h",
+    "12h",
+    "1d",
+}
+
+assert set(TIMEFRAME_TO_MINUTES.keys()) == _EXPECTED_TIMEFRAMES, (
+    f"TIMEFRAME_TO_MINUTES missing timeframes: "
+    f"{_EXPECTED_TIMEFRAMES - set(TIMEFRAME_TO_MINUTES.keys())}"
+)
+assert set(TIMEFRAME_TO_TIMEDELTA.keys()) == _EXPECTED_TIMEFRAMES, (
+    f"TIMEFRAME_TO_TIMEDELTA missing timeframes: "
+    f"{_EXPECTED_TIMEFRAMES - set(TIMEFRAME_TO_TIMEDELTA.keys())}"
+)
+assert set(TIMEFRAME_TO_BINANCE_INTERVAL.keys()) == _EXPECTED_TIMEFRAMES, (
+    f"TIMEFRAME_TO_BINANCE_INTERVAL missing timeframes: "
+    f"{_EXPECTED_TIMEFRAMES - set(TIMEFRAME_TO_BINANCE_INTERVAL.keys())}"
+)

gapless_crypto_clickhouse/utils/timestamp_format_analyzer.py

@@ -0,0 +1,256 @@
+"""Timestamp format detection and analysis for Binance data.
+
+This module provides timestamp format detection supporting both millisecond
+and microsecond precision timestamps, with comprehensive validation and
+transition tracking capabilities.
+
+Classes:
+    TimestampFormatAnalyzer: Analyzes and validates timestamp formats
+
+SLO Targets:
+    Correctness: 100% - accurate format detection for all valid timestamps
+    Observability: Complete reporting of format transitions and statistics
+    Maintainability: Single source of truth for timestamp format logic
+"""
+
+
+class TimestampFormatAnalyzer:
+    """Analyzes timestamp formats in cryptocurrency data with transition tracking.
+
+    Supports both legacy millisecond precision (13-digit) and modern microsecond
+    precision (16-digit) timestamps from Binance data. Tracks format transitions
+    and provides comprehensive statistics.
+
+    Attributes:
+        format_stats: Statistics for each detected format type
+        format_transitions: List of detected format transition points
+        current_format: Current timestamp format being processed
+
+    Examples:
+        >>> analyzer = TimestampFormatAnalyzer()
+        >>> analyzer.initialize_tracking()
+        >>> fmt, secs, valid = analyzer.analyze_timestamp_format(1609459200000, 0)
+        >>> fmt
+        'milliseconds'
+        >>> analyzer.update_format_stats(fmt, 1609459200000, 0)
+        >>> analyzer.report_format_analysis()
+        📈 COMPREHENSIVE FORMAT ANALYSIS:
+          MILLISECONDS: 1 rows (100.0%)
+        ...
+    """
+
+    def __init__(self):
+        """Initialize timestamp format analyzer."""
+        self.format_stats = {}
+        self.format_transitions = []
+        self.current_format = None
+        self._format_analysis_summary = {}
+
+    def initialize_tracking(self):
+        """Initialize format tracking state for new data processing."""
+        self.format_stats = {
+            "milliseconds": {
+                "count": 0,
+                "first_seen": None,
+                "last_seen": None,
+                "sample_values": [],
+            },
+            "microseconds": {
+                "count": 0,
+                "first_seen": None,
+                "last_seen": None,
+                "sample_values": [],
+            },
+            "unknown": {"count": 0, "errors": []},
+        }
+        self.format_transitions = []
+        self.current_format = None
+        self._format_analysis_summary = {}
+
+    def analyze_timestamp_format(self, raw_timestamp_value, csv_row_index):
+        """Comprehensive timestamp format analysis with validation.
+
+        Detects whether a timestamp is in milliseconds (13-digit) or microseconds
+        (16+ digit) format and validates the timestamp is within expected range
+        (2010-2030).
+
+        Args:
+            raw_timestamp_value: Integer timestamp value to analyze
+            csv_row_index: Row index for error reporting
+
+        Returns:
+            tuple: (detected_format_type, converted_seconds, validation_result)
+                - detected_format_type: "milliseconds", "microseconds", or "unknown"
+                - converted_seconds: Timestamp converted to seconds (float or None)
+                - validation_result: Dict with "valid" bool and optional "error_details"
+
+        Examples:
+            >>> analyzer = TimestampFormatAnalyzer()
+            >>> fmt, secs, valid = analyzer.analyze_timestamp_format(1609459200000, 0)
+            >>> fmt
+            'milliseconds'
+            >>> secs
+            1609459200.0
+            >>> valid
+            {'valid': True}
+
+            >>> fmt, secs, valid = analyzer.analyze_timestamp_format(1609459200000000, 1)
+            >>> fmt
+            'microseconds'
+            >>> secs
+            1609459200.0
+        """
+        timestamp_digit_count = len(str(raw_timestamp_value))
+
+        # Enhanced format detection logic
+        if timestamp_digit_count >= 16:  # Microseconds (16+ digits) - 2025+ format
+            detected_format_type = "microseconds"
+            converted_seconds = raw_timestamp_value / 1000000
+            timestamp_min_bound = 1262304000000000  # 2010-01-01 00:00:00 (microseconds)
+            timestamp_max_bound = 1893456000000000  # 2030-01-01 00:00:00 (microseconds)
+
+        elif timestamp_digit_count >= 10:  # Milliseconds (10-15 digits) - Legacy format
+            detected_format_type = "milliseconds"
+            converted_seconds = raw_timestamp_value / 1000
+            timestamp_min_bound = 1262304000000  # 2010-01-01 00:00:00 (milliseconds)
+            timestamp_max_bound = 1893456000000  # 2030-01-01 00:00:00 (milliseconds)
+
+        else:  # Unknown format (less than 10 digits)
+            detected_format_type = "unknown"
+            converted_seconds = None
+            timestamp_min_bound = timestamp_max_bound = None
+
+        # Enhanced validation with detailed error reporting
+        if detected_format_type == "unknown":
+            timestamp_validation_result = {
+                "valid": False,
+                "error_details": {
+                    "row_index": csv_row_index,
+                    "error_type": "unknown_timestamp_format",
+                    "timestamp_value": raw_timestamp_value,
+                    "digit_count": timestamp_digit_count,
+                    "expected_formats": "milliseconds (10-15 digits) or microseconds (16+ digits)",
+                    "raw_row": f"Timestamp too short: {timestamp_digit_count} digits",
+                },
+            }
+        elif raw_timestamp_value < timestamp_min_bound or raw_timestamp_value > timestamp_max_bound:
+            timestamp_validation_result = {
+                "valid": False,
+                "error_details": {
+                    "row_index": csv_row_index,
+                    "error_type": "invalid_timestamp_range",
+                    "timestamp_value": raw_timestamp_value,
+                    "timestamp_format": detected_format_type,
+                    "digit_count": timestamp_digit_count,
+                    "valid_range": f"{timestamp_min_bound} to {timestamp_max_bound}",
+                    "parsed_date": "out_of_range",
+                    "raw_row": f"Out of valid {detected_format_type} range (2010-2030)",
+                },
+            }
+        else:
+            timestamp_validation_result = {"valid": True}
+
+        return detected_format_type, converted_seconds, timestamp_validation_result
+
+    def update_format_stats(self, detected_timestamp_format, raw_timestamp_value, csv_row_index):
+        """Update format statistics and detect transitions.
+
+        Args:
+            detected_timestamp_format: Format type ("milliseconds", "microseconds", "unknown")
+            raw_timestamp_value: Original timestamp value
+            csv_row_index: Row index for tracking
+
+        Returns:
+            bool: True if format transition detected, False otherwise
+        """
+        transition_detected = False
+
+        # Track format transitions
+        if self.current_format is None:
+            self.current_format = detected_timestamp_format
+        elif (
+            self.current_format != detected_timestamp_format
+            and detected_timestamp_format != "unknown"
+        ):
+            self.format_transitions.append(
+                {
+                    "row_index": csv_row_index,
+                    "from_format": self.current_format,
+                    "to_format": detected_timestamp_format,
+                    "timestamp_value": raw_timestamp_value,
+                }
+            )
+            self.current_format = detected_timestamp_format
+            transition_detected = True
+
+        # Update format statistics
+        self.format_stats[detected_timestamp_format]["count"] += 1
+        if self.format_stats[detected_timestamp_format]["first_seen"] is None:
+            self.format_stats[detected_timestamp_format]["first_seen"] = csv_row_index
+        self.format_stats[detected_timestamp_format]["last_seen"] = csv_row_index
+
+        # Store sample values (first 3 per format)
+        if len(self.format_stats[detected_timestamp_format]["sample_values"]) < 3:
+            self.format_stats[detected_timestamp_format]["sample_values"].append(
+                raw_timestamp_value
+            )
+
+        return transition_detected
+
+    def report_format_analysis(self):
+        """Report comprehensive format analysis with transition detection.
+
+        Prints format statistics and transitions to console, and stores
+        analysis summary in self._format_analysis_summary for metadata.
+        """
+        total_rows = sum(stats["count"] for stats in self.format_stats.values())
+
+        print("    📈 COMPREHENSIVE FORMAT ANALYSIS:")
+
+        for format_type, stats in self.format_stats.items():
+            if stats["count"] > 0:
+                percentage = (stats["count"] / total_rows) * 100 if total_rows > 0 else 0
+                print(f"      {format_type.upper()}: {stats['count']:,} rows ({percentage:.1f}%)")
+
+                if format_type != "unknown" and stats["sample_values"]:
+                    first_sample = stats["sample_values"][0]
+                    print(
+                        f"        Sample: {first_sample} (rows {stats['first_seen']}-{stats['last_seen']})"
+                    )
+
+        # Report format transitions
+        if len(self.format_transitions) > 0:
+            print(f"    🔄 FORMAT TRANSITIONS DETECTED: {len(self.format_transitions)}")
+            for i, transition in enumerate(self.format_transitions[:3]):  # Show first 3
+                print(
+                    f"      #{i + 1}: Row {transition['row_index']} - {transition['from_format']} → {transition['to_format']}"
+                )
+                print(f"          Timestamp: {transition['timestamp_value']}")
+            if len(self.format_transitions) > 3:
+                print(f"      ... and {len(self.format_transitions) - 3} more transitions")
+        else:
+            print(
+                f"    ✅ SINGLE FORMAT: No transitions detected - consistent {self.current_format}"
+            )
+
+        # Store format analysis results for metadata
+        self._format_analysis_summary = {
+            "total_rows_analyzed": total_rows,
+            "formats_detected": {
+                fmt: stats["count"]
+                for fmt, stats in self.format_stats.items()
+                if stats["count"] > 0
+            },
+            "transitions_detected": len(self.format_transitions),
+            "transition_details": self.format_transitions,
+            "primary_format": self.current_format,
+            "format_consistency": len(self.format_transitions) == 0,
+        }
+
+    def get_format_analysis_summary(self):
+        """Get format analysis summary for metadata.
+
+        Returns:
+            dict: Format analysis summary with statistics and transitions
+        """
+        return self._format_analysis_summary

gapless_crypto_clickhouse/utils/timestamp_utils.py

@@ -0,0 +1,130 @@
+"""Simple timestamp utility functions for Binance data format conversions.
+
+This module provides lightweight utility functions for timestamp format detection
+and normalization, complementing the comprehensive TimestampFormatAnalyzer class.
+
+Binance Vision API Format Transition (2025-01-01):
+    - Spot data: Transitioned to microseconds (16 digits)
+    - Futures data: Remains milliseconds (13 digits)
+    - Target: Universal microsecond precision (DateTime64(6))
+
+Functions:
+    detect_timestamp_precision: Quick format detection (milliseconds vs microseconds)
+    normalize_timestamp_to_microseconds: Convert timestamps to microsecond precision
+
+SLO Targets:
+    Correctness: 100% - accurate conversion with no data loss
+    Maintainability: Simple functions for inline use throughout codebase
+"""
+
+
+def detect_timestamp_precision(timestamp: int) -> str:
+    """Detect timestamp precision from magnitude.
+
+    Args:
+        timestamp: Raw timestamp from Binance CSV (integer)
+
+    Returns:
+        str: "microseconds" (16+ digits) or "milliseconds" (10-15 digits)
+
+    Raises:
+        ValueError: If timestamp has unexpected digit count (<10 digits)
+
+    Examples:
+        >>> detect_timestamp_precision(1704067200000000)  # 16 digits
+        'microseconds'
+
+        >>> detect_timestamp_precision(1704067200000)  # 13 digits
+        'milliseconds'
+
+        >>> detect_timestamp_precision(123)  # Too short
+        Traceback (most recent call last):
+            ...
+        ValueError: Invalid timestamp 123: expected 10+ digits, got 3
+
+    SLO: Correctness - accurate detection for all valid Binance timestamps
+    """
+    digit_count = len(str(timestamp))
+
+    if digit_count >= 16:  # Microseconds (2025+ spot data)
+        return "microseconds"
+    elif digit_count >= 10:  # Milliseconds (legacy spot, all futures)
+        return "milliseconds"
+    else:
+        raise ValueError(
+            f"Invalid timestamp {timestamp}: expected 10+ digits, got {digit_count}. "
+            f"Valid formats: milliseconds (10-15 digits) or microseconds (16+ digits)."
+        )
+
+
+def normalize_timestamp_to_microseconds(timestamp: int, source_precision: str) -> int:
+    """Normalize timestamp to microsecond precision.
+
+    Converts millisecond timestamps to microseconds for uniform DateTime64(6) storage.
+    Microsecond timestamps are passed through unchanged.
+
+    Args:
+        timestamp: Raw timestamp from Binance CSV
+        source_precision: Detected precision ("milliseconds" or "microseconds")
+
+    Returns:
+        int: Timestamp in microseconds (DateTime64(6) compatible)
+
+    Raises:
+        ValueError: If source_precision is not "milliseconds" or "microseconds"
+
+    Examples:
+        >>> # Milliseconds → Microseconds (multiply by 1000)
+        >>> normalize_timestamp_to_microseconds(1704067200000, "milliseconds")
+        1704067200000000
+
+        >>> # Microseconds → Microseconds (no change)
+        >>> normalize_timestamp_to_microseconds(1704067200000000, "microseconds")
+        1704067200000000
+
+        >>> # Invalid precision
+        >>> normalize_timestamp_to_microseconds(1704067200, "seconds")
+        Traceback (most recent call last):
+            ...
+        ValueError: Unknown precision: seconds. Must be 'milliseconds' or 'microseconds'.
+
+    SLO: Correctness - lossless conversion with validation
+    """
+    if source_precision == "microseconds":
+        return timestamp  # Already correct precision
+    elif source_precision == "milliseconds":
+        return timestamp * 1000  # Convert ms → μs
+    else:
+        raise ValueError(
+            f"Unknown precision: {source_precision}. Must be 'milliseconds' or 'microseconds'."
+        )
+
+
+def normalize_timestamp_auto(timestamp: int) -> int:
+    """Auto-detect and normalize timestamp to microseconds.
+
+    Convenience function combining detection and normalization in one call.
+    Useful for inline conversions without explicit precision tracking.
+
+    Args:
+        timestamp: Raw timestamp from Binance CSV
+
+    Returns:
+        int: Timestamp normalized to microseconds
+
+    Raises:
+        ValueError: If timestamp is invalid (<10 digits)
+
+    Examples:
+        >>> # Auto-detect milliseconds and convert
+        >>> normalize_timestamp_auto(1704067200000)
+        1704067200000000
+
+        >>> # Auto-detect microseconds and pass through
+        >>> normalize_timestamp_auto(1704067200000000)
+        1704067200000000
+
+    SLO: Correctness - accurate auto-detection and conversion
+    """
+    precision = detect_timestamp_precision(timestamp)
+    return normalize_timestamp_to_microseconds(timestamp, precision)