gapless-crypto-clickhouse 7.1.0__py3-none-any.whl

gapless_crypto_clickhouse/gap_filling/universal_gap_filler.py

@@ -0,0 +1,757 @@
+#!/usr/bin/env python3
+"""
+Universal Gap Filler - Detects and fills ALL gaps in OHLCV CSV files
+
+This script automatically detects ALL gaps in any timeframe's CSV file and fills them
+using authentic Binance API data with full 11-column microstructure format.
+
+Unlike synthetic data approaches, this filler uses authentic Binance data
+providing complete microstructure columns for professional analysis.
+
+Key Features:
+- Auto-detects gaps by analyzing timestamp sequences
+- Uses authentic Binance API with full 11-column microstructure format
+- Handles all timeframes (1s, 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d)
+- Provides authentic order flow metrics including trade counts and taker volumes
+- Processes gaps chronologically to maintain data integrity
+- NO synthetic or estimated data - only authentic exchange data
+- API-first validation protocol using authentic Binance data exclusively
+"""
+
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import httpx
+import pandas as pd
+
+from ..utils.timeframe_constants import (
+    TIMEFRAME_TO_BINANCE_INTERVAL,
+    TIMEFRAME_TO_PYTHON_TIMEDELTA,
+    TIMEFRAME_TO_TIMEDELTA,
+)
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+class UniversalGapFiller:
+    """Universal gap detection and filling for all timeframes with authentic 11-column microstructure format.
+
+    Automatically detects and fills timestamp gaps in OHLCV CSV files using authentic
+    Binance API data. Provides complete gap detection across all timeframes with
+    professional-grade microstructure data including order flow metrics.
+
+    Unlike synthetic data generators, this gap filler exclusively uses authentic
+    Binance market data, ensuring all filled gaps contain real order flow metrics,
+    trade counts, and taker volume statistics essential for quantitative analysis.
+
+    Features:
+        - Universal gap detection for any timeframe (1s to 1d)
+        - Authentic Binance API data for gap filling (never synthetic)
+        - Complete 11-column microstructure format preservation
+        - Chronological processing for data integrity
+        - Automatic symbol extraction from filenames
+        - Batch processing for multiple files
+        - Safe atomic operations with backup/rollback
+
+    Supported Timeframes:
+        - 1s: Second-based intervals
+        - 1m, 3m, 5m, 15m, 30m: Minute-based intervals
+        - 1h, 2h, 4h, 6h, 8h, 12h: Hour-based intervals
+        - 1d: Daily intervals
+
+    Data Quality:
+        All gap-filled data maintains the same structure as original Binance data:
+        - OHLCV: Open, High, Low, Close, Volume (base asset)
+        - Timestamps: Open time, Close time
+        - Order Flow: Quote asset volume, Number of trades
+        - Taker Metrics: Taker buy base volume, Taker buy quote volume
+
+    Examples:
+        For simple gap filling, consider using the function-based API:
+
+        >>> import gapless_crypto_clickhouse as gcd
+        >>> results = gcd.fill_gaps("./data")
+        >>> print(f"Filled {results['gaps_filled']}/{results['gaps_detected']} gaps")
+
+        Advanced usage with this class for detailed control:
+
+        >>> gap_filler = UniversalGapFiller()
+        >>> gaps = gap_filler.detect_all_gaps("BTCUSDT_1h_2024-01-01_to_2024-12-31.csv", "1h")
+        >>> print(f"Found {len(gaps)} gaps")
+        >>> success = gap_filler.fill_gap(gaps[0], "BTCUSDT_1h_data.csv", "1h")
+        >>> print(f"Gap filled: {success}")
+        Found 3 gaps
+        Gap filled: True
+
+        Batch processing for directory:
+
+        >>> gap_filler = UniversalGapFiller()
+        >>> result = gap_filler.process_file("BTCUSDT_1h.csv", "1h")
+        >>> print(f"Filled {result['gaps_filled']}/{result['gaps_detected']} gaps")
+        Filled 2/3 gaps
+
+        Custom symbol processing:
+
+        >>> symbol = gap_filler.extract_symbol_from_filename("SOLUSDT_15m_data.csv")
+        >>> print(f"Extracted symbol: {symbol}")
+        Extracted symbol: SOLUSDT
+
+    Note:
+        This gap filler requires internet connectivity to fetch authentic data
+        from Binance's public API. Rate limiting is automatically handled to
+        respect API limits during gap filling operations.
+    """
+
+    # ADR-0021: API endpoint constants for spot and futures
+    SPOT_API_URL = "https://api.binance.com/api/v3/klines"
+    FUTURES_API_URL = "https://fapi.binance.com/fapi/v1/klines"
+
+    def __init__(self, instrument_type: str = "spot"):
+        """Initialize UniversalGapFiller with instrument type support.
+
+        Args:
+            instrument_type: Instrument type - "spot" or "futures-um" (default: "spot")
+
+        Raises:
+            ValueError: If instrument_type is invalid
+        """
+        # ADR-0021: Validate instrument type and set API endpoint
+        if instrument_type not in ("spot", "futures-um"):
+            raise ValueError(
+                f"Invalid instrument_type '{instrument_type}'. Must be 'spot' or 'futures-um'"
+            )
+        self.instrument_type = instrument_type
+
+        # ADR-0021: API endpoint selection based on instrument type
+        if instrument_type == "spot":
+            self.binance_base_url = self.SPOT_API_URL
+        else:  # futures-um
+            self.binance_base_url = self.FUTURES_API_URL
+
+    def extract_symbol_from_filename(self, csv_path) -> str:
+        """Extract symbol from CSV filename
+
+        Supports formats like:
+        - binance_spot_BTCUSDT-1h_20240101-20240101_v2.5.0.csv
+        - BTCUSDT_1h_data.csv
+        - ETHUSDT-4h.csv
+        """
+        # Handle both string and Path inputs
+        if isinstance(csv_path, (str, Path)):
+            path_obj = Path(csv_path)
+            filename = path_obj.name
+        else:
+            filename = str(csv_path)
+
+        # Handle gapless-crypto-data format: binance_spot_SYMBOL-timeframe_dates.csv
+        if "binance_spot_" in filename:
+            parts = filename.split("_")
+            if len(parts) >= 3:
+                symbol_part = parts[2]  # BTCUSDT-1h
+                symbol = symbol_part.split("-")[0]  # BTCUSDT
+                return symbol
+
+        # Handle simple formats: SYMBOL_timeframe or SYMBOL-timeframe
+        for separator in ["-", "_"]:
+            if separator in filename:
+                parts = filename.split(separator)
+                potential_symbol = parts[0]
+                # Check if it looks like a trading pair (ends with USDT, BTC, ETH, etc.)
+                if potential_symbol.endswith(("USDT", "BTC", "ETH", "BNB")):
+                    return potential_symbol
+
+        # Fallback: look for common trading pairs (top 20 by market cap)
+        common_symbols = [
+            "BTCUSDT",
+            "ETHUSDT",
+            "BNBUSDT",
+            "SOLUSDT",
+            "XRPUSDT",
+            "DOGEUSDT",
+            "ADAUSDT",
+            "AVAXUSDT",
+            "DOTUSDT",
+            "LINKUSDT",
+            "MATICUSDT",
+            "LTCUSDT",
+            "UNIUSDT",
+            "ATOMUSDT",
+            "FTMUSDT",
+            "NEARUSDT",
+            "ALGOUSDT",
+            "SANDUSDT",
+            "MANAUSDT",
+            "APEUSDT",
+        ]
+        filename_upper = filename.upper()
+        for symbol in common_symbols:
+            if symbol in filename_upper:
+                return symbol
+
+        # Default fallback (should not happen in practice)
+        logger.warning(
+            f"⚠️  Could not extract symbol from filename {filename}, defaulting to BTCUSDT"
+        )
+        return "BTCUSDT"
+
+    def detect_all_gaps(self, csv_path: Path, timeframe: str) -> List[Dict]:
+        """Detect ALL gaps in CSV file by analyzing timestamp sequence for 11-column format"""
+        logger.info(f"🔍 Analyzing {csv_path} for gaps...")
+
+        # Load CSV data
+        ohlcv_dataframe = pd.read_csv(csv_path, comment="#")
+        ohlcv_dataframe["date"] = pd.to_datetime(ohlcv_dataframe["date"])
+        ohlcv_dataframe = ohlcv_dataframe.sort_values("date")
+
+        # Calculate expected interval using centralized constants
+        expected_interval = TIMEFRAME_TO_PYTHON_TIMEDELTA[timeframe]
+
+        detected_gaps = []
+        for row_index in range(1, len(ohlcv_dataframe)):
+            current_time = ohlcv_dataframe.iloc[row_index]["date"]
+            previous_time = ohlcv_dataframe.iloc[row_index - 1]["date"]
+            actual_gap_duration = current_time - previous_time
+
+            if actual_gap_duration > expected_interval:
+                timestamp_gap_info = {
+                    "position": row_index,
+                    "start_time": previous_time + expected_interval,
+                    "end_time": current_time,
+                    "duration": actual_gap_duration,
+                    "expected_interval": expected_interval,
+                }
+                detected_gaps.append(timestamp_gap_info)
+                logger.info(
+                    f"   📊 Gap {len(detected_gaps)}: {timestamp_gap_info['start_time']} → {timestamp_gap_info['end_time']} ({timestamp_gap_info['duration']})"
+                )
+
+        logger.info(f"✅ Found {len(detected_gaps)} gaps in {timeframe} timeframe")
+        return detected_gaps
+
+    def fetch_binance_data(
+        self,
+        start_time: datetime,
+        end_time: datetime,
+        timeframe: str,
+        symbol: str,
+        enhanced_format: bool = False,
+    ) -> Optional[List[Dict]]:
+        """Fetch authentic microstructure data from Binance API - NO synthetic data"""
+        binance_interval = TIMEFRAME_TO_BINANCE_INTERVAL[timeframe]
+
+        # Convert to millisecond timestamps for Binance API
+        # ✅ UTC ONLY: All timestamps are UTC - no timezone conversion needed
+
+        # Convert pandas Timestamp to datetime if needed
+        if hasattr(start_time, "to_pydatetime"):
+            start_time = start_time.to_pydatetime()
+        if hasattr(end_time, "to_pydatetime"):
+            end_time = end_time.to_pydatetime()
+
+        # Simple UTC timestamp conversion - CSV timestamps are naive UTC
+        # The CSV timestamps should be interpreted as local machine time for API calls
+        # This matches how Binance API expects timestamps
+        start_timestamp_ms = int(start_time.timestamp() * 1000)
+        end_timestamp_ms = int(end_time.timestamp() * 1000)
+
+        api_request_params = {
+            "symbol": symbol,
+            "interval": binance_interval,
+            "startTime": start_timestamp_ms,
+            "endTime": end_timestamp_ms,
+            "limit": 1000,
+        }
+
+        logger.info(f"   📡 Binance API call: {api_request_params}")
+
+        try:
+            http_response = httpx.get(self.binance_base_url, params=api_request_params, timeout=30)
+            http_response.raise_for_status()
+            binance_klines_data = http_response.json()
+
+            if not binance_klines_data:
+                logger.warning("   ❌ Binance returned no data")
+                return None
+
+            # Convert Binance data to required format with authentic microstructure data
+            processed_candles = []
+            for raw_candle_data in binance_klines_data:
+                # Binance returns: [open_time, open, high, low, close, volume, close_time,
+                #                  quote_asset_volume, number_of_trades, taker_buy_base_asset_volume,
+                #                  taker_buy_quote_asset_volume, ignore]
+
+                open_time = datetime.fromtimestamp(int(raw_candle_data[0]) / 1000)
+                close_time = datetime.fromtimestamp(int(raw_candle_data[6]) / 1000)
+
+                # Only include candles within the gap period (all UTC)
+                if start_time <= open_time.replace(tzinfo=None) < end_time:
+                    # Basic OHLCV data (always included)
+                    candle_bar_data = {
+                        "timestamp": open_time.strftime("%Y-%m-%d %H:%M:%S"),
+                        "open": float(raw_candle_data[1]),
+                        "high": float(raw_candle_data[2]),
+                        "low": float(raw_candle_data[3]),
+                        "close": float(raw_candle_data[4]),
+                        "volume": float(raw_candle_data[5]),
+                    }
+
+                    # Add authentic microstructure data for enhanced format
+                    if enhanced_format:
+                        candle_bar_data.update(
+                            {
+                                "close_time": close_time.strftime("%Y-%m-%d %H:%M:%S"),
+                                "quote_asset_volume": float(raw_candle_data[7]),
+                                "number_of_trades": int(raw_candle_data[8]),
+                                "taker_buy_base_asset_volume": float(raw_candle_data[9]),
+                                "taker_buy_quote_asset_volume": float(raw_candle_data[10]),
+                            }
+                        )
+
+                    processed_candles.append(candle_bar_data)
+                    logger.info(f"   ✅ Retrieved authentic candle: {open_time}")
+
+            logger.info(f"   📈 Retrieved {len(processed_candles)} authentic candles from Binance")
+            return processed_candles
+
+        except Exception as api_exception:
+            logger.error(f"   ❌ Binance API error: {api_exception}")
+            return None
+
+    def _detect_csv_format(self, existing_data: pd.DataFrame) -> tuple[bool, bool]:
+        """Detect CSV format: enhanced (11 columns) vs legacy (6 columns).
+
+        Args:
+            existing_data: Existing CSV data as DataFrame
+
+        Returns:
+            tuple: (is_enhanced_format, is_legacy_format)
+        """
+        enhanced_columns = [
+            "date",
+            "open",
+            "high",
+            "low",
+            "close",
+            "volume",
+            "close_time",
+            "quote_asset_volume",
+            "number_of_trades",
+            "taker_buy_base_asset_volume",
+            "taker_buy_quote_asset_volume",
+        ]
+        legacy_columns = ["date", "open", "high", "low", "close", "volume"]
+
+        is_enhanced = all(col in existing_data.columns for col in enhanced_columns)
+        is_legacy = all(col in existing_data.columns for col in legacy_columns)
+
+        if is_enhanced:
+            logger.info("   🚀 Enhanced 11-column format detected")
+        elif is_legacy:
+            logger.info("   📊 Legacy 6-column format detected")
+        else:
+            logger.error(f"   ❌ Unknown CSV format. Columns: {list(existing_data.columns)}")
+
+        return is_enhanced, is_legacy
+
+    def _retrieve_api_data_with_metadata(
+        self,
+        timestamp_gap_info: Dict,
+        trading_timeframe: str,
+        extracted_symbol: str,
+        is_enhanced_format: bool,
+    ) -> tuple[Optional[List[Dict]], Dict]:
+        """Retrieve API data and create metadata tracking.
+
+        Args:
+            timestamp_gap_info: Gap information with start/end times
+            trading_timeframe: Trading timeframe (e.g., "1h")
+            extracted_symbol: Symbol extracted from filename
+            is_enhanced_format: Whether enhanced format is used
+
+        Returns:
+            tuple: (authentic_api_data, gap_fill_metadata)
+        """
+        logger.info("   🔍 Step 1: Attempting authentic Binance REST API data retrieval")
+        authentic_api_data = self.fetch_binance_data(
+            timestamp_gap_info["start_time"],
+            timestamp_gap_info["end_time"],
+            trading_timeframe,
+            extracted_symbol,
+            enhanced_format=is_enhanced_format,
+        )
+
+        # Initialize metadata
+        gap_fill_metadata = {
+            "timestamp": timestamp_gap_info["start_time"].strftime("%Y-%m-%d %H:%M:%S"),
+            "duration_hours": (
+                timestamp_gap_info["end_time"] - timestamp_gap_info["start_time"]
+            ).total_seconds()
+            / 3600,
+            "fill_method": None,
+            "data_source": None,
+            "authentic_data": False,
+            "synthetic_data": False,
+            "reason": None,
+            "ohlcv": None,
+            "microstructure_data": None,
+        }
+
+        if not authentic_api_data:
+            logger.warning("   ⚠️ Step 1 Failed: No authentic API data available")
+            logger.info("   🔍 Step 2: Checking if gap is legitimate exchange outage")
+            logger.error("   ❌ Gap filling failed: No authentic data available via API")
+            logger.info("   📋 Preserving authentic data integrity - no synthetic fill applied")
+            return None, gap_fill_metadata
+
+        logger.info(
+            f"   ✅ Step 1 Success: Retrieved {len(authentic_api_data)} authentic candles from API"
+        )
+
+        # Update metadata for successful retrieval
+        gap_fill_metadata.update(
+            {
+                "fill_method": "binance_rest_api",
+                "data_source": "https://api.binance.com/api/v3/klines",
+                "authentic_data": True,
+                "synthetic_data": False,
+                "reason": "missing_from_monthly_file_but_available_via_api",
+            }
+        )
+
+        # Add OHLCV data to metadata
+        if authentic_api_data:
+            first_candle = authentic_api_data[0]
+            gap_fill_metadata["ohlcv"] = {
+                "open": first_candle["open"],
+                "high": first_candle["high"],
+                "low": first_candle["low"],
+                "close": first_candle["close"],
+                "volume": first_candle["volume"],
+            }
+
+            if is_enhanced_format and "quote_asset_volume" in first_candle:
+                gap_fill_metadata["microstructure_data"] = {
+                    "quote_asset_volume": first_candle["quote_asset_volume"],
+                    "number_of_trades": first_candle["number_of_trades"],
+                    "taker_buy_base_asset_volume": first_candle["taker_buy_base_asset_volume"],
+                    "taker_buy_quote_asset_volume": first_candle["taker_buy_quote_asset_volume"],
+                }
+
+        return authentic_api_data, gap_fill_metadata
+
+    def _prepare_api_dataframe(
+        self,
+        authentic_api_data: List[Dict],
+        is_enhanced_format: bool,
+    ) -> pd.DataFrame:
+        """Convert API data to DataFrame and select appropriate columns.
+
+        Args:
+            authentic_api_data: Raw API data
+            is_enhanced_format: Whether to include microstructure columns
+
+        Returns:
+            DataFrame with selected columns
+        """
+        df = pd.DataFrame(authentic_api_data)
+        df["date"] = pd.to_datetime(df["timestamp"])
+
+        if is_enhanced_format:
+            # Enhanced format: include all microstructure columns
+            columns = ["date", "open", "high", "low", "close", "volume"]
+            if "close_time" in df.columns:
+                columns.extend(
+                    [
+                        "close_time",
+                        "quote_asset_volume",
+                        "number_of_trades",
+                        "taker_buy_base_asset_volume",
+                        "taker_buy_quote_asset_volume",
+                    ]
+                )
+            return df[columns]
+        else:
+            # Legacy format: only basic OHLCV columns
+            return df[["date", "open", "high", "low", "close", "volume"]]
+
+    def _filter_to_gap_period(
+        self,
+        api_dataframe: pd.DataFrame,
+        timestamp_gap_info: Dict,
+    ) -> Optional[pd.DataFrame]:
+        """Filter API data to only include timestamps within gap period.
+
+        Args:
+            api_dataframe: API data as DataFrame
+            timestamp_gap_info: Gap information with start/end times
+
+        Returns:
+            Filtered DataFrame or None if no data in gap period
+        """
+        gap_start = pd.to_datetime(timestamp_gap_info["start_time"])
+        gap_end = pd.to_datetime(timestamp_gap_info["end_time"])
+
+        time_filter = (api_dataframe["date"] >= gap_start) & (api_dataframe["date"] < gap_end)
+        filtered = api_dataframe[time_filter].copy()
+
+        if len(filtered) == 0:
+            logger.warning("   ⚠️ No authentic Binance data falls within gap period after filtering")
+            return None
+
+        logger.info(f"   📊 Filtered to {len(filtered)} authentic candles within gap period")
+        return filtered
+
+    def _merge_and_deduplicate(
+        self,
+        existing_data: pd.DataFrame,
+        filtered_api_data: pd.DataFrame,
+    ) -> pd.DataFrame:
+        """Merge existing and new data, removing duplicates.
+
+        Args:
+            existing_data: Existing CSV data
+            filtered_api_data: Filtered API data for gap period
+
+        Returns:
+            Combined DataFrame with duplicates removed
+        """
+        combined = pd.concat([existing_data, filtered_api_data], ignore_index=True)
+
+        pre_dedup = len(combined)
+        combined = combined.sort_values("date").drop_duplicates(subset=["date"], keep="first")
+        duplicates = pre_dedup - len(combined)
+
+        if duplicates > 0:
+            logger.warning(f"   ⚠️ Removed {duplicates} duplicate timestamp(s) during gap filling")
+
+        return combined
+
+    def _validate_gap_filled(
+        self,
+        combined_data: pd.DataFrame,
+        timestamp_gap_info: Dict,
+        trading_timeframe: str,
+    ) -> None:
+        """Validate that gap was actually filled.
+
+        Args:
+            combined_data: Combined DataFrame after gap filling
+            timestamp_gap_info: Gap information with start/end times
+            trading_timeframe: Trading timeframe for interval calculation
+        """
+        gap_start = pd.to_datetime(timestamp_gap_info["start_time"])
+        gap_end = pd.to_datetime(timestamp_gap_info["end_time"])
+
+        sorted_data = combined_data.sort_values("date").reset_index(drop=True)
+        remaining_gaps = []
+
+        for i in range(1, len(sorted_data)):
+            current = sorted_data.iloc[i]["date"]
+            previous = sorted_data.iloc[i - 1]["date"]
+            expected_interval = TIMEFRAME_TO_TIMEDELTA[trading_timeframe]
+            actual_diff = current - previous
+
+            if actual_diff > expected_interval:
+                # Check if overlaps with target gap
+                if (previous < gap_end) and (current > gap_start):
+                    remaining_gaps.append(f"{previous} → {current}")
+
+        if remaining_gaps:
+            logger.warning(f"   ⚠️ Gap partially filled - remaining gaps: {remaining_gaps}")
+
+    def _save_with_headers(
+        self,
+        csv_path: Path,
+        dataframe: pd.DataFrame,
+    ) -> None:
+        """Save DataFrame to CSV with header comments preserved.
+
+        Args:
+            csv_path: Path to CSV file
+            dataframe: DataFrame to save
+        """
+        # Read header comments
+        headers = []
+        with open(csv_path, "r") as f:
+            for line in f:
+                if line.startswith("#"):
+                    headers.append(line.rstrip())
+                else:
+                    break
+
+        # Write headers + data
+        with open(csv_path, "w") as f:
+            for header in headers:
+                f.write(header + "\n")
+            dataframe.to_csv(f, index=False)
+
+    def fill_gap(
+        self,
+        timestamp_gap_info: Dict,
+        csv_path: Path,
+        trading_timeframe: str,
+    ) -> bool:
+        """Fill a single gap with authentic Binance data using API-first validation protocol."""
+        logger.info(
+            f"🔧 Filling gap: {timestamp_gap_info['start_time']} → {timestamp_gap_info['end_time']}"
+        )
+        logger.info("   📋 Applying API-first validation protocol")
+
+        # Load and detect format
+        existing_data = pd.read_csv(csv_path, comment="#")
+        existing_data["date"] = pd.to_datetime(existing_data["date"])
+
+        is_enhanced, is_legacy = self._detect_csv_format(existing_data)
+        if not is_enhanced and not is_legacy:
+            return False
+
+        # Extract symbol and retrieve API data
+        extracted_symbol = self.extract_symbol_from_filename(csv_path)
+        filename = Path(csv_path).name if isinstance(csv_path, str) else csv_path.name
+        logger.info(f"   🎯 Extracted symbol: {extracted_symbol} from file: {filename}")
+
+        api_data, metadata = self._retrieve_api_data_with_metadata(
+            timestamp_gap_info, trading_timeframe, extracted_symbol, is_enhanced
+        )
+
+        if not api_data:
+            return False
+
+        # Prepare and filter API data
+        api_df = self._prepare_api_dataframe(api_data, is_enhanced)
+        filtered_df = self._filter_to_gap_period(api_df, timestamp_gap_info)
+
+        if filtered_df is None:
+            return False
+
+        # Merge and validate
+        combined = self._merge_and_deduplicate(existing_data, filtered_df)
+        self._validate_gap_filled(combined, timestamp_gap_info, trading_timeframe)
+
+        # Save results
+        self._save_with_headers(csv_path, combined)
+
+        logger.info(f"   ✅ Gap filled with {len(filtered_df)} authentic candles")
+        return True
+
+    def process_file(self, csv_path: Path, trading_timeframe: str) -> Dict:
+        """Process a single CSV file - detect and fill ALL gaps"""
+        logger.info(f"🎯 Processing {csv_path} ({trading_timeframe})")
+
+        # Detect all gaps
+        detected_gaps = self.detect_all_gaps(csv_path, trading_timeframe)
+
+        if not detected_gaps:
+            logger.info(f"   ✅ No gaps found in {trading_timeframe}")
+            return {
+                "timeframe": trading_timeframe,
+                "gaps_detected": 0,
+                "gaps_filled": 0,
+                "gaps_failed": 0,
+                "success_rate": 100.0,
+            }
+
+        # Fill each gap
+        gaps_filled_count = 0
+        gaps_failed_count = 0
+
+        for gap_index, timestamp_gap in enumerate(detected_gaps, 1):
+            logger.info(f"   🔧 Processing gap {gap_index}/{len(detected_gaps)}")
+            if self.fill_gap(timestamp_gap, csv_path, trading_timeframe):
+                gaps_filled_count += 1
+            else:
+                gaps_failed_count += 1
+
+            # Brief pause between API calls
+            if gap_index < len(detected_gaps):
+                time.sleep(1)
+
+        gap_fill_success_rate = (
+            (gaps_filled_count / len(detected_gaps)) * 100 if detected_gaps else 100.0
+        )
+
+        processing_result = {
+            "timeframe": trading_timeframe,
+            "gaps_detected": len(detected_gaps),
+            "gaps_filled": gaps_filled_count,
+            "gaps_failed": gaps_failed_count,
+            "success_rate": gap_fill_success_rate,
+        }
+
+        logger.info(
+            f"   📊 Result: {gaps_filled_count}/{len(detected_gaps)} gaps filled ({gap_fill_success_rate:.1f}%)"
+        )
+        return processing_result
+
+
+def main():
+    """Main execution function"""
+    logger.info("🚀 UNIVERSAL GAP FILLER - Fill ALL Gaps in ALL Timeframes")
+    logger.info("=" * 60)
+
+    gap_filler_instance = UniversalGapFiller()
+    sample_data_directory = Path("../sample_data")
+
+    # Define timeframes that need gap filling (exclude 4h which is perfect)
+    target_trading_timeframes = ["1m", "3m", "5m", "15m", "30m", "1h", "2h"]
+
+    processing_results = []
+
+    for trading_timeframe in target_trading_timeframes:
+        csv_file_pattern = f"binance_spot_SOLUSDT-{trading_timeframe}_*.csv"
+        matching_csv_files = list(sample_data_directory.glob(csv_file_pattern))
+
+        if not matching_csv_files:
+            logger.warning(f"❌ No CSV file found for {trading_timeframe}")
+            continue
+
+        selected_csv_file = matching_csv_files[0]  # Use first match
+        timeframe_result = gap_filler_instance.process_file(selected_csv_file, trading_timeframe)
+        processing_results.append(timeframe_result)
+
+    # Summary report
+    logger.info("\n" + "=" * 60)
+    logger.info("📊 UNIVERSAL GAP FILLING SUMMARY")
+    logger.info("=" * 60)
+
+    total_gaps_detected_count = sum(
+        result_data["gaps_detected"] for result_data in processing_results
+    )
+    total_gaps_filled_count = sum(result_data["gaps_filled"] for result_data in processing_results)
+    total_gaps_failed_count = sum(result_data["gaps_failed"] for result_data in processing_results)
+
+    for timeframe_result in processing_results:
+        status_icon = (
+            "✅"
+            if timeframe_result["success_rate"] == 100.0
+            else "⚠️"
+            if timeframe_result["success_rate"] > 0
+            else "❌"
+        )
+        logger.info(
+            f"{status_icon} {timeframe_result['timeframe']:>3}: {timeframe_result['gaps_filled']:>2}/{timeframe_result['gaps_detected']:>2} gaps filled ({timeframe_result['success_rate']:>5.1f}%)"
+        )
+
+    logger.info("-" * 60)
+    overall_success_rate = (
+        (total_gaps_filled_count / total_gaps_detected_count * 100)
+        if total_gaps_detected_count > 0
+        else 100.0
+    )
+    logger.info(
+        f"🎯 OVERALL: {total_gaps_filled_count}/{total_gaps_detected_count} gaps filled ({overall_success_rate:.1f}%)"
+    )
+    logger.info("=" * 60)
+
+    if overall_success_rate == 100.0:
+        logger.info("🎉 ALL GAPS FILLED SUCCESSFULLY! Ready for validation.")
+    else:
+        logger.warning(f"⚠️ {total_gaps_failed_count} gaps failed to fill. Manual review needed.")
+
+
+if __name__ == "__main__":
+    main()