quantmllibrary 0.1.0__py3-none-any.whl

quantml/data/futures.py

@@ -0,0 +1,254 @@
+"""
+Futures-specific data handling.
+
+Handles contract rolls, holidays, gaps, and session-based data.
+"""
+
+from typing import List, Dict, Optional, Tuple, Any
+from datetime import datetime, timedelta
+
+
+class FuturesDataHandler:
+    """
+    Handler for futures-specific data operations.
+    
+    Features:
+    - Contract roll detection and handling
+    - Holiday and partial session handling
+    - Overnight gap detection
+    - Session-based filtering (RTH vs ETH)
+    """
+    
+    def __init__(
+        self,
+        instrument: str,
+        roll_method: str = "volume"  # "volume", "open_interest", "date"
+    ):
+        """
+        Initialize futures data handler.
+        
+        Args:
+            instrument: Instrument symbol (ES, MES, NQ, MNQ)
+            roll_method: Method for detecting rolls ("volume", "open_interest", "date")
+        """
+        self.instrument = instrument
+        self.roll_method = roll_method
+        self.contract_size = self._get_contract_size(instrument)
+    
+    def _get_contract_size(self, instrument: str) -> float:
+        """Get contract multiplier for instrument."""
+        sizes = {
+            'ES': 50.0,
+            'MES': 5.0,
+            'NQ': 20.0,
+            'MNQ': 2.0,
+            'YM': 5.0,
+            'MYM': 0.5
+        }
+        return sizes.get(instrument.upper(), 50.0)
+    
+    def detect_contract_rolls(
+        self,
+        prices: List[float],
+        volumes: Optional[List[float]] = None,
+        dates: Optional[List[datetime]] = None
+    ) -> List[int]:
+        """
+        Detect contract roll dates.
+        
+        Args:
+            prices: Price data
+            volumes: Volume data (for volume-based detection)
+            dates: Date data (for date-based detection)
+        
+        Returns:
+            List of indices where rolls occur
+        """
+        roll_indices = []
+        
+        if self.roll_method == "volume":
+            if volumes is None:
+                raise ValueError("Volume data required for volume-based roll detection")
+            
+            # Detect roll when volume drops significantly
+            for i in range(1, len(volumes)):
+                if volumes[i] < volumes[i-1] * 0.3:  # Volume drops by 70%
+                    roll_indices.append(i)
+        
+        elif self.roll_method == "date":
+            if dates is None:
+                raise ValueError("Date data required for date-based roll detection")
+            
+            # Typical roll dates (third Friday of month)
+            for i, date in enumerate(dates):
+                if date.weekday() == 4 and 15 <= date.day <= 21:  # Third Friday
+                    roll_indices.append(i)
+        
+        return roll_indices
+    
+    def detect_overnight_gaps(
+        self,
+        closes: List[float],
+        opens: List[float],
+        dates: Optional[List[datetime]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Detect overnight gaps.
+        
+        Args:
+            closes: Closing prices
+            opens: Opening prices
+            dates: Date data (to identify holiday gaps)
+        
+        Returns:
+            List of gap dictionaries
+        """
+        gaps = []
+        
+        for i in range(1, len(opens)):
+            if closes[i-1] > 0:
+                gap = (opens[i] - closes[i-1]) / closes[i-1]
+                
+                # Check if holiday gap (more than 1 day between dates)
+                is_holiday = False
+                if dates and i < len(dates) and i-1 < len(dates):
+                    days_diff = (dates[i] - dates[i-1]).days
+                    is_holiday = days_diff > 1
+                
+                gaps.append({
+                    'index': i,
+                    'gap': gap,
+                    'gap_size': abs(gap),
+                    'is_holiday': is_holiday,
+                    'prev_close': closes[i-1],
+                    'current_open': opens[i]
+                })
+        
+        return gaps
+    
+    def filter_session(
+        self,
+        data: Dict[str, List[Any]],
+        session_type: str = "RTH",
+        timestamps: Optional[List[datetime]] = None
+    ) -> Dict[str, List[Any]]:
+        """
+        Filter data by trading session.
+        
+        Args:
+            data: Data dictionary
+            session_type: "RTH" (9:30-16:00 ET) or "ETH" (all hours)
+            timestamps: Timestamp data
+        
+        Returns:
+            Filtered data dictionary
+        """
+        if session_type == "ETH":
+            return data  # No filtering
+        
+        if timestamps is None:
+            # Assume all data is RTH if no timestamps
+            return data
+        
+        # RTH: 9:30 AM - 4:00 PM ET
+        filtered_indices = []
+        for i, ts in enumerate(timestamps):
+            hour = ts.hour
+            minute = ts.minute
+            
+            # Convert to ET (simplified, assumes UTC-5)
+            # In production, use proper timezone handling
+            if 9 <= hour < 16 or (hour == 9 and minute >= 30):
+                filtered_indices.append(i)
+        
+        filtered_data = {}
+        for key, values in data.items():
+            filtered_data[key] = [values[i] for i in filtered_indices]
+        
+        return filtered_data
+    
+    def handle_missing_data(
+        self,
+        data: Dict[str, List[float]],
+        method: str = "forward_fill"
+    ) -> Dict[str, List[float]]:
+        """
+        Handle missing data (NaN, None, zeros).
+        
+        Args:
+            data: Data dictionary
+            method: "forward_fill", "backward_fill", "interpolate", "drop"
+        
+        Returns:
+            Data with missing values handled
+        """
+        cleaned_data = {}
+        
+        for key, values in data.items():
+            cleaned = []
+            last_valid = None
+            
+            for val in values:
+                if val is None or (isinstance(val, float) and (val != val or val == 0.0)):
+                    # Missing or invalid
+                    if method == "forward_fill" and last_valid is not None:
+                        cleaned.append(last_valid)
+                    elif method == "drop":
+                        continue  # Skip
+                    else:
+                        cleaned.append(0.0)  # Default
+                else:
+                    cleaned.append(val)
+                    last_valid = val
+            
+            if method == "backward_fill":
+                # Fill backwards
+                for i in range(len(cleaned)-2, -1, -1):
+                    if cleaned[i] == 0.0 and cleaned[i+1] != 0.0:
+                        cleaned[i] = cleaned[i+1]
+            
+            cleaned_data[key] = cleaned
+        
+        return cleaned_data
+    
+    def align_contract_data(
+        self,
+        front_month: Dict[str, List[float]],
+        back_month: Dict[str, List[float]],
+        roll_indices: List[int]
+    ) -> Dict[str, List[float]]:
+        """
+        Align data across contract rolls.
+        
+        Args:
+            front_month: Front month contract data
+            back_month: Back month contract data
+            roll_indices: Indices where rolls occur
+        
+        Returns:
+            Aligned data dictionary
+        """
+        aligned = {}
+        
+        for key in front_month.keys():
+            if key not in back_month:
+                continue
+            
+            aligned_values = []
+            front_data = front_month[key]
+            back_data = back_month[key]
+            
+            for i in range(len(front_data)):
+                if i in roll_indices:
+                    # Use back month data after roll
+                    if i < len(back_data):
+                        aligned_values.append(back_data[i])
+                    else:
+                        aligned_values.append(front_data[i])
+                else:
+                    aligned_values.append(front_data[i])
+            
+            aligned[key] = aligned_values
+        
+        return aligned
+

quantml/data/loaders.py

@@ -0,0 +1,236 @@
+"""
+Standardized data loaders for market data.
+"""
+
+from typing import List, Optional, Dict, Any, Callable
+import os
+
+# Try to import pandas
+try:
+    import pandas as pd
+    HAS_PANDAS = True
+except ImportError:
+    HAS_PANDAS = False
+    pd = None
+
+
+def load_csv_data(
+    filepath: str,
+    price_column: str = "close",
+    volume_column: str = "volume",
+    timestamp_column: Optional[str] = "timestamp",
+    date_format: Optional[str] = None
+) -> Dict[str, List]:
+    """
+    Load market data from CSV file.
+    
+    Args:
+        filepath: Path to CSV file
+        price_column: Name of price column
+        volume_column: Name of volume column
+        timestamp_column: Name of timestamp column (optional)
+        date_format: Date format string (optional)
+    
+    Returns:
+        Dictionary with 'prices', 'volumes', and optionally 'timestamps'
+    """
+    if not HAS_PANDAS:
+        raise ImportError("pandas required for CSV loading. Install with: pip install pandas")
+    
+    if not os.path.exists(filepath):
+        raise FileNotFoundError(f"Data file not found: {filepath}")
+    
+    df = pd.read_csv(filepath)
+    
+    # Validate columns exist
+    if price_column not in df.columns:
+        raise ValueError(f"Price column '{price_column}' not found in CSV")
+    
+    if volume_column not in df.columns:
+        raise ValueError(f"Volume column '{volume_column}' not found in CSV")
+    
+    # Extract data
+    prices = df[price_column].tolist()
+    volumes = df[volume_column].tolist()
+    
+    result = {
+        'prices': prices,
+        'volumes': volumes
+    }
+    
+    # Add timestamps if available
+    if timestamp_column and timestamp_column in df.columns:
+        if date_format:
+            timestamps = pd.to_datetime(df[timestamp_column], format=date_format).tolist()
+        else:
+            timestamps = pd.to_datetime(df[timestamp_column]).tolist()
+        result['timestamps'] = timestamps
+    
+    return result
+
+
+def load_dataframe(
+    df: Any,
+    price_column: str = "close",
+    volume_column: str = "volume",
+    timestamp_column: Optional[str] = "timestamp"
+) -> Dict[str, List]:
+    """
+    Load market data from pandas DataFrame.
+    
+    Args:
+        df: Pandas DataFrame
+        price_column: Name of price column
+        volume_column: Name of volume column
+        timestamp_column: Name of timestamp column (optional)
+    
+    Returns:
+        Dictionary with 'prices', 'volumes', and optionally 'timestamps'
+    """
+    if not HAS_PANDAS:
+        raise ImportError("pandas required. Install with: pip install pandas")
+    
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("df must be a pandas DataFrame")
+    
+    # Validate columns
+    if price_column not in df.columns:
+        raise ValueError(f"Price column '{price_column}' not found")
+    
+    if volume_column not in df.columns:
+        raise ValueError(f"Volume column '{volume_column}' not found")
+    
+    # Extract data
+    prices = df[price_column].tolist()
+    volumes = df[volume_column].tolist()
+    
+    result = {
+        'prices': prices,
+        'volumes': volumes
+    }
+    
+    # Add timestamps if available
+    if timestamp_column and timestamp_column in df.columns:
+        timestamps = pd.to_datetime(df[timestamp_column]).tolist()
+        result['timestamps'] = timestamps
+    
+    return result
+
+
+class DataLoader:
+    """Generic data loader with validation."""
+    
+    def __init__(
+        self,
+        load_fn: Callable,
+        validate: bool = True,
+        handle_missing: str = "forward_fill"
+    ):
+        """
+        Initialize data loader.
+        
+        Args:
+            load_fn: Function to load data
+            validate: Whether to validate loaded data
+            handle_missing: How to handle missing values (forward_fill, drop, interpolate)
+        """
+        self.load_fn = load_fn
+        self.validate = validate
+        self.handle_missing = handle_missing
+    
+    def load(
+        self,
+        *args,
+        **kwargs
+    ) -> Dict[str, List]:
+        """
+        Load data using the configured function.
+        
+        Args:
+            *args: Arguments for load function
+            **kwargs: Keyword arguments for load function
+        
+        Returns:
+            Dictionary with loaded data
+        """
+        data = self.load_fn(*args, **kwargs)
+        
+        if self.validate:
+            from quantml.data.validators import validate_price_data
+            is_valid, errors = validate_price_data(
+                data.get('prices', []),
+                data.get('volumes'),
+                data.get('timestamps')
+            )
+            if not is_valid:
+                raise ValueError(f"Data validation failed: {errors}")
+        
+        # Handle missing values
+        if self.handle_missing != "drop":
+            data = self._handle_missing(data)
+        
+        return data
+    
+    def _handle_missing(self, data: Dict[str, List]) -> Dict[str, List]:
+        """Handle missing values in data."""
+        if self.handle_missing == "forward_fill":
+            return self._forward_fill(data)
+        elif self.handle_missing == "interpolate":
+            return self._interpolate(data)
+        else:
+            return data
+    
+    def _forward_fill(self, data: Dict[str, List]) -> Dict[str, List]:
+        """Forward fill missing values."""
+        import math
+        
+        prices = data.get('prices', [])
+        volumes = data.get('volumes', [])
+        
+        # Forward fill prices
+        last_valid_price = None
+        for i, price in enumerate(prices):
+            if price is None or (isinstance(price, float) and math.isnan(price)):
+                if last_valid_price is not None:
+                    prices[i] = last_valid_price
+            else:
+                last_valid_price = price
+        
+        # Forward fill volumes
+        if volumes:
+            last_valid_volume = None
+            for i, volume in enumerate(volumes):
+                if volume is None or (isinstance(volume, float) and math.isnan(volume)):
+                    if last_valid_volume is not None:
+                        volumes[i] = last_valid_volume
+                else:
+                    last_valid_volume = volume
+        
+        data['prices'] = prices
+        if volumes:
+            data['volumes'] = volumes
+        
+        return data
+    
+    def _interpolate(self, data: Dict[str, List]) -> Dict[str, List]:
+        """Interpolate missing values."""
+        if not HAS_PANDAS:
+            # Fallback to forward fill
+            return self._forward_fill(data)
+        
+        import math
+        
+        prices = data.get('prices', [])
+        
+        # Convert to Series and interpolate
+        prices_series = pd.Series(prices)
+        prices_series = prices_series.interpolate(method='linear')
+        data['prices'] = prices_series.tolist()
+        
+        if 'volumes' in data:
+            volumes_series = pd.Series(data['volumes'])
+            volumes_series = volumes_series.interpolate(method='linear')
+            data['volumes'] = volumes_series.tolist()
+        
+        return data
+