quantmllibrary 0.1.0__py3-none-any.whl

quantml/config/config.py

@@ -0,0 +1,259 @@
+"""
+Configuration management for QuantML experiments.
+
+Supports YAML/JSON config files and command-line argument integration.
+"""
+
+import json
+import os
+from typing import Dict, Any, Optional, List
+from dataclasses import dataclass, field, asdict
+import argparse
+
+# Try to import YAML
+try:
+    import yaml
+    HAS_YAML = True
+except ImportError:
+    HAS_YAML = False
+
+
+@dataclass
+class DataConfig:
+    """Data loading and preprocessing configuration."""
+    instrument: str = "ES"  # MES, ES, MNQ, NQ, etc.
+    start_date: str = "2015-01-01"
+    end_date: str = "2024-12-31"
+    data_source: str = "csv"  # csv, database, api
+    data_path: Optional[str] = None
+    frequency: str = "1min"  # 1min, 5min, daily
+    validate_data: bool = True
+    handle_missing: str = "forward_fill"  # forward_fill, drop, interpolate
+    cache_features: bool = True
+    feature_cache_path: str = "./cache/features"
+
+
+@dataclass
+class FeatureConfig:
+    """Feature engineering configuration."""
+    enabled_features: List[str] = field(default_factory=lambda: [
+        "lagged_price",
+        "rolling_mean",
+        "rolling_std",
+        "returns",
+        "volatility"
+    ])
+    lag_periods: List[int] = field(default_factory=lambda: [1, 5, 10, 20])
+    rolling_windows: List[int] = field(default_factory=lambda: [20, 50])
+    normalize: bool = True
+    normalization_method: str = "zscore"  # zscore, minmax, robust
+    alpha_factors: Dict[str, Any] = field(default_factory=lambda: {
+        "momentum": {"enabled": True, "lookback": 20},
+        "mean_reversion": {"enabled": True, "window": 20},
+        "volatility": {"enabled": True, "window": 20}
+    })
+
+
+@dataclass
+class ModelConfig:
+    """Model architecture and hyperparameters."""
+    model_type: str = "Linear"  # Linear, SimpleRNN, TCN
+    in_features: int = 10
+    out_features: int = 1
+    hidden_size: Optional[int] = None
+    bias: bool = True
+    dropout: float = 0.0
+    activation: Optional[str] = None
+
+
+@dataclass
+class TrainingConfig:
+    """Training configuration."""
+    optimizer: str = "Adam"  # SGD, Adam, RMSProp, etc.
+    learning_rate: float = 0.001
+    batch_size: int = 32
+    epochs: int = 100
+    loss_function: str = "mse_loss"  # mse_loss, sharpe_loss, etc.
+    early_stopping: Dict[str, Any] = field(default_factory=lambda: {
+        "enabled": True,
+        "patience": 10,
+        "min_delta": 0.0001
+    })
+    gradient_clipping: Dict[str, Any] = field(default_factory=lambda: {
+        "enabled": False,
+        "max_norm": 1.0
+    })
+    scheduler: Optional[Dict[str, Any]] = None
+    walk_forward: Dict[str, Any] = field(default_factory=lambda: {
+        "enabled": True,
+        "train_size": 500,
+        "test_size": 100,
+        "window_type": "expanding"  # expanding, rolling
+    })
+
+
+@dataclass
+class ExperimentConfig:
+    """Complete experiment configuration."""
+    name: str = "default_experiment"
+    description: str = ""
+    random_seed: int = 42
+    data: DataConfig = field(default_factory=DataConfig)
+    features: FeatureConfig = field(default_factory=FeatureConfig)
+    model: ModelConfig = field(default_factory=ModelConfig)
+    training: TrainingConfig = field(default_factory=TrainingConfig)
+    output_dir: str = "./experiments"
+    log_level: str = "INFO"
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert config to dictionary."""
+        return asdict(self)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ExperimentConfig':
+        """Create config from dictionary."""
+        # Handle nested configs
+        if 'data' in data and isinstance(data['data'], dict):
+            data['data'] = DataConfig(**data['data'])
+        if 'features' in data and isinstance(data['features'], dict):
+            data['features'] = FeatureConfig(**data['features'])
+        if 'model' in data and isinstance(data['model'], dict):
+            data['model'] = ModelConfig(**data['model'])
+        if 'training' in data and isinstance(data['training'], dict):
+            data['training'] = TrainingConfig(**data['training'])
+        return cls(**data)
+    
+    def validate(self) -> List[str]:
+        """Validate configuration and return list of errors."""
+        errors = []
+        
+        # Validate dates
+        try:
+            from datetime import datetime
+            datetime.strptime(self.data.start_date, "%Y-%m-%d")
+            datetime.strptime(self.data.end_date, "%Y-%m-%d")
+        except ValueError:
+            errors.append("Invalid date format. Use YYYY-MM-DD")
+        
+        # Validate learning rate
+        if self.training.learning_rate <= 0:
+            errors.append("Learning rate must be positive")
+        
+        # Validate batch size
+        if self.training.batch_size <= 0:
+            errors.append("Batch size must be positive")
+        
+        # Validate model features
+        if self.model.in_features <= 0:
+            errors.append("Model input features must be positive")
+        
+        return errors
+
+
+class Config:
+    """Main configuration class."""
+    
+    @staticmethod
+    def load_yaml(filepath: str) -> ExperimentConfig:
+        """Load configuration from YAML file."""
+        if not HAS_YAML:
+            raise ImportError("PyYAML not installed. Install with: pip install pyyaml")
+        
+        with open(filepath, 'r') as f:
+            data = yaml.safe_load(f)
+        
+        return ExperimentConfig.from_dict(data)
+    
+    @staticmethod
+    def load_json(filepath: str) -> ExperimentConfig:
+        """Load configuration from JSON file."""
+        with open(filepath, 'r') as f:
+            data = json.load(f)
+        
+        return ExperimentConfig.from_dict(data)
+    
+    @staticmethod
+    def save_yaml(config: ExperimentConfig, filepath: str):
+        """Save configuration to YAML file."""
+        if not HAS_YAML:
+            raise ImportError("PyYAML not installed. Install with: pyyaml")
+        
+        os.makedirs(os.path.dirname(filepath) or '.', exist_ok=True)
+        
+        with open(filepath, 'w') as f:
+            yaml.dump(config.to_dict(), f, default_flow_style=False, sort_keys=False)
+    
+    @staticmethod
+    def save_json(config: ExperimentConfig, filepath: str):
+        """Save configuration to JSON file."""
+        os.makedirs(os.path.dirname(filepath) or '.', exist_ok=True)
+        
+        with open(filepath, 'w') as f:
+            json.dump(config.to_dict(), f, indent=2)
+
+
+def load_config(filepath: str) -> ExperimentConfig:
+    """Load configuration from file (auto-detect YAML/JSON)."""
+    ext = os.path.splitext(filepath)[1].lower()
+    
+    if ext in ['.yaml', '.yml']:
+        return Config.load_yaml(filepath)
+    elif ext == '.json':
+        return Config.load_json(filepath)
+    else:
+        raise ValueError(f"Unsupported config file format: {ext}. Use .yaml, .yml, or .json")
+
+
+def save_config(config: ExperimentConfig, filepath: str):
+    """Save configuration to file (auto-detect YAML/JSON)."""
+    ext = os.path.splitext(filepath)[1].lower()
+    
+    if ext in ['.yaml', '.yml']:
+        Config.save_yaml(config, filepath)
+    elif ext == '.json':
+        Config.save_json(config, filepath)
+    else:
+        raise ValueError(f"Unsupported config file format: {ext}. Use .yaml, .yml, or .json")
+
+
+def create_argparser() -> argparse.ArgumentParser:
+    """Create argument parser with common config options."""
+    parser = argparse.ArgumentParser(description='QuantML Experiment Runner')
+    
+    parser.add_argument('--config', type=str, help='Path to config file (YAML/JSON)')
+    parser.add_argument('--instrument', type=str, help='Trading instrument (ES, MES, NQ, MNQ)')
+    parser.add_argument('--start-date', type=str, help='Start date (YYYY-MM-DD)')
+    parser.add_argument('--end-date', type=str, help='End date (YYYY-MM-DD)')
+    parser.add_argument('--model-type', type=str, help='Model type (Linear, SimpleRNN, TCN)')
+    parser.add_argument('--learning-rate', type=float, help='Learning rate')
+    parser.add_argument('--epochs', type=int, help='Number of epochs')
+    parser.add_argument('--batch-size', type=int, help='Batch size')
+    parser.add_argument('--random-seed', type=int, help='Random seed')
+    parser.add_argument('--output-dir', type=str, help='Output directory')
+    
+    return parser
+
+
+def merge_config_with_args(config: ExperimentConfig, args: argparse.Namespace) -> ExperimentConfig:
+    """Merge command-line arguments into config."""
+    if args.instrument:
+        config.data.instrument = args.instrument
+    if args.start_date:
+        config.data.start_date = args.start_date
+    if args.end_date:
+        config.data.end_date = args.end_date
+    if args.model_type:
+        config.model.model_type = args.model_type
+    if args.learning_rate:
+        config.training.learning_rate = args.learning_rate
+    if args.epochs:
+        config.training.epochs = args.epochs
+    if args.batch_size:
+        config.training.batch_size = args.batch_size
+    if args.random_seed:
+        config.random_seed = args.random_seed
+    if args.output_dir:
+        config.output_dir = args.output_dir
+    
+    return config
+

quantml/data/__init__.py

@@ -0,0 +1,33 @@
+"""
+QuantML Data Management
+
+This module provides data loading, validation, and caching utilities.
+"""
+
+from quantml.data.validators import (
+    validate_price_data,
+    validate_timestamps,
+    check_duplicates,
+    check_missing_values,
+    generate_data_quality_report,
+    DataQualityReport
+)
+
+from quantml.data.loaders import (
+    load_csv_data,
+    load_dataframe,
+    DataLoader
+)
+
+__all__ = [
+    'validate_price_data',
+    'validate_timestamps',
+    'check_duplicates',
+    'check_missing_values',
+    'generate_data_quality_report',
+    'DataQualityReport',
+    'load_csv_data',
+    'load_dataframe',
+    'DataLoader'
+]
+

quantml/data/cache.py

@@ -0,0 +1,149 @@
+"""
+Cache management utilities.
+"""
+
+from typing import Optional, Callable, Any
+from functools import wraps
+import hashlib
+import json
+import os
+
+from quantml.data.feature_store import FeatureStore
+
+
+def cached_features(
+    cache_dir: str = "./cache/features",
+    use_cache: bool = True
+):
+    """
+    Decorator to cache feature computation results.
+    
+    Args:
+        cache_dir: Cache directory
+        use_cache: Whether to use cache
+    
+    Example:
+        @cached_features(cache_dir="./cache")
+        def compute_features(data):
+            # Expensive computation
+            return features
+    """
+    def decorator(func: Callable) -> Callable:
+        store = FeatureStore(cache_dir=cache_dir)
+        
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if not use_cache:
+                return func(*args, **kwargs)
+            
+            # Generate cache key from function arguments
+            cache_key_data = {
+                'func': func.__name__,
+                'args': str(args),
+                'kwargs': json.dumps(kwargs, sort_keys=True)
+            }
+            cache_key_str = json.dumps(cache_key_data, sort_keys=True)
+            cache_key = hashlib.md5(cache_key_str.encode()).hexdigest()
+            
+            # Check cache
+            if store.cache_exists(cache_key):
+                features, _ = store.load_features(cache_key)
+                return features
+            
+            # Compute features
+            features = func(*args, **kwargs)
+            
+            # Save to cache
+            # Extract metadata from kwargs if available
+            instrument = kwargs.get('instrument', 'unknown')
+            start_date = kwargs.get('start_date', 'unknown')
+            end_date = kwargs.get('end_date', 'unknown')
+            feature_config = kwargs.get('feature_config', {})
+            
+            store.save_features(
+                features,
+                instrument,
+                start_date,
+                end_date,
+                feature_config
+            )
+            
+            return features
+        
+        return wrapper
+    return decorator
+
+
+class CacheManager:
+    """Cache manager for feature computation."""
+    
+    def __init__(self, cache_dir: str = "./cache/features"):
+        """
+        Initialize cache manager.
+        
+        Args:
+            cache_dir: Cache directory
+        """
+        self.store = FeatureStore(cache_dir=cache_dir)
+        self.cache_dir = cache_dir
+    
+    def get_or_compute(
+        self,
+        cache_key: str,
+        compute_fn: Callable,
+        *args,
+        **kwargs
+    ) -> Any:
+        """
+        Get from cache or compute.
+        
+        Args:
+            cache_key: Cache key
+            compute_fn: Function to compute if not cached
+            *args: Arguments for compute function
+            **kwargs: Keyword arguments for compute function
+        
+        Returns:
+            Cached or computed result
+        """
+        if self.store.cache_exists(cache_key):
+            features, _ = self.store.load_features(cache_key)
+            return features
+        
+        # Compute
+        result = compute_fn(*args, **kwargs)
+        
+        # Save to cache if result is features
+        if isinstance(result, list) and len(result) > 0:
+            if isinstance(result[0], list):  # List of feature vectors
+                instrument = kwargs.get('instrument', 'unknown')
+                start_date = kwargs.get('start_date', 'unknown')
+                end_date = kwargs.get('end_date', 'unknown')
+                feature_config = kwargs.get('feature_config', {})
+                
+                self.store.save_features(
+                    result,
+                    instrument,
+                    start_date,
+                    end_date,
+                    feature_config
+                )
+        
+        return result
+    
+    def clear_cache(self, pattern: Optional[str] = None):
+        """
+        Clear cache.
+        
+        Args:
+            pattern: Optional pattern to match (e.g., instrument name)
+        """
+        if pattern:
+            self.store.invalidate_cache(instrument=pattern)
+        else:
+            # Clear all
+            for filename in os.listdir(self.cache_dir):
+                filepath = os.path.join(self.cache_dir, filename)
+                if os.path.isfile(filepath):
+                    os.remove(filepath)
+

quantml/data/feature_store.py

@@ -0,0 +1,234 @@
+"""
+Feature store for caching computed features.
+
+Uses Parquet format for efficient storage and loading of large feature datasets.
+"""
+
+import os
+import hashlib
+import json
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+# Try to import pandas and pyarrow
+try:
+    import pandas as pd
+    HAS_PANDAS = True
+except ImportError:
+    HAS_PANDAS = False
+    pd = None
+
+try:
+    import pyarrow.parquet as pq
+    HAS_PARQUET = True
+except ImportError:
+    HAS_PARQUET = False
+    pq = None
+
+
+class FeatureStore:
+    """Feature caching system using Parquet format."""
+    
+    def __init__(
+        self,
+        cache_dir: str = "./cache/features",
+        use_parquet: bool = True
+    ):
+        """
+        Initialize feature store.
+        
+        Args:
+            cache_dir: Directory for cached features
+            use_parquet: Whether to use Parquet format (requires pyarrow)
+        """
+        self.cache_dir = cache_dir
+        self.use_parquet = use_parquet and HAS_PARQUET
+        
+        if not self.use_parquet and not HAS_PANDAS:
+            raise ImportError(
+                "Either pandas or pyarrow required. "
+                "Install with: pip install pandas pyarrow"
+            )
+        
+        os.makedirs(cache_dir, exist_ok=True)
+    
+    def _generate_cache_key(
+        self,
+        instrument: str,
+        start_date: str,
+        end_date: str,
+        feature_config: Dict[str, Any]
+    ) -> str:
+        """Generate cache key from configuration."""
+        key_data = {
+            'instrument': instrument,
+            'start_date': start_date,
+            'end_date': end_date,
+            'features': feature_config
+        }
+        key_str = json.dumps(key_data, sort_keys=True)
+        return hashlib.md5(key_str.encode()).hexdigest()
+    
+    def _get_cache_path(self, cache_key: str) -> str:
+        """Get cache file path."""
+        if self.use_parquet:
+            return os.path.join(self.cache_dir, f"{cache_key}.parquet")
+        else:
+            return os.path.join(self.cache_dir, f"{cache_key}.csv")
+    
+    def _get_metadata_path(self, cache_key: str) -> str:
+        """Get metadata file path."""
+        return os.path.join(self.cache_dir, f"{cache_key}_metadata.json")
+    
+    def save_features(
+        self,
+        features: List[List[float]],
+        instrument: str,
+        start_date: str,
+        end_date: str,
+        feature_config: Dict[str, Any],
+        metadata: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Save features to cache.
+        
+        Args:
+            features: List of feature vectors
+            instrument: Trading instrument
+            start_date: Start date
+            end_date: End date
+            feature_config: Feature configuration
+            metadata: Optional metadata
+        
+        Returns:
+            Cache key
+        """
+        cache_key = self._generate_cache_key(
+            instrument, start_date, end_date, feature_config
+        )
+        
+        cache_path = self._get_cache_path(cache_key)
+        metadata_path = self._get_metadata_path(cache_key)
+        
+        # Convert to DataFrame
+        if not HAS_PANDAS:
+            raise ImportError("pandas required for feature storage")
+        
+        df = pd.DataFrame(features)
+        
+        # Save features
+        if self.use_parquet:
+            df.to_parquet(cache_path, compression='snappy', index=False)
+        else:
+            df.to_csv(cache_path, index=False)
+        
+        # Save metadata
+        meta = {
+            'cache_key': cache_key,
+            'instrument': instrument,
+            'start_date': start_date,
+            'end_date': end_date,
+            'feature_config': feature_config,
+            'num_features': len(features[0]) if features else 0,
+            'num_samples': len(features),
+            'created_at': datetime.now().isoformat(),
+            'metadata': metadata or {}
+        }
+        
+        with open(metadata_path, 'w') as f:
+            json.dump(meta, f, indent=2)
+        
+        return cache_key
+    
+    def load_features(self, cache_key: str) -> tuple:
+        """
+        Load features from cache.
+        
+        Args:
+            cache_key: Cache key
+        
+        Returns:
+            Tuple of (features, metadata)
+        """
+        cache_path = self._get_cache_path(cache_key)
+        metadata_path = self._get_metadata_path(cache_key)
+        
+        if not os.path.exists(cache_path):
+            raise FileNotFoundError(f"Cache not found: {cache_key}")
+        
+        # Load features
+        if not HAS_PANDAS:
+            raise ImportError("pandas required for feature loading")
+        
+        if self.use_parquet:
+            df = pd.read_parquet(cache_path)
+        else:
+            df = pd.read_csv(cache_path)
+        
+        features = df.values.tolist()
+        
+        # Load metadata
+        metadata = {}
+        if os.path.exists(metadata_path):
+            with open(metadata_path, 'r') as f:
+                metadata = json.load(f)
+        
+        return features, metadata
+    
+    def cache_exists(self, cache_key: str) -> bool:
+        """Check if cache exists."""
+        cache_path = self._get_cache_path(cache_key)
+        return os.path.exists(cache_path)
+    
+    def invalidate_cache(
+        self,
+        instrument: Optional[str] = None,
+        cache_key: Optional[str] = None
+    ):
+        """
+        Invalidate cache entries.
+        
+        Args:
+            instrument: Invalidate all caches for this instrument
+            cache_key: Invalidate specific cache key
+        """
+        if cache_key:
+            cache_path = self._get_cache_path(cache_key)
+            metadata_path = self._get_metadata_path(cache_key)
+            
+            if os.path.exists(cache_path):
+                os.remove(cache_path)
+            if os.path.exists(metadata_path):
+                os.remove(metadata_path)
+        
+        elif instrument:
+            # Remove all caches for instrument
+            for filename in os.listdir(self.cache_dir):
+                if filename.endswith('_metadata.json'):
+                    metadata_path = os.path.join(self.cache_dir, filename)
+                    try:
+                        with open(metadata_path, 'r') as f:
+                            meta = json.load(f)
+                            if meta.get('instrument') == instrument:
+                                cache_key = meta.get('cache_key')
+                                if cache_key:
+                                    self.invalidate_cache(cache_key=cache_key)
+                    except Exception:
+                        pass
+    
+    def list_caches(self) -> List[Dict[str, Any]]:
+        """List all cached features."""
+        caches = []
+        
+        for filename in os.listdir(self.cache_dir):
+            if filename.endswith('_metadata.json'):
+                metadata_path = os.path.join(self.cache_dir, filename)
+                try:
+                    with open(metadata_path, 'r') as f:
+                        meta = json.load(f)
+                        caches.append(meta)
+                except Exception:
+                    pass
+        
+        return caches
+