quantmllibrary 0.1.0__py3-none-any.whl

quantml/experiments/logger.py

@@ -0,0 +1,208 @@
+"""
+CSV/JSON logger for experiment tracking.
+"""
+
+import csv
+import json
+from typing import Dict, Any, List, Optional
+from pathlib import Path
+from datetime import datetime
+
+
+class CSVExperimentLogger:
+    """
+    CSV-based experiment logger.
+    
+    Logs experiments to a CSV file for easy comparison and analysis.
+    """
+    
+    def __init__(self, log_file: str = "experiments_log.csv"):
+        """
+        Initialize CSV logger.
+        
+        Args:
+            log_file: Path to CSV log file
+        """
+        self.log_file = Path(log_file)
+        self.fieldnames = [
+            'timestamp',
+            'experiment_id',
+            'model_type',
+            'optimizer',
+            'learning_rate',
+            'batch_size',
+            'epochs',
+            'dataset_version',
+            'feature_set',
+            'random_seed',
+            'ic',
+            'rank_ic',
+            'sharpe_ratio',
+            'total_return',
+            'max_drawdown',
+            'win_rate',
+            'n_trades',
+            'notes'
+        ]
+    
+    def log_experiment(
+        self,
+        experiment_id: str,
+        config: Dict[str, Any],
+        results: Dict[str, Any],
+        notes: Optional[str] = None
+    ):
+        """
+        Log an experiment to CSV.
+        
+        Args:
+            experiment_id: Unique experiment identifier
+            config: Experiment configuration
+            results: Experiment results
+            notes: Optional notes
+        """
+        file_exists = self.log_file.exists()
+        
+        with open(self.log_file, 'a', newline='') as f:
+            writer = csv.DictWriter(f, fieldnames=self.fieldnames)
+            
+            if not file_exists:
+                writer.writeheader()
+            
+            row = {
+                'timestamp': datetime.now().isoformat(),
+                'experiment_id': experiment_id,
+                'model_type': config.get('model', {}).get('model_type', ''),
+                'optimizer': config.get('training', {}).get('optimizer', ''),
+                'learning_rate': config.get('training', {}).get('learning_rate', ''),
+                'batch_size': config.get('training', {}).get('batch_size', ''),
+                'epochs': config.get('training', {}).get('epochs', ''),
+                'dataset_version': config.get('data', {}).get('dataset_version', ''),
+                'feature_set': ','.join(config.get('features', {}).get('enabled_features', [])),
+                'random_seed': config.get('random_seed', ''),
+                'ic': results.get('alpha_metrics', {}).get('ic', ''),
+                'rank_ic': results.get('alpha_metrics', {}).get('rank_ic', ''),
+                'sharpe_ratio': results.get('backtest_results', {}).get('sharpe_ratio', ''),
+                'total_return': results.get('backtest_results', {}).get('total_return', ''),
+                'max_drawdown': results.get('backtest_results', {}).get('max_drawdown', ''),
+                'win_rate': results.get('backtest_results', {}).get('win_rate', ''),
+                'n_trades': results.get('backtest_results', {}).get('n_trades', ''),
+                'notes': notes or ''
+            }
+            
+            writer.writerow(row)
+    
+    def load_experiments(self) -> List[Dict[str, Any]]:
+        """
+        Load all experiments from CSV.
+        
+        Returns:
+            List of experiment dictionaries
+        """
+        if not self.log_file.exists():
+            return []
+        
+        experiments = []
+        with open(self.log_file, 'r') as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                experiments.append(row)
+        
+        return experiments
+    
+    def get_best_experiment(self, metric: str = 'sharpe_ratio') -> Optional[Dict[str, Any]]:
+        """
+        Get best experiment by metric.
+        
+        Args:
+            metric: Metric to optimize (default: 'sharpe_ratio')
+        
+        Returns:
+            Best experiment dictionary or None
+        """
+        experiments = self.load_experiments()
+        
+        if not experiments:
+            return None
+        
+        # Filter valid values
+        valid_exps = []
+        for exp in experiments:
+            try:
+                value = float(exp.get(metric, 0))
+                if value != 0:
+                    valid_exps.append((exp, value))
+            except (ValueError, TypeError):
+                continue
+        
+        if not valid_exps:
+            return None
+        
+        # Sort by metric (descending)
+        valid_exps.sort(key=lambda x: x[1], reverse=True)
+        return valid_exps[0][0]
+
+
+class JSONExperimentLogger:
+    """
+    JSON-based experiment logger.
+    
+    Logs experiments to JSON files for detailed tracking.
+    """
+    
+    def __init__(self, log_dir: str = "experiments"):
+        """
+        Initialize JSON logger.
+        
+        Args:
+            log_dir: Directory to store JSON logs
+        """
+        self.log_dir = Path(log_dir)
+        self.log_dir.mkdir(parents=True, exist_ok=True)
+    
+    def log_experiment(
+        self,
+        experiment_id: str,
+        config: Dict[str, Any],
+        results: Dict[str, Any],
+        metadata: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Log experiment to JSON.
+        
+        Args:
+            experiment_id: Unique experiment identifier
+            config: Experiment configuration
+            results: Experiment results
+            metadata: Optional metadata
+        """
+        log_data = {
+            'experiment_id': experiment_id,
+            'timestamp': datetime.now().isoformat(),
+            'config': config,
+            'results': results,
+            'metadata': metadata or {}
+        }
+        
+        log_file = self.log_dir / f"{experiment_id}.json"
+        with open(log_file, 'w') as f:
+            json.dump(log_data, f, indent=2, default=str)
+    
+    def load_experiment(self, experiment_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Load experiment from JSON.
+        
+        Args:
+            experiment_id: Experiment identifier
+        
+        Returns:
+            Experiment data or None
+        """
+        log_file = self.log_dir / f"{experiment_id}.json"
+        
+        if not log_file.exists():
+            return None
+        
+        with open(log_file, 'r') as f:
+            return json.load(f)
+

quantml/experiments/results.py

@@ -0,0 +1,158 @@
+"""
+Experiment results comparison and analysis.
+"""
+
+import json
+from typing import Dict, Any, List, Optional
+from pathlib import Path
+
+
+def compare_experiments(experiment_dirs: List[str]) -> Dict[str, Any]:
+    """
+    Compare multiple experiment runs.
+    
+    Args:
+        experiment_dirs: List of experiment directory paths
+    
+    Returns:
+        Comparison dictionary
+    """
+    experiments = []
+    
+    for exp_dir in experiment_dirs:
+        results_path = Path(exp_dir) / 'results.json'
+        metadata_path = Path(exp_dir) / 'metadata.json'
+        
+        results = {}
+        metadata = {}
+        
+        if results_path.exists():
+            with open(results_path, 'r') as f:
+                results = json.load(f)
+        
+        if metadata_path.exists():
+            with open(metadata_path, 'r') as f:
+                metadata = json.load(f)
+        
+        experiments.append({
+            'dir': str(exp_dir),
+            'metadata': metadata,
+            'results': results
+        })
+    
+    # Extract key metrics
+    comparison = {
+        'n_experiments': len(experiments),
+        'experiments': []
+    }
+    
+    for exp in experiments:
+        alpha_metrics = exp['results'].get('alpha_metrics', {})
+        backtest_results = exp['results'].get('backtest_results', {})
+        
+        comparison['experiments'].append({
+            'experiment_id': Path(exp['dir']).name,
+            'model_type': exp['metadata'].get('model_type', ''),
+            'optimizer': exp['metadata'].get('optimizer', ''),
+            'learning_rate': exp['metadata'].get('learning_rate', ''),
+            'ic': alpha_metrics.get('ic', 0.0),
+            'rank_ic': alpha_metrics.get('rank_ic', 0.0),
+            'sharpe_ratio': backtest_results.get('sharpe_ratio', 0.0),
+            'total_return': backtest_results.get('total_return', 0.0),
+            'max_drawdown': backtest_results.get('max_drawdown', 0.0),
+            'win_rate': backtest_results.get('win_rate', 0.0)
+        })
+    
+    return comparison
+
+
+def generate_summary_table(comparison: Dict[str, Any]) -> str:
+    """
+    Generate summary table for paper.
+    
+    Args:
+        comparison: Comparison dictionary
+    
+    Returns:
+        Formatted table string
+    """
+    lines = []
+    lines.append("Experiment Comparison")
+    lines.append("=" * 100)
+    lines.append(f"{'Experiment':<20} {'Model':<10} {'IC':<8} {'Sharpe':<8} {'Return':<10} {'MaxDD':<8}")
+    lines.append("-" * 100)
+    
+    for exp in comparison['experiments']:
+        lines.append(
+            f"{exp['experiment_id']:<20} "
+            f"{exp['model_type']:<10} "
+            f"{exp['ic']:>7.4f} "
+            f"{exp['sharpe_ratio']:>7.4f} "
+            f"{exp['total_return']*100:>9.2f}% "
+            f"{exp['max_drawdown']:>7.4f}"
+        )
+    
+    lines.append("=" * 100)
+    
+    return "\n".join(lines)
+
+
+def export_for_paper(comparison: Dict[str, Any], output_path: str, format: str = 'json'):
+    """
+    Export results for paper.
+    
+    Args:
+        comparison: Comparison dictionary
+        output_path: Output file path
+        format: Export format ('json', 'csv', 'latex')
+    """
+    if format == 'json':
+        with open(output_path, 'w') as f:
+            json.dump(comparison, f, indent=2, default=str)
+    
+    elif format == 'csv':
+        import csv
+        with open(output_path, 'w', newline='') as f:
+            writer = csv.DictWriter(f, fieldnames=[
+                'experiment_id', 'model_type', 'optimizer', 'ic', 'sharpe_ratio',
+                'total_return', 'max_drawdown'
+            ])
+            writer.writeheader()
+            for exp in comparison['experiments']:
+                writer.writerow({
+                    'experiment_id': exp['experiment_id'],
+                    'model_type': exp['model_type'],
+                    'optimizer': exp['optimizer'],
+                    'ic': exp['ic'],
+                    'sharpe_ratio': exp['sharpe_ratio'],
+                    'total_return': exp['total_return'],
+                    'max_drawdown': exp['max_drawdown']
+                })
+    
+    elif format == 'latex':
+        # Generate LaTeX table
+        lines = []
+        lines.append("\\begin{table}[h]")
+        lines.append("\\centering")
+        lines.append("\\begin{tabular}{lcccc}")
+        lines.append("\\hline")
+        lines.append("Experiment & Model & IC & Sharpe & Return \\\\")
+        lines.append("\\hline")
+        
+        for exp in comparison['experiments']:
+            lines.append(
+                f"{exp['experiment_id']} & "
+                f"{exp['model_type']} & "
+                f"{exp['ic']:.4f} & "
+                f"{exp['sharpe_ratio']:.4f} & "
+                f"{exp['total_return']*100:.2f}\\% \\\\"
+            )
+        
+        lines.append("\\hline")
+        lines.append("\\end{tabular}")
+        lines.append("\\caption{Experiment Results}")
+        lines.append("\\end{table}")
+        
+        with open(output_path, 'w') as f:
+            f.write("\n".join(lines))
+

quantml/experiments/tracker.py

@@ -0,0 +1,223 @@
+"""
+Enhanced experiment tracker with dataset versioning and results comparison.
+"""
+
+import os
+import json
+import csv
+from typing import Dict, Any, Optional, List
+from datetime import datetime
+from pathlib import Path
+
+
+class ExperimentTracker:
+    """
+    Enhanced experiment tracker.
+    
+    Tracks:
+    - Dataset version
+    - Feature set version
+    - Hyperparameters
+    - Random seed
+    - Model architecture
+    - Run date/time with git hash
+    - Results
+    """
+    
+    def __init__(self, experiment_dir: str):
+        """
+        Initialize experiment tracker.
+        
+        Args:
+            experiment_dir: Directory to store experiment data
+        """
+        self.experiment_dir = Path(experiment_dir)
+        self.experiment_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.metadata = {}
+        self.results = {}
+        self.start_time = datetime.now()
+    
+    def log_config(self, config: Dict[str, Any]):
+        """Log experiment configuration."""
+        self.metadata['config'] = config
+        self.metadata['random_seed'] = config.get('random_seed')
+        self.metadata['model_type'] = config.get('model', {}).get('model_type')
+        self.metadata['optimizer'] = config.get('training', {}).get('optimizer')
+        self.metadata['learning_rate'] = config.get('training', {}).get('learning_rate')
+    
+    def log_dataset_version(self, dataset_path: str, version: Optional[str] = None):
+        """
+        Log dataset version.
+        
+        Args:
+            dataset_path: Path to dataset
+            version: Optional version string (if None, uses file modification time)
+        """
+        if version is None:
+            # Use file modification time as version
+            if os.path.exists(dataset_path):
+                mtime = os.path.getmtime(dataset_path)
+                version = datetime.fromtimestamp(mtime).isoformat()
+            else:
+                version = "unknown"
+        
+        self.metadata['dataset_path'] = dataset_path
+        self.metadata['dataset_version'] = version
+    
+    def log_feature_set(self, features: List[str], version: Optional[str] = None):
+        """
+        Log feature set.
+        
+        Args:
+            features: List of feature names
+            version: Optional version string (hash of feature list)
+        """
+        if version is None:
+            # Create hash from feature list
+            import hashlib
+            feature_str = ','.join(sorted(features))
+            version = hashlib.md5(feature_str.encode()).hexdigest()[:8]
+        
+        self.metadata['features'] = features
+        self.metadata['feature_set_version'] = version
+    
+    def log_model_architecture(self, model_config: Dict[str, Any]):
+        """Log model architecture details."""
+        self.metadata['model_architecture'] = model_config
+    
+    def log_git_hash(self):
+        """Log current git commit hash."""
+        try:
+            import subprocess
+            result = subprocess.run(
+                ['git', 'rev-parse', 'HEAD'],
+                capture_output=True,
+                text=True,
+                cwd=Path(__file__).parent.parent.parent
+            )
+            if result.returncode == 0:
+                self.metadata['git_hash'] = result.stdout.strip()
+        except Exception:
+            self.metadata['git_hash'] = "unknown"
+    
+    def log_results(self, results: Dict[str, Any]):
+        """Log experiment results."""
+        self.results = results
+        self.metadata['results'] = results
+    
+    def save(self):
+        """Save experiment data to disk."""
+        self.metadata['start_time'] = self.start_time.isoformat()
+        self.metadata['end_time'] = datetime.now().isoformat()
+        
+        # Save metadata
+        metadata_path = self.experiment_dir / 'metadata.json'
+        with open(metadata_path, 'w') as f:
+            json.dump(self.metadata, f, indent=2, default=str)
+        
+        # Save results
+        if self.results:
+            results_path = self.experiment_dir / 'results.json'
+            with open(results_path, 'w') as f:
+                json.dump(self.results, f, indent=2, default=str)
+        
+        # Append to CSV log
+        self._append_to_csv_log()
+    
+    def _append_to_csv_log(self):
+        """Append experiment to CSV log for comparison."""
+        csv_path = self.experiment_dir.parent / 'experiments_log.csv'
+        
+        # Check if CSV exists
+        file_exists = csv_path.exists()
+        
+        with open(csv_path, 'a', newline='') as f:
+            writer = csv.DictWriter(f, fieldnames=[
+                'experiment_id',
+                'start_time',
+                'model_type',
+                'optimizer',
+                'learning_rate',
+                'dataset_version',
+                'feature_set_version',
+                'git_hash',
+                'ic',
+                'sharpe_ratio',
+                'total_return',
+                'experiment_dir'
+            ])
+            
+            if not file_exists:
+                writer.writeheader()
+            
+            row = {
+                'experiment_id': self.experiment_dir.name,
+                'start_time': self.metadata.get('start_time', ''),
+                'model_type': self.metadata.get('model_type', ''),
+                'optimizer': self.metadata.get('optimizer', ''),
+                'learning_rate': self.metadata.get('learning_rate', ''),
+                'dataset_version': self.metadata.get('dataset_version', ''),
+                'feature_set_version': self.metadata.get('feature_set_version', ''),
+                'git_hash': self.metadata.get('git_hash', ''),
+                'ic': self.results.get('alpha_metrics', {}).get('ic', ''),
+                'sharpe_ratio': self.results.get('backtest_results', {}).get('sharpe_ratio', ''),
+                'total_return': self.results.get('backtest_results', {}).get('total_return', ''),
+                'experiment_dir': str(self.experiment_dir)
+            }
+            
+            writer.writerow(row)
+
+
+def compare_experiments(experiment_dirs: List[str]) -> Dict[str, Any]:
+    """
+    Compare multiple experiments.
+    
+    Args:
+        experiment_dirs: List of experiment directory paths
+    
+    Returns:
+        Comparison dictionary
+    """
+    experiments = []
+    
+    for exp_dir in experiment_dirs:
+        metadata_path = Path(exp_dir) / 'metadata.json'
+        results_path = Path(exp_dir) / 'results.json'
+        
+        if metadata_path.exists():
+            with open(metadata_path, 'r') as f:
+                metadata = json.load(f)
+        else:
+            metadata = {}
+        
+        if results_path.exists():
+            with open(results_path, 'r') as f:
+                results = json.load(f)
+        else:
+            results = {}
+        
+        experiments.append({
+            'dir': exp_dir,
+            'metadata': metadata,
+            'results': results
+        })
+    
+    # Create comparison
+    comparison = {
+        'n_experiments': len(experiments),
+        'experiments': []
+    }
+    
+    for exp in experiments:
+        comparison['experiments'].append({
+            'dir': exp['dir'],
+            'model_type': exp['metadata'].get('model_type'),
+            'optimizer': exp['metadata'].get('optimizer'),
+            'ic': exp['results'].get('alpha_metrics', {}).get('ic'),
+            'sharpe': exp['results'].get('backtest_results', {}).get('sharpe_ratio'),
+            'return': exp['results'].get('backtest_results', {}).get('total_return')
+        })
+    
+    return comparison
+

quantml/features/__init__.py

@@ -0,0 +1,25 @@
+"""
+QuantML Feature Registry
+
+Plugin-based feature system for easy feature addition and configuration.
+"""
+
+from quantml.features.base import BaseFeature, Feature
+from quantml.features.registry import FeatureRegistry, register_feature
+from quantml.features.gap_features import OvernightGapFeature, GapSizeFeature
+from quantml.features.volume_features import VolumeRegimeFeature, VolumeShockFeature
+from quantml.features.volatility_features import VolatilityRegimeFeature, RealizedVolatilityFeature
+
+__all__ = [
+    'BaseFeature',
+    'Feature',
+    'FeatureRegistry',
+    'register_feature',
+    'OvernightGapFeature',
+    'GapSizeFeature',
+    'VolumeRegimeFeature',
+    'VolumeShockFeature',
+    'VolatilityRegimeFeature',
+    'RealizedVolatilityFeature'
+]
+