quantmllibrary 0.1.0__py3-none-any.whl

quantml/utils/ops_cpu.py

@@ -0,0 +1,231 @@
+"""
+CPU-optimized operations for QuantML.
+
+This module provides low-level CPU-optimized implementations of operations.
+It uses NumPy when available for better performance, with pure Python fallbacks.
+"""
+
+from typing import List, Union, Any
+import math
+
+# Try to import NumPy for optimized operations
+try:
+    import numpy as np
+    HAS_NUMPY = True
+except ImportError:
+    HAS_NUMPY = False
+
+
+def add_cpu(a: Union[List, Any], b: Union[List, Any]) -> List:
+    """
+    CPU-optimized addition.
+    
+    Uses NumPy if available, otherwise falls back to pure Python.
+    
+    Args:
+        a: First operand
+        b: Second operand
+    
+    Returns:
+        Result of addition
+    """
+    if HAS_NUMPY:
+        a_arr = np.array(a)
+        b_arr = np.array(b)
+        result = (a_arr + b_arr).tolist()
+        return result
+    else:
+        return _add_pure_python(a, b)
+
+
+def mul_cpu(a: Union[List, Any], b: Union[List, Any]) -> List:
+    """CPU-optimized multiplication."""
+    if HAS_NUMPY:
+        a_arr = np.array(a)
+        b_arr = np.array(b)
+        result = (a_arr * b_arr).tolist()
+        return result
+    else:
+        return _mul_pure_python(a, b)
+
+
+def matmul_cpu(a: Union[List, Any], b: Union[List, Any]) -> List:
+    """CPU-optimized matrix multiplication."""
+    if HAS_NUMPY:
+        a_arr = np.array(a)
+        b_arr = np.array(b)
+        result = np.matmul(a_arr, b_arr).tolist()
+        return result
+    else:
+        return _matmul_pure_python(a, b)
+
+
+def sum_cpu(a: Union[List, Any], axis: int = None) -> Union[float, List]:
+    """CPU-optimized sum."""
+    if HAS_NUMPY:
+        a_arr = np.array(a)
+        result = np.sum(a_arr, axis=axis)
+        if isinstance(result, np.ndarray):
+            return result.tolist()
+        return float(result)
+    else:
+        return _sum_pure_python(a, axis)
+
+
+def mean_cpu(a: Union[List, Any], axis: int = None) -> Union[float, List]:
+    """CPU-optimized mean."""
+    if HAS_NUMPY:
+        a_arr = np.array(a)
+        result = np.mean(a_arr, axis=axis)
+        if isinstance(result, np.ndarray):
+            return result.tolist()
+        return float(result)
+    else:
+        return _mean_pure_python(a, axis)
+
+
+def std_cpu(a: Union[List, Any], axis: int = None) -> Union[float, List]:
+    """CPU-optimized standard deviation."""
+    if HAS_NUMPY:
+        a_arr = np.array(a)
+        result = np.std(a_arr, axis=axis)
+        if isinstance(result, np.ndarray):
+            return result.tolist()
+        return float(result)
+    else:
+        return _std_pure_python(a, axis)
+
+
+# Pure Python implementations (fallbacks)
+
+def _add_pure_python(a: Any, b: Any) -> List:
+    """Pure Python addition."""
+    if isinstance(a, list) and isinstance(b, list):
+        if isinstance(a[0], list) and isinstance(b[0], list):
+            return [[a[i][j] + b[i][j] for j in range(len(a[i]))] 
+                    for i in range(len(a))]
+        else:
+            return [a[i] + b[i] for i in range(len(a))]
+    elif isinstance(a, list):
+        if isinstance(a[0], list):
+            return [[a[i][j] + b for j in range(len(a[i]))] 
+                    for i in range(len(a))]
+        return [a[i] + b for i in range(len(a))]
+    elif isinstance(b, list):
+        if isinstance(b[0], list):
+            return [[a + b[i][j] for j in range(len(b[i]))] 
+                    for i in range(len(b))]
+        return [a + b[i] for i in range(len(b))]
+    else:
+        return [[a + b]]
+
+
+def _mul_pure_python(a: Any, b: Any) -> List:
+    """Pure Python multiplication."""
+    if isinstance(a, list) and isinstance(b, list):
+        if isinstance(a[0], list) and isinstance(b[0], list):
+            return [[a[i][j] * b[i][j] for j in range(len(a[i]))] 
+                    for i in range(len(a))]
+        else:
+            return [a[i] * b[i] for i in range(len(a))]
+    elif isinstance(a, list):
+        if isinstance(a[0], list):
+            return [[a[i][j] * b for j in range(len(a[i]))] 
+                    for i in range(len(a))]
+        return [a[i] * b for i in range(len(a))]
+    elif isinstance(b, list):
+        if isinstance(b[0], list):
+            return [[a * b[i][j] for j in range(len(b[i]))] 
+                    for i in range(len(b))]
+        return [a * b[i] for i in range(len(b))]
+    else:
+        return [[a * b]]
+
+
+def _matmul_pure_python(a: List, b: List) -> List:
+    """Pure Python matrix multiplication."""
+    # Ensure 2D
+    a_2d = a if isinstance(a[0], list) else [a]
+    b_2d = b if isinstance(b[0], list) else [b]
+    
+    m, n = len(a_2d), len(a_2d[0])
+    n2, p = len(b_2d), len(b_2d[0])
+    
+    if n != n2:
+        raise ValueError(f"Incompatible dimensions: {m}x{n} x {n2}x{p}")
+    
+    result = [[sum(a_2d[i][k] * b_2d[k][j] for k in range(n)) 
+              for j in range(p)] 
+             for i in range(m)]
+    return result
+
+
+def _sum_pure_python(a: Any, axis: int = None) -> Union[float, List]:
+    """Pure Python sum."""
+    if axis is None:
+        if isinstance(a, list):
+            if isinstance(a[0], list):
+                return sum(sum(row) for row in a)
+            return sum(a)
+        return float(a)
+    elif axis == 0:
+        if isinstance(a[0], list):
+            return [sum(a[i][j] for i in range(len(a))) 
+                   for j in range(len(a[0]))]
+        return [sum(a)]
+    elif axis == 1:
+        if isinstance(a[0], list):
+            return [sum(row) for row in a]
+        return a
+    else:
+        raise ValueError(f"Invalid axis: {axis}")
+
+
+def _mean_pure_python(a: Any, axis: int = None) -> Union[float, List]:
+    """Pure Python mean."""
+    s = _sum_pure_python(a, axis)
+    if axis is None:
+        count = 1.0
+        if isinstance(a, list):
+            if isinstance(a[0], list):
+                count = len(a) * len(a[0])
+            else:
+                count = len(a)
+        return s / count if count > 0 else 0.0
+    elif axis == 0:
+        count = len(a) if isinstance(a[0], list) else 1.0
+        if isinstance(s, list):
+            return [x / count for x in s]
+        return s / count
+    else:
+        count = len(a[0]) if isinstance(a[0], list) else len(a)
+        if isinstance(s, list):
+            return [x / count for x in s]
+        return s / count
+
+
+def _std_pure_python(a: Any, axis: int = None) -> Union[float, List]:
+    """Pure Python standard deviation."""
+    m = _mean_pure_python(a, axis)
+    
+    # Compute variance
+    if axis is None:
+        if isinstance(a, list):
+            if isinstance(a[0], list):
+                diff_sq = sum((a[i][j] - m) ** 2 
+                             for i in range(len(a)) 
+                             for j in range(len(a[i])))
+                count = len(a) * len(a[0])
+            else:
+                diff_sq = sum((a[i] - m) ** 2 for i in range(len(a)))
+                count = len(a)
+        else:
+            diff_sq = (a - m) ** 2
+            count = 1.0
+        var = diff_sq / count if count > 0 else 0.0
+        return math.sqrt(var)
+    else:
+        # For axis-specific std, use simplified approach
+        # Full implementation would be more complex
+        return m  # Placeholder - full implementation needed
+

quantml/utils/profiling.py

@@ -0,0 +1,364 @@
+"""
+Profiling and performance measurement utilities.
+
+This module provides tools for measuring latency and performance of operations,
+essential for optimizing quant trading systems.
+"""
+
+import time
+import functools
+import sys
+from typing import Callable, Any, Optional, Dict, List
+from collections import defaultdict
+
+# Try to import psutil for memory tracking
+try:
+    import psutil
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+    psutil = None
+
+
+def timing(func: Callable) -> Callable:
+    """
+    Decorator to measure execution time of a function.
+    
+    The decorator prints the execution time and returns the result.
+    
+    Args:
+        func: Function to time
+    
+    Returns:
+        Wrapped function that measures execution time
+    
+    Examples:
+        >>> @timing
+        >>> def my_function():
+        >>>     # ... do work ...
+        >>>     return result
+        >>> result = my_function()  # Prints: "my_function took 0.123 seconds"
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.perf_counter()
+        result = func(*args, **kwargs)
+        end = time.perf_counter()
+        elapsed = end - start
+        print(f"{func.__name__} took {elapsed:.6f} seconds")
+        return result
+    return wrapper
+
+
+def measure_latency(func: Callable, *args, **kwargs) -> tuple:
+    """
+    Measure the latency of a function call.
+    
+    Args:
+        func: Function to measure
+        *args: Positional arguments for function
+        **kwargs: Keyword arguments for function
+    
+    Returns:
+        Tuple of (result, latency_in_seconds)
+    
+    Examples:
+        >>> result, latency = measure_latency(my_function, arg1, arg2)
+        >>> print(f"Latency: {latency * 1000:.2f} ms")
+    """
+    start = time.perf_counter()
+    result = func(*args, **kwargs)
+    end = time.perf_counter()
+    latency = end - start
+    return result, latency
+
+
+def measure_latency_microseconds(func: Callable, *args, **kwargs) -> tuple:
+    """
+    Measure latency in microseconds (useful for HFT).
+    
+    Args:
+        func: Function to measure
+        *args: Positional arguments
+        **kwargs: Keyword arguments
+    
+    Returns:
+        Tuple of (result, latency_in_microseconds)
+    """
+    result, latency_seconds = measure_latency(func, *args, **kwargs)
+    latency_us = latency_seconds * 1_000_000
+    return result, latency_us
+
+
+class PerformanceProfiler:
+    """
+    Performance profiler for tracking multiple function calls.
+    
+    Tracks statistics like mean, min, max latency for multiple calls.
+    
+    Attributes:
+        stats: Dictionary mapping function names to statistics
+    
+    Examples:
+        >>> profiler = PerformanceProfiler()
+        >>> for _ in range(100):
+        >>>     profiler.record('my_func', measure_latency(my_function))
+        >>> print(profiler.get_stats('my_func'))
+    """
+    
+    def __init__(self):
+        """Initialize profiler."""
+        self.stats = defaultdict(list)
+    
+    def record(self, name: str, latency: float):
+        """
+        Record a latency measurement.
+        
+        Args:
+            name: Function or operation name
+            latency: Latency in seconds
+        """
+        self.stats[name].append(latency)
+    
+    def get_stats(self, name: str) -> Optional[dict]:
+        """
+        Get statistics for a function.
+        
+        Args:
+            name: Function name
+        
+        Returns:
+            Dictionary with mean, min, max, count, or None if no data
+        """
+        if name not in self.stats or len(self.stats[name]) == 0:
+            return None
+        
+        latencies = self.stats[name]
+        return {
+            'mean': sum(latencies) / len(latencies),
+            'min': min(latencies),
+            'max': max(latencies),
+            'count': len(latencies),
+            'total': sum(latencies)
+        }
+    
+    def print_stats(self, name: Optional[str] = None):
+        """
+        Print statistics for one or all functions.
+        
+        Args:
+            name: Optional function name, or None for all
+        """
+        if name is not None:
+            stats = self.get_stats(name)
+            if stats:
+                print(f"{name}:")
+                print(f"  Mean: {stats['mean']*1000:.3f} ms")
+                print(f"  Min:  {stats['min']*1000:.3f} ms")
+                print(f"  Max:  {stats['max']*1000:.3f} ms")
+                print(f"  Count: {stats['count']}")
+        else:
+            for func_name in self.stats:
+                self.print_stats(func_name)
+    
+    def clear(self):
+        """Clear all recorded statistics."""
+        self.stats.clear()
+
+
+def benchmark(func: Callable, n_iterations: int = 100, *args, **kwargs) -> dict:
+    """
+    Benchmark a function over multiple iterations.
+    
+    Args:
+        func: Function to benchmark
+        n_iterations: Number of iterations
+        *args: Positional arguments
+        **kwargs: Keyword arguments
+    
+    Returns:
+        Dictionary with benchmark statistics
+    
+    Examples:
+        >>> stats = benchmark(my_function, n_iterations=1000, arg1, arg2)
+        >>> print(f"Mean latency: {stats['mean']*1000:.2f} ms")
+    """
+    latencies = []
+    for _ in range(n_iterations):
+        _, latency = measure_latency(func, *args, **kwargs)
+        latencies.append(latency)
+    
+    return {
+        'mean': sum(latencies) / len(latencies),
+        'min': min(latencies),
+        'max': max(latencies),
+        'median': sorted(latencies)[len(latencies) // 2],
+        'p95': sorted(latencies)[int(len(latencies) * 0.95)],
+        'p99': sorted(latencies)[int(len(latencies) * 0.99)],
+        'count': len(latencies)
+    }
+
+
+def get_memory_usage() -> Dict[str, float]:
+    """
+    Get current memory usage.
+    
+    Returns:
+        Dictionary with memory usage in MB
+    """
+    if HAS_PSUTIL:
+        process = psutil.Process()
+        mem_info = process.memory_info()
+        return {
+            'rss_mb': mem_info.rss / 1024 / 1024,  # Resident Set Size
+            'vms_mb': mem_info.vms / 1024 / 1024,  # Virtual Memory Size
+            'percent': process.memory_percent()
+        }
+    else:
+        # Fallback using sys
+        try:
+            import resource
+            mem_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
+            return {'rss_mb': mem_mb, 'vms_mb': 0.0, 'percent': 0.0}
+        except (ImportError, AttributeError):
+            return {'rss_mb': 0.0, 'vms_mb': 0.0, 'percent': 0.0}
+
+
+class PipelineProfiler:
+    """
+    Profiler for entire pipeline (data loading, feature generation, training).
+    
+    Tracks timing and memory usage for each stage.
+    """
+    
+    def __init__(self):
+        """Initialize pipeline profiler."""
+        self.stages = {}
+        self.memory_snapshots = []
+    
+    def start_stage(self, stage_name: str):
+        """Start timing a stage."""
+        self.stages[stage_name] = {
+            'start_time': time.perf_counter(),
+            'start_memory': get_memory_usage()
+        }
+    
+    def end_stage(self, stage_name: str):
+        """End timing a stage."""
+        if stage_name not in self.stages:
+            return
+        
+        end_time = time.perf_counter()
+        end_memory = get_memory_usage()
+        
+        start_info = self.stages[stage_name]
+        duration = end_time - start_info['start_time']
+        
+        memory_delta = {
+            'rss_mb': end_memory['rss_mb'] - start_info['start_memory']['rss_mb'],
+            'vms_mb': end_memory['vms_mb'] - start_info['start_memory']['vms_mb']
+        }
+        
+        self.stages[stage_name] = {
+            'duration': duration,
+            'memory_delta': memory_delta,
+            'peak_memory': end_memory
+        }
+    
+    def get_report(self) -> Dict[str, Any]:
+        """Get profiling report."""
+        total_time = sum(s.get('duration', 0) for s in self.stages.values())
+        
+        return {
+            'stages': self.stages,
+            'total_time': total_time,
+            'bottleneck': self._identify_bottleneck()
+        }
+    
+    def _identify_bottleneck(self) -> Optional[str]:
+        """Identify the slowest stage."""
+        if not self.stages:
+            return None
+        
+        max_duration = 0
+        bottleneck = None
+        
+        for stage_name, info in self.stages.items():
+            duration = info.get('duration', 0)
+            if duration > max_duration:
+                max_duration = duration
+                bottleneck = stage_name
+        
+        return bottleneck
+    
+    def print_report(self):
+        """Print profiling report."""
+        report = self.get_report()
+        
+        print("\n" + "=" * 70)
+        print("Pipeline Profiling Report")
+        print("=" * 70)
+        
+        for stage_name, info in report['stages'].items():
+            duration = info.get('duration', 0)
+            memory_delta = info.get('memory_delta', {})
+            
+            print(f"\n{stage_name}:")
+            print(f"  Duration: {duration:.4f} seconds ({duration*1000:.2f} ms)")
+            print(f"  Memory Delta: {memory_delta.get('rss_mb', 0):.2f} MB")
+        
+        print(f"\nTotal Time: {report['total_time']:.4f} seconds")
+        if report['bottleneck']:
+            print(f"Bottleneck: {report['bottleneck']} ({report['stages'][report['bottleneck']]['duration']:.4f}s)")
+        print("=" * 70 + "\n")
+
+
+def profile_training_loop(
+    trainer,
+    n_epochs: int,
+    log_interval: int = 10
+) -> Dict[str, Any]:
+    """
+    Profile a training loop.
+    
+    Args:
+        trainer: Trainer instance
+        n_epochs: Number of epochs
+        log_interval: Log every N epochs
+    
+    Returns:
+        Training profile dictionary
+    """
+    profiler = PipelineProfiler()
+    epoch_times = []
+    
+    profiler.start_stage('total_training')
+    
+    for epoch in range(n_epochs):
+        epoch_start = time.perf_counter()
+        
+        # Train epoch (assuming trainer has train_epoch method)
+        if hasattr(trainer, 'train_epoch'):
+            trainer.train_epoch()
+        else:
+            # Fallback: assume train_step is called externally
+            pass
+        
+        epoch_time = time.perf_counter() - epoch_start
+        epoch_times.append(epoch_time)
+        
+        if (epoch + 1) % log_interval == 0:
+            avg_time = sum(epoch_times[-log_interval:]) / log_interval
+            print(f"Epoch {epoch+1}/{n_epochs}: {avg_time:.4f}s per epoch")
+    
+    profiler.end_stage('total_training')
+    
+    return {
+        'total_time': sum(epoch_times),
+        'avg_epoch_time': sum(epoch_times) / len(epoch_times),
+        'min_epoch_time': min(epoch_times),
+        'max_epoch_time': max(epoch_times),
+        'epoch_times': epoch_times,
+        'memory': get_memory_usage()
+    }
+