quantmllibrary 0.1.0__py3-none-any.whl

quantml/training/gradient_clipping.py

@@ -0,0 +1,206 @@
+"""
+Gradient clipping utilities for quant training.
+
+Provides various gradient clipping strategies to stabilize training.
+"""
+
+from typing import Union, Optional
+from quantml.tensor import Tensor
+
+# Try to import NumPy
+try:
+    import numpy as np
+    HAS_NUMPY = True
+except ImportError:
+    HAS_NUMPY = False
+    np = None
+
+
+def clip_grad_norm(parameters: list, max_norm: float, norm_type: float = 2.0) -> float:
+    """
+    Clip gradients by norm.
+    
+    Args:
+        parameters: List of parameters with gradients
+        max_norm: Maximum norm value
+        norm_type: Type of norm (2.0 for L2, 1.0 for L1, etc.)
+    
+    Returns:
+        Total norm before clipping
+    """
+    if HAS_NUMPY:
+        total_norm = 0.0
+        for param in parameters:
+            if param.grad is not None:
+                grad = param.grad
+                if isinstance(grad, np.ndarray):
+                    param_norm = np.linalg.norm(grad, ord=norm_type)
+                else:
+                    grad_arr = np.array(grad, dtype=np.float64)
+                    param_norm = np.linalg.norm(grad_arr, ord=norm_type)
+                total_norm += param_norm ** norm_type
+        
+        total_norm = total_norm ** (1.0 / norm_type)
+        clip_coef = max_norm / (total_norm + 1e-6)
+        
+        if clip_coef < 1.0:
+            for param in parameters:
+                if param.grad is not None:
+                    grad = param.grad
+                    if isinstance(grad, np.ndarray):
+                        param.grad = grad * clip_coef
+                    else:
+                        grad_arr = np.array(grad, dtype=np.float64)
+                        param.grad = (grad_arr * clip_coef).tolist()
+        
+        return float(total_norm)
+    else:
+        # Fallback to list-based computation
+        total_norm = 0.0
+        for param in parameters:
+            if param.grad is not None:
+                grad = param.grad
+                if isinstance(grad, list):
+                    if isinstance(grad[0], list):
+                        param_norm = sum(sum(x ** norm_type for x in row) for row in grad) ** (1.0 / norm_type)
+                    else:
+                        param_norm = sum(x ** norm_type for x in grad) ** (1.0 / norm_type)
+                else:
+                    param_norm = abs(grad) ** norm_type
+                total_norm += param_norm ** norm_type
+        
+        total_norm = total_norm ** (1.0 / norm_type)
+        clip_coef = max_norm / (total_norm + 1e-6)
+        
+        if clip_coef < 1.0:
+            for param in parameters:
+                if param.grad is not None:
+                    grad = param.grad
+                    if isinstance(grad, list):
+                        if isinstance(grad[0], list):
+                            param.grad = [[x * clip_coef for x in row] for row in grad]
+                        else:
+                            param.grad = [x * clip_coef for x in grad]
+                    else:
+                        param.grad = grad * clip_coef
+        
+        return float(total_norm)
+
+
+def clip_grad_value(parameters: list, clip_value: float):
+    """
+    Clip gradients by value.
+    
+    Args:
+        parameters: List of parameters with gradients
+        clip_value: Maximum absolute value for gradients
+    """
+    if HAS_NUMPY:
+        for param in parameters:
+            if param.grad is not None:
+                grad = param.grad
+                if isinstance(grad, np.ndarray):
+                    param.grad = np.clip(grad, -clip_value, clip_value)
+                else:
+                    grad_arr = np.array(grad, dtype=np.float64)
+                    param.grad = np.clip(grad_arr, -clip_value, clip_value).tolist()
+    else:
+        for param in parameters:
+            if param.grad is not None:
+                grad = param.grad
+                if isinstance(grad, list):
+                    if isinstance(grad[0], list):
+                        param.grad = [[max(-clip_value, min(clip_value, x)) for x in row] for row in grad]
+                    else:
+                        param.grad = [max(-clip_value, min(clip_value, x)) for x in grad]
+                else:
+                    param.grad = max(-clip_value, min(clip_value, grad))
+
+
+class GradientNormClipper:
+    """Gradient clipper that clips by norm."""
+    
+    def __init__(self, max_norm: float, norm_type: float = 2.0):
+        """
+        Initialize gradient norm clipper.
+        
+        Args:
+            max_norm: Maximum norm value
+            norm_type: Type of norm (2.0 for L2)
+        """
+        self.max_norm = max_norm
+        self.norm_type = norm_type
+    
+    def __call__(self, parameters: list) -> float:
+        """Clip gradients and return total norm."""
+        return clip_grad_norm(parameters, self.max_norm, self.norm_type)
+
+
+class GradientValueClipper:
+    """Gradient clipper that clips by value."""
+    
+    def __init__(self, clip_value: float):
+        """
+        Initialize gradient value clipper.
+        
+        Args:
+            clip_value: Maximum absolute value
+        """
+        self.clip_value = clip_value
+    
+    def __call__(self, parameters: list):
+        """Clip gradients by value."""
+        clip_grad_value(parameters, self.clip_value)
+
+
+class AdaptiveClipper:
+    """Adaptive gradient clipper based on gradient statistics."""
+    
+    def __init__(self, percentile: float = 95.0, factor: float = 2.0):
+        """
+        Initialize adaptive clipper.
+        
+        Args:
+            percentile: Percentile to use for clipping threshold
+            factor: Factor to multiply percentile by
+        """
+        self.percentile = percentile
+        self.factor = factor
+    
+    def __call__(self, parameters: list):
+        """Clip gradients adaptively."""
+        if HAS_NUMPY:
+            all_grads = []
+            for param in parameters:
+                if param.grad is not None:
+                    grad = param.grad
+                    if isinstance(grad, np.ndarray):
+                        all_grads.extend(grad.flatten())
+                    else:
+                        grad_arr = np.array(grad, dtype=np.float64)
+                        all_grads.extend(grad_arr.flatten())
+            
+            if all_grads:
+                threshold = np.percentile(np.abs(all_grads), self.percentile) * self.factor
+                clip_grad_value(parameters, threshold)
+        else:
+            # Fallback
+            all_grads = []
+            for param in parameters:
+                if param.grad is not None:
+                    grad = param.grad
+                    if isinstance(grad, list):
+                        if isinstance(grad[0], list):
+                            all_grads.extend([x for row in grad for x in row])
+                        else:
+                            all_grads.extend(grad)
+                    else:
+                        all_grads.append(grad)
+            
+            if all_grads:
+                abs_grads = [abs(g) for g in all_grads]
+                abs_grads.sort()
+                idx = int(len(abs_grads) * self.percentile / 100.0)
+                threshold = abs_grads[min(idx, len(abs_grads) - 1)] * self.factor
+                clip_grad_value(parameters, threshold)
+

quantml/training/losses.py

@@ -0,0 +1,248 @@
+"""
+Quant-specific loss functions for alpha generation.
+
+This module provides loss functions optimized for quantitative trading,
+including Sharpe ratio loss, quantile loss, and information ratio loss.
+"""
+
+from typing import Union, List, Optional
+from quantml.tensor import Tensor
+from quantml import ops
+
+# Try to import NumPy
+try:
+    import numpy as np
+    HAS_NUMPY = True
+except ImportError:
+    HAS_NUMPY = False
+    np = None
+
+
+def mse_loss(pred: Tensor, target: Tensor) -> Tensor:
+    """
+    Mean Squared Error loss.
+    
+    Args:
+        pred: Predictions
+        target: Targets
+    
+    Returns:
+        MSE loss tensor
+    """
+    diff = ops.sub(pred, target)
+    return ops.mean(ops.mul(diff, diff))
+
+
+def mae_loss(pred: Tensor, target: Tensor) -> Tensor:
+    """
+    Mean Absolute Error loss.
+    
+    Args:
+        pred: Predictions
+        target: Targets
+    
+    Returns:
+        MAE loss tensor
+    """
+    diff = ops.sub(pred, target)
+    return ops.mean(ops.abs(diff))
+
+
+def quantile_loss(pred: Tensor, target: Tensor, quantile: float = 0.5) -> Tensor:
+    """
+    Quantile loss (pinball loss) for robust regression.
+    
+    Useful for predicting percentiles and handling outliers.
+    
+    Args:
+        pred: Predictions
+        target: Targets
+        quantile: Quantile level (0.0 to 1.0, default: 0.5 for median)
+    
+    Returns:
+        Quantile loss tensor
+    """
+    if not 0.0 <= quantile <= 1.0:
+        raise ValueError("quantile must be between 0.0 and 1.0")
+    
+    diff = ops.sub(pred, target)
+    
+    # L_quantile = max(quantile * diff, (quantile - 1) * diff)
+    pos_part = ops.mul(diff, quantile)
+    neg_part = ops.mul(diff, quantile - 1.0)
+    
+    # Element-wise maximum
+    loss = ops.maximum(pos_part, neg_part)
+    return ops.mean(loss)
+
+
+def sharpe_loss(pred: Tensor, target: Tensor, risk_free_rate: float = 0.0) -> Tensor:
+    """
+    Negative Sharpe ratio as loss (to maximize Sharpe ratio).
+    
+    This loss function directly optimizes for risk-adjusted returns.
+    
+    Args:
+        pred: Predictions (returns)
+        target: Targets (actual returns)
+        risk_free_rate: Risk-free rate
+    
+    Returns:
+        Negative Sharpe ratio (to minimize)
+    """
+    # Use predictions as portfolio returns
+    returns = pred
+    
+    # Calculate mean and std
+    mean_ret = ops.mean(returns)
+    std_ret = ops.std(returns)
+    
+    # Sharpe = (mean - rf) / std
+    # We want to maximize Sharpe, so minimize negative Sharpe
+    numerator = ops.sub(mean_ret, risk_free_rate)
+    sharpe = ops.div(numerator, ops.add(std_ret, 1e-8))  # Add small epsilon
+    
+    # Return negative to minimize (maximize Sharpe)
+    return ops.mul(sharpe, -1.0)
+
+
+def information_ratio_loss(pred: Tensor, target: Tensor) -> Tensor:
+    """
+    Negative Information Ratio as loss.
+    
+    Information Ratio = mean(alpha) / std(alpha), where alpha = pred - target
+    
+    Args:
+        pred: Predictions
+        target: Targets
+    
+    Returns:
+        Negative Information Ratio
+    """
+    # Alpha = prediction error (we want to minimize this)
+    alpha = ops.sub(pred, target)
+    
+    mean_alpha = ops.mean(alpha)
+    std_alpha = ops.std(alpha)
+    
+    # IR = mean / std
+    ir = ops.div(mean_alpha, ops.add(std_alpha, 1e-8))
+    
+    # Return negative to minimize (maximize IR)
+    return ops.mul(ir, -1.0)
+
+
+def huber_loss(pred: Tensor, target: Tensor, delta: float = 1.0) -> Tensor:
+    """
+    Huber loss: combines MSE and MAE, robust to outliers.
+    
+    Args:
+        pred: Predictions
+        target: Targets
+        delta: Threshold parameter
+    
+    Returns:
+        Huber loss tensor
+    """
+    diff = ops.sub(pred, target)
+    abs_diff = ops.abs(diff)
+    
+    # L_huber = 0.5 * diff^2 if |diff| <= delta, else delta * (|diff| - 0.5*delta)
+    # We'll use a smooth approximation
+    squared = ops.mul(ops.mul(diff, diff), 0.5)
+    linear = ops.sub(ops.mul(abs_diff, delta), ops.mul(delta * delta, 0.5))
+    
+    # Use relu to switch between squared and linear
+    # Simplified: use squared for small errors, linear for large
+    loss = ops.add(squared, ops.mul(ops.relu(ops.sub(abs_diff, delta)), ops.sub(linear, squared)))
+    
+    return ops.mean(loss)
+
+
+def asymmetric_loss(pred: Tensor, target: Tensor, asymmetry: float = 1.0) -> Tensor:
+    """
+    Asymmetric loss: penalizes over-prediction and under-prediction differently.
+    
+    Useful when false positives and false negatives have different costs.
+    
+    Args:
+        pred: Predictions
+        target: Targets
+        asymmetry: Asymmetry factor (>1 penalizes over-prediction more)
+    
+    Returns:
+        Asymmetric loss tensor
+    """
+    diff = ops.sub(pred, target)
+    
+    # L_asym = asymmetry * diff^2 if diff > 0, else diff^2
+    diff_sq = ops.mul(diff, diff)
+    
+    # Weight over-predictions more
+    over_pred = ops.mul(ops.relu(diff), ops.mul(diff_sq, asymmetry - 1.0))
+    loss = ops.add(diff_sq, over_pred)
+    
+    return ops.mean(loss)
+
+
+def max_drawdown_loss(pred: Tensor, target: Tensor) -> Tensor:
+    """
+    Loss based on maximum drawdown of cumulative returns.
+    
+    This encourages predictions that lead to smoother equity curves.
+    
+    Args:
+        pred: Predictions (returns)
+        target: Targets (actual returns)
+    
+    Returns:
+        Maximum drawdown (to minimize)
+    """
+    # Use predictions as returns
+    returns = pred
+    
+    # For simplicity, we'll compute a proxy using variance
+    # A full implementation would compute actual drawdown
+    # This is a simplified version
+    mean_ret = ops.mean(returns)
+    std_ret = ops.std(returns)
+    
+    # Proxy: higher variance -> higher drawdown risk
+    # We want to minimize this
+    return std_ret
+
+
+def combined_quant_loss(pred: Tensor, target: Tensor, 
+                       mse_weight: float = 0.5,
+                       sharpe_weight: float = 0.3,
+                       drawdown_weight: float = 0.2) -> Tensor:
+    """
+    Combined loss function for quant trading.
+    
+    Combines MSE, Sharpe ratio, and drawdown considerations.
+    
+    Args:
+        pred: Predictions
+        target: Targets
+        mse_weight: Weight for MSE component
+        sharpe_weight: Weight for Sharpe component
+        drawdown_weight: Weight for drawdown component
+    
+    Returns:
+        Combined loss tensor
+    """
+    mse = mse_loss(pred, target)
+    sharpe = sharpe_loss(pred, target)
+    dd = max_drawdown_loss(pred, target)
+    
+    # Normalize components (simplified)
+    combined = ops.add(
+        ops.mul(mse, mse_weight),
+        ops.add(
+            ops.mul(sharpe, sharpe_weight),
+            ops.mul(dd, drawdown_weight)
+        )
+    )
+    
+    return combined
+

quantml/training/lr_finder.py

@@ -0,0 +1,127 @@
+"""
+Learning rate finder for optimal LR discovery.
+
+Implements learning rate range test to find optimal learning rate.
+"""
+
+from typing import List, Optional, Callable, Any
+from quantml.tensor import Tensor
+import math
+
+
+class LRFinder:
+    """
+    Learning rate finder using range test.
+    
+    Finds optimal learning rate by testing a range of values
+    and identifying where loss decreases most rapidly.
+    """
+    
+    def __init__(
+        self,
+        model: Any,
+        optimizer: Any,
+        loss_fn: Callable,
+        min_lr: float = 1e-7,
+        max_lr: float = 10.0,
+        num_iterations: int = 100
+    ):
+        """
+        Initialize LR finder.
+        
+        Args:
+            model: Model to test
+            optimizer: Optimizer
+            loss_fn: Loss function
+            min_lr: Minimum learning rate to test
+            max_lr: Maximum learning rate to test
+            num_iterations: Number of iterations to run
+        """
+        self.model = model
+        self.optimizer = optimizer
+        self.loss_fn = loss_fn
+        self.min_lr = min_lr
+        self.max_lr = max_lr
+        self.num_iterations = num_iterations
+        self.lrs = []
+        self.losses = []
+    
+    def range_test(self, x: Tensor, y: Tensor) -> tuple:
+        """
+        Run learning rate range test.
+        
+        Args:
+            x: Input features
+            y: Targets
+        
+        Returns:
+            Tuple of (lrs, losses) lists
+        """
+        self.lrs = []
+        self.losses = []
+        
+        # Calculate LR multiplier
+        lr_mult = (self.max_lr / self.min_lr) ** (1.0 / self.num_iterations)
+        
+        original_lr = self.optimizer.lr
+        
+        for i in range(self.num_iterations):
+            # Set learning rate
+            current_lr = self.min_lr * (lr_mult ** i)
+            self.optimizer.lr = current_lr
+            self.lrs.append(current_lr)
+            
+            # Forward pass
+            pred = self.model.forward(x)
+            loss = self.loss_fn(pred, y)
+            
+            # Backward pass
+            if loss.requires_grad:
+                loss.backward()
+                self.optimizer.step()
+                self.model.zero_grad()
+            
+            # Record loss
+            loss_val = self._get_value(loss)
+            self.losses.append(loss_val)
+        
+        # Restore original LR
+        self.optimizer.lr = original_lr
+        
+        return self.lrs, self.losses
+    
+    def suggest_lr(self) -> float:
+        """
+        Suggest optimal learning rate based on range test.
+        
+        Returns:
+            Suggested learning rate
+        """
+        if not self.losses:
+            return self.min_lr
+        
+        # Find steepest descent point
+        # Look for point with maximum negative gradient
+        best_idx = 0
+        best_grad = float('-inf')
+        
+        for i in range(1, len(self.losses) - 1):
+            # Calculate gradient (negative of loss change)
+            grad = -(self.losses[i+1] - self.losses[i-1]) / (self.lrs[i+1] - self.lrs[i-1])
+            if grad > best_grad and self.losses[i] < self.losses[0] * 0.5:
+                best_grad = grad
+                best_idx = i
+        
+        if best_idx > 0:
+            return self.lrs[best_idx]
+        return self.min_lr * 10.0  # Default suggestion
+    
+    def _get_value(self, tensor: Tensor) -> float:
+        """Extract scalar value from tensor."""
+        data = tensor.data
+        if isinstance(data, list):
+            if isinstance(data[0], list):
+                return float(data[0][0])
+            return float(data[0])
+        return float(data)
+