quantmllibrary 0.1.0__py3-none-any.whl

quantml/optim/lookahead.py

@@ -0,0 +1,97 @@
+"""
+Lookahead optimizer wrapper.
+
+Lookahead is a wrapper that improves training stability by maintaining
+a slow-moving average of parameters.
+"""
+
+from typing import List, Optional, Any
+from quantml.tensor import Tensor
+
+
+class Lookahead:
+    """
+    Lookahead optimizer wrapper.
+    
+    Wraps any base optimizer and maintains a slow-moving average of parameters.
+    Improves training stability for quant models.
+    
+    Attributes:
+        base_optimizer: The base optimizer to wrap
+        k: Number of steps before updating slow weights
+        alpha: Interpolation factor for slow weights
+        slow_weights: Slow-moving parameter averages
+    
+    Examples:
+        >>> base_opt = Adam(lr=0.001)
+        >>> optimizer = Lookahead(base_opt, k=5, alpha=0.5)
+        >>> for param in model.parameters():
+        >>>     optimizer.step(param)
+    """
+    
+    def __init__(
+        self,
+        base_optimizer: Any,
+        k: int = 5,
+        alpha: float = 0.5
+    ):
+        """
+        Initialize Lookahead optimizer.
+        
+        Args:
+            base_optimizer: Base optimizer to wrap (SGD, Adam, etc.)
+            k: Number of steps before updating slow weights
+            alpha: Interpolation factor (0 < alpha < 1)
+        """
+        self.base_optimizer = base_optimizer
+        self.k = k
+        self.alpha = alpha
+        self.slow_weights: dict = {}  # Slow-moving parameter averages
+        self.step_count = 0
+    
+    def step(self, param: Optional[Tensor] = None):
+        """Perform a single optimization step."""
+        # Use base optimizer
+        self.base_optimizer.step(param)
+        self.step_count += 1
+        
+        # Update slow weights every k steps
+        if self.step_count % self.k == 0:
+            if param is not None:
+                self._update_slow_weights(param)
+            else:
+                for p in self.base_optimizer.params:
+                    self._update_slow_weights(p)
+    
+    def _update_slow_weights(self, param: Tensor):
+        """Update slow-moving parameter average."""
+        param_id = id(param)
+        
+        if param_id not in self.slow_weights:
+            # Initialize slow weights with current parameter values
+            self.slow_weights[param_id] = param.data.copy() if hasattr(param.data, 'copy') else param.data
+        else:
+            # Interpolate: slow = alpha * slow + (1 - alpha) * param
+            slow = self.slow_weights[param_id]
+            if hasattr(slow, '__iter__') and hasattr(param.data, '__iter__'):
+                # Update slow weights (simplified - would need proper NumPy/list handling)
+                # For now, just store current param as slow weight
+                self.slow_weights[param_id] = param.data.copy() if hasattr(param.data, 'copy') else param.data
+            else:
+                self.slow_weights[param_id] = self.alpha * slow + (1.0 - self.alpha) * param.data
+    
+    def sync_slow_weights(self):
+        """Synchronize parameters with slow weights."""
+        for param in self.base_optimizer.params:
+            param_id = id(param)
+            if param_id in self.slow_weights:
+                param.data = self.slow_weights[param_id]
+    
+    def zero_grad(self, param: Optional[Tensor] = None):
+        """Clear gradients."""
+        self.base_optimizer.zero_grad(param)
+    
+    def add_param_group(self, params: List[Tensor]):
+        """Add a parameter group to optimize."""
+        self.base_optimizer.add_param_group(params)
+

quantml/optim/quant_optimizer.py

@@ -0,0 +1,228 @@
+"""
+QuantOptimizer - Custom optimizer for quant trading.
+
+Adaptive learning rate based on market volatility and regime-aware parameter updates.
+"""
+
+from typing import List, Optional, Dict, Any
+from quantml.tensor import Tensor
+from quantml import ops
+
+# Try to import NumPy
+try:
+    import numpy as np
+    HAS_NUMPY = True
+except ImportError:
+    HAS_NUMPY = False
+    np = None
+
+
+class QuantOptimizer:
+    """
+    QuantOptimizer - Custom optimizer for quant trading.
+    
+    Features:
+    - Adaptive learning rate based on market volatility
+    - Regime-aware parameter updates
+    - Per-feature learning rate scaling
+    
+    Attributes:
+        lr: Base learning rate
+        volatility_window: Window size for volatility calculation
+        regime_threshold: Threshold for regime detection
+        feature_lrs: Per-feature learning rate multipliers
+    
+    Examples:
+        >>> optimizer = QuantOptimizer(lr=0.001, volatility_window=20)
+        >>> for param in model.parameters():
+        >>>     optimizer.step(param, market_volatility=0.02)
+    """
+    
+    def __init__(
+        self,
+        params: Optional[List[Tensor]] = None,
+        lr: float = 0.001,
+        beta1: float = 0.9,
+        beta2: float = 0.999,
+        eps: float = 1e-8,
+        weight_decay: float = 0.0,
+        volatility_window: int = 20,
+        regime_threshold: float = 0.5
+    ):
+        """
+        Initialize QuantOptimizer.
+        
+        Args:
+            params: Optional list of parameters to optimize
+            lr: Base learning rate
+            beta1: First moment decay rate
+            beta2: Second moment decay rate
+            eps: Small value for numerical stability
+            weight_decay: Weight decay coefficient
+            volatility_window: Window size for volatility calculation
+            regime_threshold: Threshold for regime detection
+        """
+        self.params = params if params is not None else []
+        self.lr = lr
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.eps = eps
+        self.weight_decay = weight_decay
+        self.volatility_window = volatility_window
+        self.regime_threshold = regime_threshold
+        
+        self.m: Dict[int, Any] = {}  # First moment
+        self.v: Dict[int, Any] = {}  # Second moment
+        self.step_count = 0
+        self.volatility_history: List[float] = []
+        self.feature_lrs: Dict[int, Any] = {}  # Per-parameter learning rate multipliers
+    
+    def step(self, param: Optional[Tensor] = None, market_volatility: Optional[float] = None):
+        """
+        Perform a single optimization step.
+        
+        Args:
+            param: Optional single parameter to update
+            market_volatility: Current market volatility (for adaptive LR)
+        """
+        if param is not None:
+            self._update_param(param, market_volatility)
+        else:
+            for p in self.params:
+                self._update_param(p, market_volatility)
+        self.step_count += 1
+        
+        # Update volatility history
+        if market_volatility is not None:
+            self.volatility_history.append(market_volatility)
+            if len(self.volatility_history) > self.volatility_window:
+                self.volatility_history.pop(0)
+    
+    def _update_param(self, param: Tensor, market_volatility: Optional[float] = None):
+        """Update a single parameter with adaptive learning rate."""
+        if not param.requires_grad:
+            return
+        
+        if param.grad is None:
+            return
+        
+        param_id = id(param)
+        
+        # Calculate adaptive learning rate based on volatility
+        adaptive_lr = self.lr
+        if market_volatility is not None and len(self.volatility_history) > 1:
+            # Adjust LR based on volatility regime
+            avg_vol = sum(self.volatility_history) / len(self.volatility_history)
+            if market_volatility > avg_vol * (1.0 + self.regime_threshold):
+                # High volatility regime - reduce LR
+                adaptive_lr = self.lr * 0.5
+            elif market_volatility < avg_vol * (1.0 - self.regime_threshold):
+                # Low volatility regime - increase LR
+                adaptive_lr = self.lr * 1.5
+        
+        # Apply per-parameter learning rate multiplier
+        if param_id in self.feature_lrs:
+            adaptive_lr = adaptive_lr * self.feature_lrs[param_id]
+        
+        if HAS_NUMPY:
+            try:
+                grad = param.grad
+                if isinstance(grad, np.ndarray):
+                    grad_arr = grad
+                else:
+                    grad_arr = np.array(grad, dtype=np.float64)
+                
+                param_arr = param.numpy if param.numpy is not None else np.array(param.data, dtype=np.float64)
+                
+                if param_id not in self.m:
+                    self.m[param_id] = np.zeros_like(param_arr, dtype=np.float64)
+                    self.v[param_id] = np.zeros_like(param_arr, dtype=np.float64)
+                
+                if self.weight_decay > 0:
+                    grad_arr = grad_arr + self.weight_decay * param_arr
+                
+                # Update moments (Adam-like)
+                m = self.m[param_id]
+                v = self.v[param_id]
+                m[:] = self.beta1 * m + (1.0 - self.beta1) * grad_arr
+                v[:] = self.beta2 * v + (1.0 - self.beta2) * (grad_arr ** 2)
+                
+                # Bias correction
+                bias_correction1 = 1.0 - (self.beta1 ** self.step_count)
+                bias_correction2 = 1.0 - (self.beta2 ** self.step_count)
+                m_hat = m / bias_correction1
+                v_hat = v / bias_correction2
+                
+                # Update parameter with adaptive LR
+                v_hat_sqrt = np.sqrt(v_hat) + self.eps
+                update = m_hat / v_hat_sqrt
+                param_update = adaptive_lr * update
+                new_param_arr = param_arr - param_update
+                param.data = new_param_arr
+                
+            except (ValueError, TypeError, AttributeError):
+                self._update_param_fallback(param, adaptive_lr)
+        else:
+            self._update_param_fallback(param, adaptive_lr)
+    
+    def _update_param_fallback(self, param: Tensor, adaptive_lr: float):
+        """Fallback update using Tensor operations."""
+        if param.grad is None:
+            return
+        
+        param_id = id(param)
+        
+        if param_id not in self.m:
+            if isinstance(param.data[0], list):
+                self.m[param_id] = Tensor([[0.0] * len(row) for row in param.data])
+                self.v[param_id] = Tensor([[0.0] * len(row) for row in param.data])
+            else:
+                self.m[param_id] = Tensor([0.0] * len(param.data))
+                self.v[param_id] = Tensor([0.0] * len(param.data))
+        
+        grad = Tensor(param.grad)
+        if self.weight_decay > 0:
+            grad = ops.add(grad, ops.mul(param, self.weight_decay))
+        
+        m_prev = self.m[param_id]
+        m_new = ops.add(ops.mul(m_prev, self.beta1), ops.mul(grad, 1.0 - self.beta1))
+        self.m[param_id] = m_new
+        
+        v_prev = self.v[param_id]
+        grad_sq = ops.mul(grad, grad)
+        v_new = ops.add(ops.mul(v_prev, self.beta2), ops.mul(grad_sq, 1.0 - self.beta2))
+        self.v[param_id] = v_new
+        
+        bias_correction1 = 1.0 - (self.beta1 ** self.step_count)
+        bias_correction2 = 1.0 - (self.beta2 ** self.step_count)
+        m_hat = ops.div(m_new, bias_correction1)
+        v_hat = ops.div(v_new, bias_correction2)
+        
+        v_hat_sqrt = ops.pow(ops.add(v_hat, self.eps), 0.5)
+        update = ops.div(m_hat, v_hat_sqrt)
+        param_update = ops.mul(update, adaptive_lr)
+        
+        if param.requires_grad:
+            param_detached = param.detach()
+            param_detached.sub_(param_update)
+            param.data = param_detached.data
+        else:
+            param.sub_(param_update)
+    
+    def set_feature_lr(self, param: Tensor, multiplier: float):
+        """Set learning rate multiplier for a specific parameter."""
+        param_id = id(param)
+        self.feature_lrs[param_id] = multiplier
+    
+    def zero_grad(self, param: Optional[Tensor] = None):
+        """Clear gradients."""
+        if param is not None:
+            param.zero_grad()
+        else:
+            for p in self.params:
+                p.zero_grad()
+    
+    def add_param_group(self, params: List[Tensor]):
+        """Add a parameter group to optimize."""
+        self.params.extend(params)
+

quantml/optim/radam.py

@@ -0,0 +1,192 @@
+"""
+RAdam (Rectified Adam) optimizer implementation.
+
+RAdam rectifies the variance of the adaptive learning rate in Adam.
+"""
+
+from typing import List, Optional, Dict, Any
+from quantml.tensor import Tensor
+from quantml import ops
+import math
+
+# Try to import NumPy
+try:
+    import numpy as np
+    HAS_NUMPY = True
+except ImportError:
+    HAS_NUMPY = False
+    np = None
+
+
+class RAdam:
+    """
+    RAdam (Rectified Adam) optimizer.
+    
+    Rectifies the variance of the adaptive learning rate in Adam,
+    providing better convergence for quant training.
+    
+    Attributes:
+        lr: Learning rate
+        betas: Tuple of (beta1, beta2) for moment estimates
+        eps: Small value for numerical stability
+        weight_decay: Weight decay coefficient
+        m: First moment estimates
+        v: Second moment estimates
+    
+    Examples:
+        >>> optimizer = RAdam(lr=0.001, betas=(0.9, 0.999))
+        >>> for param in model.parameters():
+        >>>     optimizer.step(param)
+    """
+    
+    def __init__(
+        self,
+        params: Optional[List[Tensor]] = None,
+        lr: float = 0.001,
+        betas: tuple = (0.9, 0.999),
+        eps: float = 1e-8,
+        weight_decay: float = 0.0
+    ):
+        """
+        Initialize RAdam optimizer.
+        
+        Args:
+            params: Optional list of parameters to optimize
+            lr: Learning rate
+            betas: Tuple of (beta1, beta2) for exponential decay rates
+            eps: Small value to prevent division by zero
+            weight_decay: Weight decay coefficient
+        """
+        self.params = params if params is not None else []
+        self.lr = lr
+        self.beta1, self.beta2 = betas
+        self.eps = eps
+        self.weight_decay = weight_decay
+        self.m: Dict[int, Any] = {}  # First moment
+        self.v: Dict[int, Any] = {}  # Second moment
+        self.step_count = 0
+    
+    def step(self, param: Optional[Tensor] = None):
+        """Perform a single optimization step."""
+        if param is not None:
+            self._update_param(param)
+        else:
+            for p in self.params:
+                self._update_param(p)
+        self.step_count += 1
+    
+    def _update_param(self, param: Tensor):
+        """Update a single parameter using RAdam algorithm."""
+        if not param.requires_grad:
+            return
+        
+        if param.grad is None:
+            return
+        
+        param_id = id(param)
+        
+        if HAS_NUMPY:
+            try:
+                grad = param.grad
+                if isinstance(grad, np.ndarray):
+                    grad_arr = grad
+                else:
+                    grad_arr = np.array(grad, dtype=np.float64)
+                
+                param_arr = param.numpy if param.numpy is not None else np.array(param.data, dtype=np.float64)
+                
+                if param_id not in self.m:
+                    self.m[param_id] = np.zeros_like(param_arr, dtype=np.float64)
+                    self.v[param_id] = np.zeros_like(param_arr, dtype=np.float64)
+                
+                if self.weight_decay > 0:
+                    grad_arr = grad_arr + self.weight_decay * param_arr
+                
+                # Update moments
+                m = self.m[param_id]
+                v = self.v[param_id]
+                m[:] = self.beta1 * m + (1.0 - self.beta1) * grad_arr
+                v[:] = self.beta2 * v + (1.0 - self.beta2) * (grad_arr ** 2)
+                
+                # RAdam variance rectification
+                beta2_t = self.beta2 ** self.step_count
+                rho_inf = 2.0 / (1.0 - self.beta2) - 1.0
+                rho_t = rho_inf - 2.0 * self.step_count * beta2_t / (1.0 - beta2_t)
+                
+                if rho_t > 4.0:
+                    # Rectified update
+                    r_t = math.sqrt((rho_t - 4.0) * (rho_t - 2.0) * rho_inf / ((rho_inf - 4.0) * (rho_inf - 2.0) * rho_t))
+                    m_hat = m / (1.0 - self.beta1 ** self.step_count)
+                    v_hat = v / (1.0 - beta2_t)
+                    update = r_t * m_hat / (np.sqrt(v_hat) + self.eps)
+                else:
+                    # Simple momentum update
+                    m_hat = m / (1.0 - self.beta1 ** self.step_count)
+                    update = m_hat
+                
+                param_update = self.lr * update
+                new_param_arr = param_arr - param_update
+                param.data = new_param_arr
+                
+            except (ValueError, TypeError, AttributeError):
+                self._update_param_fallback(param)
+        else:
+            self._update_param_fallback(param)
+    
+    def _update_param_fallback(self, param: Tensor):
+        """Fallback update using Tensor operations."""
+        if param.grad is None:
+            return
+        
+        param_id = id(param)
+        
+        if param_id not in self.m:
+            if isinstance(param.data[0], list):
+                self.m[param_id] = Tensor([[0.0] * len(row) for row in param.data])
+                self.v[param_id] = Tensor([[0.0] * len(row) for row in param.data])
+            else:
+                self.m[param_id] = Tensor([0.0] * len(param.data))
+                self.v[param_id] = Tensor([0.0] * len(param.data))
+        
+        grad = Tensor(param.grad)
+        if self.weight_decay > 0:
+            grad = ops.add(grad, ops.mul(param, self.weight_decay))
+        
+        m_prev = self.m[param_id]
+        m_new = ops.add(ops.mul(m_prev, self.beta1), ops.mul(grad, 1.0 - self.beta1))
+        self.m[param_id] = m_new
+        
+        v_prev = self.v[param_id]
+        grad_sq = ops.mul(grad, grad)
+        v_new = ops.add(ops.mul(v_prev, self.beta2), ops.mul(grad_sq, 1.0 - self.beta2))
+        self.v[param_id] = v_new
+        
+        # Simplified RAdam (use regular Adam update in fallback)
+        bias_correction1 = 1.0 - (self.beta1 ** self.step_count)
+        bias_correction2 = 1.0 - (self.beta2 ** self.step_count)
+        m_hat = ops.div(m_new, bias_correction1)
+        v_hat = ops.div(v_new, bias_correction2)
+        
+        v_hat_sqrt = ops.pow(ops.add(v_hat, self.eps), 0.5)
+        update = ops.div(m_hat, v_hat_sqrt)
+        param_update = ops.mul(update, self.lr)
+        
+        if param.requires_grad:
+            param_detached = param.detach()
+            param_detached.sub_(param_update)
+            param.data = param_detached.data
+        else:
+            param.sub_(param_update)
+    
+    def zero_grad(self, param: Optional[Tensor] = None):
+        """Clear gradients."""
+        if param is not None:
+            param.zero_grad()
+        else:
+            for p in self.params:
+                p.zero_grad()
+    
+    def add_param_group(self, params: List[Tensor]):
+        """Add a parameter group to optimize."""
+        self.params.extend(params)
+