dfm-python 0.4.7__tar.gz → 0.4.8__tar.gz

{dfm_python-0.4.7 → dfm_python-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.4.7
+Version: 0.4.8
 Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme

{dfm_python-0.4.7 → dfm_python-0.4.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dfm-python"
-version = "0.4.7"
+version = "0.4.8"
 description = "Dynamic Factor Model (DFM) estimation and nowcasting in Python"
 readme = "README.md"
 requires-python = ">=3.10"

{dfm_python-0.4.7 → dfm_python-0.4.8}/src/dfm_python/__init__.py

@@ -65,7 +65,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.4.7"
+__version__ = "0.4.8"
 
 # ============================================================================
 # PUBLIC API DEFINITION

{dfm_python-0.4.7 → dfm_python-0.4.8}/src/dfm_python/ssm/em.py

@@ -48,7 +48,7 @@ from typing import Tuple, Optional, Dict, Any
 from dataclasses import dataclass
 from ..logger import get_logger
 from .kalman import KalmanFilter
-from .utils import ensure_positive_definite, ensure_symmetric
+from .utils import ensure_positive_definite, ensure_symmetric, cap_max_eigenval
 
 _logger = get_logger(__name__)
 
@@ -108,6 +108,10 @@ class EMAlgorithm(nn.Module):
             self.kalman = kalman
         self.register_buffer('regularization_scale', torch.tensor(regularization_scale))
     
+    def _cap_max_eigenval(self, M: torch.Tensor, max_eigenval: float = 1e6) -> torch.Tensor:
+        """Cap maximum eigenvalue of matrix to prevent numerical explosion."""
+        return cap_max_eigenval(M, max_eigenval=max_eigenval, warn=False)
+    
     def forward(
         self,
         params: EMStepParams
@@ -241,8 +245,27 @@ class EMAlgorithm(nn.Module):
             # Transpose y to (T, N) for batch operations
             # Use y_for_mstep (NaN replaced with 0) for M-step
             sum_yEZ = torch.sum(y_for_mstep.T[:, :, None] * EZ[:, None, :], dim=0)  # (N, m)
-            # Compute sum_EZZ = sum_t E[Z_t Z_t^T]
-            sum_EZZ = torch.sum(EZZ, dim=0)
+            
+            # CRITICAL FIX: Remove NaN/Inf from EZZ before computing sum_EZZ
+            # This prevents ill-conditioned sum_EZZ caused by corrupted EZZ from smoother
+            # MATLAB doesn't check for NaN/Inf, but we need to handle it explicitly
+            EZZ_clean = EZZ.clone()
+            nan_inf_mask = torch.isnan(EZZ_clean) | torch.isinf(EZZ_clean)
+            if torch.any(nan_inf_mask):
+                # Replace NaN/Inf with zero (exclude corrupted time steps from sum)
+                EZZ_clean[nan_inf_mask] = 0.0
+                corrupted_count = torch.sum(nan_inf_mask).item()
+                total_count = EZZ_clean.numel()
+                _logger.warning(
+                    f"EM algorithm: EZZ contains {corrupted_count}/{total_count} NaN/Inf values "
+                    f"({corrupted_count/total_count:.1%}). Excluding corrupted time steps from sum_EZZ calculation."
+                )
+            
+            # Compute sum_EZZ = sum_t E[Z_t Z_t^T] (only from valid time steps)
+            sum_EZZ = torch.sum(EZZ_clean, dim=0)
+            
+            # Cap maximum eigenvalue to prevent condition number explosion
+            sum_EZZ = self._cap_max_eigenval(sum_EZZ, max_eigenval=1e6)
             
             # ROOT CAUSE FIX: Adaptive regularization based on condition number
             # Fixed regularization (1e-6) becomes insufficient as condition number grows
@@ -268,22 +291,38 @@ class EMAlgorithm(nn.Module):
                         reg_scale = base_reg_scale
                 else:
                     # All eigenvalues are near-zero, use stronger regularization
-                    reg_scale = base_reg_scale * 100
+                    # Ensure minimum regularization (1e-3) for numerical stability
+                    reg_scale = max(base_reg_scale * 100, 1e-3)
                     _logger.warning(
                         f"EM algorithm: sum_EZZ has no valid eigenvalues, "
-                        f"using strong regularization: {reg_scale:.2e}"
+                        f"using strong regularization: {reg_scale:.2e} (min=1e-3, base={base_reg_scale:.2e})"
                     )
             except (RuntimeError, ValueError) as e:
-                # Eigendecomposition failed, use stronger regularization
-                reg_scale = base_reg_scale * 10
+                # CRITICAL FIX: Eigendecomposition failed, use stronger regularization
+                # Ensure minimum regularization (1e-3) even if base_reg_scale * 10 is smaller
+                # This is the Ridge Regression approach: eigendecomposition failure means
+                # the matrix is too ill-conditioned, so we need strong regularization
+                reg_scale = max(base_reg_scale * 10, 1e-3)
                 _logger.warning(
                     f"EM algorithm: Failed to compute condition number for sum_EZZ ({e}), "
-                    f"using increased regularization: {reg_scale:.2e}"
+                    f"using strong regularization: {reg_scale:.2e} (min=1e-3, base={base_reg_scale:.2e})"
                 )
             
             # Apply adaptive regularization
             sum_EZZ_reg = sum_EZZ + torch.eye(m, device=device, dtype=dtype) * reg_scale
-            C_new = torch.linalg.solve(sum_EZZ_reg.T, sum_yEZ.T).T
+            
+            # CRITICAL FIX: Use pseudo-inverse as fallback when solve fails
+            # Original MATLAB uses pinv() in smoother (line 1039, 1064)
+            # This handles singular/ill-conditioned matrices more robustly
+            try:
+                C_new = torch.linalg.solve(sum_EZZ_reg.T, sum_yEZ.T).T
+            except RuntimeError as e:
+                # Fallback to pseudo-inverse (original MATLAB: pinv() in smoother)
+                _logger.warning(
+                    f"EM algorithm: torch.linalg.solve failed for C matrix update ({e}), "
+                    f"using pseudo-inverse fallback (original MATLAB: pinv() in smoother)"
+                )
+                C_new = (torch.linalg.pinv(sum_EZZ_reg.T) @ sum_yEZ.T).T
             
             # Check for NaN in C_new after solve (indicates numerical instability)
             if torch.any(torch.isnan(C_new)):
@@ -294,11 +333,25 @@ class EMAlgorithm(nn.Module):
                     f"after solve operation. This indicates numerical instability. "
                     f"Possible causes: singular matrix, extreme values, or insufficient regularization."
                 )
-                # Set NaN columns to zero instead of leaving them as NaN
-                nan_cols = torch.any(torch.isnan(C_new), dim=0)
-                if torch.any(nan_cols):
-                    C_new[:, nan_cols] = 0.0
-                    _logger.warning(f"Set {torch.sum(nan_cols).item()} NaN columns in C matrix to zero.")
+                # CRITICAL FIX: Preserve previous iteration values instead of setting to zero
+                # This allows gradual improvement and preserves information from previous iterations
+                nan_mask = torch.isnan(C_new)
+                if torch.any(nan_mask):
+                    # Check if previous C matrix is valid (no NaN)
+                    if not torch.any(torch.isnan(params.C)):
+                        C_new[nan_mask] = params.C[nan_mask]
+                        # Only log if significant NaN ratio to reduce log spam
+                        if nan_ratio > 0.1:  # Only log if >10% NaN
+                            _logger.warning(
+                                f"Preserved {torch.sum(nan_mask).item()} NaN values in C matrix from previous iteration "
+                                f"(total NaN: {nan_count}/{C_new.numel()}, {nan_ratio:.1%})"
+                            )
+                    else:
+                        # If previous C also has NaN, set to zero as last resort
+                        C_new[nan_mask] = 0.0
+                        _logger.warning(
+                            f"Previous C matrix also contains NaN. Set {torch.sum(nan_mask).item()} NaN values to zero."
+                        )
             
             # Normalize C columns (factor loadings)
             for j in range(m):

{dfm_python-0.4.7 → dfm_python-0.4.8}/src/dfm_python/ssm/kalman.py

@@ -36,6 +36,7 @@ from .utils import (
     clean_matrix,
     safe_inverse,
     safe_determinant,
+    cap_max_eigenval,
 )
 
 _logger = get_logger(__name__)
@@ -343,6 +344,8 @@ class KalmanFilter(nn.Module):
                 # Fallback: use previous covariance with regularization
                 V = Vu + torch.eye(V.shape[0], device=device, dtype=dtype) * 1e-6
             
+            # Cap maximum eigenvalue to prevent explosion
+            V = self._cap_max_eigenval(V, max_eigenval=1e6)
             # Ensure V is real, symmetric, and positive semi-definite (MATLAB: V = 0.5 * (V+V'))
             V = ensure_real_and_symmetric(V)
             V = self._ensure_cov_stable(V, min_eigenval=self.min_eigenval.item(), ensure_real=True)
@@ -356,13 +359,24 @@ class KalmanFilter(nn.Module):
                 Zu = Z
                 Vu = V
             else:
-                # CRITICAL FIX: Adaptive regularization based on missing data ratio
+                # CRITICAL FIX: Adaptive regularization based on missing data ratio and n_obs
                 # When data is sparse (many masked values), increase regularization to prevent numerical instability
                 missing_ratio = 1.0 - (len(Y_t) / k)  # Ratio of missing observations
                 # Scale regularization: base regularization * (1 + missing_ratio * 10)
                 # This means: 0% missing -> base reg, 50% missing -> 6x reg, 100% missing -> 11x reg
                 adaptive_reg = self.inv_regularization.item() * (1.0 + missing_ratio * 10.0)
                 
+                # CRITICAL FIX: Additional regularization based on n_obs (number of observed series)
+                # Log analysis shows F matrix inversion becomes unstable when n_obs >= 30
+                # Scale regularization: n_obs >= 30 -> additional factor (more generic threshold)
+                n_obs = len(Y_t)
+                if n_obs >= 30:
+                    n_obs_factor = 1.0 + (n_obs - 29) * 0.1  # 30 -> 1.1x, 33 -> 1.4x, 39 -> 2.0x
+                    adaptive_reg = adaptive_reg * n_obs_factor
+                    # Only log once per iteration (first time step) to reduce log spam
+                    if t == 0:
+                        _logger.debug(f"kalman_filter_forward: n_obs={n_obs} >= 30, applying n_obs_factor={n_obs_factor:.2f}, adaptive_reg={adaptive_reg:.2e}")
+                
                 # Steps for variance and population regression coefficients:
                 # Var(c_t*Z_t + e_t) = c_t Var(Z) c_t' + Var(e) = c_t*V*c_t' + R
                 VC = V @ C_t.T
@@ -378,6 +392,26 @@ class KalmanFilter(nn.Module):
                     diag_R_t = torch.clamp(diag_R_t, min=self.min_diagonal_variance.item() * (1.0 + missing_ratio * 5.0))
                     R_t = torch.diag(diag_R_t)
                 
+                # ROOT CAUSE ANALYSIS: Compute and log F matrix condition number before capping
+                # This helps identify when F becomes ill-conditioned
+                try:
+                    eigenvals_F = torch.linalg.eigvalsh(F)
+                    eigenvals_F = eigenvals_F[eigenvals_F > 1e-12]
+                    if len(eigenvals_F) > 0:
+                        max_eig_F = torch.max(eigenvals_F).item()
+                        min_eig_F = torch.min(eigenvals_F).item()
+                        cond_F = max_eig_F / min_eig_F if min_eig_F > 1e-12 else float('inf')
+                        # Only log first 5 steps or when severely ill-conditioned to reduce log spam
+                        if cond_F > 1e10 or t < 5:
+                            _logger.debug(
+                                f"kalman_filter_forward: t={t}, n_obs={n_obs}, F condition={cond_F:.2e}, "
+                                f"eigenvals=[{min_eig_F:.2e}, {max_eig_F:.2e}], adaptive_reg={adaptive_reg:.2e}"
+                            )
+                except (RuntimeError, ValueError):
+                    pass  # Skip if eigendecomposition fails
+                
+                # Cap maximum eigenvalue to prevent condition number explosion
+                F = self._cap_max_eigenval(F, max_eigenval=1e6)
                 # Ensure F is real, symmetric, and positive semi-definite
                 # Use adaptive minimum eigenvalue based on missing data ratio
                 min_eig_adaptive = self.min_eigenval.item() * (1.0 + missing_ratio * 5.0)
@@ -394,6 +428,15 @@ class KalmanFilter(nn.Module):
                 # This handles GPU numerical stability issues (singular matrix errors)
                 iF = self._safe_inv(F, regularization=adaptive_reg, use_pinv_fallback=True)
                 
+                # ROOT CAUSE ANALYSIS: Check if inversion was successful (only log warnings, not every occurrence)
+                if not self._check_finite(iF, f"iF (inverse of F) at t={t}"):
+                    # Only log first occurrence to reduce log spam
+                    if t < 5:
+                        _logger.warning(
+                            f"kalman_filter_forward: iF contains NaN/Inf at t={t}, n_obs={n_obs}, "
+                            f"adaptive_reg={adaptive_reg:.2e}, missing_ratio={missing_ratio:.2f}"
+                        )
+                
                 # Matrix of population regression coefficients (Kalman gain)
                 VCF = VC @ iF
                 
@@ -431,6 +474,8 @@ class KalmanFilter(nn.Module):
                     
                     # MATLAB-style: Ensure symmetry immediately after update (V = 0.5 * (V+V'))
                     Vu = ensure_real_and_symmetric(Vu)
+                    # Cap maximum eigenvalue to prevent explosion
+                    Vu = self._cap_max_eigenval(Vu, max_eigenval=1e6)
                     
                     # Clean NaN/Inf before stabilization
                     if not self._check_finite(Vu, f"Vu at t={t}"):
@@ -830,6 +875,10 @@ class KalmanFilter(nn.Module):
             min_eigenval = self.min_eigenval.item()
         return ensure_covariance_stable(M, min_eigenval=min_eigenval, ensure_real=ensure_real)
     
+    def _cap_max_eigenval(self, M: torch.Tensor, max_eigenval: float = 1e6) -> torch.Tensor:
+        """Cap maximum eigenvalue of matrix to prevent numerical explosion."""
+        return cap_max_eigenval(M, max_eigenval=max_eigenval, warn=False)
+    
     def _clean_matrix(
         self,
         M: torch.Tensor, 

{dfm_python-0.4.7 → dfm_python-0.4.8}/src/dfm_python/ssm/utils.py

@@ -97,6 +97,51 @@ def ensure_symmetric(tensor: "torch.Tensor") -> "torch.Tensor":
     return 0.5 * (tensor + tensor.T)
 
 
+def cap_max_eigenval(
+    M: "torch.Tensor", 
+    max_eigenval: float = 1e6,
+    warn: bool = True
+) -> "torch.Tensor":
+    """Cap maximum eigenvalue of matrix to prevent numerical explosion.
+    
+    Parameters
+    ----------
+    M : torch.Tensor
+        Matrix to cap (square matrix)
+    max_eigenval : float, default 1e6
+        Maximum allowed eigenvalue
+    warn : bool, default True
+        Whether to log warnings
+        
+    Returns
+    -------
+    torch.Tensor
+        Matrix with capped eigenvalues
+    """
+    if M.numel() == 0 or M.shape[0] == 0:
+        return M
+    
+    try:
+        eigenvals = torch.linalg.eigvalsh(M)
+        max_eig = torch.max(eigenvals)
+        
+        if max_eig > max_eigenval:
+            # Scale matrix to cap maximum eigenvalue
+            scale_factor = max_eigenval / max_eig
+            M = M * scale_factor
+            M = ensure_symmetric(M)
+            if warn:
+                _logger.warning(
+                    f"Matrix maximum eigenvalue capped: {max_eig:.2e} -> {max_eigenval:.2e} "
+                    f"(scale_factor={scale_factor:.2e})"
+                )
+    except (RuntimeError, ValueError):
+        # If eigendecomposition fails, return matrix as-is
+        pass
+    
+    return M
+
+
 def ensure_real_and_symmetric(tensor: "torch.Tensor") -> "torch.Tensor":
     """Ensure matrix is real and symmetric.