dfm-python 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

dfm_python/models/dfm/dfm.py

@@ -539,13 +539,43 @@ class DFM(BaseFactorModel):
         
         # Fallback: compute smoothed factors (should rarely happen if EM completed successfully)
         _logger.debug("Computing smoothed factors (cache not available)")
-        kalman_final = self._create_kalman_filter()
-        y_masked = np.ma.masked_invalid(self.data_processed)
-        # Use filter_and_smooth() with compute_loglik=False to save time (log-likelihood already known)
-        # This prevents SVD convergence failures with ill-conditioned covariance matrices
-        smoothed_state_means, _, _, _ = kalman_final.filter_and_smooth(y_masked, compute_loglik=False)
-        
-        return smoothed_state_means
+        try:
+            kalman_final = self._create_kalman_filter()
+            y_masked = np.ma.masked_invalid(self.data_processed)
+            # Use filter_and_smooth() with compute_loglik=False to save time (log-likelihood already known)
+            # This prevents SVD convergence failures with ill-conditioned covariance matrices
+            smoothed_state_means, _, _, _ = kalman_final.filter_and_smooth(y_masked, compute_loglik=False)
+            return smoothed_state_means
+        except (ValueError, FloatingPointError, OverflowError) as e:
+            if "infs or NaNs" in str(e) or "overflow" in str(e).lower() or "not contain infs" in str(e):
+                # Parameters are too unstable to recompute factors
+                # This can happen if EM converged to numerically unstable parameters
+                # Raise a clear error with diagnostic information
+                param_status = {}
+                if self.training_state is not None:
+                    for name, param in [
+                        ('A', self.training_state.A),
+                        ('C', self.training_state.C),
+                        ('Q', self.training_state.Q),
+                        ('R', self.training_state.R),
+                        ('V_0', self.training_state.V_0),
+                    ]:
+                        if param is not None:
+                            n_inf = np.sum(np.isinf(param))
+                            n_nan = np.sum(np.isnan(param))
+                            if n_inf > 0 or n_nan > 0:
+                                param_status[name] = f"{n_inf} Inf, {n_nan} NaN"
+                
+                error_msg = (
+                    f"Cannot compute smoothed factors: final parameters contain non-finite values. "
+                    f"This indicates numerical instability in the fitted model. "
+                    f"Parameter status: {param_status if param_status else 'all finite'}. "
+                    f"Consider: (1) reducing model complexity (fewer factors/blocks), "
+                    f"(2) increasing regularization, (3) checking data scaling."
+                )
+                raise NumericalError(error_msg, details=f"Original error: {type(e).__name__}: {e}") from e
+            else:
+                raise
     
     def get_result(self) -> DFMResult:
         """Extract DFMResult from trained model.

dfm_python/ssm/kalman.py

@@ -384,44 +384,146 @@ class DFMKalmanFilter:
                     f"observation_offsets shape mismatch: expected ({N},), got {observation_offsets_final.shape}"
                 )
             
+            # Validate parameters for NaN/Inf before calling _filter (prevents cryptic errors)
+            A = self._pykalman.transition_matrices
+            C = self._pykalman.observation_matrices
+            Q = self._pykalman.transition_covariance
+            R = self._pykalman.observation_covariance
+            Z_0 = self._pykalman.initial_state_mean
+            V_0 = self._pykalman.initial_state_covariance
+            
+            # Check for non-finite values in parameters
+            param_checks = {
+                'A (transition)': A,
+                'C (observation)': C,
+                'Q (process noise)': Q,
+                'R (observation noise)': R,
+                'Z_0 (initial state mean)': Z_0,
+                'V_0 (initial state cov)': V_0,
+            }
+            non_finite_params = []
+            for name, param in param_checks.items():
+                if param is not None and not np.isfinite(param).all():
+                    n_inf = np.sum(np.isinf(param)) if param is not None else 0
+                    n_nan = np.sum(np.isnan(param)) if param is not None else 0
+                    non_finite_params.append(f"{name}: {n_inf} Inf, {n_nan} NaN")
+            
+            if non_finite_params:
+                # Try to recover by stabilizing parameters
+                _logger.warning(
+                    f"    Filter: Non-finite values detected in parameters: {', '.join(non_finite_params)}. "
+                    f"Attempting to stabilize before filtering..."
+                )
+                # Stabilize all covariance matrices
+                if Q is not None and not np.isfinite(Q).all():
+                    Q = ensure_symmetric(Q)
+                    Q = np.where(np.isfinite(Q), Q, 0.0)
+                    Q = cap_max_eigenval(Q, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
+                if R is not None and not np.isfinite(R).all():
+                    R = ensure_symmetric(R)
+                    R = np.where(np.isfinite(R), R, np.diag(np.diag(R)))  # Preserve diagonal structure
+                    # Ensure R diagonal is positive
+                    R_diag = np.diag(R)
+                    R_diag = np.where(np.isfinite(R_diag) & (R_diag > 0), R_diag, 1.0)  # Fallback to 1.0
+                    np.fill_diagonal(R, R_diag)
+                    R = cap_max_eigenval(R, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
+                if V_0 is not None and not np.isfinite(V_0).all():
+                    V_0 = ensure_symmetric(V_0)
+                    V_0 = np.where(np.isfinite(V_0), V_0, 0.0)
+                    V_0 = cap_max_eigenval(V_0, max_eigenval=MAX_EIGENVALUE / 10, symmetric=True, warn=False)
+                if A is not None and not np.isfinite(A).all():
+                    A = np.where(np.isfinite(A), A, 0.0)
+                if C is not None and not np.isfinite(C).all():
+                    C = np.where(np.isfinite(C), C, 0.0)
+                if Z_0 is not None and not np.isfinite(Z_0).all():
+                    Z_0 = np.where(np.isfinite(Z_0), Z_0, 0.0)
+            
             # Option 4: Try filter with current parameters, retry with more aggressive stabilization if overflow
             try:
                 return _filter(
-                    self._pykalman.transition_matrices,
-                    self._pykalman.observation_matrices,
-                    self._pykalman.transition_covariance,
-                    self._pykalman.observation_covariance,
+                    A,
+                    C,
+                    Q,
+                    R,
                     transition_offsets_final,
                     observation_offsets_final,
-                    self._pykalman.initial_state_mean,
-                    self._pykalman.initial_state_covariance,
+                    Z_0,
+                    V_0,
                     observations
                 )
             except (ValueError, FloatingPointError, OverflowError) as e:
                 if "infs or NaNs" in str(e) or "overflow" in str(e).lower():
                     _logger.warning(
                         f"    Filter: Overflow detected in _filter ({type(e).__name__}: {e}). "
-                        f"Retrying with more aggressively stabilized V_0..."
+                        f"Retrying with more aggressively stabilized parameters..."
                     )
-                    # Retry with more aggressively stabilized V_0
-                    V_0_retry = self._pykalman.initial_state_covariance.copy()
-                    # Cap at smaller value for retry
+                    # Retry with more aggressively stabilized parameters
+                    V_0_retry = V_0.copy() if V_0 is not None else self._pykalman.initial_state_covariance.copy()
+                    V_0_retry = ensure_symmetric(V_0_retry)
+                    V_0_retry = np.where(np.isfinite(V_0_retry), V_0_retry, 0.0)
                     V_0_retry = cap_max_eigenval(V_0_retry, max_eigenval=MAX_EIGENVALUE / 10, symmetric=True, warn=False)
-                    # Also ensure Q is well-bounded
-                    Q_retry = self._pykalman.transition_covariance.copy()
+                    
+                    Q_retry = Q.copy() if Q is not None else self._pykalman.transition_covariance.copy()
+                    Q_retry = ensure_symmetric(Q_retry)
+                    Q_retry = np.where(np.isfinite(Q_retry), Q_retry, 0.0)
                     Q_retry = cap_max_eigenval(Q_retry, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
                     
-                    return _filter(
-                        self._pykalman.transition_matrices,
-                        self._pykalman.observation_matrices,
-                        Q_retry,
-                        self._pykalman.observation_covariance,
-                        transition_offsets_final,
-                        observation_offsets_final,
-                        self._pykalman.initial_state_mean,
-                        V_0_retry,
-                        observations
-                    )
+                    # Also stabilize R (critical: R appears in predicted_observation_covariance = C P C^T + R)
+                    R_retry = R.copy() if R is not None else self._pykalman.observation_covariance.copy()
+                    R_retry = ensure_symmetric(R_retry)
+                    R_retry = np.where(np.isfinite(R_retry), R_retry, np.diag(np.diag(R_retry)))
+                    # Ensure R diagonal is positive and finite
+                    R_diag = np.diag(R_retry)
+                    R_diag = np.where(np.isfinite(R_diag) & (R_diag > 0), R_diag, 1.0)
+                    np.fill_diagonal(R_retry, R_diag)
+                    R_retry = cap_max_eigenval(R_retry, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
+                    
+                    # Also stabilize A if it has eigenvalues > 1 (causes unbounded growth in prediction step)
+                    # P_{t|t-1} = A P_{t-1|t-1} A^T + Q grows unbounded if A has eigenvalues > 1
+                    A_retry = A.copy() if A is not None else self._pykalman.transition_matrices.copy()
+                    if A_retry is not None:
+                        A_retry = np.where(np.isfinite(A_retry), A_retry, 0.0)
+                        # Check if A has eigenvalues > 1 (unstable)
+                        try:
+                            eigvals = np.linalg.eigvals(A_retry)
+                            max_eig = np.max(np.abs(eigvals))  # Use absolute value for complex eigenvalues
+                            if max_eig > 0.99:  # Close to or above 1.0
+                                _logger.warning(
+                                    f"    Filter: A has max |eigenvalue| {max_eig:.2e} > 0.99, capping to 0.99 to prevent unbounded growth"
+                                )
+                                A_retry = cap_max_eigenval(A_retry, max_eigenval=0.99, symmetric=False, warn=False)
+                        except (np.linalg.LinAlgError, ValueError):
+                            # If eigendecomposition fails, just ensure finite and scale down
+                            _logger.warning("    Filter: Could not compute A eigenvalues, scaling A by 0.9 as safety measure")
+                            A_retry = A_retry * 0.9
+                    else:
+                        A_retry = A
+                    
+                    try:
+                        return _filter(
+                            A_retry,
+                            C,
+                            Q_retry,
+                            R_retry,
+                            transition_offsets_final,
+                            observation_offsets_final,
+                            Z_0,
+                            V_0_retry,
+                            observations
+                        )
+                    except (ValueError, FloatingPointError, OverflowError) as e2:
+                        # Even retry failed - parameters are too unstable
+                        _logger.error(
+                            f"    Filter: Retry also failed with {type(e2).__name__}: {e2}. "
+                            f"Parameters are too unstable to filter. This indicates severe numerical issues."
+                        )
+                        raise NumericalError(
+                            "Kalman filter failed even after parameter stabilization. "
+                            "Model parameters contain extreme values that cannot be filtered. "
+                            "This typically indicates: (1) model is too complex for the data, "
+                            "(2) data scaling issues, (3) numerical instability in EM convergence.",
+                            details=f"Original error: {type(e).__name__}: {e}, Retry error: {type(e2).__name__}: {e2}"
+                        ) from e2
                 else:
                     raise
         

{dfm_python-0.6.0.dist-info → dfm_python-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.6.0
+Version: 0.6.1
 Summary: Dynamic Factor Model (DFM) estimation and forecasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme

{dfm_python-0.6.0.dist-info → dfm_python-0.6.1.dist-info}/RECORD

@@ -31,7 +31,7 @@ dfm_python/models/__init__.py,sha256=exbooS9GFktHEMEc3mXhgZDosVDwEpeBOrC5v6wBF4I
 dfm_python/models/base.py,sha256=tJR9XD-mvokT9e2YKD7la2eo2clA1n0QwosXWCcZHY4,2504
 dfm_python/models/ddfm/ddfm.py,sha256=Ff0oCZZw-lcbofwCFD7FWaA1I-lKvJdCXnmlTysfPgw,65355
 dfm_python/models/dfm/__init__.py,sha256=6Lwt2R-xPRgWDrFL7txyA-HMp_CJGoFr7qHe91n65xE,1074
-dfm_python/models/dfm/dfm.py,sha256=ELwkuyZIaXEMkHGmRMDCODOpaPkxvGwz1XzM2dIAHuI,46087
+dfm_python/models/dfm/dfm.py,sha256=UhaJ5v-2TBfRzXZ0NSF0vysiFe473_WgcwKT6sRSSn8,47889
 dfm_python/models/dfm/initialization.py,sha256=bWQKfrA4B3_KMoW4dXAzRNZDSvyEH4iQOdbR5znnmH0,55247
 dfm_python/models/dfm/mixed_freq.py,sha256=S5nl7r0G8MMdmFHzJpfc0imHsjUeaOA4gHu_gltSq80,16148
 dfm_python/models/dfm/tent.py,sha256=lNB1TKwb9a5E66TsnFDtTsN3FpU4ar-OeuFC8h1Ar04,15436
@@ -41,7 +41,7 @@ dfm_python/numeric/stability.py,sha256=9qk9RcXm0mppMdwYH7FPhBHXAp3xfbGkToA_n2crX
 dfm_python/numeric/statistic.py,sha256=Y9rrv18OKBiholy-R5Bz9tzGCkaziMn0Jk4Ca_KS08Y,14429
 dfm_python/numeric/validator.py,sha256=9DInNp-DplvM4NoniE3tAS1JidMTGO4k4UMD272-h7s,25891
 dfm_python/ssm/__init__.py,sha256=AmmiJPlAy1bctM9AVso7eWKdPMrUbCfAZQHgqZv-KLY,784
-dfm_python/ssm/kalman.py,sha256=ZHw1PdUje9NH9Pv2Ol1-QywfM6goWI43Z8rg_FysTwk,41863
+dfm_python/ssm/kalman.py,sha256=wQYj_SMRcer8BPNUPKCQMXTJ6JZ1bAvmLV2zZN7GSlw,48235
 dfm_python/utils/__init__.py,sha256=JbtEviPIgUJQANv-FXCUQBvyVMULduZK8aY-yOFIQJU,4740
 dfm_python/utils/checkpoint.py,sha256=npdYsNg74KE7wUtU25Wuqs846oXfwYxUJcEzKnh-k6o,11201
 dfm_python/utils/errors.py,sha256=BKMjYldXjdFg-O_pIBo1-WITInNXvNrHk-6IyFHMf-k,1861
@@ -50,6 +50,6 @@ dfm_python/utils/loss.py,sha256=1KcYwFJmY9l-Uh6Sk-dFpJ2kgMfm0u41Gmgu-5NzaAs,3878
 dfm_python/utils/metric.py,sha256=AC_TnJ3eIXiF_maNNexSHzXpLv5Hrf1W88X5pJx9C-Q,15327
 dfm_python/utils/misc.py,sha256=aPUNTRLhKUUiqEWT1tOiV5AJUck_qzbENA0OK3uCdII,11159
 dfm_python/utils/validation.py,sha256=NFpTS4mq36IDDLmbj4crJGoWLLz_BdlS0tKo96BEoNw,1456
-dfm_python-0.6.0.dist-info/METADATA,sha256=7W-bwVDcO2xYvFM_UKHW4g9BANbFIeElsJKo8xeqGM0,2269
-dfm_python-0.6.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dfm_python-0.6.0.dist-info/RECORD,,
+dfm_python-0.6.1.dist-info/METADATA,sha256=aG5pgOcarTHAPzXgkw3qCyuNenEnTBgu46tirm_4iuM,2269
+dfm_python-0.6.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dfm_python-0.6.1.dist-info/RECORD,,