dfm-python 0.6.0__tar.gz → 0.6.11__tar.gz

{dfm_python-0.6.0 → dfm_python-0.6.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.6.0
+Version: 0.6.11
 Summary: Dynamic Factor Model (DFM) estimation and forecasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme

{dfm_python-0.6.0 → dfm_python-0.6.11}/config/dfm_finance.yaml

@@ -7,7 +7,7 @@
 
 # EM Algorithm Parameters
 threshold: 1.0e-05  # EM convergence threshold
-max_iter: 50  # Maximum EM iterations
+max_iter: 12  # Maximum EM iterations
 
 # Data Handling
 clock: d  # Daily clock frequency (finance.csv uses daily index via date_id)

{dfm_python-0.6.0 → dfm_python-0.6.11}/config/dfm_macro.yaml

@@ -6,7 +6,7 @@
 
 # EM Algorithm Parameters
 threshold: 1.0e-05  # EM convergence threshold
-max_iter: 50  # Maximum EM iterations
+max_iter: 12  # Maximum EM iterations
 
 # Numerical Stability Parameters
 clip_ar_coefficients: true

dfm_python-0.6.11/failure_reports/finance_failure_20260127_183841.json

@@ -0,0 +1,22 @@
+{
+  "experiment": "finance",
+  "step": "Step 2: Creating Dataset",
+  "timestamp": "20260127_183841",
+  "error_type": "AttributeError",
+  "error_message": "'DFMDataset' object has no attribute 'obs_dim'",
+  "traceback": "Traceback (most recent call last):\n  File \"/data/nowcasting-kr/dfm-python/tutorial/tutorial_finance_dfm.py\", line 150, in main\n    print(f\"   Dataset created: obs_dim={dataset.obs_dim}, state_dim={dataset.state_dim if hasattr(dataset, 'state_dim') else 'N/A'}\")\n                                         ^^^^^^^^^^^^^^^\nAttributeError: 'DFMDataset' object has no attribute 'obs_dim'\n",
+  "context": {
+    "config_max_iter": 12,
+    "config_blocks": {
+      "Block_ALL": null,
+      "Block_EI": null,
+      "Block_MD": null,
+      "Block_PSV": null
+    },
+    "data_shape": [
+      1000,
+      99
+    ],
+    "filtered_series_count": 27
+  }
+}

dfm_python-0.6.11/failure_reports/macro_failure_20260127_183842.json

@@ -0,0 +1,15 @@
+{
+  "experiment": "macro",
+  "step": "Step 2: Creating Dataset",
+  "timestamp": "20260127_183842",
+  "error_type": "AttributeError",
+  "error_message": "'DFMDataset' object has no attribute 'obs_dim'",
+  "traceback": "Traceback (most recent call last):\n  File \"/data/nowcasting-kr/dfm-python/tutorial/tutorial_macro_dfm.py\", line 123, in main\n    print(f\"   Dataset created: obs_dim={dataset.obs_dim}, state_dim={dataset.state_dim if hasattr(dataset, 'state_dim') else 'N/A'}\")\n                                         ^^^^^^^^^^^^^^^\nAttributeError: 'DFMDataset' object has no attribute 'obs_dim'\n",
+  "context": {
+    "config_max_iter": 12,
+    "data_shape": [
+      2538,
+      15
+    ]
+  }
+}

{dfm_python-0.6.0 → dfm_python-0.6.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dfm-python"
-version = "0.6.0"
+version = "0.6.11"
 description = "Dynamic Factor Model (DFM) estimation and forecasting in Python"
 requires-python = ">=3.10"
 license = {text = "MIT"}

{dfm_python-0.6.0 → dfm_python-0.6.11}/src/dfm_python/models/dfm/dfm.py

@@ -539,13 +539,43 @@ class DFM(BaseFactorModel):
         
         # Fallback: compute smoothed factors (should rarely happen if EM completed successfully)
         _logger.debug("Computing smoothed factors (cache not available)")
-        kalman_final = self._create_kalman_filter()
-        y_masked = np.ma.masked_invalid(self.data_processed)
-        # Use filter_and_smooth() with compute_loglik=False to save time (log-likelihood already known)
-        # This prevents SVD convergence failures with ill-conditioned covariance matrices
-        smoothed_state_means, _, _, _ = kalman_final.filter_and_smooth(y_masked, compute_loglik=False)
-        
-        return smoothed_state_means
+        try:
+            kalman_final = self._create_kalman_filter()
+            y_masked = np.ma.masked_invalid(self.data_processed)
+            # Use filter_and_smooth() with compute_loglik=False to save time (log-likelihood already known)
+            # This prevents SVD convergence failures with ill-conditioned covariance matrices
+            smoothed_state_means, _, _, _ = kalman_final.filter_and_smooth(y_masked, compute_loglik=False)
+            return smoothed_state_means
+        except (ValueError, FloatingPointError, OverflowError) as e:
+            if "infs or NaNs" in str(e) or "overflow" in str(e).lower() or "not contain infs" in str(e):
+                # Parameters are too unstable to recompute factors
+                # This can happen if EM converged to numerically unstable parameters
+                # Raise a clear error with diagnostic information
+                param_status = {}
+                if self.training_state is not None:
+                    for name, param in [
+                        ('A', self.training_state.A),
+                        ('C', self.training_state.C),
+                        ('Q', self.training_state.Q),
+                        ('R', self.training_state.R),
+                        ('V_0', self.training_state.V_0),
+                    ]:
+                        if param is not None:
+                            n_inf = np.sum(np.isinf(param))
+                            n_nan = np.sum(np.isnan(param))
+                            if n_inf > 0 or n_nan > 0:
+                                param_status[name] = f"{n_inf} Inf, {n_nan} NaN"
+                
+                error_msg = (
+                    f"Cannot compute smoothed factors: final parameters contain non-finite values. "
+                    f"This indicates numerical instability in the fitted model. "
+                    f"Parameter status: {param_status if param_status else 'all finite'}. "
+                    f"Consider: (1) reducing model complexity (fewer factors/blocks), "
+                    f"(2) increasing regularization, (3) checking data scaling."
+                )
+                raise NumericalError(error_msg, details=f"Original error: {type(e).__name__}: {e}") from e
+            else:
+                raise
     
     def get_result(self) -> DFMResult:
         """Extract DFMResult from trained model.

{dfm_python-0.6.0 → dfm_python-0.6.11}/src/dfm_python/ssm/kalman.py

@@ -384,44 +384,146 @@ class DFMKalmanFilter:
                     f"observation_offsets shape mismatch: expected ({N},), got {observation_offsets_final.shape}"
                 )
             
+            # Validate parameters for NaN/Inf before calling _filter (prevents cryptic errors)
+            A = self._pykalman.transition_matrices
+            C = self._pykalman.observation_matrices
+            Q = self._pykalman.transition_covariance
+            R = self._pykalman.observation_covariance
+            Z_0 = self._pykalman.initial_state_mean
+            V_0 = self._pykalman.initial_state_covariance
+            
+            # Check for non-finite values in parameters
+            param_checks = {
+                'A (transition)': A,
+                'C (observation)': C,
+                'Q (process noise)': Q,
+                'R (observation noise)': R,
+                'Z_0 (initial state mean)': Z_0,
+                'V_0 (initial state cov)': V_0,
+            }
+            non_finite_params = []
+            for name, param in param_checks.items():
+                if param is not None and not np.isfinite(param).all():
+                    n_inf = np.sum(np.isinf(param)) if param is not None else 0
+                    n_nan = np.sum(np.isnan(param)) if param is not None else 0
+                    non_finite_params.append(f"{name}: {n_inf} Inf, {n_nan} NaN")
+            
+            if non_finite_params:
+                # Try to recover by stabilizing parameters
+                _logger.warning(
+                    f"    Filter: Non-finite values detected in parameters: {', '.join(non_finite_params)}. "
+                    f"Attempting to stabilize before filtering..."
+                )
+                # Stabilize all covariance matrices
+                if Q is not None and not np.isfinite(Q).all():
+                    Q = ensure_symmetric(Q)
+                    Q = np.where(np.isfinite(Q), Q, 0.0)
+                    Q = cap_max_eigenval(Q, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
+                if R is not None and not np.isfinite(R).all():
+                    R = ensure_symmetric(R)
+                    R = np.where(np.isfinite(R), R, np.diag(np.diag(R)))  # Preserve diagonal structure
+                    # Ensure R diagonal is positive
+                    R_diag = np.diag(R)
+                    R_diag = np.where(np.isfinite(R_diag) & (R_diag > 0), R_diag, 1.0)  # Fallback to 1.0
+                    np.fill_diagonal(R, R_diag)
+                    R = cap_max_eigenval(R, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
+                if V_0 is not None and not np.isfinite(V_0).all():
+                    V_0 = ensure_symmetric(V_0)
+                    V_0 = np.where(np.isfinite(V_0), V_0, 0.0)
+                    V_0 = cap_max_eigenval(V_0, max_eigenval=MAX_EIGENVALUE / 10, symmetric=True, warn=False)
+                if A is not None and not np.isfinite(A).all():
+                    A = np.where(np.isfinite(A), A, 0.0)
+                if C is not None and not np.isfinite(C).all():
+                    C = np.where(np.isfinite(C), C, 0.0)
+                if Z_0 is not None and not np.isfinite(Z_0).all():
+                    Z_0 = np.where(np.isfinite(Z_0), Z_0, 0.0)
+            
             # Option 4: Try filter with current parameters, retry with more aggressive stabilization if overflow
             try:
                 return _filter(
-                    self._pykalman.transition_matrices,
-                    self._pykalman.observation_matrices,
-                    self._pykalman.transition_covariance,
-                    self._pykalman.observation_covariance,
+                    A,
+                    C,
+                    Q,
+                    R,
                     transition_offsets_final,
                     observation_offsets_final,
-                    self._pykalman.initial_state_mean,
-                    self._pykalman.initial_state_covariance,
+                    Z_0,
+                    V_0,
                     observations
                 )
             except (ValueError, FloatingPointError, OverflowError) as e:
                 if "infs or NaNs" in str(e) or "overflow" in str(e).lower():
                     _logger.warning(
                         f"    Filter: Overflow detected in _filter ({type(e).__name__}: {e}). "
-                        f"Retrying with more aggressively stabilized V_0..."
+                        f"Retrying with more aggressively stabilized parameters..."
                     )
-                    # Retry with more aggressively stabilized V_0
-                    V_0_retry = self._pykalman.initial_state_covariance.copy()
-                    # Cap at smaller value for retry
+                    # Retry with more aggressively stabilized parameters
+                    V_0_retry = V_0.copy() if V_0 is not None else self._pykalman.initial_state_covariance.copy()
+                    V_0_retry = ensure_symmetric(V_0_retry)
+                    V_0_retry = np.where(np.isfinite(V_0_retry), V_0_retry, 0.0)
                     V_0_retry = cap_max_eigenval(V_0_retry, max_eigenval=MAX_EIGENVALUE / 10, symmetric=True, warn=False)
-                    # Also ensure Q is well-bounded
-                    Q_retry = self._pykalman.transition_covariance.copy()
+                    
+                    Q_retry = Q.copy() if Q is not None else self._pykalman.transition_covariance.copy()
+                    Q_retry = ensure_symmetric(Q_retry)
+                    Q_retry = np.where(np.isfinite(Q_retry), Q_retry, 0.0)
                     Q_retry = cap_max_eigenval(Q_retry, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
                     
-                    return _filter(
-                        self._pykalman.transition_matrices,
-                        self._pykalman.observation_matrices,
-                        Q_retry,
-                        self._pykalman.observation_covariance,
-                        transition_offsets_final,
-                        observation_offsets_final,
-                        self._pykalman.initial_state_mean,
-                        V_0_retry,
-                        observations
-                    )
+                    # Also stabilize R (critical: R appears in predicted_observation_covariance = C P C^T + R)
+                    R_retry = R.copy() if R is not None else self._pykalman.observation_covariance.copy()
+                    R_retry = ensure_symmetric(R_retry)
+                    R_retry = np.where(np.isfinite(R_retry), R_retry, np.diag(np.diag(R_retry)))
+                    # Ensure R diagonal is positive and finite
+                    R_diag = np.diag(R_retry)
+                    R_diag = np.where(np.isfinite(R_diag) & (R_diag > 0), R_diag, 1.0)
+                    np.fill_diagonal(R_retry, R_diag)
+                    R_retry = cap_max_eigenval(R_retry, max_eigenval=MAX_EIGENVALUE, symmetric=True, warn=False)
+                    
+                    # Also stabilize A if it has eigenvalues > 1 (causes unbounded growth in prediction step)
+                    # P_{t|t-1} = A P_{t-1|t-1} A^T + Q grows unbounded if A has eigenvalues > 1
+                    A_retry = A.copy() if A is not None else self._pykalman.transition_matrices.copy()
+                    if A_retry is not None:
+                        A_retry = np.where(np.isfinite(A_retry), A_retry, 0.0)
+                        # Check if A has eigenvalues > 1 (unstable)
+                        try:
+                            eigvals = np.linalg.eigvals(A_retry)
+                            max_eig = np.max(np.abs(eigvals))  # Use absolute value for complex eigenvalues
+                            if max_eig > 0.99:  # Close to or above 1.0
+                                _logger.warning(
+                                    f"    Filter: A has max |eigenvalue| {max_eig:.2e} > 0.99, capping to 0.99 to prevent unbounded growth"
+                                )
+                                A_retry = cap_max_eigenval(A_retry, max_eigenval=0.99, symmetric=False, warn=False)
+                        except (np.linalg.LinAlgError, ValueError):
+                            # If eigendecomposition fails, just ensure finite and scale down
+                            _logger.warning("    Filter: Could not compute A eigenvalues, scaling A by 0.9 as safety measure")
+                            A_retry = A_retry * 0.9
+                    else:
+                        A_retry = A
+                    
+                    try:
+                        return _filter(
+                            A_retry,
+                            C,
+                            Q_retry,
+                            R_retry,
+                            transition_offsets_final,
+                            observation_offsets_final,
+                            Z_0,
+                            V_0_retry,
+                            observations
+                        )
+                    except (ValueError, FloatingPointError, OverflowError) as e2:
+                        # Even retry failed - parameters are too unstable
+                        _logger.error(
+                            f"    Filter: Retry also failed with {type(e2).__name__}: {e2}. "
+                            f"Parameters are too unstable to filter. This indicates severe numerical issues."
+                        )
+                        raise NumericalError(
+                            "Kalman filter failed even after parameter stabilization. "
+                            "Model parameters contain extreme values that cannot be filtered. "
+                            "This typically indicates: (1) model is too complex for the data, "
+                            "(2) data scaling issues, (3) numerical instability in EM convergence.",
+                            details=f"Original error: {type(e).__name__}: {e}, Retry error: {type(e2).__name__}: {e2}"
+                        ) from e2
                 else:
                     raise
         

dfm_python-0.6.11/tutorial/tutorial_exchange_dfm.py

@@ -0,0 +1,205 @@
+"""Tutorial: DFM for Exchange Data
+
+This tutorial demonstrates the complete workflow for training and prediction
+using exchange rate data.
+"""
+
+import sys
+from pathlib import Path
+import traceback
+import json
+from datetime import datetime
+
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root / "src"))
+
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+from dfm_python import DFM, DFMDataset
+from dfm_python.config import DFMConfig
+import hydra
+from omegaconf import DictConfig, OmegaConf
+
+def save_failure_report(step: str, error: Exception, context: dict, project_root: Path):
+    """Save detailed failure report to file."""
+    reports_dir = project_root / "failure_reports"
+    reports_dir.mkdir(exist_ok=True)
+    
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    report_file = reports_dir / f"exchange_failure_{timestamp}.json"
+    
+    report = {
+        "experiment": "exchange",
+        "step": step,
+        "timestamp": timestamp,
+        "error_type": type(error).__name__,
+        "error_message": str(error),
+        "traceback": traceback.format_exc(),
+        "context": context
+    }
+    
+    with open(report_file, 'w') as f:
+        json.dump(report, f, indent=2, default=str)
+    
+    print(f"\n[FAILURE] Error report saved to: {report_file}")
+    return report_file
+
+def check_model_state(model, step: str):
+    """Check model state for numerical issues and return diagnostics."""
+    diagnostics = {}
+    
+    if model is None:
+        return diagnostics
+    
+    try:
+        if hasattr(model, 'training_state') and model.training_state is not None:
+            state = model.training_state
+            diagnostics['training_state'] = {}
+            
+            for param_name in ['A', 'C', 'Q', 'R', 'V_0']:
+                if hasattr(state, param_name):
+                    param = getattr(state, param_name)
+                    if param is not None:
+                        diagnostics['training_state'][param_name] = {
+                            'shape': param.shape if hasattr(param, 'shape') else None,
+                            'has_nan': bool(np.any(np.isnan(param))) if isinstance(param, np.ndarray) else None,
+                            'has_inf': bool(np.any(np.isinf(param))) if isinstance(param, np.ndarray) else None,
+                            'min': float(np.nanmin(param)) if isinstance(param, np.ndarray) else None,
+                            'max': float(np.nanmax(param)) if isinstance(param, np.ndarray) else None,
+                        }
+        
+        if hasattr(model, 'result') and model.result is not None:
+            result = model.result
+            diagnostics['result'] = {
+                'converged': result.converged if hasattr(result, 'converged') else None,
+                'num_iter': result.num_iter if hasattr(result, 'num_iter') else None,
+                'loglik': float(result.loglik) if hasattr(result, 'loglik') and result.loglik is not None else None,
+            }
+        
+        if hasattr(model, 'data_processed') and model.data_processed is not None:
+            data = model.data_processed
+            diagnostics['data_processed'] = {
+                'shape': data.shape if hasattr(data, 'shape') else None,
+                'has_nan': bool(np.any(np.isnan(data))) if isinstance(data, np.ndarray) else None,
+                'has_inf': bool(np.any(np.isinf(data))) if isinstance(data, np.ndarray) else None,
+            }
+    except Exception as e:
+        diagnostics['diagnostic_error'] = f"Failed to collect diagnostics: {str(e)}"
+    
+    return diagnostics
+
+@hydra.main(version_base=None, config_path="../config", config_name="dfm_exchange")
+def main(cfg: DictConfig) -> None:
+    print("=" * 80)
+    print("DFM Tutorial: Exchange Data")
+    print("=" * 80)
+    
+    start_time = datetime.now()
+    model = None
+    dataset = None
+    config = None
+    df = None
+    
+    try:
+        print("\n[Step 1] Loading data...")
+        try:
+            df = pd.read_csv(project_root / "data" / "exchange_rate.csv")
+            if "date" in df.columns:
+                df["date"] = pd.to_datetime(df["date"])
+            print(f"   Data shape: {df.shape}")
+        except Exception as e:
+            context = {
+                "data_shape": df.shape if df is not None else None,
+                "data_columns": list(df.columns) if df is not None else None,
+            }
+            save_failure_report("Step 1: Loading data", e, context, project_root)
+            raise
+
+        try:
+            print("\n[Step 2] Creating Dataset...")
+            config = DFMConfig.from_dict(OmegaConf.to_container(cfg, resolve=True))
+            dataset = DFMDataset(config=config, data=df, time_index="date")
+            obs_dim = len(dataset.variables.columns)
+            print(f"   Dataset created: obs_dim={obs_dim}")
+        except Exception as e:
+            context = {
+                "config_max_iter": cfg.get('max_iter', None),
+                "data_shape": df.shape if df is not None else None,
+            }
+            save_failure_report("Step 2: Creating Dataset", e, context, project_root)
+            raise
+
+        try:
+            print("\n[Step 3] Training DFM model...")
+            model = DFM(dataset=dataset, config=config, scaler=StandardScaler())
+            model.fit()
+
+            result = model.result
+            print(f"   Converged: {result.converged}, Iterations: {result.num_iter}")
+            
+            # Check for numerical issues after training
+            diagnostics = check_model_state(model, "after_training")
+            if diagnostics.get('training_state'):
+                for param_name, param_info in diagnostics['training_state'].items():
+                    if param_info.get('has_nan') or param_info.get('has_inf'):
+                        print(f"   WARNING: {param_name} contains NaN or Inf!")
+        except Exception as e:
+            context = {
+                "config_max_iter": cfg.get('max_iter', None),
+                "dataset_obs_dim": dataset.obs_dim if dataset is not None else None,
+                "elapsed_time": str(datetime.now() - start_time),
+            }
+            diagnostics = check_model_state(model, "during_training")
+            context.update(diagnostics)
+            save_failure_report("Step 3: Training DFM model", e, context, project_root)
+            raise
+
+        try:
+            print("\n[Step 4] Making predictions...")
+            X_forecast, Z_forecast = model.predict(horizon=6)
+            print(f"   Forecast shape: {X_forecast.shape}")
+            
+            # Check forecast for issues
+            if np.any(np.isnan(X_forecast)) or np.any(np.isinf(X_forecast)):
+                print(f"   WARNING: Forecast contains NaN or Inf!")
+        except Exception as e:
+            context = {
+                "result_converged": result.converged if 'result' in locals() else None,
+                "result_iterations": result.num_iter if 'result' in locals() else None,
+            }
+            diagnostics = check_model_state(model, "during_prediction")
+            context.update(diagnostics)
+            save_failure_report("Step 4: Making predictions", e, context, project_root)
+            raise
+
+        try:
+            print("\n[Step 5] Saving model...")
+            model_path = project_root / "models" / "dfm_exchange.pkl"
+            model_path.parent.mkdir(exist_ok=True)
+            model.save(model_path)
+            print(f"   Model saved to: {model_path}")
+        except Exception as e:
+            context = {
+                "model_path": str(model_path) if 'model_path' in locals() else None,
+                "result_converged": result.converged if 'result' in locals() else None,
+            }
+            diagnostics = check_model_state(model, "during_saving")
+            context.update(diagnostics)
+            save_failure_report("Step 5: Saving model", e, context, project_root)
+            raise
+
+        print("\n" + result.summary())
+        
+        elapsed = datetime.now() - start_time
+        print(f"\n[SUCCESS] Tutorial completed in {elapsed}")
+        
+    except Exception as e:
+        elapsed = datetime.now() - start_time
+        print(f"\n[FAILURE] Tutorial failed after {elapsed}")
+        print(f"Error: {type(e).__name__}: {str(e)}")
+        raise
+
+
+if __name__ == "__main__":
+    main()