dfm-python 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

dfm_python/__init__.py

@@ -50,7 +50,7 @@ Example (Low-level API - For advanced usage):
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.2.6"
+__version__ = "0.2.8"
 
 from .config import DFMConfig, SeriesConfig, BlockConfig, Params, DEFAULT_GLOBAL_BLOCK_NAME
 from .config import (

dfm_python/core/diagnostics.py

@@ -4,12 +4,10 @@ from typing import Optional, Dict, Any, Tuple, TYPE_CHECKING
 import numpy as np
 import logging
 
-try:
-    import pandas as pd
-    PANDAS_AVAILABLE = True
-except ImportError:
-    PANDAS_AVAILABLE = False
-    pd = None
+import pandas as pd
+from scipy.linalg import orthogonal_procrustes
+
+PANDAS_AVAILABLE = True
 
 from ..config import DFMConfig
 
@@ -96,6 +94,217 @@ def calculate_rmse(actual: np.ndarray, predicted: np.ndarray,
     return rmse_overall, rmse_per_series
 
 
+def evaluate_factor_estimation(
+    true_factors: np.ndarray,
+    estimated_factors: np.ndarray,
+    use_procrustes: bool = True
+) -> Dict[str, Any]:
+    """Evaluate how well factors are estimated (synthetic-data diagnostic).
+    
+    This function compares known ``true_factors`` with ``estimated_factors``
+    (typically ``result.Z[:, :num_factors]``) and returns:
+    
+    - Per-factor correlations (before any rotation)
+    - Optionally Procrustes-rotated factors and an overall correlation
+    
+    Parameters
+    ----------
+    true_factors : np.ndarray
+        Array of shape (T, k) with generated factors used to build the data.
+    estimated_factors : np.ndarray
+        Array of shape (T, k_est) with estimated factors. Only the first
+        ``k = min(k, k_est)`` columns are compared.
+    use_procrustes : bool, default True
+        If True and SciPy is available, apply orthogonal Procrustes rotation
+        to align estimated factors with true factors before computing the
+        overall correlation.
+    
+    Returns
+    -------
+    Dict[str, Any]
+        {
+          'num_factors': int,
+          'correlation_per_factor': np.ndarray (k,),
+          'overall_correlation': float or None,
+          'rotation_matrix': np.ndarray (k, k) or None,
+          'aligned_factors': np.ndarray (T, k) or None,
+        }
+    
+    Notes
+    -----
+    - This is primarily intended for synthetic-data experiments where the
+      true latent factors are known.
+    - Correlations are computed factor-by-factor, ignoring any time points
+      with NaN values.
+    - Procrustes rotation is useful because factors are only identified up
+      to sign, scale (if not constrained), and rotation in general.
+    """
+    if true_factors.ndim != 2 or estimated_factors.ndim != 2:
+        raise ValueError(
+            f"true_factors and estimated_factors must be 2D, "
+            f"got shapes {true_factors.shape} and {estimated_factors.shape}"
+        )
+    
+    T_true, k_true = true_factors.shape
+    T_est, k_est = estimated_factors.shape
+    if T_true != T_est:
+        raise ValueError(
+            f"Time dimension must match, got {T_true} and {T_est}"
+        )
+    
+    k = min(k_true, k_est)
+    if k == 0:
+        raise ValueError("At least one factor is required for comparison.")
+    
+    F_true = np.asarray(true_factors[:, :k], dtype=float)
+    F_est = np.asarray(estimated_factors[:, :k], dtype=float)
+    
+    # Per-factor correlations (no rotation)
+    corr_per_factor = np.full(k, np.nan, dtype=float)
+    for j in range(k):
+        mask_j = np.isfinite(F_true[:, j]) & np.isfinite(F_est[:, j])
+        if np.sum(mask_j) > 1:
+            corr = np.corrcoef(F_true[mask_j, j], F_est[mask_j, j])[0, 1]
+            corr_per_factor[j] = float(corr)
+    
+    rotation_matrix = None
+    aligned_factors = None
+    overall_corr = None
+    
+    if use_procrustes:
+        # Center factors before Procrustes to focus on shape, not level.
+        F_true_centered = F_true - np.nanmean(F_true, axis=0, keepdims=True)
+        F_est_centered = F_est - np.nanmean(F_est, axis=0, keepdims=True)
+        
+        # Replace any remaining NaNs with 0 before Procrustes (rare in synthetic tests)
+        F_true_clean = np.where(np.isfinite(F_true_centered), F_true_centered, 0.0)
+        F_est_clean = np.where(np.isfinite(F_est_centered), F_est_centered, 0.0)
+        
+        try:
+            # Find R such that F_est_clean @ R ≈ F_true_clean
+            R, _ = orthogonal_procrustes(F_est_clean, F_true_clean)  # type: ignore[arg-type]
+            rotation_matrix = R
+            aligned_factors = F_est @ R
+            
+            mask_all = np.isfinite(F_true) & np.isfinite(aligned_factors)
+            if np.any(mask_all):
+                overall_corr = float(
+                    np.corrcoef(
+                        F_true[mask_all].ravel(),
+                        aligned_factors[mask_all].ravel()
+                    )[0, 1]
+                )
+        except Exception as e:
+            _logger.debug(f"Procrustes rotation failed in evaluate_factor_estimation: {e}")
+    
+    return {
+        "num_factors": k,
+        "correlation_per_factor": corr_per_factor,
+        "overall_correlation": overall_corr,
+        "rotation_matrix": rotation_matrix,
+        "aligned_factors": aligned_factors,
+    }
+
+
+def evaluate_loading_estimation(
+    true_loadings: np.ndarray,
+    estimated_loadings: np.ndarray,
+    rotation_matrix: Optional[np.ndarray] = None
+) -> Dict[str, Any]:
+    """Evaluate loading matrix estimation accuracy (synthetic-data diagnostic).
+    
+    Parameters
+    ----------
+    true_loadings : np.ndarray
+        Array of shape (N, k_true) with the loadings used to generate data.
+    estimated_loadings : np.ndarray
+        Array of shape (N, k_est) with estimated loadings from the model
+        (typically ``result.C[:, :num_factors]``).
+    rotation_matrix : np.ndarray, optional
+        Orthogonal rotation matrix of shape (k_est, k_rot) returned by
+        :func:`evaluate_factor_estimation`. If provided, the estimated
+        loadings are rotated as ``estimated_loadings @ rotation_matrix``
+        before comparison.
+    
+    Returns
+    -------
+    Dict[str, Any]
+        {
+          'num_factors': int,
+          'correlation_per_factor': np.ndarray (k,),
+          'rmse_per_series': np.ndarray (N,),
+          'overall_rmse': float,
+          'aligned_loadings': np.ndarray (N, k),
+        }
+    
+    Notes
+    -----
+    - RMSE is computed per series (row-wise) across factors.
+    - This function assumes synthetic data where ``true_loadings`` are known.
+    """
+    if true_loadings.ndim != 2 or estimated_loadings.ndim != 2:
+        raise ValueError(
+            f"true_loadings and estimated_loadings must be 2D, "
+            f"got shapes {true_loadings.shape} and {estimated_loadings.shape}"
+        )
+    
+    N_true, k_true = true_loadings.shape
+    N_est, k_est = estimated_loadings.shape
+    if N_true != N_est:
+        raise ValueError(
+            f"Number of series (rows) must match, got {N_true} and {N_est}"
+        )
+    
+    # Apply rotation if provided (align loadings with rotated factors)
+    L_est = np.asarray(estimated_loadings, dtype=float)
+    if rotation_matrix is not None:
+        if rotation_matrix.ndim != 2 or rotation_matrix.shape[0] != k_est:
+            raise ValueError(
+                f"rotation_matrix shape {rotation_matrix.shape} incompatible "
+                f"with estimated_loadings shape {estimated_loadings.shape}"
+            )
+        L_est = L_est @ rotation_matrix
+    
+    k = min(k_true, L_est.shape[1])
+    if k == 0:
+        raise ValueError("At least one factor/loading column is required.")
+    
+    L_true = np.asarray(true_loadings[:, :k], dtype=float)
+    L_est_k = np.asarray(L_est[:, :k], dtype=float)
+    
+    # Per-factor loading correlations
+    corr_per_factor = np.full(k, np.nan, dtype=float)
+    for j in range(k):
+        mask_j = np.isfinite(L_true[:, j]) & np.isfinite(L_est_k[:, j])
+        if np.sum(mask_j) > 1:
+            corr = np.corrcoef(L_true[mask_j, j], L_est_k[mask_j, j])[0, 1]
+            corr_per_factor[j] = float(corr)
+    
+    # RMSE per series (row-wise across factors)
+    diff = L_true - L_est_k
+    mask = np.isfinite(L_true) & np.isfinite(L_est_k)
+    rmse_per_series = np.full(N_true, np.nan, dtype=float)
+    for i in range(N_true):
+        mask_i = mask[i, :]
+        if np.any(mask_i):
+            rmse_per_series[i] = float(
+                np.sqrt(np.mean(diff[i, mask_i] ** 2))
+            )
+    
+    if np.any(mask):
+        overall_rmse = float(np.sqrt(np.mean(diff[mask] ** 2)))
+    else:
+        overall_rmse = float("nan")
+    
+    return {
+        "num_factors": k,
+        "correlation_per_factor": corr_per_factor,
+        "rmse_per_series": rmse_per_series,
+        "overall_rmse": overall_rmse,
+        "aligned_loadings": L_est_k,
+    }
+
+
 def _display_dfm_tables(Res: DFMResult, config: DFMConfig, nQ: int) -> None:
     """Display DFM estimation output tables.
     

dfm_python/core/em.py

@@ -809,8 +809,8 @@ def em_step(params: EMStepParams) -> Tuple[np.ndarray, np.ndarray, np.ndarray, n
                     lambda_ridge = (trace_val / size_val) * 1e-5  # Scale factor
                     EZZ_lag_sub = EZZ_lag_sub + lambda_ridge * np.eye(size_val)
                     # Recompute condition number after regularization
-                    eigenvals = np.linalg.eigvals(EZZ_lag_sub)
-                    cond_num = (np.max(eigenvals) / max(np.min(eigenvals), 1e-12)) if np.max(eigenvals) > 0 else 1.0
+                eigenvals = np.linalg.eigvals(EZZ_lag_sub)
+                cond_num = (np.max(eigenvals) / max(np.min(eigenvals), 1e-12)) if np.max(eigenvals) > 0 else 1.0
                 EZZ_lag_inv = pinv(EZZ_lag_sub, rcond=1e-8) if cond_num > 1e12 else inv(EZZ_lag_sub)
             except _NUMERICAL_EXCEPTIONS:
                 EZZ_lag_inv = pinv(EZZ_lag_sub)
@@ -1247,33 +1247,39 @@ def em_step(params: EMStepParams) -> Tuple[np.ndarray, np.ndarray, np.ndarray, n
     # Q cap to prevent explosion: maximum allowed innovation variance for factors
     Q_cap_factor = safe_get_attr(config, "max_innovation_variance", 1.0) if config is not None else 1.0
     
-    # Always normalize C for clock-frequency factors to prevent scale issues
-    # Even if there are slower-frequency series, we can still normalize clock-frequency factors
-    # (tent weight constraints only apply to slower-frequency parts, not clock-frequency parts)
+    # Normalize C for clock-frequency factors to prevent scale issues
+    # Only normalize if C norm is in reasonable range (not too small, not too large)
+    # This prevents Q from becoming too small (when C norm is small) or too large (when C norm is large)
     if num_clock_factors > 0:
-        # Only normalize if there are no slower-frequency series
-        # (to avoid violating tent weight constraints)
-        for j in range(num_clock_factors):
-            norm = np.linalg.norm(C_new[:, j])
-            if norm > 1e-8:
-                # Improved normalization: clip C norm before normalization to prevent Q explosion
-                # If C norm is too large, we cap it to prevent Q from becoming extreme
-                C_norm_cap = safe_get_attr(config, "max_loading_norm", 10.0) if config is not None else 10.0
-                norm_clipped = min(norm, C_norm_cap)
-                
-                # Normalize C column (using clipped norm to prevent extreme Q)
-                C_new[:, j] = C_new[:, j] / norm
-                
-                # Adjust Q to preserve variance, but use clipped norm to prevent explosion
-                # This is a compromise: we normalize C fully but limit Q adjustment
-                Q_adjustment = norm_clipped ** 2
-                Q_new[j, j] = Q_new[j, j] * Q_adjustment
-                
-                # Off-diagonal: Q[j,k] and Q[k,j] *= norm_clipped (for k != j)
-                for k in range(Q_new.shape[0]):
-                    if k != j:
-                        Q_new[j, k] = Q_new[j, k] * norm_clipped
-                        Q_new[k, j] = Q_new[k, j] * norm_clipped
+        # Check if there are slower-frequency series (tent weight constraints)
+        has_slower_freq = (frequencies is not None and 
+                          any(freq != clock for freq in frequencies if freq is not None))
+        
+        # Only normalize if no slower-frequency series (to avoid violating tent weight constraints)
+        if not has_slower_freq:
+            for j in range(num_clock_factors):
+                norm = np.linalg.norm(C_new[:, j])
+                if norm > 1e-8:
+                    # Only normalize if norm is in reasonable range
+                    # Too small norm (< 0.1) would make Q too small
+                    # Too large norm (> 10) would make Q too large
+                    C_norm_min = 0.1  # Minimum norm to normalize
+                    C_norm_cap = safe_get_attr(config, "max_loading_norm", 10.0) if config is not None else 10.0
+                    
+                    if C_norm_min <= norm <= C_norm_cap:
+                        # Normalize C column
+                        C_new[:, j] = C_new[:, j] / norm
+                        
+                        # Adjust Q to preserve variance: Q[j,j] *= norm^2
+                        Q_adjustment = norm ** 2
+                        Q_new[j, j] = Q_new[j, j] * Q_adjustment
+                        
+                        # Off-diagonal: Q[j,k] and Q[k,j] *= norm (for k != j)
+                        for k in range(Q_new.shape[0]):
+                            if k != j:
+                                Q_new[j, k] = Q_new[j, k] * norm
+                                Q_new[k, j] = Q_new[k, j] * norm
+                    # If norm is too small or too large, skip normalization to preserve stability
         
         # Re-apply Q floor and cap after C normalization to prevent extreme values
         # C normalization can make Q very large (if C norm was large) or very small (if C norm was small)

{dfm_python-0.2.6.dist-info → dfm_python-0.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.2.6
+Version: 0.2.8
 Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme
@@ -709,12 +709,20 @@ Demonstrates:
 
 ## Project Status
 
-**Version**: 0.2.6  
+**Version**: 0.2.7  
 **Status**: Stable and production-ready  
 **Python**: 3.10+  
 **PyPI**: https://pypi.org/project/dfm-python/
 
-### Recent Improvements (v0.2.6)
+### Recent Improvements (v0.2.7)
+
+- ✅ **Fixed C normalization issue**: C normalization now only applies when C norm is in reasonable range (0.1 ~ 10)
+  - Prevents Q from becoming too small when C norm is very small
+  - Prevents factor values from becoming extreme
+- ✅ **Improved stability**: Conditional C normalization prevents pathological factor paths
+- ✅ **Better diagnostics**: Added variance guard binding checks in tutorial
+
+### Previous Improvements (v0.2.6)
 
 - ✅ **Q matrix cap**: Factor innovation variances bounded above by 1.0 to prevent explosion
 - ✅ **R matrix cap**: Observation error variances bounded above by 10.0 for standardized data
@@ -758,7 +766,7 @@ If you use `dfm-python` in your research, please cite:
   author = {DFM Python Contributors},
   year = {2025},
   url = {https://pypi.org/project/dfm-python/},
-  version = {0.2.6}
+  version = {0.2.7}
 }
 ```
 

{dfm_python-0.2.6.dist-info → dfm_python-0.2.8.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dfm_python/__init__.py,sha256=arYgbDHYx9jgEjCZ1JMKZO4cZewBc1j-iXi_g6SDu4o,5589
+dfm_python/__init__.py,sha256=oz0p9JreUg6D9SfU2vUAFMRwf7JwHaa3dQDy87YHtIw,5589
 dfm_python/api.py,sha256=-mmV7aaXOAqVW9HPhTEut7T6wN1EyW0rS3J0y-z-MRs,16221
 dfm_python/config.py,sha256=bBiKfoETohNVJ3brRbY1330C1nQROybCR3co9iagIKs,46978
 dfm_python/config_sources.py,sha256=6LTK7k5WJpnayMmtxtYwZ9SQi_hGXoz-Vg4sRGTabI0,21798
@@ -8,11 +8,11 @@ dfm_python/kalman.py,sha256=EXhloLbpzy7GKG9KFrSZt9pRp0y3CZyvycPVbvUWxq8,18074
 dfm_python/news.py,sha256=iXbJnsvH5FgKToPzKnqIpTosrZu-QHXBNHjsqwMYrHo,32583
 dfm_python/utils.py,sha256=qOAJm2KG71yP2a29iJVDGOC-ZUS_HPL5oAo16EgNXA4,15709
 dfm_python/core/__init__.py,sha256=oDrhAoc1W_ODohgufQ-gN0uri8-vZI0EXsF0dysth_A,331
-dfm_python/core/diagnostics.py,sha256=2Cm8XIN5jE9jL3FssEUmZNQy12FPuKvYs1k3bYKAySw,20224
-dfm_python/core/em.py,sha256=nNLL179l_SYSowxtnZBHFpEDR_54FOU_tNyn8XaVwnU,72357
+dfm_python/core/diagnostics.py,sha256=1ninWcQlWhJmfUZq9kky7IQoRf_6fI_IlVzzSb7apRQ,28085
+dfm_python/core/em.py,sha256=jbtsFHynMDtblZTtlzgodX04e86BBB9hPn20MMnV3w0,72705
 dfm_python/core/helpers.py,sha256=XJvWWEZlUqozBfFNcENKgbOOFkwlm-tGwO-u0LnAf-E,31541
 dfm_python/core/numeric.py,sha256=IcIwcKtMhIoJ5-7Mz6UNYwTUtEa5W_ZMnQtF87v0SSc,26431
-dfm_python-0.2.6.dist-info/METADATA,sha256=DnwocbXu0WC8ruBF1GSU2PQINCNSYjxMlGrQlRXVWs0,28159
-dfm_python-0.2.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dfm_python-0.2.6.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
-dfm_python-0.2.6.dist-info/RECORD,,
+dfm_python-0.2.8.dist-info/METADATA,sha256=wY69TZa4fK9I4zY4t7hQUJksWuMCyxtGUS0Nr_C3SPk,28599
+dfm_python-0.2.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dfm_python-0.2.8.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
+dfm_python-0.2.8.dist-info/RECORD,,