dfm-python 0.2.7__py3-none-any.whl → 0.2.8__py3-none-any.whl

dfm_python/__init__.py

@@ -50,7 +50,7 @@ Example (Low-level API - For advanced usage):
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.2.7"
+__version__ = "0.2.8"
 
 from .config import DFMConfig, SeriesConfig, BlockConfig, Params, DEFAULT_GLOBAL_BLOCK_NAME
 from .config import (

dfm_python/core/diagnostics.py

@@ -4,12 +4,10 @@ from typing import Optional, Dict, Any, Tuple, TYPE_CHECKING
 import numpy as np
 import logging
 
-try:
-    import pandas as pd
-    PANDAS_AVAILABLE = True
-except ImportError:
-    PANDAS_AVAILABLE = False
-    pd = None
+import pandas as pd
+from scipy.linalg import orthogonal_procrustes
+
+PANDAS_AVAILABLE = True
 
 from ..config import DFMConfig
 
@@ -96,6 +94,217 @@ def calculate_rmse(actual: np.ndarray, predicted: np.ndarray,
     return rmse_overall, rmse_per_series
 
 
+def evaluate_factor_estimation(
+    true_factors: np.ndarray,
+    estimated_factors: np.ndarray,
+    use_procrustes: bool = True
+) -> Dict[str, Any]:
+    """Evaluate how well factors are estimated (synthetic-data diagnostic).
+    
+    This function compares known ``true_factors`` with ``estimated_factors``
+    (typically ``result.Z[:, :num_factors]``) and returns:
+    
+    - Per-factor correlations (before any rotation)
+    - Optionally Procrustes-rotated factors and an overall correlation
+    
+    Parameters
+    ----------
+    true_factors : np.ndarray
+        Array of shape (T, k) with generated factors used to build the data.
+    estimated_factors : np.ndarray
+        Array of shape (T, k_est) with estimated factors. Only the first
+        ``k = min(k, k_est)`` columns are compared.
+    use_procrustes : bool, default True
+        If True and SciPy is available, apply orthogonal Procrustes rotation
+        to align estimated factors with true factors before computing the
+        overall correlation.
+    
+    Returns
+    -------
+    Dict[str, Any]
+        {
+          'num_factors': int,
+          'correlation_per_factor': np.ndarray (k,),
+          'overall_correlation': float or None,
+          'rotation_matrix': np.ndarray (k, k) or None,
+          'aligned_factors': np.ndarray (T, k) or None,
+        }
+    
+    Notes
+    -----
+    - This is primarily intended for synthetic-data experiments where the
+      true latent factors are known.
+    - Correlations are computed factor-by-factor, ignoring any time points
+      with NaN values.
+    - Procrustes rotation is useful because factors are only identified up
+      to sign, scale (if not constrained), and rotation in general.
+    """
+    if true_factors.ndim != 2 or estimated_factors.ndim != 2:
+        raise ValueError(
+            f"true_factors and estimated_factors must be 2D, "
+            f"got shapes {true_factors.shape} and {estimated_factors.shape}"
+        )
+    
+    T_true, k_true = true_factors.shape
+    T_est, k_est = estimated_factors.shape
+    if T_true != T_est:
+        raise ValueError(
+            f"Time dimension must match, got {T_true} and {T_est}"
+        )
+    
+    k = min(k_true, k_est)
+    if k == 0:
+        raise ValueError("At least one factor is required for comparison.")
+    
+    F_true = np.asarray(true_factors[:, :k], dtype=float)
+    F_est = np.asarray(estimated_factors[:, :k], dtype=float)
+    
+    # Per-factor correlations (no rotation)
+    corr_per_factor = np.full(k, np.nan, dtype=float)
+    for j in range(k):
+        mask_j = np.isfinite(F_true[:, j]) & np.isfinite(F_est[:, j])
+        if np.sum(mask_j) > 1:
+            corr = np.corrcoef(F_true[mask_j, j], F_est[mask_j, j])[0, 1]
+            corr_per_factor[j] = float(corr)
+    
+    rotation_matrix = None
+    aligned_factors = None
+    overall_corr = None
+    
+    if use_procrustes:
+        # Center factors before Procrustes to focus on shape, not level.
+        F_true_centered = F_true - np.nanmean(F_true, axis=0, keepdims=True)
+        F_est_centered = F_est - np.nanmean(F_est, axis=0, keepdims=True)
+        
+        # Replace any remaining NaNs with 0 before Procrustes (rare in synthetic tests)
+        F_true_clean = np.where(np.isfinite(F_true_centered), F_true_centered, 0.0)
+        F_est_clean = np.where(np.isfinite(F_est_centered), F_est_centered, 0.0)
+        
+        try:
+            # Find R such that F_est_clean @ R ≈ F_true_clean
+            R, _ = orthogonal_procrustes(F_est_clean, F_true_clean)  # type: ignore[arg-type]
+            rotation_matrix = R
+            aligned_factors = F_est @ R
+            
+            mask_all = np.isfinite(F_true) & np.isfinite(aligned_factors)
+            if np.any(mask_all):
+                overall_corr = float(
+                    np.corrcoef(
+                        F_true[mask_all].ravel(),
+                        aligned_factors[mask_all].ravel()
+                    )[0, 1]
+                )
+        except Exception as e:
+            _logger.debug(f"Procrustes rotation failed in evaluate_factor_estimation: {e}")
+    
+    return {
+        "num_factors": k,
+        "correlation_per_factor": corr_per_factor,
+        "overall_correlation": overall_corr,
+        "rotation_matrix": rotation_matrix,
+        "aligned_factors": aligned_factors,
+    }
+
+
+def evaluate_loading_estimation(
+    true_loadings: np.ndarray,
+    estimated_loadings: np.ndarray,
+    rotation_matrix: Optional[np.ndarray] = None
+) -> Dict[str, Any]:
+    """Evaluate loading matrix estimation accuracy (synthetic-data diagnostic).
+    
+    Parameters
+    ----------
+    true_loadings : np.ndarray
+        Array of shape (N, k_true) with the loadings used to generate data.
+    estimated_loadings : np.ndarray
+        Array of shape (N, k_est) with estimated loadings from the model
+        (typically ``result.C[:, :num_factors]``).
+    rotation_matrix : np.ndarray, optional
+        Orthogonal rotation matrix of shape (k_est, k_rot) returned by
+        :func:`evaluate_factor_estimation`. If provided, the estimated
+        loadings are rotated as ``estimated_loadings @ rotation_matrix``
+        before comparison.
+    
+    Returns
+    -------
+    Dict[str, Any]
+        {
+          'num_factors': int,
+          'correlation_per_factor': np.ndarray (k,),
+          'rmse_per_series': np.ndarray (N,),
+          'overall_rmse': float,
+          'aligned_loadings': np.ndarray (N, k),
+        }
+    
+    Notes
+    -----
+    - RMSE is computed per series (row-wise) across factors.
+    - This function assumes synthetic data where ``true_loadings`` are known.
+    """
+    if true_loadings.ndim != 2 or estimated_loadings.ndim != 2:
+        raise ValueError(
+            f"true_loadings and estimated_loadings must be 2D, "
+            f"got shapes {true_loadings.shape} and {estimated_loadings.shape}"
+        )
+    
+    N_true, k_true = true_loadings.shape
+    N_est, k_est = estimated_loadings.shape
+    if N_true != N_est:
+        raise ValueError(
+            f"Number of series (rows) must match, got {N_true} and {N_est}"
+        )
+    
+    # Apply rotation if provided (align loadings with rotated factors)
+    L_est = np.asarray(estimated_loadings, dtype=float)
+    if rotation_matrix is not None:
+        if rotation_matrix.ndim != 2 or rotation_matrix.shape[0] != k_est:
+            raise ValueError(
+                f"rotation_matrix shape {rotation_matrix.shape} incompatible "
+                f"with estimated_loadings shape {estimated_loadings.shape}"
+            )
+        L_est = L_est @ rotation_matrix
+    
+    k = min(k_true, L_est.shape[1])
+    if k == 0:
+        raise ValueError("At least one factor/loading column is required.")
+    
+    L_true = np.asarray(true_loadings[:, :k], dtype=float)
+    L_est_k = np.asarray(L_est[:, :k], dtype=float)
+    
+    # Per-factor loading correlations
+    corr_per_factor = np.full(k, np.nan, dtype=float)
+    for j in range(k):
+        mask_j = np.isfinite(L_true[:, j]) & np.isfinite(L_est_k[:, j])
+        if np.sum(mask_j) > 1:
+            corr = np.corrcoef(L_true[mask_j, j], L_est_k[mask_j, j])[0, 1]
+            corr_per_factor[j] = float(corr)
+    
+    # RMSE per series (row-wise across factors)
+    diff = L_true - L_est_k
+    mask = np.isfinite(L_true) & np.isfinite(L_est_k)
+    rmse_per_series = np.full(N_true, np.nan, dtype=float)
+    for i in range(N_true):
+        mask_i = mask[i, :]
+        if np.any(mask_i):
+            rmse_per_series[i] = float(
+                np.sqrt(np.mean(diff[i, mask_i] ** 2))
+            )
+    
+    if np.any(mask):
+        overall_rmse = float(np.sqrt(np.mean(diff[mask] ** 2)))
+    else:
+        overall_rmse = float("nan")
+    
+    return {
+        "num_factors": k,
+        "correlation_per_factor": corr_per_factor,
+        "rmse_per_series": rmse_per_series,
+        "overall_rmse": overall_rmse,
+        "aligned_loadings": L_est_k,
+    }
+
+
 def _display_dfm_tables(Res: DFMResult, config: DFMConfig, nQ: int) -> None:
     """Display DFM estimation output tables.
     

dfm_python/core/em.py

@@ -949,55 +949,55 @@ def em_step(params: EMStepParams) -> Tuple[np.ndarray, np.ndarray, np.ndarray, n
         # Keep C fixed for idio components (not updated in M-step)
     else:
         # Fallback to old approach for backward compatibility
-    i_subset = slice(idio_start_idx, idio_start_idx + n_idio)
-    i_subset_slice = slice(i_subset.start, i_subset.stop)
-    Z_idio = Zsmooth[i_subset_slice, 1:] if idio_start_idx < Zsmooth.shape[1] else np.zeros((n_idio, T))
-    n_idio_actual = Z_idio.shape[0] if Z_idio.shape[0] > 0 else n_idio
-    expected_idio_current_sq = np.sum(Z_idio**2, axis=1)
-    vsmooth_idio_block = vsmooth[i_subset_slice, :, :][:, i_subset_slice, 1:]
-    vsmooth_idio_sum = np.sum(vsmooth_idio_block, axis=2)
-    if vsmooth_idio_sum.ndim == 2 and vsmooth_idio_sum.shape[0] >= n_idio_actual and vsmooth_idio_sum.shape[1] >= n_idio_actual:
-        vsmooth_idio_diag = np.diag(vsmooth_idio_sum[:n_idio_actual, :n_idio_actual])
-    else:
-        vsmooth_idio_diag = np.zeros(n_idio_actual)
-    expected_idio_current_sq = expected_idio_current_sq + vsmooth_idio_diag
-    Z_idio_lag = Zsmooth[i_subset_slice, :-1]
-    expected_idio_lag_sq = np.sum(Z_idio_lag**2, axis=1)
-    vsmooth_lag_block = vsmooth[i_subset_slice, :, :][:, i_subset_slice, :-1]
-    vsmooth_lag_sum = np.sum(vsmooth_lag_block, axis=2)
-    if vsmooth_lag_sum.ndim == 2 and vsmooth_lag_sum.shape[0] >= n_idio_actual and vsmooth_lag_sum.shape[1] >= n_idio_actual:
-        vsmooth_lag_diag = np.diag(vsmooth_lag_sum[:n_idio_actual, :n_idio_actual])
-    else:
-        vsmooth_lag_diag = np.zeros(n_idio_actual)
-    expected_idio_lag_sq = expected_idio_lag_sq + vsmooth_lag_diag
-    min_cols = min(Z_idio.shape[1], Z_idio_lag.shape[1])
-    expected_idio_cross = np.sum(Z_idio[:, :min_cols] * Z_idio_lag[:, :min_cols], axis=1)
-    vvsmooth_block = vvsmooth[i_subset_slice, :, :][:, i_subset_slice, :]
-    vvsmooth_sum = np.sum(vvsmooth_block, axis=2)
-    if vvsmooth_sum.ndim == 2 and vvsmooth_sum.shape[0] >= n_idio_actual and vvsmooth_sum.shape[1] >= n_idio_actual:
-        vvsmooth_diag = np.diag(vvsmooth_sum[:n_idio_actual, :n_idio_actual])
-    else:
-        vvsmooth_diag = np.zeros(n_idio_actual)
-    expected_idio_cross = expected_idio_cross + vvsmooth_diag
-    ar_coeffs_diag, _ = _estimate_ar_coefficient(expected_idio_cross, expected_idio_lag_sq, vsmooth_sum=vsmooth_lag_diag)
-    A_block_idio = np.diag(ar_coeffs_diag)
-    innovation_cov_diag = (np.maximum(expected_idio_current_sq, 0.0) - ar_coeffs_diag * expected_idio_cross) / T
-    innovation_cov_diag = np.maximum(innovation_cov_diag, MIN_INNOVATION_VARIANCE)
-    Q_block_idio = np.diag(innovation_cov_diag)
-    i_subset_size = i_subset.stop - i_subset.start
-    if n_idio_actual == i_subset_size:
-        A_new[i_subset, i_subset] = A_block_idio
-        Q_new[i_subset, i_subset] = Q_block_idio
-    elif n_idio_actual < i_subset_size:
-        A_new[i_subset.start:i_subset.start + n_idio_actual, i_subset.start:i_subset.start + n_idio_actual] = A_block_idio
-        Q_new[i_subset.start:i_subset.start + n_idio_actual, i_subset.start:i_subset.start + n_idio_actual] = Q_block_idio
-    else:
-        A_new[i_subset, i_subset] = A_block_idio[:i_subset_size, :i_subset_size]
-        Q_new[i_subset, i_subset] = Q_block_idio[:i_subset_size, :i_subset_size]
-    vsmooth_sub = vsmooth[i_subset_slice, :, :][:, i_subset_slice, 0]
-    vsmooth_diag = np.diag(vsmooth_sub[:n_idio_actual, :n_idio_actual]) if vsmooth_sub.ndim == 2 else np.zeros(n_idio_actual)
-    for idx in range(min(n_idio_actual, i_subset_size)):
-        V_0_new[i_subset.start + idx, i_subset.start + idx] = vsmooth_diag[idx] if idx < len(vsmooth_diag) else 0.0
+        i_subset = slice(idio_start_idx, idio_start_idx + n_idio)
+        i_subset_slice = slice(i_subset.start, i_subset.stop)
+        Z_idio = Zsmooth[i_subset_slice, 1:] if idio_start_idx < Zsmooth.shape[1] else np.zeros((n_idio, T))
+        n_idio_actual = Z_idio.shape[0] if Z_idio.shape[0] > 0 else n_idio
+        expected_idio_current_sq = np.sum(Z_idio**2, axis=1)
+        vsmooth_idio_block = vsmooth[i_subset_slice, :, :][:, i_subset_slice, 1:]
+        vsmooth_idio_sum = np.sum(vsmooth_idio_block, axis=2)
+        if vsmooth_idio_sum.ndim == 2 and vsmooth_idio_sum.shape[0] >= n_idio_actual and vsmooth_idio_sum.shape[1] >= n_idio_actual:
+            vsmooth_idio_diag = np.diag(vsmooth_idio_sum[:n_idio_actual, :n_idio_actual])
+        else:
+            vsmooth_idio_diag = np.zeros(n_idio_actual)
+        expected_idio_current_sq = expected_idio_current_sq + vsmooth_idio_diag
+        Z_idio_lag = Zsmooth[i_subset_slice, :-1]
+        expected_idio_lag_sq = np.sum(Z_idio_lag**2, axis=1)
+        vsmooth_lag_block = vsmooth[i_subset_slice, :, :][:, i_subset_slice, :-1]
+        vsmooth_lag_sum = np.sum(vsmooth_lag_block, axis=2)
+        if vsmooth_lag_sum.ndim == 2 and vsmooth_lag_sum.shape[0] >= n_idio_actual and vsmooth_lag_sum.shape[1] >= n_idio_actual:
+            vsmooth_lag_diag = np.diag(vsmooth_lag_sum[:n_idio_actual, :n_idio_actual])
+        else:
+            vsmooth_lag_diag = np.zeros(n_idio_actual)
+        expected_idio_lag_sq = expected_idio_lag_sq + vsmooth_lag_diag
+        min_cols = min(Z_idio.shape[1], Z_idio_lag.shape[1])
+        expected_idio_cross = np.sum(Z_idio[:, :min_cols] * Z_idio_lag[:, :min_cols], axis=1)
+        vvsmooth_block = vvsmooth[i_subset_slice, :, :][:, i_subset_slice, :]
+        vvsmooth_sum = np.sum(vvsmooth_block, axis=2)
+        if vvsmooth_sum.ndim == 2 and vvsmooth_sum.shape[0] >= n_idio_actual and vvsmooth_sum.shape[1] >= n_idio_actual:
+            vvsmooth_diag = np.diag(vvsmooth_sum[:n_idio_actual, :n_idio_actual])
+        else:
+            vvsmooth_diag = np.zeros(n_idio_actual)
+        expected_idio_cross = expected_idio_cross + vvsmooth_diag
+        ar_coeffs_diag, _ = _estimate_ar_coefficient(expected_idio_cross, expected_idio_lag_sq, vsmooth_sum=vsmooth_lag_diag)
+        A_block_idio = np.diag(ar_coeffs_diag)
+        innovation_cov_diag = (np.maximum(expected_idio_current_sq, 0.0) - ar_coeffs_diag * expected_idio_cross) / T
+        innovation_cov_diag = np.maximum(innovation_cov_diag, MIN_INNOVATION_VARIANCE)
+        Q_block_idio = np.diag(innovation_cov_diag)
+        i_subset_size = i_subset.stop - i_subset.start
+        if n_idio_actual == i_subset_size:
+            A_new[i_subset, i_subset] = A_block_idio
+            Q_new[i_subset, i_subset] = Q_block_idio
+        elif n_idio_actual < i_subset_size:
+            A_new[i_subset.start:i_subset.start + n_idio_actual, i_subset.start:i_subset.start + n_idio_actual] = A_block_idio
+            Q_new[i_subset.start:i_subset.start + n_idio_actual, i_subset.start:i_subset.start + n_idio_actual] = Q_block_idio
+        else:
+            A_new[i_subset, i_subset] = A_block_idio[:i_subset_size, :i_subset_size]
+            Q_new[i_subset, i_subset] = Q_block_idio[:i_subset_size, :i_subset_size]
+        vsmooth_sub = vsmooth[i_subset_slice, :, :][:, i_subset_slice, 0]
+        vsmooth_diag = np.diag(vsmooth_sub[:n_idio_actual, :n_idio_actual]) if vsmooth_sub.ndim == 2 else np.zeros(n_idio_actual)
+        for idx in range(min(n_idio_actual, i_subset_size)):
+            V_0_new[i_subset.start + idx, i_subset.start + idx] = vsmooth_diag[idx] if idx < len(vsmooth_diag) else 0.0
 
     Z_0 = Zsmooth[0, :].copy()
     nanY = np.isnan(y)

dfm_python/core/numeric.py

@@ -605,14 +605,14 @@ def _safe_determinant(M: np.ndarray, use_logdet: bool = True) -> float:
                     if not np.isfinite(log_det) or sign <= 0:
                         return 0.0
                     # Avoid overflow in exp
-                if log_det > 700:
-                    _logger.debug("_safe_determinant: log_det too large, returning 0.0")
-                    return 0.0
-                with warnings.catch_warnings():
-                    warnings.filterwarnings('ignore', category=RuntimeWarning)
+                    if log_det > 700:
+                        _logger.debug("_safe_determinant: log_det too large, returning 0.0")
+                        return 0.0
+                    with warnings.catch_warnings():
+                        warnings.filterwarnings('ignore', category=RuntimeWarning)
                         det = np.exp(log_det)
-                if np.isfinite(det):
-                    return float(det)
+                    if np.isfinite(det):
+                        return float(det)
                 except Exception:
                     pass
         except (np.linalg.LinAlgError, ValueError, OverflowError, RuntimeWarning):

{dfm_python-0.2.7.dist-info → dfm_python-0.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.2.7
+Version: 0.2.8
 Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme

{dfm_python-0.2.7.dist-info → dfm_python-0.2.8.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dfm_python/__init__.py,sha256=YYhRgV773DAFbaTrfF6LVF45rmbR8SpsH6LOYtDAHnU,5589
+dfm_python/__init__.py,sha256=oz0p9JreUg6D9SfU2vUAFMRwf7JwHaa3dQDy87YHtIw,5589
 dfm_python/api.py,sha256=-mmV7aaXOAqVW9HPhTEut7T6wN1EyW0rS3J0y-z-MRs,16221
 dfm_python/config.py,sha256=bBiKfoETohNVJ3brRbY1330C1nQROybCR3co9iagIKs,46978
 dfm_python/config_sources.py,sha256=6LTK7k5WJpnayMmtxtYwZ9SQi_hGXoz-Vg4sRGTabI0,21798
@@ -8,11 +8,11 @@ dfm_python/kalman.py,sha256=EXhloLbpzy7GKG9KFrSZt9pRp0y3CZyvycPVbvUWxq8,18074
 dfm_python/news.py,sha256=iXbJnsvH5FgKToPzKnqIpTosrZu-QHXBNHjsqwMYrHo,32583
 dfm_python/utils.py,sha256=qOAJm2KG71yP2a29iJVDGOC-ZUS_HPL5oAo16EgNXA4,15709
 dfm_python/core/__init__.py,sha256=oDrhAoc1W_ODohgufQ-gN0uri8-vZI0EXsF0dysth_A,331
-dfm_python/core/diagnostics.py,sha256=2Cm8XIN5jE9jL3FssEUmZNQy12FPuKvYs1k3bYKAySw,20224
-dfm_python/core/em.py,sha256=ltMVW1izAVmMwwYKf3GIFLAP-nLAgGy9zdOenuK-SiU,72509
+dfm_python/core/diagnostics.py,sha256=1ninWcQlWhJmfUZq9kky7IQoRf_6fI_IlVzzSb7apRQ,28085
+dfm_python/core/em.py,sha256=jbtsFHynMDtblZTtlzgodX04e86BBB9hPn20MMnV3w0,72705
 dfm_python/core/helpers.py,sha256=XJvWWEZlUqozBfFNcENKgbOOFkwlm-tGwO-u0LnAf-E,31541
-dfm_python/core/numeric.py,sha256=__uwDgRj3TV4S0vXhNc_93kk8U6ewtW8smk0OBZLbdg,26403
-dfm_python-0.2.7.dist-info/METADATA,sha256=ZeeKWsk4BkRbihDNjkLkOq-DofODB9RhwG7dxMRIbQI,28599
-dfm_python-0.2.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dfm_python-0.2.7.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
-dfm_python-0.2.7.dist-info/RECORD,,
+dfm_python/core/numeric.py,sha256=IcIwcKtMhIoJ5-7Mz6UNYwTUtEa5W_ZMnQtF87v0SSc,26431
+dfm_python-0.2.8.dist-info/METADATA,sha256=wY69TZa4fK9I4zY4t7hQUJksWuMCyxtGUS0Nr_C3SPk,28599
+dfm_python-0.2.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dfm_python-0.2.8.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
+dfm_python-0.2.8.dist-info/RECORD,,