dfm-python 0.1.5__py3-none-any.whl → 0.1.6__py3-none-any.whl

dfm_python/__init__.py

@@ -30,13 +30,13 @@ Example:
 For detailed documentation, see the README.md file.
 """
 
-__version__ = "0.1.4"
+__version__ = "0.1.6"
 
 from .config import DFMConfig
 from .data_loader import (
     load_config, load_config_from_yaml, load_config_from_csv, load_data, transform_data
 )
-from .dfm import DFMResult, dfm, calculate_rmse
+from .dfm import DFMResult, dfm, calculate_rmse, diagnose_series, print_series_diagnosis
 from .kalman import run_kf, skf, fis, miss_data
 from .news import update_nowcast, news_dfm, para_const
 
@@ -51,7 +51,7 @@ __all__ = [
     'ModelConfig',  # Alias for DFMConfig
     'load_config', 'load_config_from_yaml', 'load_config_from_csv',
     'load_data', 'transform_data',
-    'DFMResult', 'dfm', 'calculate_rmse',
+    'DFMResult', 'dfm', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
     'run_kf', 'skf', 'fis', 'miss_data',
     'update_nowcast', 'news_dfm', 'para_const',
     # Deprecated aliases

dfm_python/dfm.py

@@ -646,6 +646,20 @@ class DFMResult:
         how many factors are in each block structure.
     p : int
         Number of lags in the autoregressive structure of factors. Typically p=1.
+    rmse : float, optional
+        Overall RMSE on original scale (averaged across all series).
+    rmse_per_series : np.ndarray, optional
+        RMSE per series on original scale (N,).
+    rmse_std : float, optional
+        Overall RMSE on standardized scale (averaged across all series).
+    rmse_std_per_series : np.ndarray, optional
+        RMSE per series on standardized scale (N,).
+    converged : bool, optional
+        Whether EM algorithm converged.
+    num_iter : int, optional
+        Number of EM iterations performed.
+    loglik : float, optional
+        Final log-likelihood value.
     
     Examples
     --------
@@ -673,8 +687,10 @@ class DFMResult:
     converged: bool = False  # Whether EM algorithm converged
     num_iter: int = 0     # Number of iterations completed
     loglik: float = -np.inf  # Final log-likelihood
-    rmse: Optional[float] = None  # Root Mean Squared Error (averaged across all series)
-    rmse_per_series: Optional[np.ndarray] = None  # RMSE per series (N,)
+    rmse: Optional[float] = None  # Overall RMSE (original scale)
+    rmse_per_series: Optional[np.ndarray] = None  # RMSE per series (original scale)
+    rmse_std: Optional[float] = None  # Overall RMSE (standardized scale)
+    rmse_std_per_series: Optional[np.ndarray] = None  # RMSE per series (standardized scale)
 
 
 def em_converged(loglik: float, previous_loglik: float, threshold: float = 1e-4,
@@ -1729,12 +1745,11 @@ def em_step(y: np.ndarray, A: np.ndarray, C: np.ndarray, Q: np.ndarray,
     num_blocks = blocks.shape[1]
     
     # Get config defaults if not provided
+    # Create a minimal config object with defaults for backward compatibility
+    # We use a simple object to avoid DFMConfig validation which requires series
     if config is None:
-        # Create a minimal config with defaults for backward compatibility
-        from dfm_python.config import DFMConfig
-        config = DFMConfig(
-            series=[],
-            block_names=[],
+        from types import SimpleNamespace
+        config = SimpleNamespace(
             clip_ar_coefficients=True,
             ar_clip_min=-0.99,
             ar_clip_max=0.99,
@@ -2953,6 +2968,12 @@ def dfm(X: np.ndarray, config: DFMConfig, threshold: Optional[float] = None, max
     # Only use non-missing observations for comparison
     rmse_overall, rmse_per_series = calculate_rmse(X, X_sm, mask=None)
     
+    # Calculate standardized RMSE for better comparison across series
+    # Standardize both actual and predicted using the same Wx and Mx
+    X_std = (X - Mx_clean) / np.where(Wx_clean != 0, Wx_clean, 1.0)
+    x_sm_std = x_sm  # Already standardized
+    rmse_std_overall, rmse_std_per_series = calculate_rmse(X_std, x_sm_std, mask=None)
+    
     # Create result structure
     Res = DFMResult(
         x_sm=x_sm,
@@ -2972,7 +2993,9 @@ def dfm(X: np.ndarray, config: DFMConfig, threshold: Optional[float] = None, max
         num_iter=num_iter,
         loglik=loglik,
         rmse=rmse_overall,
-        rmse_per_series=rmse_per_series
+        rmse_per_series=rmse_per_series,
+        rmse_std=rmse_std_overall,
+        rmse_std_per_series=rmse_std_per_series
     )
     
     # Display output tables (optional - can be disabled for cleaner output)
@@ -3099,22 +3122,243 @@ def _display_dfm_tables(Res: DFMResult, config, nQ: int) -> None:
         # Table 8: Model Fit Statistics (RMSE)
         if Res.rmse is not None and not np.isnan(Res.rmse):
             print('Table 8: Model Fit Statistics')
-            print(f'  Overall RMSE (averaged across all series): {Res.rmse:.6f}')
+            print(f'  Overall RMSE (original scale): {Res.rmse:.6f}')
+            if Res.rmse_std is not None and not np.isnan(Res.rmse_std):
+                print(f'  Overall RMSE (standardized scale): {Res.rmse_std:.6f}')
             if Res.rmse_per_series is not None and len(Res.rmse_per_series) > 0:
-                print('\n  RMSE per Series:')
+                print('\n  RMSE per Series (Original Scale):')
                 try:
                     series_names = config.SeriesName if hasattr(config, 'SeriesName') else [f"Series_{i}" for i in range(len(Res.rmse_per_series))]
                     for i, (name, rmse_val) in enumerate(zip(series_names, Res.rmse_per_series)):
                         if not np.isnan(rmse_val):
-                            print(f'    {name:40s}: {rmse_val:.6f}')
+                            # Calculate RMSE as percentage of mean if available
+                            mean_val = Res.Mx[i] if i < len(Res.Mx) else np.nan
+                            if not np.isnan(mean_val) and abs(mean_val) > 1e-6:
+                                pct = 100.0 * rmse_val / abs(mean_val)
+                                print(f'    {name:40s}: {rmse_val:.6f} ({pct:.2f}% of mean)')
+                            else:
+                                print(f'    {name:40s}: {rmse_val:.6f}')
                 except Exception:
                     # Fallback if series names not available
                     for i, rmse_val in enumerate(Res.rmse_per_series):
                         if not np.isnan(rmse_val):
                             print(f'    Series {i:3d}: {rmse_val:.6f}')
+            
+            # Show standardized RMSE per series
+            if Res.rmse_std_per_series is not None and len(Res.rmse_std_per_series) > 0:
+                print('\n  RMSE per Series (Standardized Scale):')
+                try:
+                    series_names = config.SeriesName if hasattr(config, 'SeriesName') else [f"Series_{i}" for i in range(len(Res.rmse_std_per_series))]
+                    for i, (name, rmse_std_val) in enumerate(zip(series_names, Res.rmse_std_per_series)):
+                        if not np.isnan(rmse_std_val):
+                            print(f'    {name:40s}: {rmse_std_val:.6f} std dev')
+                except Exception:
+                    # Fallback if series names not available
+                    for i, rmse_std_val in enumerate(Res.rmse_std_per_series):
+                        if not np.isnan(rmse_std_val):
+                            print(f'    Series {i:3d}: {rmse_std_val:.6f} std dev')
+            
+            # Diagnostic warnings for high RMSE
+            if Res.rmse_per_series is not None and Res.Mx is not None:
+                print('\n  Diagnostic Warnings:')
+                try:
+                    series_names = config.SeriesName if hasattr(config, 'SeriesName') else [f"Series_{i}" for i in range(len(Res.rmse_per_series))]
+                    warnings_count = 0
+                    for i, (name, rmse_val) in enumerate(zip(series_names, Res.rmse_per_series)):
+                        if not np.isnan(rmse_val) and i < len(Res.Mx):
+                            mean_val = Res.Mx[i]
+                            std_val = Res.Wx[i] if i < len(Res.Wx) else np.nan
+                            if not np.isnan(mean_val) and abs(mean_val) > 1e-6:
+                                pct_of_mean = 100.0 * rmse_val / abs(mean_val)
+                                # Warn if RMSE > 50% of mean or > 10 standard deviations
+                                if pct_of_mean > 50.0 or (not np.isnan(std_val) and rmse_val > 10.0 * std_val):
+                                    warnings_count += 1
+                                    if warnings_count <= 5:  # Limit to first 5 warnings
+                                        print(f'    ⚠ {name:40s}: RMSE is {pct_of_mean:.1f}% of mean')
+                                        if not np.isnan(std_val):
+                                            print(f'      (RMSE={rmse_val:.2e}, Mean={mean_val:.2e}, Std={std_val:.2e})')
+                    if warnings_count > 5:
+                        print(f'    ... and {warnings_count - 5} more series with high RMSE')
+                except Exception:
+                    pass
             print('\n\n\n')
         
     except Exception as e:
         print(f'Error displaying tables: {e}')
         pass
 
+
+def diagnose_series(Res: DFMResult, config: DFMConfig, series_name: Optional[str] = None, 
+                    series_idx: Optional[int] = None) -> Dict[str, Any]:
+    """Diagnose model fit issues for a specific series.
+    
+    This function provides detailed diagnostics to help identify why a series
+    may have high RMSE, including factor loadings, standardization values,
+    and reconstruction errors.
+    
+    Parameters
+    ----------
+    Res : DFMResult
+        DFM estimation results
+    config : DFMConfig
+        DFM configuration
+    series_name : str, optional
+        Name of the series to diagnose. If provided, series_idx is ignored.
+    series_idx : int, optional
+        Index of the series to diagnose (0-based). Used if series_name is not provided.
+        
+    Returns
+    -------
+    dict
+        Dictionary containing diagnostic information:
+        - 'series_name': Name of the series
+        - 'series_idx': Index of the series
+        - 'rmse_original': RMSE on original scale
+        - 'rmse_standardized': RMSE on standardized scale
+        - 'mean': Series mean (Mx)
+        - 'std': Series standard deviation (Wx)
+        - 'rmse_pct_of_mean': RMSE as percentage of mean
+        - 'rmse_in_std_devs': RMSE in standard deviations
+        - 'factor_loadings': Factor loadings for this series
+        - 'max_loading': Maximum absolute loading
+        - 'loading_sum_sq': Sum of squared loadings
+        - 'reconstruction_error_mean': Mean reconstruction error
+        - 'reconstruction_error_std': Std of reconstruction error
+        
+    Examples
+    --------
+    >>> Res = dfm(X, config)
+    >>> diag = diagnose_series(Res, config, series_name="Real GDP")
+    >>> print(f"Standardized RMSE: {diag['rmse_standardized']:.4f}")
+    """
+    # Find series index
+    if series_name is not None:
+        try:
+            series_names = config.SeriesName if hasattr(config, 'SeriesName') else []
+            if series_name in series_names:
+                series_idx = series_names.index(series_name)
+            else:
+                # Try case-insensitive match
+                series_idx = next((i for i, name in enumerate(series_names) 
+                                 if name.lower() == series_name.lower()), None)
+                if series_idx is None:
+                    raise ValueError(f"Series '{series_name}' not found in configuration")
+        except (AttributeError, ValueError) as e:
+            raise ValueError(f"Cannot find series '{series_name}': {e}")
+    
+    if series_idx is None:
+        raise ValueError("Must provide either series_name or series_idx")
+    
+    if series_idx < 0 or series_idx >= Res.C.shape[0]:
+        raise ValueError(f"Series index {series_idx} out of range [0, {Res.C.shape[0]})")
+    
+    # Get series name
+    try:
+        series_names = config.SeriesName if hasattr(config, 'SeriesName') else []
+        name = series_names[series_idx] if series_idx < len(series_names) else f"Series_{series_idx}"
+    except (AttributeError, IndexError):
+        name = f"Series_{series_idx}"
+    
+    # Get RMSE values
+    rmse_original = None
+    rmse_standardized = None
+    if Res.rmse_per_series is not None and series_idx < len(Res.rmse_per_series):
+        rmse_original = Res.rmse_per_series[series_idx]
+    if Res.rmse_std_per_series is not None and series_idx < len(Res.rmse_std_per_series):
+        rmse_standardized = Res.rmse_std_per_series[series_idx]
+    
+    # Get standardization values
+    mean_val = Res.Mx[series_idx] if series_idx < len(Res.Mx) else np.nan
+    std_val = Res.Wx[series_idx] if series_idx < len(Res.Wx) else np.nan
+    
+    # Calculate RMSE metrics
+    rmse_pct_of_mean = None
+    rmse_in_std_devs = None
+    if not np.isnan(rmse_original) and not np.isnan(mean_val) and abs(mean_val) > 1e-6:
+        rmse_pct_of_mean = 100.0 * rmse_original / abs(mean_val)
+    if not np.isnan(rmse_original) and not np.isnan(std_val) and std_val > 1e-6:
+        rmse_in_std_devs = rmse_original / std_val
+    
+    # Get factor loadings
+    factor_loadings = Res.C[series_idx, :] if series_idx < Res.C.shape[0] else np.array([])
+    max_loading = np.max(np.abs(factor_loadings)) if len(factor_loadings) > 0 else np.nan
+    loading_sum_sq = np.sum(factor_loadings ** 2) if len(factor_loadings) > 0 else np.nan
+    
+    # Calculate reconstruction errors (if we have original data)
+    reconstruction_error_mean = None
+    reconstruction_error_std = None
+    # Note: We don't have access to original X here, so we can't calculate this
+    # This would require passing X to the function or storing it in Res
+    
+    return {
+        'series_name': name,
+        'series_idx': series_idx,
+        'rmse_original': rmse_original,
+        'rmse_standardized': rmse_standardized,
+        'mean': mean_val,
+        'std': std_val,
+        'rmse_pct_of_mean': rmse_pct_of_mean,
+        'rmse_in_std_devs': rmse_in_std_devs,
+        'factor_loadings': factor_loadings,
+        'max_loading': max_loading,
+        'loading_sum_sq': loading_sum_sq,
+        'reconstruction_error_mean': reconstruction_error_mean,
+        'reconstruction_error_std': reconstruction_error_std
+    }
+
+
+def print_series_diagnosis(Res: DFMResult, config: DFMConfig, 
+                          series_name: Optional[str] = None, 
+                          series_idx: Optional[int] = None) -> None:
+    """Print a formatted diagnosis report for a specific series.
+    
+    Parameters
+    ----------
+    Res : DFMResult
+        DFM estimation results
+    config : DFMConfig
+        DFM configuration
+    series_name : str, optional
+        Name of the series to diagnose
+    series_idx : int, optional
+        Index of the series to diagnose
+        
+    Examples
+    --------
+    >>> Res = dfm(X, config)
+    >>> print_series_diagnosis(Res, config, series_name="Real GDP")
+    """
+    diag = diagnose_series(Res, config, series_name=series_name, series_idx=series_idx)
+    
+    print(f"\n{'='*70}")
+    print(f"DIAGNOSTIC REPORT: {diag['series_name']}")
+    print(f"{'='*70}\n")
+    
+    print("RMSE Statistics:")
+    print(f"  Original scale:     {diag['rmse_original']:.6e}" if diag['rmse_original'] is not None else "  Original scale:     N/A")
+    print(f"  Standardized scale: {diag['rmse_standardized']:.6f} std dev" if diag['rmse_standardized'] is not None else "  Standardized scale: N/A")
+    if diag['rmse_pct_of_mean'] is not None:
+        print(f"  As % of mean:       {diag['rmse_pct_of_mean']:.2f}%")
+    if diag['rmse_in_std_devs'] is not None:
+        print(f"  In std deviations: {diag['rmse_in_std_devs']:.2f}x")
+    
+    print("\nStandardization Values:")
+    print(f"  Mean:  {diag['mean']:.6e}" if not np.isnan(diag['mean']) else "  Mean:  N/A")
+    print(f"  Std:   {diag['std']:.6e}" if not np.isnan(diag['std']) else "  Std:   N/A")
+    
+    print("\nFactor Loadings:")
+    if len(diag['factor_loadings']) > 0:
+        print(f"  Number of loadings: {len(diag['factor_loadings'])}")
+        print(f"  Max absolute:       {diag['max_loading']:.6f}" if not np.isnan(diag['max_loading']) else "  Max absolute:       N/A")
+        print(f"  Sum of squares:     {diag['loading_sum_sq']:.6f}" if not np.isnan(diag['loading_sum_sq']) else "  Sum of squares:     N/A")
+        # Show top 5 loadings
+        abs_loadings = np.abs(diag['factor_loadings'])
+        top_indices = np.argsort(abs_loadings)[-5:][::-1]
+        print(f"  Top 5 loadings:")
+        for idx in top_indices:
+            print(f"    Factor {idx:3d}: {diag['factor_loadings'][idx]:8.4f}")
+    else:
+        print("  No loadings available")
+    
+    print(f"\n{'='*70}\n")
+

{dfm_python-0.1.5.dist-info → dfm_python-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.1.5
+Version: 0.1.6
 Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme
@@ -555,7 +555,7 @@ If you use this package in your research, please cite:
 @software{dfm-python,
   title = {dfm-python: Dynamic Factor Models for Nowcasting and Forecasting},
   author = {DFM Python Contributors},
-  year = {2024},
+  year = {2025},
   url = {https://pypi.org/project/dfm-python/},
   version = {0.1.5}
 }

{dfm_python-0.1.5.dist-info → dfm_python-0.1.6.dist-info}/RECORD

@@ -1,13 +1,13 @@
-dfm_python/__init__.py,sha256=XePi1tPUqRiHwSJeJ7D-Dqz_qdFtIVBxKBB23d85cu0,2333
+dfm_python/__init__.py,sha256=f7MFOpB9MmPPjX58bvlLWB4ujuIAWmc_whPN1np3be4,2419
 dfm_python/config.py,sha256=QI-kdEQIkiDahN7lwoGr6PUVeNXW4MiZQkOEr5xypzs,26283
 dfm_python/data_loader.py,sha256=0pNH4YyOgGsgcYZJL60OhC4QA-iTvjFrsq8I-WEF1rk,20585
-dfm_python/dfm.py,sha256=2qlSq6tAWvkaS6X713Ktycu3gfFlwKDoFrHyNKusmPE,143215
+dfm_python/dfm.py,sha256=zNkT-iVEqNvMyjVmfGdCn5Txa9w8ehGGhV2ujNkieps,155260
 dfm_python/kalman.py,sha256=vaIpTiaV3EcZW3SIa0gxkKtYqboffu1wAr3-7yhKdIc,17308
 dfm_python/news.py,sha256=AEs-Z7UoOzvw7TN22AKsglVnYlI4JgNB7ittEcdGU2M,31583
 dfm_python/utils/__init__.py,sha256=nfFHSKJGUzXoeleoqrWqzuQriDU79o9xUhfG3VYy2sI,659
 dfm_python/utils/aggregation.py,sha256=Z0gbGticxMs2GLxG4pD-eOR-7aHd-S5aeHo5S4OlrKc,10270
 dfm_python/utils/data_utils.py,sha256=zUpIBD1_ydyZLXvVxl6OVW4AM43edEgGs3thE8687e0,6805
-dfm_python-0.1.5.dist-info/METADATA,sha256=KEbsNS-TFaBOvlXQy4cDmcLS6Nl5VBiEX8rmZ3z2gMA,19086
-dfm_python-0.1.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dfm_python-0.1.5.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
-dfm_python-0.1.5.dist-info/RECORD,,
+dfm_python-0.1.6.dist-info/METADATA,sha256=WFMfZB8RwXbmUKjBUBYVoAwM1zloKRjB6LxJmFtW7_0,19086
+dfm_python-0.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dfm_python-0.1.6.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
+dfm_python-0.1.6.dist-info/RECORD,,