dfm-python 0.5.57__tar.gz → 0.5.58__tar.gz

{dfm_python-0.5.57 → dfm_python-0.5.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.5.57
+Version: 0.5.58
 Summary: Dynamic Factor Model (DFM) estimation and forecasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme

{dfm_python-0.5.57 → dfm_python-0.5.58}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dfm-python"
-version = "0.5.57"
+version = "0.5.58"
 description = "Dynamic Factor Model (DFM) estimation and forecasting in Python"
 readme = "README.md"
 requires-python = ">=3.10"

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/__init__.py

@@ -80,7 +80,6 @@ __version__ = "0.5.56"
 
 # Configuration (from config/ subpackage)
 from .config import (
-    # DEFAULT_BLOCK_NAME,  # Removed to avoid circular import - import directly from functional.dfm_block
     ConfigSource, YamlSource,
     make_config_source,
 )

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/config/__init__.py

@@ -86,9 +86,8 @@ except ImportError:
     compute_idio_lengths = None
 
 try:
-    from ..numeric.tent import get_tent_weights, get_agg_structure, group_by_freq
+    from ..numeric.tent import get_agg_structure, group_by_freq
 except ImportError:
-    get_tent_weights = None
     get_agg_structure = None
     group_by_freq = None
 from .constants import FREQUENCY_HIERARCHY, PERIODS_PER_YEAR
@@ -119,7 +118,6 @@ __all__ = [
     'PERIODS_PER_YEAR',
     'get_periods_per_year',
     'compute_idio_lengths',
-    'get_tent_weights',
     'get_agg_structure',
     'group_by_freq',
     # Type definitions (from types.py)

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/config/adapter.py

@@ -552,6 +552,11 @@ def detect_config_type(data: Dict[str, Any]) -> str:
     if model_type == MODEL_TYPE_DFM:
         return MODEL_TYPE_DFM
     
+    # Priority check: blocks parameter indicates DFM (highest priority)
+    # DFM is the only model type that uses blocks structure
+    if 'blocks' in data and data.get('blocks'):
+        return MODEL_TYPE_DFM
+    
     # Check for model-specific parameters
     keys = set(data.keys())
     if keys & _KDFM_PARAMS:

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/config/constants.py

@@ -387,22 +387,6 @@ FREQUENCY_HIERARCHY: Dict[str, int] = {
 # For frequency gaps larger than this, the missing data approach is used instead
 MAX_TENT_SIZE: int = 12
 
-# Deterministic tent weights lookup for supported frequency pairs
-# Format: (slower_freq, faster_freq) -> tent_weights_array
-# These weights define how slower-frequency series aggregate clock-frequency factors
-TENT_WEIGHTS_LOOKUP: Dict[Tuple[str, str], np.ndarray] = {
-    ('q', 'm'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: quarterly -> monthly
-    ('sa', 'm'): np.array([1, 2, 3, 4, 3, 2, 1]),             # 7 periods: semi-annual -> monthly
-    ('a', 'm'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: annual -> monthly
-    ('m', 'w'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: monthly -> weekly
-    ('q', 'w'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: quarterly -> weekly
-    ('sa', 'w'): np.array([1, 2, 3, 4, 3, 2, 1]),             # 7 periods: semi-annual -> weekly
-    ('a', 'w'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: annual -> weekly
-    ('sa', 'q'): np.array([1, 2, 1]),                         # 3 periods: semi-annual -> quarterly
-    ('a', 'q'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: annual -> quarterly
-    ('a', 'sa'): np.array([1, 2, 1]),                         # 3 periods: annual -> semi-annual
-}
-
 # ============================================================================
 # Export all constants
 # ============================================================================
@@ -582,6 +566,5 @@ __all__ = [
     # Frequency hierarchy and tent kernels
     'FREQUENCY_HIERARCHY',
     'MAX_TENT_SIZE',
-    'TENT_WEIGHTS_LOOKUP',
 ]
 

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/config/schema/block.py

@@ -90,6 +90,7 @@ class BlockStructure:
     _cached_n_idio_M: Optional[int] = field(default=None, init=False, repr=False)
     _cached_c_idio_indicator: Optional[np.ndarray] = field(default=None, init=False, repr=False)
     _cached_rp1: Optional[int] = field(default=None, init=False, repr=False)
+    _cached_N: Optional[int] = field(default=None, init=False, repr=False)  # N value used for caching
     
     def is_valid(self) -> bool:
         """Check if block structure is valid (all required fields are not None)."""
@@ -119,4 +120,5 @@ class BlockStructure:
         self._cached_n_idio_M = None
         self._cached_c_idio_indicator = None
         self._cached_rp1 = None
+        self._cached_N = None
 

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/config/schema/model.py

@@ -286,12 +286,19 @@ class DFMConfig(BaseModelConfig):
     # Damped Updates: If provided (not None), automatically enables damping and always warns
     damping_factor: Optional[float] = None  # Damping factor (0.8 = 80% new, 20% old). If None, no damping. If provided, damping enabled and warnings always shown.
     
-    # Idiosyncratic Component Parameters (auto-detected from frequencies)
+    # Idiosyncratic Component Parameters
     idio_rho0: float = DEFAULT_IDIO_RHO0  # Initial AR coefficient for idiosyncratic components (default: 0.1)
     idio_min_var: float = MIN_DIAGONAL_VARIANCE  # Minimum variance for idiosyncratic innovation covariance (defaults to MIN_DIAGONAL_VARIANCE)
-    # Note: augment_idio and augment_idio_slow are auto-detected from frequency configuration
+    # Note: augment_idio and augment_idio_slow are INTERNALLY auto-detected (not configurable)
     # - If all series use clock frequency: augment_idio=False, augment_idio_slow=False
-    # - If mixed frequencies detected: augment_idio=True, augment_idio_slow=True (auto-enabled)
+    # - If mixed frequencies detected: augment_idio=True, augment_idio_slow=True (required for tent kernel)
+    # These are properties, not config fields - automatically determined from frequency configuration
+    
+    # Kalman Filter Type
+    use_cholesky_filter: bool = False  # Use CholeskyKalmanFilter for better numerical stability (default: False)
+    
+    # Tent Kernel Weights (for mixed-frequency aggregation)
+    tent_weights: Optional[Union[Dict[str, List[float]], Dict[str, np.ndarray]]] = None  # Required for mixed-frequency data: tent weights for frequency pairs. Format: {'freq_pair': [weights]} or {'freq': [weights]}. Example: {'m:w': [1, 2, 1]} or {'m': [1, 2, 1]}. Must be specified in config for all slower-frequency pairs.
     
     def __post_init__(self):
         """Validate blocks structure and derive block properties."""
@@ -446,6 +453,8 @@ class DFMConfig(BaseModelConfig):
             'data_clip': data.get('data_clip', None),
             'regularization': data.get('regularization', None),
             'damping_factor': data.get('damping_factor', None),
+            'use_cholesky_filter': data.get('use_cholesky_filter', False),
+            'tent_weights': data.get('tent_weights', None),
         }
         
         result = base_params.copy()
@@ -524,6 +533,11 @@ class DDFMConfig(BaseModelConfig):
     seed: Optional[int] = None  # Random seed for reproducibility
     lags_input: int = 0  # Number of lags of inputs on encoder (default 0, matching original TensorFlow DDFM)
     
+    # Interpolation parameters (for handling missing values)
+    interpolation_method: str = 'linear'  # Interpolation method: 'linear', 'spline', 'cubic', etc. (default: 'linear' for stability)
+    interpolation_limit: Optional[int] = 10  # Maximum consecutive NaNs to interpolate (default: 10, prevents extreme extrapolation)
+    interpolation_limit_direction: str = 'both'  # Direction to fill: 'forward', 'backward', or 'both' (default: 'both')
+    
     
     # ========================================================================
     # Factory Methods
@@ -550,6 +564,9 @@ class DDFMConfig(BaseModelConfig):
             'disp': DEFAULT_DISP,
             'seed': None,
             'lags_input': 0,  # Number of lags (default 0, matching original TensorFlow)
+            'interpolation_method': 'linear',  # Interpolation method (default: 'linear' for stability)
+            'interpolation_limit': 10,  # Maximum consecutive NaNs to interpolate (default: 10)
+            'interpolation_limit_direction': 'both',  # Direction to fill (default: 'both')
         })
         # Map 'epochs' from config to 'n_mc_samples' for clarity (backward compatibility)
         # Always remove 'epochs' if present (even if n_mc_samples is also present)

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/dataset/ddfm_dataset.py

@@ -21,7 +21,7 @@ from ..config.constants import DEFAULT_TORCH_DTYPE
 class DDFMDataset(Dataset):
     """Dataset for DDFM training.
     
-    Scales target series if scaler provided. Feature series are used as-is.
+    Scales target series if scaler provided. Feature series can also be scaled if provided.
     
     Parameters
     ----------
@@ -33,6 +33,8 @@ class DDFMDataset(Dataset):
         Target series column names to scale. If None or empty, all columns will be used as targets.
     target_scaler : StandardScaler | RobustScaler | MinMaxScaler, optional
         Scaler instance to scale target series. If None, no scaling.
+    feature_scaler : StandardScaler | RobustScaler | MinMaxScaler, optional
+        Scaler instance to scale feature series. If None, no scaling.
     """
     
     def __init__(
@@ -41,6 +43,7 @@ class DDFMDataset(Dataset):
         time_idx: str,
         target_series: Optional[List[str]] = None,
         target_scaler: Optional[Union[StandardScaler, RobustScaler, MinMaxScaler]] = None,
+        feature_scaler: Optional[Union[StandardScaler, RobustScaler, MinMaxScaler]] = None,
     ):
         if _has_polars and isinstance(data, pl.DataFrame):
             data = data.to_pandas()
@@ -70,10 +73,31 @@ class DDFMDataset(Dataset):
         self.target_series = target_series_list
         self.data_original = data.copy()
         self.target_scaler = target_scaler
+        self.feature_scaler = feature_scaler
         
         y = data[target_series_list]
         X = data.drop(columns=target_series_list)
         
+        # Optionally scale feature series to avoid scale dominance in encoder input
+        if feature_scaler is not None and not X.empty:
+            x_mean = X.mean().abs().max()
+            x_std = (X.std() - 1.0).abs().max()
+            x_is_already_scaled = x_mean < 0.1 and x_std < 0.1
+            x_scaler_is_fitted = (
+                hasattr(feature_scaler, 'mean_') and
+                feature_scaler.mean_ is not None and
+                hasattr(feature_scaler, 'scale_') and
+                feature_scaler.scale_ is not None
+            )
+            if not x_is_already_scaled:
+                if not x_scaler_is_fitted:
+                    self.feature_scaler.fit(X.values)
+                X_scaled = self.feature_scaler.transform(X.values)
+                X = pd.DataFrame(X_scaled, index=X.index, columns=X.columns)
+            else:
+                if not x_scaler_is_fitted:
+                    self.feature_scaler.fit(X.values)
+        
         # Check if data is already standardized (mean≈0, std≈1) to avoid double scaling
         # If data is already scaled, we still need target_scaler for inverse transformation
         # but we should NOT scale again
@@ -83,17 +107,58 @@ class DDFMDataset(Dataset):
             y_std = (y.std() - 1.0).abs().max()
             is_already_scaled = y_mean < 0.1 and y_std < 0.1
             
+            # Check if scaler is already fitted
+            scaler_is_fitted = (
+                hasattr(target_scaler, 'mean_') and 
+                target_scaler.mean_ is not None and
+                hasattr(target_scaler, 'scale_') and
+                target_scaler.scale_ is not None
+            )
+            
             if is_already_scaled:
                 # Data is already standardized - don't scale again
-                # But still fit the scaler if not already fitted (for inverse transformation)
+                # IMPORTANT: If scaler is already fitted, do NOT refit it on already-scaled data
                 # The scaler should have been fitted on original unscaled data by the caller
-                if not hasattr(target_scaler, 'mean_') or target_scaler.mean_ is None:
-                    # If scaler not fitted, fit it on current data (assumes caller passed unscaled data)
+                # Refitting on already-scaled data would break inverse transformation
+                if not scaler_is_fitted:
+                    # Only fit if scaler is not already fitted
+                    # This assumes caller passed unscaled data (edge case)
+                    import warnings
+                    warnings.warn(
+                        "DDFMDataset: Data appears already scaled but scaler is not fitted. "
+                        "Fitting scaler on already-scaled data may break inverse transformation. "
+                        "Ensure scaler is fitted on original unscaled data before passing to DDFMDataset.",
+                        UserWarning
+                    )
                     self.target_scaler.fit(y.values)
                 # Don't transform - data is already scaled
             else:
                 # Data is not standardized - apply scaling
-                self.target_scaler.fit(y.values)
+                # If scaler is already fitted, DO NOT refit (preserve training scaler).
+                if scaler_is_fitted:
+                    # Check if scaler was fitted on data with similar scale
+                    # If mean/scale are very different, warn user but keep scaler.
+                    scaler_mean = np.abs(target_scaler.mean_)
+                    scaler_scale = target_scaler.scale_
+                    data_mean = np.abs(y.mean().values)
+                    data_std = y.std().values
+                    
+                    # Check for significant mismatch (more than 2x difference)
+                    mean_mismatch = np.any((data_mean > 2 * scaler_mean) | (scaler_mean > 2 * data_mean))
+                    scale_mismatch = np.any((data_std > 2 * scaler_scale) | (scaler_scale > 2 * data_std))
+                    
+                    if mean_mismatch or scale_mismatch:
+                        import warnings
+                        warnings.warn(
+                            "DDFMDataset: Scaler appears to be fitted on data with different scale. "
+                            "Using existing scaler to preserve inverse transformation. "
+                            "Ensure scaler consistency across train/test.",
+                            UserWarning
+                        )
+                else:
+                    # Scaler not fitted - fit on current data
+                    self.target_scaler.fit(y.values)
+                
                 y_scaled = self.target_scaler.transform(y.values)
                 y = pd.DataFrame(y_scaled, index=y.index, columns=y.columns)
         
@@ -184,13 +249,14 @@ class DDFMDataset(Dataset):
         Returns
         -------
         DDFMDataset
-            New dataset with same time_idx, target_series, target_scaler.
+            New dataset with same time_idx, target_series, target_scaler, feature_scaler.
         """
         return cls(
             data=new_data,
             time_idx=dataset.time_idx,
             target_series=dataset.target_series,
-            target_scaler=dataset.target_scaler
+            target_scaler=dataset.target_scaler,
+            feature_scaler=getattr(dataset, 'feature_scaler', None)
         )
     
     def create_autoencoder_dataset(

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/dataset/dfm_dataset.py

@@ -14,10 +14,9 @@ if TYPE_CHECKING:
 
 from ..logger import get_logger
 from ..config import DFMConfig
-from ..numeric.tent import get_tent_weights, generate_R_mat
+from ..numeric.tent import generate_R_mat
 from ..config.constants import (
     FREQUENCY_HIERARCHY,
-    TENT_WEIGHTS_LOOKUP,
     DEFAULT_HIERARCHY_VALUE,
     DEFAULT_CLOCK_FREQUENCY,
     DEFAULT_NAN_METHOD,
@@ -368,26 +367,72 @@ class DFMDataset:
             self.config._update_idio_flags_from_frequencies(frequencies_list)
         clock_hierarchy = FREQUENCY_HIERARCHY.get(clock, DEFAULT_HIERARCHY_VALUE)
         
-        # Validate frequency pairs
-        missing_pairs = [
-            (freq, clock) for freq in frequencies_set
-                    if FREQUENCY_HIERARCHY.get(freq, DEFAULT_HIERARCHY_VALUE) > clock_hierarchy and get_tent_weights(freq, clock) is None
-        ]
+        # Validate frequency pairs (tent weights must be in config)
+        config_tent_weights = getattr(self.config, 'tent_weights', None)
+        missing_pairs = []
+        for freq in frequencies_set:
+            if FREQUENCY_HIERARCHY.get(freq, DEFAULT_HIERARCHY_VALUE) > clock_hierarchy:
+                # Check if tent weights available in config
+                has_config_weights = False
+                if config_tent_weights is not None:
+                    pair_key = f"{freq}:{clock}"
+                    has_config_weights = (pair_key in config_tent_weights) or (freq in config_tent_weights)
+                
+                if not has_config_weights:
+                    missing_pairs.append((freq, clock))
+        
         if missing_pairs:
             raise DataValidationError(
-                f"Mixed-frequency data detected but the following frequency pairs are not in TENT_WEIGHTS_LOOKUP: {missing_pairs}. "
-                f"Available pairs: {list(TENT_WEIGHTS_LOOKUP.keys())}. "
-                f"Either add the missing pairs to TENT_WEIGHTS_LOOKUP or ensure all series use clock frequency.",
+                f"Mixed-frequency data detected but no tent weights found in config for frequency pairs: {missing_pairs}. "
+                f"Please specify tent_weights in config (e.g., tent_weights: {{'m:w': [1, 2, 1]}}). "
+                f"Alternatively, ensure all series use clock frequency.",
                 details=f"Frequency pairs without tent weights: {missing_pairs}"
             )
         
-        # Get aggregation structure
+        # Get aggregation structure (tent weights must be in config)
         tent_weights_dict = {}
+        config_tent_weights = getattr(self.config, 'tent_weights', None)
+        
+        if config_tent_weights is None:
+            slower_freqs = [freq for freq in frequencies_set 
+                           if FREQUENCY_HIERARCHY.get(freq, DEFAULT_HIERARCHY_VALUE) > clock_hierarchy]
+            if slower_freqs:
+                raise DataValidationError(
+                    f"Mixed-frequency data detected (slower frequencies: {slower_freqs}) but no tent_weights "
+                    f"specified in config. Please specify tent_weights in config for all frequency pairs.",
+                    details=f"Slower frequencies: {slower_freqs}, clock: {clock}"
+                )
+        
         for freq in frequencies_set:
             if FREQUENCY_HIERARCHY.get(freq, DEFAULT_HIERARCHY_VALUE) > clock_hierarchy:
-                tent_w = get_tent_weights(freq, clock)
+                tent_w = None
+                
+                # Get tent weights from config (required)
+                if config_tent_weights is not None:
+                    # Try both formats: 'freq:clock' (e.g., 'm:w') or just 'freq' (e.g., 'm')
+                    pair_key = f"{freq}:{clock}"
+                    if pair_key in config_tent_weights:
+                        tent_w = config_tent_weights[pair_key]
+                    elif freq in config_tent_weights:
+                        tent_w = config_tent_weights[freq]
+                
+                # Tent weights must be in config
+                if tent_w is None:
+                    pair_key = f"{freq}:{clock}"
+                    raise DataValidationError(
+                        f"Tent weights not found in config for frequency pair ({freq}, {clock}). "
+                        f"Please specify tent_weights in config, e.g., tent_weights: {{'{pair_key}': [1, 2, 1]}}",
+                        details=f"Frequency pair: ({freq}, {clock})"
+                    )
+                
+                # Convert to numpy array if found
                 if tent_w is not None:
-                    tent_weights_dict[freq] = np.array(tent_w, dtype=np.float32)
+                    if isinstance(tent_w, list):
+                        tent_weights_dict[freq] = np.array(tent_w, dtype=np.float32)
+                    elif isinstance(tent_w, np.ndarray):
+                        tent_weights_dict[freq] = tent_w.astype(np.float32)
+                    else:
+                        tent_weights_dict[freq] = np.array(tent_w, dtype=np.float32)
         
         # Validate: DFM supports only clock + one slower frequency
         if len(tent_weights_dict) > 1:
@@ -398,6 +443,19 @@ class DFMDataset:
                 details=f"Slower frequencies detected: {slower_freqs}, clock: {clock}"
             )
         
+        # Warn if slower frequency detected but no tent weights found
+        slower_freqs_detected = [freq for freq in frequencies_set 
+                                  if FREQUENCY_HIERARCHY.get(freq, DEFAULT_HIERARCHY_VALUE) > clock_hierarchy]
+        if slower_freqs_detected and not tent_weights_dict:
+            missing_pairs = [(freq, clock) for freq in slower_freqs_detected]
+            from ..logger import get_logger
+            _logger = get_logger(__name__)
+            _logger.warning(
+                f"Mixed-frequency data detected but no tent weights found in config for pairs: {missing_pairs}. "
+                f"Tent weights must be specified in config. "
+                f"This may cause errors in model initialization."
+            )
+        
         # Generate constraint matrices if needed
         R_mat = None
         q = None

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/encoder/pca.py

@@ -153,26 +153,76 @@ class PCAEncoder(BaseEncoder):
             
             # Use SVD for efficient low-rank PCA (NumPy equivalent of torch.pca_lowrank)
             try:
+                # Check for NaN/Inf values before SVD
+                if np.any(~np.isfinite(X_centered)):
+                    raise ValueError("Input contains non-finite values")
+                
+                # Check for zero variance columns (can cause SVD issues)
+                col_stds = np.std(X_centered, axis=0)
+                if np.any(col_stds < 1e-10):
+                    _logger.warning(
+                        f"PCA: Some columns have near-zero variance (min_std={col_stds.min():.2e}). "
+                        "This may cause SVD convergence issues. Consider removing constant columns."
+                    )
+                
                 # Use truncated SVD for efficiency
-                U, S, Vt = np.linalg.svd(X_centered, full_matrices=False)
+                # Use hermitian=False explicitly for clarity
+                U, S, Vt = np.linalg.svd(X_centered, full_matrices=False, hermitian=False)
+                
+                # Validate SVD results
+                if not np.all(np.isfinite(S)):
+                    raise ValueError("SVD produced non-finite singular values")
+                
                 # Take top n_components
-                U = U[:, :self.n_components]
-                S = S[:self.n_components]
-                Vt = Vt[:self.n_components, :]
+                n_components_actual = min(self.n_components, len(S))
+                if n_components_actual < self.n_components:
+                    _logger.warning(
+                        f"PCA: Requested {self.n_components} components but only {len(S)} available. "
+                        f"Using {n_components_actual} components."
+                    )
+                
+                U = U[:, :n_components_actual]
+                S = S[:n_components_actual]
+                Vt = Vt[:n_components_actual, :]
                 
                 # Vt contains the principal components (eigenvectors) as rows
                 # Transpose to get (N x n_components)
                 self.eigenvectors = Vt.T
                 # Convert singular values to eigenvalues
                 self.eigenvalues = S ** 2
+                
             except (ValueError, np.linalg.LinAlgError) as e:
                 # Fallback: compute covariance and use eigendecomposition
-                _logger.warning(f"PCA SVD failed, falling back to eigendecomposition. Error: {type(e).__name__}")
-                T = X_centered.shape[0]
-                self.cov_matrix = (X_centered.T @ X_centered) / (T - 1)
-                self.eigenvalues, self.eigenvectors = compute_principal_components(
-                    self.cov_matrix, self.n_components, block_idx=self.block_idx
+                error_msg = str(e) if str(e) else type(e).__name__
+                _logger.warning(
+                    f"PCA SVD failed ({error_msg}), falling back to eigendecomposition. "
+                    "This may be due to numerical instability or ill-conditioned data."
                 )
+                
+                # Clean data for covariance computation
+                X_centered_clean = np.nan_to_num(X_centered, nan=0.0, posinf=0.0, neginf=0.0)
+                T = X_centered_clean.shape[0]
+                
+                # Compute covariance with regularization for stability
+                cov_raw = (X_centered_clean.T @ X_centered_clean) / max(T - 1, 1)
+                
+                # Add small regularization to diagonal for numerical stability
+                reg = 1e-8 * np.eye(cov_raw.shape[0], dtype=cov_raw.dtype)
+                self.cov_matrix = cov_raw + reg
+                
+                try:
+                    self.eigenvalues, self.eigenvectors = compute_principal_components(
+                        self.cov_matrix, self.n_components, block_idx=self.block_idx
+                    )
+                except (ValueError, np.linalg.LinAlgError) as e2:
+                    # If eigendecomposition also fails, raise with helpful message
+                    raise RuntimeError(
+                        f"PCA failed: Both SVD and eigendecomposition failed. "
+                        f"Original error: {error_msg}, Eigendecomposition error: {str(e2)}. "
+                        "This may indicate severely ill-conditioned data. "
+                        "Consider: (1) Removing constant/near-constant columns, "
+                        "(2) Checking for outliers, (3) Using fewer components."
+                    ) from e2
         
         return self
     

{dfm_python-0.5.57 → dfm_python-0.5.58}/src/dfm_python/functional/dfm_block.py

@@ -8,8 +8,8 @@ This module provides functions for initializing DFM blocks, including:
 """
 
 import numpy as np
-from typing import Optional, Tuple, Dict, Any, List
-from ..numeric.tent import get_tent_weights, generate_tent_weights
+from typing import Optional, Tuple, Dict, Any, List, Callable
+from ..numeric.tent import generate_tent_weights
 from ..numeric.stability import create_scaled_identity
 from ..logger import get_logger
 from ..config.constants import (
@@ -176,7 +176,8 @@ def initialize_block_loadings(
     N: int,
     max_lag_size: int,
     matrix_regularization: Optional[float] = None,
-    dtype: type = np.float32
+    dtype: type = np.float32,
+    impute_func: Optional[callable] = None
 ) -> Tuple[np.ndarray, np.ndarray]:
     """Initialize loadings for a block (clock frequency PCA + slower frequency constrained OLS).
     
@@ -210,6 +211,9 @@ def initialize_block_loadings(
         Regularization for matrix operations
     dtype : type, default np.float32
         Data type
+    impute_func : Optional[Callable[[np.ndarray], np.ndarray]], optional
+        Function to impute NaN values for initialization fallback.
+        Only used when insufficient non-NaN observations for regression.
         
     Returns
     -------
@@ -310,10 +314,17 @@ def initialize_block_loadings(
             series_data = data_with_nans[tent_kernel_size:, series_idx_int]
             non_nan_mask = ~np.isnan(series_data)
             
-            # Use clean data if insufficient non-NaN values
+            # Use imputed data if insufficient non-NaN values (following FRBNY pattern)
             min_required = slower_freq_factors.shape[1] + 2
             if np.sum(non_nan_mask) < min_required:
-                series_data = data_for_extraction[tent_kernel_size:, series_idx_int]
+                # Try data_for_extraction first (may already be imputed)
+                series_data_attempt = data_for_extraction[tent_kernel_size:, series_idx_int]
+                if np.sum(~np.isnan(series_data_attempt)) < min_required and impute_func is not None:
+                    # Fallback: use imputation function (only for initialization)
+                    series_data = impute_func(series_data)
+                    _logger.debug(f"      Using imputed data for series {series_idx_int} (insufficient observations)")
+                else:
+                    series_data = series_data_attempt
                 non_nan_mask = np.ones(len(series_data), dtype=bool)
             
             slower_freq_factors_clean = slower_freq_factors[tent_kernel_size:][non_nan_mask, :]
@@ -341,6 +352,10 @@ def initialize_block_loadings(
                     regularization=reg,
                     dtype=dtype
                 )
+                # Validate loadings are finite
+                if np.any(~np.isfinite(loadings_constrained)):
+                    _logger.warning(f"Computed loadings contain non-finite values for series {series_idx_int}. Skipping.")
+                    continue
                 C_i[series_idx_int, :num_factors * tent_kernel_size] = loadings_constrained
             except (np.linalg.LinAlgError, ValueError) as e:
                 _logger.warning(f"Failed to compute constrained loadings for series {series_idx_int}: {e}. Skipping.")