dfm-python 0.5.60__py3-none-any.whl → 0.6.0__py3-none-any.whl

dfm_python/config/adapter.py

@@ -508,18 +508,16 @@ def make_config_source(
 # ============================================================================
 
 # Model type constants (exported for use in schema/model.py)
-MODEL_TYPE_KDFM = 'kdfm'
+# NOTE: KDFM is legacy and has been removed from this codebase.
 MODEL_TYPE_DDFM = 'ddfm'
 MODEL_TYPE_DFM = 'dfm'
 
 # Model type detection patterns
-_KDFM_TYPE_ALIASES = {'kdfm', 'kernelized'}
 _DDFM_TYPE_ALIASES = {'ddfm', 'deep'}
-_KDFM_PARAMS = {'ar_order', 'ma_order', 'structural_method', 'structural_reg_weight'}
 _DDFM_PARAMS = {'encoder_layers', 'epochs', 'learning_rate', 'batch_size'}
 
 def detect_config_type(data: Dict[str, Any]) -> str:
-    """Detect config type (DFM, DDFM, or KDFM) from data dictionary.
+    """Detect config type (DFM or DDFM) from data dictionary.
     
     Parameters
     ----------
@@ -529,7 +527,7 @@ def detect_config_type(data: Dict[str, Any]) -> str:
     Returns
     -------
     str
-        'kdfm', 'ddfm', or 'dfm'
+        'ddfm' or 'dfm'
         
     Raises
     ------
@@ -544,8 +542,6 @@ def detect_config_type(data: Dict[str, Any]) -> str:
     model_type = data.get('model_type', '').lower()
     
     # Check explicit model type
-    if model_type in _KDFM_TYPE_ALIASES:
-        return MODEL_TYPE_KDFM
     if model_type in _DDFM_TYPE_ALIASES:
         return MODEL_TYPE_DDFM
     if model_type == MODEL_TYPE_DFM:
@@ -558,8 +554,6 @@ def detect_config_type(data: Dict[str, Any]) -> str:
     
     # Check for model-specific parameters
     keys = set(data.keys())
-    if keys & _KDFM_PARAMS:
-        return MODEL_TYPE_KDFM
     if any(k.startswith('ddfm_') for k in keys) or (keys & _DDFM_PARAMS):
         return MODEL_TYPE_DDFM
     

dfm_python/config/constants.py

@@ -35,6 +35,29 @@ DEFAULT_VARIANCE_FALLBACK = 1.0  # Default variance fallback value for numerical
 # Maximum eigenvalues
 MAX_EIGENVALUE = 1e3  # Maximum eigenvalue cap (reduced to 1e3 to improve condition number: 1e3/1e-6 = 1e9)
 
+# ============================================================================
+# Performance and Numerical Health Guardrails
+# ============================================================================
+
+# Time budgets (hard limits to prevent pathological runs)
+# Single constraint: 10 minutes for all datasets
+MAX_ESTEP_SECONDS = 600.0  # Maximum E-step time (Kalman filter + smoother) - 10 minutes
+MAX_MSTEP_SECONDS = 600.0  # Maximum M-step time (parameter updates) - 10 minutes
+MAX_EM_ITERATION_SECONDS = 1200.0  # Maximum total EM iteration time - 20 minutes
+MAX_FIRST_ITERATION_SECONDS = 600.0  # Maximum first iteration E-step time (diagnostic) - 10 minutes
+
+# Numerical health thresholds (fail fast on pathological regimes)
+# Note: Condition number check is more lenient during initialization (first iteration)
+MAX_CONDITION_NUMBER_SMOOTHER = 1e8  # Maximum condition number before refusing to smooth (increased from 1e7)
+MAX_CONDITION_NUMBER_INIT = 1e10  # Maximum condition number during initialization (more lenient)
+MAX_STABILIZATION_AMOUNT = 1e-3  # Maximum stabilization amount before error (increased from 1e-4 to 0.1% diagonal loading)
+MAX_STABILIZATION_AMOUNT_INIT = 5e-3  # Maximum stabilization during initialization (more lenient)
+MAX_STD_UNSCALED = 100.0  # Maximum standard deviation for unscaled data (indicates missing preprocessing)
+MIN_STD_SCALED = 1e-4  # Minimum standard deviation for scaled data (indicates degenerate series)
+
+# Fallback tracking
+MAX_FALLBACKS_PER_ITERATION = 0  # Maximum allowed fallbacks per iteration (0 = fail on any fallback)
+
 # Eigenvalue stability thresholds
 DEFAULT_EIGENVALUE_MAX_MAGNITUDE = 1.0  # Default maximum eigenvalue magnitude for stability checks
 DEFAULT_EIGENVALUE_WARN_THRESHOLD = 0.99  # Default warning threshold for near-unstable eigenvalues
@@ -566,5 +589,17 @@ __all__ = [
     # Frequency hierarchy and tent kernels
     'FREQUENCY_HIERARCHY',
     'MAX_TENT_SIZE',
+    # Performance and numerical health guardrails
+    'MAX_ESTEP_SECONDS',
+    'MAX_MSTEP_SECONDS',
+    'MAX_EM_ITERATION_SECONDS',
+    'MAX_FIRST_ITERATION_SECONDS',
+    'MAX_CONDITION_NUMBER_SMOOTHER',
+    'MAX_CONDITION_NUMBER_INIT',
+    'MAX_STABILIZATION_AMOUNT',
+    'MAX_STABILIZATION_AMOUNT_INIT',
+    'MAX_STD_UNSCALED',
+    'MIN_STD_SCALED',
+    'MAX_FALLBACKS_PER_ITERATION',
 ]
 

dfm_python/config/schema/model.py

@@ -418,16 +418,32 @@ class DFMConfig(BaseModelConfig):
             series_ids = list(self.frequency.keys()) if columns is None else columns
             
             # Build blocks array from block series lists (N x B matrix)
-            block_series_sets = {
-                name: set(self.blocks[name].get('series', []))
-                for name in self.block_names
-            }
+            # Default global block behavior:
+            # If config omitted blocks.series (empty list) and we have runtime columns,
+            # interpret a single empty Block_Global as "all series in this block".
+            block_series_sets: Dict[str, set] = {}
+            default_global_applied = False
+            for name in self.block_names:
+                series_list = self.blocks[name].get('series', [])
+                if (
+                    columns is not None
+                    and (series_list is None or len(series_list) == 0)
+                    and len(self.block_names) == 1
+                    and name == "Block_Global"
+                ):
+                    block_series_sets[name] = set(series_ids)
+                    default_global_applied = True
+                else:
+                    block_series_sets[name] = set(series_list or [])
+
             blocks_list = [
                 [1 if series_id in block_series_sets[name] else 0 for name in self.block_names]
                 for series_id in series_ids
             ]
             
             self._cached_blocks = np.array(blocks_list, dtype=int)
+
+            # Quick sanity on membership counts for the first block (if any)
         return self._cached_blocks
     
     @classmethod
@@ -463,14 +479,14 @@ class DFMConfig(BaseModelConfig):
         return result
     
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> Union['DFMConfig', 'DDFMConfig', 'KDFMConfig']:
-        """Create DFMConfig, DDFMConfig, or KDFMConfig from dictionary.
+    def from_dict(cls, data: Dict[str, Any]) -> Union['DFMConfig', 'DDFMConfig']:
+        """Create DFMConfig or DDFMConfig from dictionary.
         
         Expected format: {'frequency': {'column_name': 'frequency'}, 'blocks': {...}, ...}
         
         Also accepts estimation parameters: threshold, max_iter, etc.
         """
-        from ...config.adapter import detect_config_type, MODEL_TYPE_DDFM, MODEL_TYPE_KDFM, _normalize_blocks_dict
+        from ...config.adapter import detect_config_type, MODEL_TYPE_DDFM, _normalize_blocks_dict
         from ...utils.errors import ConfigurationError
         
         # Extract base params (handles frequency conversion from series if needed)
@@ -482,14 +498,24 @@ class DFMConfig(BaseModelConfig):
         if config_type == MODEL_TYPE_DDFM:
             return DDFMConfig(**base_params, **DDFMConfig._extract_ddfm(data))
         
-        if config_type == MODEL_TYPE_KDFM:
-            return KDFMConfig(**base_params, **KDFMConfig._extract_kdfm(data))
-        
         # Handle blocks for DFM
         from ...config.adapter import _raise_config_error, _is_dict_like
         blocks_dict = data.get('blocks', {})
         if not blocks_dict:
-            _raise_config_error("blocks dict is required for DFM config")
+            # Default behavior: single global block.
+            # This enables minimal configs like:
+            #   clock: d
+            #   num_factors: 2
+            # and lets the model bind series names from the dataset at runtime.
+            default_num_factors = int(data.get('num_factors', 1) or 1)
+            if default_num_factors < 1:
+                _raise_config_error(f"num_factors must be >= 1, got {default_num_factors}")
+            blocks_dict = {
+                "Block_Global": {
+                    "num_factors": default_num_factors,
+                    "series": [],  # to be bound from dataset columns
+                }
+            }
         if not _is_dict_like(blocks_dict):
             _raise_config_error(f"blocks must be a dict, got {type(blocks_dict)}")
         
@@ -600,72 +626,6 @@ class DDFMConfig(BaseModelConfig):
         )
 
 
-@dataclass
-class KDFMConfig(BaseModelConfig):
-    """KDFM configuration dataclass.
-    
-    This dataclass contains all configuration parameters for the KDFM model.
-    It inherits from BaseModelConfig and adds KDFM-specific parameters.
-    
-    Note: KDFM does not use blocks structure (unlike DFM). Only frequency dict is needed.
-    """
-    # VARMA parameters
-    ar_order: int = DEFAULT_KDFM_AR_ORDER  # VAR order p
-    ma_order: int = DEFAULT_KDFM_MA_ORDER  # MA order q (0 = pure VAR)
-    
-    # Structural identification
-    structural_method: str = 'cholesky'  # 'cholesky', 'full', 'low_rank'
-    structural_rank: Optional[int] = None  # For low-rank parameterization
-    
-    # Training parameters (use constants for defaults)
-    learning_rate: float = DEFAULT_LEARNING_RATE
-    max_epochs: int = DEFAULT_MAX_EPOCHS
-    batch_size: int = DEFAULT_BATCH_SIZE
-    weight_decay: float = DEFAULT_REGULARIZATION_SCALE
-    grad_clip_val: float = DEFAULT_GRAD_CLIP_VAL
-    
-    # Regularization
-    structural_reg_weight: float = DEFAULT_STRUCTURAL_REG_WEIGHT  # Weight for structural loss
-    use_regularization: bool = True
-    regularization_scale: float = DEFAULT_REGULARIZATION_SCALE
-    
-    # ========================================================================
-    # Factory Methods
-    # ========================================================================
-    
-    @classmethod
-    def _extract_kdfm(cls, data: Dict[str, Any]) -> Dict[str, Any]:
-        """Extract KDFM-specific parameters from config dict."""
-        # Don't extract base params here - they're already in base_params from from_dict
-        kdfm_params = cls._extract_params(data, {
-            'ar_order': DEFAULT_KDFM_AR_ORDER,
-            'ma_order': DEFAULT_KDFM_MA_ORDER,
-            'structural_method': 'cholesky',
-            'structural_rank': None,
-            'learning_rate': DEFAULT_LEARNING_RATE,
-            'max_epochs': DEFAULT_MAX_EPOCHS,
-            'batch_size': DEFAULT_BATCH_SIZE,
-            'weight_decay': DEFAULT_REGULARIZATION_SCALE,
-            'grad_clip_val': DEFAULT_GRAD_CLIP_VAL,
-            'structural_reg_weight': DEFAULT_STRUCTURAL_REG_WEIGHT,
-            'use_regularization': True,
-            'regularization_scale': DEFAULT_REGULARIZATION_SCALE,
-        })
-        return kdfm_params
-    
-    @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> 'KDFMConfig':
-        """Create KDFMConfig from dictionary (delegates to DFMConfig.from_dict for type detection)."""
-        result = DFMConfig.from_dict(data)
-        if isinstance(result, KDFMConfig):
-            return result
-        from ..utils.errors import ConfigurationError
-        raise ConfigurationError(
-            f"Expected KDFMConfig but got {type(result).__name__}",
-            details=f"Result type: {type(result).__name__}, expected: KDFMConfig"
-        )
-
-
 # ============================================================================
 # Validation Functions
 # ============================================================================

dfm_python/dataset/dfm_dataset.py

@@ -22,6 +22,7 @@ if TYPE_CHECKING:
 
 from ..logger import get_logger
 from ..config import DFMConfig
+from ..config.constants import FREQUENCY_HIERARCHY, DEFAULT_HIERARCHY_VALUE
 from ..dataset.time import TimeIndex
 from ..utils.errors import DataValidationError, ConfigurationError
 
@@ -139,9 +140,13 @@ class DFMDataset:
         
         # Get all available columns (numeric only)
         numeric_columns = list(X_df.select_dtypes(include=[np.number]).columns)
+        
+        # Reorder columns for mixed-frequency models: clock-frequency first, then slower-frequency
+        numeric_columns = self._reorder_columns_for_mixed_freq(numeric_columns, X_df)
+        
         self._columns = numeric_columns
         
-        # Store DataFrame with time index (numeric columns only)
+        # Store DataFrame with time index (numeric columns only, properly ordered)
         self.data_df = X_df[numeric_columns]
     
     @property
@@ -154,6 +159,88 @@ class DFMDataset:
         """Get time index."""
         return self._time_index
     
+    def _reorder_columns_for_mixed_freq(
+        self, 
+        columns: List[str], 
+        data_df: pd.DataFrame
+    ) -> List[str]:
+        """Reorder columns for mixed-frequency models.
+        
+        Ensures columns are ordered as [clock-frequency series | slower-frequency series]
+        as required by the mixed-frequency model implementation.
+        
+        Parameters
+        ----------
+        columns : List[str]
+            Current column list
+        data_df : pd.DataFrame
+            DataFrame with data (for validation)
+            
+        Returns
+        -------
+        List[str]
+            Reordered column list (clock-frequency first, then slower-frequency)
+        """
+        # Check if config has frequency mapping
+        if not self.config.frequency:
+            return columns  # No frequency mapping, return as-is
+        
+        # Get clock frequency
+        clock = self.config.clock
+        
+        # Build frequency dict from config
+        # Handle both grouped format {'w': [series1, ...], 'm': [series2, ...]}
+        # and individual format {'series1': 'w', 'series2': 'm', ...}
+        frequency_dict = {}
+        if isinstance(self.config.frequency, dict):
+            for key, value in self.config.frequency.items():
+                if isinstance(value, list):
+                    # Grouped format: {'w': [series1, ...], 'm': [series2, ...]}
+                    for series in value:
+                        if series in columns:
+                            frequency_dict[series] = key
+                else:
+                    # Individual format: {'series1': 'w', 'series2': 'm', ...}
+                    if key in columns:
+                        frequency_dict[key] = value
+        
+        # If no frequency mapping found for any columns, return as-is
+        if not frequency_dict:
+            return columns
+        
+        # Get clock hierarchy value
+        clock_hierarchy = FREQUENCY_HIERARCHY.get(clock, DEFAULT_HIERARCHY_VALUE)
+        
+        # Separate columns by frequency
+        clock_cols = []
+        slower_cols = []
+        
+        for col in columns:
+            freq = frequency_dict.get(col)
+            if freq is None:
+                # Column not in frequency mapping - assume clock frequency
+                clock_cols.append(col)
+            else:
+                freq_hierarchy = FREQUENCY_HIERARCHY.get(freq, DEFAULT_HIERARCHY_VALUE)
+                if freq_hierarchy <= clock_hierarchy:
+                    # Clock frequency or same as clock
+                    clock_cols.append(col)
+                else:
+                    # Slower frequency
+                    slower_cols.append(col)
+        
+        # Reorder: clock-frequency first, then slower-frequency
+        reordered = clock_cols + slower_cols
+        
+        # Log if reordering occurred
+        if reordered != columns:
+            _logger.info(
+                f"Reordered columns for mixed-frequency model: "
+                f"{len(clock_cols)} clock-frequency, {len(slower_cols)} slower-frequency"
+            )
+        
+        return reordered
+    
     def _validate(self) -> None:
         """Validate dataset state and data integrity.
         

dfm_python/encoder/pca.py

@@ -56,8 +56,11 @@ def compute_principal_components(
     
     n_series = cov_matrix.shape[0]
     
+    
+    
     try:
         eigenvalues, eigenvectors = np.linalg.eigh(cov_matrix)
+        
         # Sort by absolute value, descending
         sort_idx = np.argsort(np.abs(eigenvalues))[::-1][:n_components]
         eigenvalues_sorted = eigenvalues[sort_idx]