dfm-python 0.5.56__py3-none-any.whl → 0.5.57__py3-none-any.whl

dfm_python/config/__init__.py

@@ -10,7 +10,7 @@ Note: Series are specified via frequency dict mapping column names to frequencie
 from .schema import (
     BaseModelConfig, DFMConfig, DDFMConfig,
     BaseResult, DFMResult, DDFMResult,
-    DFMFitParams, DDFMFitParams, FitParams,  # FitParams is backward compatibility alias
+    DFMStateSpaceParams, DDFMStateSpaceParams,
 )
 # DEFAULT_BLOCK_NAME is imported lazily where needed to avoid circular imports
 from .constants import (
@@ -46,8 +46,6 @@ from .types import (
     ForecastResult,
     CoefficientMatrix,
     CovarianceMatrix,
-    IRFArray,
-    CompanionMatrix,
     SeriesID,
     Frequency,
     DatasetName,
@@ -58,7 +56,6 @@ from .types import (
     ResultDict,
     ForecastDict,
     MetricsDict,
-    IRFResultDict,
     CheckpointDict,
     ConfigDict,
     Device,
@@ -107,9 +104,8 @@ __all__ = [
     # 'DEFAULT_BLOCK_NAME',  # Removed to avoid circular import - import directly from functional.dfm_block
     # Model-specific configs (from schema.py)
     'DFMConfig', 'DDFMConfig',
-    # Parameter overrides
-    'DFMFitParams', 'DDFMFitParams',
-    'FitParams',  # Backward compatibility alias
+    # State-space parameters
+    'DFMStateSpaceParams', 'DDFMStateSpaceParams',
     # Model-specific results (from results.py)
     'DFMResult', 'DDFMResult',
     # Utilities
@@ -139,8 +135,6 @@ __all__ = [
     'ForecastResult',
     'CoefficientMatrix',
     'CovarianceMatrix',
-    'IRFArray',
-    'CompanionMatrix',
     'SeriesID',
     'Frequency',
     'DatasetName',
@@ -151,7 +145,6 @@ __all__ = [
     'ResultDict',
     'ForecastDict',
     'MetricsDict',
-    'IRFResultDict',
     'CheckpointDict',
     'ConfigDict',
     'Device',

dfm_python/config/adapter.py

@@ -118,7 +118,12 @@ def _extract_frequency_dict(
     data: Dict[str, Any],
     clock: str = DEFAULT_CLOCK_FREQUENCY
 ) -> Optional[Dict[str, str]]:
-    """Extract frequency dict from data, handling both new API and legacy formats.
+    """Extract frequency dict from data, handling multiple formats.
+    
+    Supports:
+    1. Grouped format: {'w': [series1, series2, ...], 'm': [series3, ...]}
+    2. Individual format: {'series1': 'w', 'series2': 'm', ...}
+    3. Legacy format: {'series': [{'series_id': ..., 'frequency': ...}, ...]}
     
     Parameters
     ----------
@@ -132,12 +137,63 @@ def _extract_frequency_dict(
     Optional[Dict[str, str]]
         Frequency dict mapping column names to frequencies, or None if not found
     """
-    if 'frequency' in data:
-        # New API: frequency dict directly
-        return data['frequency']
-    elif 'series' in data:
-        # Legacy: convert series list/dict to frequency dict
-        return _convert_series_to_frequency_dict(data['series'], clock)
+    if 'frequency' not in data:
+        # Legacy: check for 'series' key
+        if 'series' in data:
+            return _convert_series_to_frequency_dict(data['series'], clock)
+        return None
+    
+    freq_data = data['frequency']
+    
+    # Check if it's in grouped format: {'w': [...], 'm': [...]}
+    if _is_dict_like(freq_data):
+        from .constants import VALID_FREQUENCIES
+        
+        # Check if all keys are valid frequency codes (grouped format)
+        keys_are_frequencies = all(
+            isinstance(k, str) and k in VALID_FREQUENCIES 
+            for k in freq_data.keys()
+        )
+        
+        # Check if values are lists (grouped format)
+        values_are_lists = all(
+            isinstance(v, (list, tuple)) for v in freq_data.values()
+        )
+        
+        if keys_are_frequencies and values_are_lists:
+            # Grouped format: convert to individual format
+            result = {}
+            for freq, series_list in freq_data.items():
+                if not isinstance(series_list, (list, tuple)):
+                    _raise_config_error(
+                        f"Frequency '{freq}' must map to a list of series names, got {type(series_list)}"
+                    )
+                for series_name in series_list:
+                    if not isinstance(series_name, str):
+                        _raise_config_error(
+                            f"Series names must be strings, got {type(series_name)} in frequency '{freq}'"
+                        )
+                    if series_name in result:
+                        _raise_config_error(
+                            f"Series '{series_name}' appears in multiple frequency groups"
+                        )
+                    result[series_name] = freq
+            return result
+        else:
+            # Individual format: already in correct format, validate structure
+            result = {}
+            for series_name, freq in freq_data.items():
+                if not isinstance(series_name, str):
+                    _raise_config_error(
+                        f"Frequency dict keys must be strings (series names), got {type(series_name)}"
+                    )
+                if not isinstance(freq, str):
+                    _raise_config_error(
+                        f"Frequency dict values must be strings (frequency codes), got {type(freq)} for series '{series_name}'"
+                    )
+                result[series_name] = freq
+            return result
+    
     return None
 
 

dfm_python/config/constants.py

@@ -23,7 +23,7 @@ DEFAULT_MIN_DELTA = 1e-6  # Minimum change for improvement
 # ============================================================================
 
 # Minimum eigenvalues and variances
-MIN_EIGENVALUE = 1e-8  # Minimum eigenvalue for positive definite matrices
+MIN_EIGENVALUE = 1e-6  # Minimum eigenvalue for positive definite matrices (increased from 1e-8 for better conditioning)
 MIN_DIAGONAL_VARIANCE = 1e-8  # Minimum variance for diagonal elements
 MIN_OBSERVATION_NOISE = 1e-4  # Minimum observation noise for measurement error (used in EM updates)
 DEFAULT_DDFM_OBSERVATION_NOISE = 1e-15  # Default observation noise for DDFM state-space (matches original TensorFlow)
@@ -33,7 +33,7 @@ MIN_STD_FOR_SCALE_CHECK = 1e-10  # Minimum standard deviation for scale ratio co
 DEFAULT_VARIANCE_FALLBACK = 1.0  # Default variance fallback value for numerical stability
 
 # Maximum eigenvalues
-MAX_EIGENVALUE = 1e6  # Maximum eigenvalue cap
+MAX_EIGENVALUE = 1e3  # Maximum eigenvalue cap (reduced to 1e3 to improve condition number: 1e3/1e-6 = 1e9)
 
 # Eigenvalue stability thresholds
 DEFAULT_EIGENVALUE_MAX_MAGNITUDE = 1.0  # Default maximum eigenvalue magnitude for stability checks
@@ -217,6 +217,7 @@ DEFAULT_START_DATE = datetime(2000, 1, 1)
 # Default window size for DDFM
 DEFAULT_WINDOW_SIZE = 100
 DEFAULT_TENT_KERNEL_SIZE = 5  # Default tent kernel size for slower-frequency series aggregation
+DEFAULT_TENT_KERNEL_REGULARIZATION_MULTIPLIER = 100.0  # Regularization multiplier for tent kernel series (handles ill-conditioning)
 # Note: DEFAULT_BATCH_SIZE is defined above (line 88) as 32 for general neural networks
 # Use DEFAULT_DDFM_WINDOW_SIZE (100) for DDFM-specific window size (time-step batch size)
 
@@ -284,7 +285,7 @@ DEFAULT_PROGRESS_LOG_INTERVAL = 5  # Default progress logging interval for EM al
 # Precision Defaults
 # ============================================================================
 
-DEFAULT_DTYPE = np.float32  # Default numpy dtype for arrays
+DEFAULT_DTYPE = np.float64  # Default numpy dtype for arrays (float64 for better numerical stability)
 
 # PyTorch dtype (matches DEFAULT_DTYPE)
 try:
@@ -518,7 +519,6 @@ __all__ = [
     'MIN_VARIABLES',
     'MIN_SHAPE_FOR_AR2',
     'MIN_DDFM_TIME_STEPS',
-    'MIN_SHAPE_FOR_AR2',
     'MIN_DDFM_DATASET_SIZE_WARNING',
     'MIN_ITER_FOR_DELTA_COMPUTATION',
     'DEFAULT_MIN_ITER_FOR_CONVERGENCE_CHECK',
@@ -546,6 +546,7 @@ __all__ = [
     'DEFAULT_LOG_INTERVAL',
     'DEFAULT_PROGRESS_LOG_INTERVAL',
     'DEFAULT_TENT_KERNEL_SIZE',
+    'DEFAULT_TENT_KERNEL_REGULARIZATION_MULTIPLIER',
     'MAX_ERROR_ITEMS',
     # IRF
     'DEFAULT_IRF_HORIZON',

dfm_python/config/schema/__init__.py

@@ -9,15 +9,11 @@ Note: Series are specified via frequency dict mapping column names to frequencie
 
 from .model import BaseModelConfig, DFMConfig, DDFMConfig
 from .results import BaseResult, DFMResult, DDFMResult
-from .params import DFMFitParams, DDFMFitParams
-
-# Backward compatibility alias
-FitParams = DFMFitParams
+from .params import DFMStateSpaceParams, DDFMStateSpaceParams
 
 __all__ = [
     'BaseModelConfig', 'DFMConfig', 'DDFMConfig',
     'BaseResult', 'DFMResult', 'DDFMResult',
-    'DFMFitParams', 'DDFMFitParams',
-    'FitParams',  # Backward compatibility alias
+    'DFMStateSpaceParams', 'DDFMStateSpaceParams',
 ]
 

dfm_python/config/schema/block.py

@@ -5,8 +5,8 @@ parameters in EM algorithm, replacing long conditional checks with a single
 optional parameter.
 """
 
-from dataclasses import dataclass
-from typing import Optional, Dict
+from dataclasses import dataclass, field
+from typing import Optional, Dict, List
 import numpy as np
 
 
@@ -26,7 +26,10 @@ class BlockStructure:
     p : int
         VAR lag order
     p_plus_one : int
-        p + 1 (state dimension per factor)
+        State dimension per factor. This is equal to max_lag_size = max(p + 1, tent_kernel_size).
+        The name "p_plus_one" is used for backward compatibility with the EM algorithm interface.
+        In the model code, this is called "max_lag_size" to more clearly indicate it accounts
+        for tent kernel size when tent_kernel_size > p + 1.
     n_clock_freq : int
         Number of clock-frequency series
     idio_indicator : np.ndarray
@@ -39,6 +42,30 @@ class BlockStructure:
         Number of slower-frequency series
     tent_weights_dict : dict, optional
         Dictionary mapping frequency pairs to tent weights
+    
+    Cached indices (computed once, reused across EM iterations):
+    _cached_unique_blocks : List[np.ndarray], optional
+        Unique block patterns (cached)
+    _cached_unique_indices : List[int], optional
+        Indices of unique block patterns (cached)
+    _cached_bl_idxM : List[np.ndarray], optional
+        Clock-frequency factor loadings indices per unique block pattern (cached)
+    _cached_bl_idxQ : List[np.ndarray], optional
+        Slower-frequency factor loadings indices per unique block pattern (cached)
+    _cached_R_con : np.ndarray, optional
+        Block diagonal constraint matrix (cached)
+    _cached_q_con : np.ndarray, optional
+        Constraint vector (cached)
+    _cached_total_factor_dim : int, optional
+        Total factor state dimension (cached)
+    _cached_idio_indicator_M : np.ndarray, optional
+        Clock-frequency idiosyncratic component indicator (cached)
+    _cached_n_idio_M : int, optional
+        Number of clock-frequency idiosyncratic components (cached)
+    _cached_c_idio_indicator : np.ndarray, optional
+        Cumulative sum of idiosyncratic component indicator (cached)
+    _cached_rp1 : int, optional
+        Start of idiosyncratic components in state space (cached)
     """
     blocks: np.ndarray
     r: np.ndarray
@@ -51,6 +78,19 @@ class BlockStructure:
     n_slower_freq: Optional[int] = None
     tent_weights_dict: Optional[Dict[str, np.ndarray]] = None
     
+    # Cached indices (computed once, reused across EM iterations)
+    _cached_unique_blocks: Optional[List[np.ndarray]] = field(default=None, init=False, repr=False)
+    _cached_unique_indices: Optional[List[int]] = field(default=None, init=False, repr=False)
+    _cached_bl_idxM: Optional[List[np.ndarray]] = field(default=None, init=False, repr=False)
+    _cached_bl_idxQ: Optional[List[np.ndarray]] = field(default=None, init=False, repr=False)
+    _cached_R_con: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_q_con: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_total_factor_dim: Optional[int] = field(default=None, init=False, repr=False)
+    _cached_idio_indicator_M: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_n_idio_M: Optional[int] = field(default=None, init=False, repr=False)
+    _cached_c_idio_indicator: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_rp1: Optional[int] = field(default=None, init=False, repr=False)
+    
     def is_valid(self) -> bool:
         """Check if block structure is valid (all required fields are not None)."""
         return (
@@ -61,4 +101,22 @@ class BlockStructure:
             and self.n_clock_freq is not None
             and self.idio_indicator is not None
         )
+    
+    def has_cached_indices(self) -> bool:
+        """Check if block structure indices are cached."""
+        return self._cached_unique_blocks is not None
+    
+    def clear_cache(self) -> None:
+        """Clear all cached indices to ensure clean state for new training runs."""
+        self._cached_unique_blocks = None
+        self._cached_unique_indices = None
+        self._cached_bl_idxM = None
+        self._cached_bl_idxQ = None
+        self._cached_R_con = None
+        self._cached_q_con = None
+        self._cached_total_factor_dim = None
+        self._cached_idio_indicator_M = None
+        self._cached_n_idio_M = None
+        self._cached_c_idio_indicator = None
+        self._cached_rp1 = None
 

dfm_python/config/schema/model.py

@@ -91,7 +91,9 @@ class BaseModelConfig:
     - Data preprocessing (missing data handling)
     
     Series Configuration:
-    - Provide `frequency` dict: {'column_name': 'frequency_code'} to specify per-series frequencies
+    - Provide `frequency` dict in one of two formats:
+      1. Grouped format: {'w': [series1, series2, ...], 'm': [series3, ...]} (recommended for large configs)
+      2. Individual format: {'series1': 'w', 'series2': 'm', ...} (backward compatible)
     - If `frequency` is None, all columns will use `clock` frequency
     - If a column is missing from `frequency` dict, it will use `clock` frequency
     - When data is loaded, missing columns in `frequency` dict are automatically added with `clock` frequency
@@ -103,7 +105,14 @@ class BaseModelConfig:
     
     Examples
     --------
-    >>> # With explicit frequency mapping
+    >>> # With grouped frequency mapping (recommended for large configs)
+    >>> config = DFMConfig(
+    ...     frequency={'q': ['gdp'], 'm': ['unemployment', 'interest_rate']},
+    ...     clock='m',
+    ...     blocks={...}
+    ... )
+    >>> 
+    >>> # With individual frequency mapping (backward compatible)
     >>> config = DFMConfig(
     ...     frequency={'gdp': 'q', 'unemployment': 'm', 'interest_rate': 'm'},
     ...     clock='m',

dfm_python/config/schema/params.py

@@ -1,37 +1,199 @@
 """Model parameters for DFM and DDFM.
 
-This module defines dataclasses for storing fit-time parameters and state-space parameters.
+This module defines dataclasses for storing state-space parameters and model structure.
 """
 
 from dataclasses import dataclass
 import numpy as np
-from typing import Optional
+from typing import Optional, Any
 
 
 @dataclass
-class DFMFitParams:
-    """Fit-time parameter overrides for DFM estimation.
+class DFMStateSpaceParams:
+    """State-space model parameters for DFM (fitted during model training).
     
-    This dataclass allows overriding configuration parameters at fit time
-    without modifying the configuration object itself.
+    These parameters are computed during DFM.fit() and represent the fitted
+    state-space model structure. Contains only the state-space parameters,
+    not training metadata (loglik, num_iter, converged) which are stored
+    separately in checkpoint dicts.
     
-    Based on DFM.fit() usage, these parameters are passed to the EM algorithm.
+    Parameters
+    ----------
+    A : np.ndarray
+        Transition matrix (m x m) - VAR dynamics for factors
+    C : np.ndarray
+        Observation matrix (N x m) - factor loadings
+    Q : np.ndarray
+        Process noise covariance (m x m) - innovation covariance
+    R : np.ndarray
+        Observation noise covariance (N x N) - typically diagonal
+    Z_0 : np.ndarray
+        Initial state mean (m,) - initial factor values
+    V_0 : np.ndarray
+        Initial state covariance (m x m) - initial uncertainty
+    """
+    A: np.ndarray  # Transition matrix (m x m)
+    C: np.ndarray  # Observation matrix (N x m)
+    Q: np.ndarray  # Process noise covariance (m x m)
+    R: np.ndarray  # Observation noise covariance (N x N)
+    Z_0: np.ndarray  # Initial state mean (m,)
+    V_0: np.ndarray  # Initial state covariance (m x m)
+    
+    @classmethod
+    def from_model(cls, model: Any) -> 'DFMStateSpaceParams':
+        """Create DFMStateSpaceParams from DFM model instance.
+        
+        Parameters
+        ----------
+        model : DFM
+            DFM model instance with fitted parameters
+            
+        Returns
+        -------
+        DFMStateSpaceParams
+            State-space parameters dataclass
+        """
+        # Get parameters from model (stored after fit)
+        # First check if training_state is already DFMStateSpaceParams (new format)
+        if hasattr(model, 'training_state') and model.training_state is not None:
+            if isinstance(model.training_state, cls):
+                # Already a DFMStateSpaceParams instance
+                return model.training_state
+            # Backward compatibility: If old format training_state (has A, C, Q, R, Z_0, V_0 attributes)
+            if hasattr(model.training_state, 'A'):
+                return cls(
+                    A=model.training_state.A,
+                    C=model.training_state.C,
+                    Q=model.training_state.Q,
+                    R=model.training_state.R,
+                    Z_0=model.training_state.Z_0,
+                    V_0=model.training_state.V_0
+                )
+        
+        # Get from model attributes (after _update_parameters is called)
+        return cls(
+            A=getattr(model, 'A', None),
+            C=getattr(model, 'C', None),
+            Q=getattr(model, 'Q', None),
+            R=getattr(model, 'R', None),
+            Z_0=getattr(model, 'Z_0', None),
+            V_0=getattr(model, 'V_0', None)
+        )
+    
+    def apply_to_model(self, model: Any) -> None:
+        """Apply state-space parameters to DFM model instance.
+        
+        Parameters
+        ----------
+        model : DFM
+            DFM model instance to update
+        """
+        model._update_parameters(
+            self.A, self.C, self.Q, self.R, self.Z_0, self.V_0
+        )
+
+
+@dataclass
+class DFMModelState:
+    """DFM model structure and mixed-frequency parameters.
+    
+    Consolidates model structure and mixed-frequency state for checkpointing.
+    This allows save/load methods to be simplified by using a single dataclass.
     
     Attributes
     ----------
-    max_iter : int, optional
-        Maximum EM iterations (overrides config.max_iter)
-        Used in: kalman_filter.em(max_iter=...)
-    threshold : float, optional
-        Convergence threshold (overrides config.threshold)
-        Used in: kalman_filter.em(threshold=...)
-    regularization_scale : float, optional
-        Regularization scale for matrix operations (overrides EM config)
-        Note: This affects EMConfig.regularization and EMConfig.matrix_regularization
+    num_factors : int
+        Number of factors
+    r : np.ndarray
+        Number of factors per block
+    p : int
+        AR lag order
+    blocks : np.ndarray
+        Block structure array
+    mixed_freq : bool, optional
+        Whether mixed-frequency data is used
+    constraint_matrix : np.ndarray, optional
+        Constraint matrix for tent kernel aggregation
+    constraint_vector : np.ndarray, optional
+        Constraint vector for tent kernel aggregation
+    n_slower_freq : int
+        Number of slower-frequency series
+    n_clock_freq : int, optional
+        Number of clock-frequency series
+    tent_weights_dict : dict, optional
+        Dictionary of tent weights by frequency
+    frequencies : np.ndarray, optional
+        Frequency array for each series
+    idio_indicator : np.ndarray, optional
+        Indicator for idiosyncratic components
+    max_lag_size : int, optional
+        Maximum lag size for state dimension
     """
-    max_iter: Optional[int] = None
-    threshold: Optional[float] = None
-    regularization_scale: Optional[float] = None
+    num_factors: int
+    r: np.ndarray
+    p: int
+    blocks: np.ndarray
+    mixed_freq: Optional[bool] = None
+    constraint_matrix: Optional[np.ndarray] = None
+    constraint_vector: Optional[np.ndarray] = None
+    n_slower_freq: int = 0
+    n_clock_freq: Optional[int] = None
+    tent_weights_dict: Optional[dict] = None
+    frequencies: Optional[np.ndarray] = None
+    idio_indicator: Optional[np.ndarray] = None
+    max_lag_size: Optional[int] = None
+    
+    @classmethod
+    def from_model(cls, model: Any) -> 'DFMModelState':
+        """Create DFMModelState from DFM model instance.
+        
+        Parameters
+        ----------
+        model : DFM
+            DFM model instance
+            
+        Returns
+        -------
+        DFMModelState
+            Model state dataclass
+        """
+        return cls(
+            num_factors=model.num_factors,
+            r=model.r,
+            p=model.p,
+            blocks=model.blocks,
+            mixed_freq=getattr(model, '_mixed_freq', None),
+            constraint_matrix=getattr(model, '_constraint_matrix', None),
+            constraint_vector=getattr(model, '_constraint_vector', None),
+            n_slower_freq=getattr(model, '_n_slower_freq', 0),
+            n_clock_freq=getattr(model, '_n_clock_freq', None),
+            tent_weights_dict=getattr(model, '_tent_weights_dict', None),
+            frequencies=getattr(model, '_frequencies', None),
+            idio_indicator=getattr(model, '_idio_indicator', None),
+            max_lag_size=getattr(model, '_max_lag_size', None)
+        )
+    
+    def apply_to_model(self, model: Any) -> None:
+        """Apply this state to DFM model instance.
+        
+        Parameters
+        ----------
+        model : DFM
+            DFM model instance to update
+        """
+        model.num_factors = self.num_factors
+        model.r = self.r
+        model.p = self.p
+        model.blocks = self.blocks
+        model._mixed_freq = self.mixed_freq
+        model._constraint_matrix = self.constraint_matrix
+        model._constraint_vector = self.constraint_vector
+        model._n_slower_freq = self.n_slower_freq
+        model._n_clock_freq = self.n_clock_freq
+        model._tent_weights_dict = self.tent_weights_dict
+        model._frequencies = self.frequencies
+        model._idio_indicator = self.idio_indicator
+        model._max_lag_size = self.max_lag_size
 
 @dataclass
 class DDFMTrainingState:
@@ -62,13 +224,36 @@ class DDFMTrainingState:
     eps: Optional[np.ndarray] = None
     factors: Optional[np.ndarray] = None
     last_neurons: Optional[np.ndarray] = None
+    
+    def sync_from_model(self, model: Any) -> 'DDFMTrainingState':
+        """Update this dataclass from model instance attributes.
+        
+        Parameters
+        ----------
+        model : DDFM
+            DDFM model instance
+            
+        Returns
+        -------
+        DDFMTrainingState
+            Self (for chaining)
+        """
+        if hasattr(model, '_has_factors') and model._has_factors:
+            self.factors = getattr(model, 'factors', None)
+            self.eps = getattr(model, 'eps', None)
+            self.last_neurons = getattr(model, 'last_neurons', None)
+            self.num_iter = getattr(model, '_num_iter', 0)
+            self.loss_now = getattr(model, 'loss_now', None)
+            self.converged = getattr(model, '_converged', False)
+        return self
 
 @dataclass
-class DDFMFitParams:
-    """State-space model parameters for DDFM (created during fit).
+class DDFMStateSpaceParams:
+    """State-space model parameters for DDFM (fitted during model training).
     
     These parameters are computed during DDFM.build_state_space() and represent
-    the fitted state-space model structure. Naming follows original paper conventions.
+    the fitted state-space model structure. Similar to DFMStateSpaceParams, this contains
+    the fitted state-space parameters but for DDFM. Naming follows original paper conventions.
     
     Parameters
     ----------