dfm-python 0.5.56__py3-none-any.whl → 0.5.58__py3-none-any.whl

dfm_python/__init__.py

@@ -80,7 +80,6 @@ __version__ = "0.5.56"
 
 # Configuration (from config/ subpackage)
 from .config import (
-    # DEFAULT_BLOCK_NAME,  # Removed to avoid circular import - import directly from functional.dfm_block
     ConfigSource, YamlSource,
     make_config_source,
 )

dfm_python/config/__init__.py

@@ -10,7 +10,7 @@ Note: Series are specified via frequency dict mapping column names to frequencie
 from .schema import (
     BaseModelConfig, DFMConfig, DDFMConfig,
     BaseResult, DFMResult, DDFMResult,
-    DFMFitParams, DDFMFitParams, FitParams,  # FitParams is backward compatibility alias
+    DFMStateSpaceParams, DDFMStateSpaceParams,
 )
 # DEFAULT_BLOCK_NAME is imported lazily where needed to avoid circular imports
 from .constants import (
@@ -46,8 +46,6 @@ from .types import (
     ForecastResult,
     CoefficientMatrix,
     CovarianceMatrix,
-    IRFArray,
-    CompanionMatrix,
     SeriesID,
     Frequency,
     DatasetName,
@@ -58,7 +56,6 @@ from .types import (
     ResultDict,
     ForecastDict,
     MetricsDict,
-    IRFResultDict,
     CheckpointDict,
     ConfigDict,
     Device,
@@ -89,9 +86,8 @@ except ImportError:
     compute_idio_lengths = None
 
 try:
-    from ..numeric.tent import get_tent_weights, get_agg_structure, group_by_freq
+    from ..numeric.tent import get_agg_structure, group_by_freq
 except ImportError:
-    get_tent_weights = None
     get_agg_structure = None
     group_by_freq = None
 from .constants import FREQUENCY_HIERARCHY, PERIODS_PER_YEAR
@@ -107,9 +103,8 @@ __all__ = [
     # 'DEFAULT_BLOCK_NAME',  # Removed to avoid circular import - import directly from functional.dfm_block
     # Model-specific configs (from schema.py)
     'DFMConfig', 'DDFMConfig',
-    # Parameter overrides
-    'DFMFitParams', 'DDFMFitParams',
-    'FitParams',  # Backward compatibility alias
+    # State-space parameters
+    'DFMStateSpaceParams', 'DDFMStateSpaceParams',
     # Model-specific results (from results.py)
     'DFMResult', 'DDFMResult',
     # Utilities
@@ -123,7 +118,6 @@ __all__ = [
     'PERIODS_PER_YEAR',
     'get_periods_per_year',
     'compute_idio_lengths',
-    'get_tent_weights',
     'get_agg_structure',
     'group_by_freq',
     # Type definitions (from types.py)
@@ -139,8 +133,6 @@ __all__ = [
     'ForecastResult',
     'CoefficientMatrix',
     'CovarianceMatrix',
-    'IRFArray',
-    'CompanionMatrix',
     'SeriesID',
     'Frequency',
     'DatasetName',
@@ -151,7 +143,6 @@ __all__ = [
     'ResultDict',
     'ForecastDict',
     'MetricsDict',
-    'IRFResultDict',
     'CheckpointDict',
     'ConfigDict',
     'Device',

dfm_python/config/adapter.py

@@ -118,7 +118,12 @@ def _extract_frequency_dict(
     data: Dict[str, Any],
     clock: str = DEFAULT_CLOCK_FREQUENCY
 ) -> Optional[Dict[str, str]]:
-    """Extract frequency dict from data, handling both new API and legacy formats.
+    """Extract frequency dict from data, handling multiple formats.
+    
+    Supports:
+    1. Grouped format: {'w': [series1, series2, ...], 'm': [series3, ...]}
+    2. Individual format: {'series1': 'w', 'series2': 'm', ...}
+    3. Legacy format: {'series': [{'series_id': ..., 'frequency': ...}, ...]}
     
     Parameters
     ----------
@@ -132,12 +137,63 @@ def _extract_frequency_dict(
     Optional[Dict[str, str]]
         Frequency dict mapping column names to frequencies, or None if not found
     """
-    if 'frequency' in data:
-        # New API: frequency dict directly
-        return data['frequency']
-    elif 'series' in data:
-        # Legacy: convert series list/dict to frequency dict
-        return _convert_series_to_frequency_dict(data['series'], clock)
+    if 'frequency' not in data:
+        # Legacy: check for 'series' key
+        if 'series' in data:
+            return _convert_series_to_frequency_dict(data['series'], clock)
+        return None
+    
+    freq_data = data['frequency']
+    
+    # Check if it's in grouped format: {'w': [...], 'm': [...]}
+    if _is_dict_like(freq_data):
+        from .constants import VALID_FREQUENCIES
+        
+        # Check if all keys are valid frequency codes (grouped format)
+        keys_are_frequencies = all(
+            isinstance(k, str) and k in VALID_FREQUENCIES 
+            for k in freq_data.keys()
+        )
+        
+        # Check if values are lists (grouped format)
+        values_are_lists = all(
+            isinstance(v, (list, tuple)) for v in freq_data.values()
+        )
+        
+        if keys_are_frequencies and values_are_lists:
+            # Grouped format: convert to individual format
+            result = {}
+            for freq, series_list in freq_data.items():
+                if not isinstance(series_list, (list, tuple)):
+                    _raise_config_error(
+                        f"Frequency '{freq}' must map to a list of series names, got {type(series_list)}"
+                    )
+                for series_name in series_list:
+                    if not isinstance(series_name, str):
+                        _raise_config_error(
+                            f"Series names must be strings, got {type(series_name)} in frequency '{freq}'"
+                        )
+                    if series_name in result:
+                        _raise_config_error(
+                            f"Series '{series_name}' appears in multiple frequency groups"
+                        )
+                    result[series_name] = freq
+            return result
+        else:
+            # Individual format: already in correct format, validate structure
+            result = {}
+            for series_name, freq in freq_data.items():
+                if not isinstance(series_name, str):
+                    _raise_config_error(
+                        f"Frequency dict keys must be strings (series names), got {type(series_name)}"
+                    )
+                if not isinstance(freq, str):
+                    _raise_config_error(
+                        f"Frequency dict values must be strings (frequency codes), got {type(freq)} for series '{series_name}'"
+                    )
+                result[series_name] = freq
+            return result
+    
     return None
 
 
@@ -496,6 +552,11 @@ def detect_config_type(data: Dict[str, Any]) -> str:
     if model_type == MODEL_TYPE_DFM:
         return MODEL_TYPE_DFM
     
+    # Priority check: blocks parameter indicates DFM (highest priority)
+    # DFM is the only model type that uses blocks structure
+    if 'blocks' in data and data.get('blocks'):
+        return MODEL_TYPE_DFM
+    
     # Check for model-specific parameters
     keys = set(data.keys())
     if keys & _KDFM_PARAMS:

dfm_python/config/constants.py

@@ -23,7 +23,7 @@ DEFAULT_MIN_DELTA = 1e-6  # Minimum change for improvement
 # ============================================================================
 
 # Minimum eigenvalues and variances
-MIN_EIGENVALUE = 1e-8  # Minimum eigenvalue for positive definite matrices
+MIN_EIGENVALUE = 1e-6  # Minimum eigenvalue for positive definite matrices (increased from 1e-8 for better conditioning)
 MIN_DIAGONAL_VARIANCE = 1e-8  # Minimum variance for diagonal elements
 MIN_OBSERVATION_NOISE = 1e-4  # Minimum observation noise for measurement error (used in EM updates)
 DEFAULT_DDFM_OBSERVATION_NOISE = 1e-15  # Default observation noise for DDFM state-space (matches original TensorFlow)
@@ -33,7 +33,7 @@ MIN_STD_FOR_SCALE_CHECK = 1e-10  # Minimum standard deviation for scale ratio co
 DEFAULT_VARIANCE_FALLBACK = 1.0  # Default variance fallback value for numerical stability
 
 # Maximum eigenvalues
-MAX_EIGENVALUE = 1e6  # Maximum eigenvalue cap
+MAX_EIGENVALUE = 1e3  # Maximum eigenvalue cap (reduced to 1e3 to improve condition number: 1e3/1e-6 = 1e9)
 
 # Eigenvalue stability thresholds
 DEFAULT_EIGENVALUE_MAX_MAGNITUDE = 1.0  # Default maximum eigenvalue magnitude for stability checks
@@ -217,6 +217,7 @@ DEFAULT_START_DATE = datetime(2000, 1, 1)
 # Default window size for DDFM
 DEFAULT_WINDOW_SIZE = 100
 DEFAULT_TENT_KERNEL_SIZE = 5  # Default tent kernel size for slower-frequency series aggregation
+DEFAULT_TENT_KERNEL_REGULARIZATION_MULTIPLIER = 100.0  # Regularization multiplier for tent kernel series (handles ill-conditioning)
 # Note: DEFAULT_BATCH_SIZE is defined above (line 88) as 32 for general neural networks
 # Use DEFAULT_DDFM_WINDOW_SIZE (100) for DDFM-specific window size (time-step batch size)
 
@@ -284,7 +285,7 @@ DEFAULT_PROGRESS_LOG_INTERVAL = 5  # Default progress logging interval for EM al
 # Precision Defaults
 # ============================================================================
 
-DEFAULT_DTYPE = np.float32  # Default numpy dtype for arrays
+DEFAULT_DTYPE = np.float64  # Default numpy dtype for arrays (float64 for better numerical stability)
 
 # PyTorch dtype (matches DEFAULT_DTYPE)
 try:
@@ -386,22 +387,6 @@ FREQUENCY_HIERARCHY: Dict[str, int] = {
 # For frequency gaps larger than this, the missing data approach is used instead
 MAX_TENT_SIZE: int = 12
 
-# Deterministic tent weights lookup for supported frequency pairs
-# Format: (slower_freq, faster_freq) -> tent_weights_array
-# These weights define how slower-frequency series aggregate clock-frequency factors
-TENT_WEIGHTS_LOOKUP: Dict[Tuple[str, str], np.ndarray] = {
-    ('q', 'm'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: quarterly -> monthly
-    ('sa', 'm'): np.array([1, 2, 3, 4, 3, 2, 1]),             # 7 periods: semi-annual -> monthly
-    ('a', 'm'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: annual -> monthly
-    ('m', 'w'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: monthly -> weekly
-    ('q', 'w'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: quarterly -> weekly
-    ('sa', 'w'): np.array([1, 2, 3, 4, 3, 2, 1]),             # 7 periods: semi-annual -> weekly
-    ('a', 'w'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: annual -> weekly
-    ('sa', 'q'): np.array([1, 2, 1]),                         # 3 periods: semi-annual -> quarterly
-    ('a', 'q'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: annual -> quarterly
-    ('a', 'sa'): np.array([1, 2, 1]),                         # 3 periods: annual -> semi-annual
-}
-
 # ============================================================================
 # Export all constants
 # ============================================================================
@@ -518,7 +503,6 @@ __all__ = [
     'MIN_VARIABLES',
     'MIN_SHAPE_FOR_AR2',
     'MIN_DDFM_TIME_STEPS',
-    'MIN_SHAPE_FOR_AR2',
     'MIN_DDFM_DATASET_SIZE_WARNING',
     'MIN_ITER_FOR_DELTA_COMPUTATION',
     'DEFAULT_MIN_ITER_FOR_CONVERGENCE_CHECK',
@@ -546,6 +530,7 @@ __all__ = [
     'DEFAULT_LOG_INTERVAL',
     'DEFAULT_PROGRESS_LOG_INTERVAL',
     'DEFAULT_TENT_KERNEL_SIZE',
+    'DEFAULT_TENT_KERNEL_REGULARIZATION_MULTIPLIER',
     'MAX_ERROR_ITEMS',
     # IRF
     'DEFAULT_IRF_HORIZON',
@@ -581,6 +566,5 @@ __all__ = [
     # Frequency hierarchy and tent kernels
     'FREQUENCY_HIERARCHY',
     'MAX_TENT_SIZE',
-    'TENT_WEIGHTS_LOOKUP',
 ]
 

dfm_python/config/schema/__init__.py

@@ -9,15 +9,11 @@ Note: Series are specified via frequency dict mapping column names to frequencie
 
 from .model import BaseModelConfig, DFMConfig, DDFMConfig
 from .results import BaseResult, DFMResult, DDFMResult
-from .params import DFMFitParams, DDFMFitParams
-
-# Backward compatibility alias
-FitParams = DFMFitParams
+from .params import DFMStateSpaceParams, DDFMStateSpaceParams
 
 __all__ = [
     'BaseModelConfig', 'DFMConfig', 'DDFMConfig',
     'BaseResult', 'DFMResult', 'DDFMResult',
-    'DFMFitParams', 'DDFMFitParams',
-    'FitParams',  # Backward compatibility alias
+    'DFMStateSpaceParams', 'DDFMStateSpaceParams',
 ]
 

dfm_python/config/schema/block.py

@@ -5,8 +5,8 @@ parameters in EM algorithm, replacing long conditional checks with a single
 optional parameter.
 """
 
-from dataclasses import dataclass
-from typing import Optional, Dict
+from dataclasses import dataclass, field
+from typing import Optional, Dict, List
 import numpy as np
 
 
@@ -26,7 +26,10 @@ class BlockStructure:
     p : int
         VAR lag order
     p_plus_one : int
-        p + 1 (state dimension per factor)
+        State dimension per factor. This is equal to max_lag_size = max(p + 1, tent_kernel_size).
+        The name "p_plus_one" is used for backward compatibility with the EM algorithm interface.
+        In the model code, this is called "max_lag_size" to more clearly indicate it accounts
+        for tent kernel size when tent_kernel_size > p + 1.
     n_clock_freq : int
         Number of clock-frequency series
     idio_indicator : np.ndarray
@@ -39,6 +42,30 @@ class BlockStructure:
         Number of slower-frequency series
     tent_weights_dict : dict, optional
         Dictionary mapping frequency pairs to tent weights
+    
+    Cached indices (computed once, reused across EM iterations):
+    _cached_unique_blocks : List[np.ndarray], optional
+        Unique block patterns (cached)
+    _cached_unique_indices : List[int], optional
+        Indices of unique block patterns (cached)
+    _cached_bl_idxM : List[np.ndarray], optional
+        Clock-frequency factor loadings indices per unique block pattern (cached)
+    _cached_bl_idxQ : List[np.ndarray], optional
+        Slower-frequency factor loadings indices per unique block pattern (cached)
+    _cached_R_con : np.ndarray, optional
+        Block diagonal constraint matrix (cached)
+    _cached_q_con : np.ndarray, optional
+        Constraint vector (cached)
+    _cached_total_factor_dim : int, optional
+        Total factor state dimension (cached)
+    _cached_idio_indicator_M : np.ndarray, optional
+        Clock-frequency idiosyncratic component indicator (cached)
+    _cached_n_idio_M : int, optional
+        Number of clock-frequency idiosyncratic components (cached)
+    _cached_c_idio_indicator : np.ndarray, optional
+        Cumulative sum of idiosyncratic component indicator (cached)
+    _cached_rp1 : int, optional
+        Start of idiosyncratic components in state space (cached)
     """
     blocks: np.ndarray
     r: np.ndarray
@@ -51,6 +78,20 @@ class BlockStructure:
     n_slower_freq: Optional[int] = None
     tent_weights_dict: Optional[Dict[str, np.ndarray]] = None
     
+    # Cached indices (computed once, reused across EM iterations)
+    _cached_unique_blocks: Optional[List[np.ndarray]] = field(default=None, init=False, repr=False)
+    _cached_unique_indices: Optional[List[int]] = field(default=None, init=False, repr=False)
+    _cached_bl_idxM: Optional[List[np.ndarray]] = field(default=None, init=False, repr=False)
+    _cached_bl_idxQ: Optional[List[np.ndarray]] = field(default=None, init=False, repr=False)
+    _cached_R_con: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_q_con: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_total_factor_dim: Optional[int] = field(default=None, init=False, repr=False)
+    _cached_idio_indicator_M: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_n_idio_M: Optional[int] = field(default=None, init=False, repr=False)
+    _cached_c_idio_indicator: Optional[np.ndarray] = field(default=None, init=False, repr=False)
+    _cached_rp1: Optional[int] = field(default=None, init=False, repr=False)
+    _cached_N: Optional[int] = field(default=None, init=False, repr=False)  # N value used for caching
+    
     def is_valid(self) -> bool:
         """Check if block structure is valid (all required fields are not None)."""
         return (
@@ -61,4 +102,23 @@ class BlockStructure:
             and self.n_clock_freq is not None
             and self.idio_indicator is not None
         )
+    
+    def has_cached_indices(self) -> bool:
+        """Check if block structure indices are cached."""
+        return self._cached_unique_blocks is not None
+    
+    def clear_cache(self) -> None:
+        """Clear all cached indices to ensure clean state for new training runs."""
+        self._cached_unique_blocks = None
+        self._cached_unique_indices = None
+        self._cached_bl_idxM = None
+        self._cached_bl_idxQ = None
+        self._cached_R_con = None
+        self._cached_q_con = None
+        self._cached_total_factor_dim = None
+        self._cached_idio_indicator_M = None
+        self._cached_n_idio_M = None
+        self._cached_c_idio_indicator = None
+        self._cached_rp1 = None
+        self._cached_N = None
 

dfm_python/config/schema/model.py

@@ -91,7 +91,9 @@ class BaseModelConfig:
     - Data preprocessing (missing data handling)
     
     Series Configuration:
-    - Provide `frequency` dict: {'column_name': 'frequency_code'} to specify per-series frequencies
+    - Provide `frequency` dict in one of two formats:
+      1. Grouped format: {'w': [series1, series2, ...], 'm': [series3, ...]} (recommended for large configs)
+      2. Individual format: {'series1': 'w', 'series2': 'm', ...} (backward compatible)
     - If `frequency` is None, all columns will use `clock` frequency
     - If a column is missing from `frequency` dict, it will use `clock` frequency
     - When data is loaded, missing columns in `frequency` dict are automatically added with `clock` frequency
@@ -103,7 +105,14 @@ class BaseModelConfig:
     
     Examples
     --------
-    >>> # With explicit frequency mapping
+    >>> # With grouped frequency mapping (recommended for large configs)
+    >>> config = DFMConfig(
+    ...     frequency={'q': ['gdp'], 'm': ['unemployment', 'interest_rate']},
+    ...     clock='m',
+    ...     blocks={...}
+    ... )
+    >>> 
+    >>> # With individual frequency mapping (backward compatible)
     >>> config = DFMConfig(
     ...     frequency={'gdp': 'q', 'unemployment': 'm', 'interest_rate': 'm'},
     ...     clock='m',
@@ -277,12 +286,19 @@ class DFMConfig(BaseModelConfig):
     # Damped Updates: If provided (not None), automatically enables damping and always warns
     damping_factor: Optional[float] = None  # Damping factor (0.8 = 80% new, 20% old). If None, no damping. If provided, damping enabled and warnings always shown.
     
-    # Idiosyncratic Component Parameters (auto-detected from frequencies)
+    # Idiosyncratic Component Parameters
     idio_rho0: float = DEFAULT_IDIO_RHO0  # Initial AR coefficient for idiosyncratic components (default: 0.1)
     idio_min_var: float = MIN_DIAGONAL_VARIANCE  # Minimum variance for idiosyncratic innovation covariance (defaults to MIN_DIAGONAL_VARIANCE)
-    # Note: augment_idio and augment_idio_slow are auto-detected from frequency configuration
+    # Note: augment_idio and augment_idio_slow are INTERNALLY auto-detected (not configurable)
     # - If all series use clock frequency: augment_idio=False, augment_idio_slow=False
-    # - If mixed frequencies detected: augment_idio=True, augment_idio_slow=True (auto-enabled)
+    # - If mixed frequencies detected: augment_idio=True, augment_idio_slow=True (required for tent kernel)
+    # These are properties, not config fields - automatically determined from frequency configuration
+    
+    # Kalman Filter Type
+    use_cholesky_filter: bool = False  # Use CholeskyKalmanFilter for better numerical stability (default: False)
+    
+    # Tent Kernel Weights (for mixed-frequency aggregation)
+    tent_weights: Optional[Union[Dict[str, List[float]], Dict[str, np.ndarray]]] = None  # Required for mixed-frequency data: tent weights for frequency pairs. Format: {'freq_pair': [weights]} or {'freq': [weights]}. Example: {'m:w': [1, 2, 1]} or {'m': [1, 2, 1]}. Must be specified in config for all slower-frequency pairs.
     
     def __post_init__(self):
         """Validate blocks structure and derive block properties."""
@@ -437,6 +453,8 @@ class DFMConfig(BaseModelConfig):
             'data_clip': data.get('data_clip', None),
             'regularization': data.get('regularization', None),
             'damping_factor': data.get('damping_factor', None),
+            'use_cholesky_filter': data.get('use_cholesky_filter', False),
+            'tent_weights': data.get('tent_weights', None),
         }
         
         result = base_params.copy()
@@ -515,6 +533,11 @@ class DDFMConfig(BaseModelConfig):
     seed: Optional[int] = None  # Random seed for reproducibility
     lags_input: int = 0  # Number of lags of inputs on encoder (default 0, matching original TensorFlow DDFM)
     
+    # Interpolation parameters (for handling missing values)
+    interpolation_method: str = 'linear'  # Interpolation method: 'linear', 'spline', 'cubic', etc. (default: 'linear' for stability)
+    interpolation_limit: Optional[int] = 10  # Maximum consecutive NaNs to interpolate (default: 10, prevents extreme extrapolation)
+    interpolation_limit_direction: str = 'both'  # Direction to fill: 'forward', 'backward', or 'both' (default: 'both')
+    
     
     # ========================================================================
     # Factory Methods
@@ -541,6 +564,9 @@ class DDFMConfig(BaseModelConfig):
             'disp': DEFAULT_DISP,
             'seed': None,
             'lags_input': 0,  # Number of lags (default 0, matching original TensorFlow)
+            'interpolation_method': 'linear',  # Interpolation method (default: 'linear' for stability)
+            'interpolation_limit': 10,  # Maximum consecutive NaNs to interpolate (default: 10)
+            'interpolation_limit_direction': 'both',  # Direction to fill (default: 'both')
         })
         # Map 'epochs' from config to 'n_mc_samples' for clarity (backward compatibility)
         # Always remove 'epochs' if present (even if n_mc_samples is also present)