dfm-python 0.5.59__py3-none-any.whl → 0.5.60__py3-none-any.whl

dfm_python/__init__.py

@@ -32,7 +32,9 @@ Example:
     >>> df_processed = df[[col for col in df.columns if col != 'date']]
     >>> 
     >>> # Step 2: Create Dataset
-    >>> dataset = DFMDataset(config_path='config/default.yaml', data=df_processed)
+    >>> from dfm_python.config import DFMConfig
+    >>> config = DFMConfig.from_yaml('config/default.yaml')
+    >>> dataset = DFMDataset(config=config, data=df_processed)
     >>> 
     >>> # Step 3: Create model and load config
     >>> model = DFM()
@@ -61,7 +63,7 @@ Note: DFMConfig uses frequency dict to specify series (column names -> frequenci
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.5.56"
+__version__ = "0.5.60"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -102,7 +104,10 @@ from .models.base import BaseFactorModel
 from .models.dfm import DFM
 
 # DDFM high-level API (PyTorch is mandatory)
-from .models.ddfm import DDFM
+try:
+    from .models.ddfm.ddfm import DDFM
+except ImportError:
+    DDFM = None
 
 __all__ = [
     # Core classes

dfm_python/config/__init__.py

@@ -81,12 +81,7 @@ from .types import (
 )
 # Import lazily to avoid circular dependencies
 try:
-    from ..numeric.builder import compute_idio_lengths
-except ImportError:
-    compute_idio_lengths = None
-
-try:
-    from ..numeric.tent import get_agg_structure, group_by_freq
+    from ..models.dfm.tent import get_agg_structure, group_by_freq
 except ImportError:
     get_agg_structure = None
     group_by_freq = None
@@ -100,7 +95,7 @@ def get_periods_per_year(frequency: str) -> int:
 __all__ = [
     # Base classes
     'BaseModelConfig', 'BaseResult',
-    # 'DEFAULT_BLOCK_NAME',  # Removed to avoid circular import - import directly from functional.dfm_block
+    # 'DEFAULT_BLOCK_NAME',  # Removed to avoid circular import
     # Model-specific configs (from schema.py)
     'DFMConfig', 'DDFMConfig',
     # State-space parameters
@@ -117,7 +112,6 @@ __all__ = [
     'FREQUENCY_HIERARCHY',
     'PERIODS_PER_YEAR',
     'get_periods_per_year',
-    'compute_idio_lengths',
     'get_agg_structure',
     'group_by_freq',
     # Type definitions (from types.py)

dfm_python/config/adapter.py

@@ -303,7 +303,6 @@ class YamlSource:
         _EXCLUDED_KEYS = {
             'defaults', '_target_', '_recursive_', '_convert_', 
             'series', 'blocks', 'data', 'output', 'description', 'name', 'target', 'model',
-            'nan_method', 'nan_k'  # Internal constants, not user-configurable
         }
         main_settings = {k: v for k, v in cfg_dict.items() if k not in _EXCLUDED_KEYS}
         

dfm_python/config/schema/model.py

@@ -65,8 +65,6 @@ from ..constants import (
     MIN_EIGENVALUE,
     MAX_EIGENVALUE,
     MIN_DIAGONAL_VARIANCE,
-    DEFAULT_NAN_METHOD,
-    DEFAULT_NAN_K,
     DEFAULT_CLOCK_FREQUENCY,
     DEFAULT_KDFM_AR_ORDER,
     DEFAULT_KDFM_MA_ORDER,
@@ -138,7 +136,6 @@ class BaseModelConfig:
     # ========================================================================
     clock: str = 'm'  # Required: Base frequency for latent factors (global clock): 'd', 'w', 'm', 'q', 'sa', 'a' (defaults to 'm' for monthly)
     target_scaler: Optional[ScalerType] = None  # Fitted sklearn scaler instance (StandardScaler, RobustScaler, etc.) for target series only. Must be a fitted scaler object (call .fit() on target data first). Pass scaler object directly, not string. Feature series are assumed to be manually preprocessed. If None, target series are assumed to be already in the desired scale.
-    # Note: nan_method and nan_k are internal constants (DEFAULT_NAN_METHOD, DEFAULT_NAN_K) used during initialization only
     
     def __post_init__(self):
         """Validate basic model structure.
@@ -294,9 +291,6 @@ class DFMConfig(BaseModelConfig):
     # - If mixed frequencies detected: augment_idio=True, augment_idio_slow=True (required for tent kernel)
     # These are properties, not config fields - automatically determined from frequency configuration
     
-    # Kalman Filter Type
-    use_cholesky_filter: bool = False  # Use CholeskyKalmanFilter for better numerical stability (default: False)
-    
     # Tent Kernel Weights (for mixed-frequency aggregation)
     tent_weights: Optional[Union[Dict[str, List[float]], Dict[str, np.ndarray]]] = None  # Required for mixed-frequency data: tent weights for frequency pairs. Format: {'freq_pair': [weights]} or {'freq': [weights]}. Example: {'m:w': [1, 2, 1]} or {'m': [1, 2, 1]}. Must be specified in config for all slower-frequency pairs.
     
@@ -379,7 +373,11 @@ class DFMConfig(BaseModelConfig):
         object.__setattr__(self, '_augment_idio_slow', is_mixed_freq)
     
     def to_em_config(self) -> 'EMConfig':
-        """Create EMConfig from DFMConfig consolidated parameters."""
+        """Create EMConfig from DFMConfig consolidated parameters.
+        
+        Automatically passes all numerical stability parameters (regularization, ar_clip,
+        damping_factor, data_clip) from DFMConfig to EMConfig for use in EM algorithm.
+        """
         from ...functional.em import EMConfig
         from ..constants import DEFAULT_REGULARIZATION, VAR_STABILITY_THRESHOLD
         
@@ -397,6 +395,10 @@ class DFMConfig(BaseModelConfig):
             regularization=reg_scale,
             min_norm=min_eigenval,
             max_eigenval=max_eigenval,
+            # Pass numerical stability parameters from DFMConfig
+            ar_clip=self.ar_clip,
+            damping_factor=self.damping_factor,
+            data_clip=self.data_clip,
             # Other parameters use defaults from EMConfig
         )
     
@@ -453,7 +455,6 @@ class DFMConfig(BaseModelConfig):
             'data_clip': data.get('data_clip', None),
             'regularization': data.get('regularization', None),
             'damping_factor': data.get('damping_factor', None),
-            'use_cholesky_filter': data.get('use_cholesky_filter', False),
             'tent_weights': data.get('tent_weights', None),
         }
         

dfm_python/config/schema/params.py

@@ -8,97 +8,12 @@ import numpy as np
 from typing import Optional, Any
 
 
-@dataclass
-class DFMStateSpaceParams:
-    """State-space model parameters for DFM (fitted during model training).
-    
-    These parameters are computed during DFM.fit() and represent the fitted
-    state-space model structure. Contains only the state-space parameters,
-    not training metadata (loglik, num_iter, converged) which are stored
-    separately in checkpoint dicts.
-    
-    Parameters
-    ----------
-    A : np.ndarray
-        Transition matrix (m x m) - VAR dynamics for factors
-    C : np.ndarray
-        Observation matrix (N x m) - factor loadings
-    Q : np.ndarray
-        Process noise covariance (m x m) - innovation covariance
-    R : np.ndarray
-        Observation noise covariance (N x N) - typically diagonal
-    Z_0 : np.ndarray
-        Initial state mean (m,) - initial factor values
-    V_0 : np.ndarray
-        Initial state covariance (m x m) - initial uncertainty
-    """
-    A: np.ndarray  # Transition matrix (m x m)
-    C: np.ndarray  # Observation matrix (N x m)
-    Q: np.ndarray  # Process noise covariance (m x m)
-    R: np.ndarray  # Observation noise covariance (N x N)
-    Z_0: np.ndarray  # Initial state mean (m,)
-    V_0: np.ndarray  # Initial state covariance (m x m)
-    
-    @classmethod
-    def from_model(cls, model: Any) -> 'DFMStateSpaceParams':
-        """Create DFMStateSpaceParams from DFM model instance.
-        
-        Parameters
-        ----------
-        model : DFM
-            DFM model instance with fitted parameters
-            
-        Returns
-        -------
-        DFMStateSpaceParams
-            State-space parameters dataclass
-        """
-        # Get parameters from model (stored after fit)
-        # First check if training_state is already DFMStateSpaceParams (new format)
-        if hasattr(model, 'training_state') and model.training_state is not None:
-            if isinstance(model.training_state, cls):
-                # Already a DFMStateSpaceParams instance
-                return model.training_state
-            # Backward compatibility: If old format training_state (has A, C, Q, R, Z_0, V_0 attributes)
-            if hasattr(model.training_state, 'A'):
-                return cls(
-                    A=model.training_state.A,
-                    C=model.training_state.C,
-                    Q=model.training_state.Q,
-                    R=model.training_state.R,
-                    Z_0=model.training_state.Z_0,
-                    V_0=model.training_state.V_0
-                )
-        
-        # Get from model attributes (after _update_parameters is called)
-        return cls(
-            A=getattr(model, 'A', None),
-            C=getattr(model, 'C', None),
-            Q=getattr(model, 'Q', None),
-            R=getattr(model, 'R', None),
-            Z_0=getattr(model, 'Z_0', None),
-            V_0=getattr(model, 'V_0', None)
-        )
-    
-    def apply_to_model(self, model: Any) -> None:
-        """Apply state-space parameters to DFM model instance.
-        
-        Parameters
-        ----------
-        model : DFM
-            DFM model instance to update
-        """
-        model._update_parameters(
-            self.A, self.C, self.Q, self.R, self.Z_0, self.V_0
-        )
-
-
 @dataclass
 class DFMModelState:
-    """DFM model structure and mixed-frequency parameters.
+    """DFM model state including structure, mixed-frequency parameters, and state-space parameters.
     
-    Consolidates model structure and mixed-frequency state for checkpointing.
-    This allows save/load methods to be simplified by using a single dataclass.
+    Consolidates all DFM model state for checkpointing: model structure, mixed-frequency
+    configuration, and fitted state-space parameters.
     
     Attributes
     ----------
@@ -128,6 +43,18 @@ class DFMModelState:
         Indicator for idiosyncratic components
     max_lag_size : int, optional
         Maximum lag size for state dimension
+    A : np.ndarray, optional
+        Transition matrix (m x m) - VAR dynamics for factors
+    C : np.ndarray, optional
+        Observation matrix (N x m) - factor loadings
+    Q : np.ndarray, optional
+        Process noise covariance (m x m) - innovation covariance
+    R : np.ndarray, optional
+        Observation noise covariance (N x N) - typically diagonal
+    Z_0 : np.ndarray, optional
+        Initial state mean (m,) - initial factor values
+    V_0 : np.ndarray, optional
+        Initial state covariance (m x m) - initial uncertainty
     """
     num_factors: int
     r: np.ndarray
@@ -142,6 +69,13 @@ class DFMModelState:
     frequencies: Optional[np.ndarray] = None
     idio_indicator: Optional[np.ndarray] = None
     max_lag_size: Optional[int] = None
+    # State-space parameters (fitted during training)
+    A: Optional[np.ndarray] = None
+    C: Optional[np.ndarray] = None
+    Q: Optional[np.ndarray] = None
+    R: Optional[np.ndarray] = None
+    Z_0: Optional[np.ndarray] = None
+    V_0: Optional[np.ndarray] = None
     
     @classmethod
     def from_model(cls, model: Any) -> 'DFMModelState':
@@ -157,6 +91,27 @@ class DFMModelState:
         DFMModelState
             Model state dataclass
         """
+        # Get state-space parameters from training_state if available
+        A = C = Q = R = Z_0 = V_0 = None
+        if hasattr(model, 'training_state') and model.training_state is not None:
+            # Handle both old DFMStateSpaceParams format and new DFMModelState format
+            ts = model.training_state
+            if hasattr(ts, 'A'):
+                A = ts.A
+                C = ts.C
+                Q = ts.Q
+                R = ts.R
+                Z_0 = ts.Z_0
+                V_0 = ts.V_0
+        else:
+            # Fallback: get from model attributes directly
+            A = getattr(model, 'A', None)
+            C = getattr(model, 'C', None)
+            Q = getattr(model, 'Q', None)
+            R = getattr(model, 'R', None)
+            Z_0 = getattr(model, 'Z_0', None)
+            V_0 = getattr(model, 'V_0', None)
+        
         return cls(
             num_factors=model.num_factors,
             r=model.r,
@@ -170,7 +125,13 @@ class DFMModelState:
             tent_weights_dict=getattr(model, '_tent_weights_dict', None),
             frequencies=getattr(model, '_frequencies', None),
             idio_indicator=getattr(model, '_idio_indicator', None),
-            max_lag_size=getattr(model, '_max_lag_size', None)
+            max_lag_size=getattr(model, '_max_lag_size', None),
+            A=A,
+            C=C,
+            Q=Q,
+            R=R,
+            Z_0=Z_0,
+            V_0=V_0
         )
     
     def apply_to_model(self, model: Any) -> None:
@@ -194,6 +155,12 @@ class DFMModelState:
         model._frequencies = self.frequencies
         model._idio_indicator = self.idio_indicator
         model._max_lag_size = self.max_lag_size
+        
+        # Apply state-space parameters if available
+        if self.A is not None and self.C is not None and self.Q is not None:
+            model._update_parameters(
+                self.A, self.C, self.Q, self.R, self.Z_0, self.V_0
+            )
 
 @dataclass
 class DDFMTrainingState:
@@ -277,3 +244,6 @@ class DDFMStateSpaceParams:
     H: np.ndarray  # Observation matrix (N x m) - theta in paper
     R: np.ndarray  # Observation noise covariance (N x N)
 
+# Backward compatibility alias
+DFMStateSpaceParams = DFMModelState
+

dfm_python/dataset/ddfm_dataset.py

@@ -29,21 +29,29 @@ class DDFMDataset(Dataset):
         Input data. Target series will be scaled if scaler provided.
     time_idx : str
         Time index column name.
+    covariates : List[str], optional
+        Series to exclude from targets (used for factor extraction but not forecasted).
+        If None, all series are targets (default).
     target_series : List[str], optional
-        Target series column names to scale. If None or empty, all columns will be used as targets.
-    target_scaler : StandardScaler | RobustScaler | MinMaxScaler, optional
+        Target series column names (deprecated, use covariates instead).
+        If None or empty, all columns will be used as targets.
+    scaler : StandardScaler | RobustScaler | MinMaxScaler, optional
         Scaler instance to scale target series. If None, no scaling.
     feature_scaler : StandardScaler | RobustScaler | MinMaxScaler, optional
         Scaler instance to scale feature series. If None, no scaling.
+    target_scaler : StandardScaler | RobustScaler | MinMaxScaler, optional
+        Deprecated: use scaler instead.
     """
     
     def __init__(
         self,
         data: Union[pd.DataFrame, PolarsDataFrame],
         time_idx: str,
-        target_series: Optional[List[str]] = None,
-        target_scaler: Optional[Union[StandardScaler, RobustScaler, MinMaxScaler]] = None,
+        covariates: Optional[List[str]] = None,
+        target_series: Optional[List[str]] = None,  # Deprecated: use covariates instead
+        scaler: Optional[Union[StandardScaler, RobustScaler, MinMaxScaler]] = None,
         feature_scaler: Optional[Union[StandardScaler, RobustScaler, MinMaxScaler]] = None,
+        target_scaler: Optional[Union[StandardScaler, RobustScaler, MinMaxScaler]] = None,  # Deprecated: use scaler instead
     ):
         if _has_polars and isinstance(data, pl.DataFrame):
             data = data.to_pandas()
@@ -59,24 +67,65 @@ class DDFMDataset(Dataset):
         else:
             self.time_index = data.index
         
-        # If target_series is None or empty, use all columns as targets
-        if target_series is None or len(target_series) == 0:
-            target_series_list = list(data.columns)
-        else:
-            target_series_list = list(target_series)
-            missing_cols = [col for col in target_series_list if col not in data.columns]
+        # Get all available columns (excluding time_idx if it's a column)
+        all_columns = [col for col in data.columns if col != time_idx] if time_idx in data.columns else list(data.columns)
+        
+        # Handle backward compatibility: if target_series is provided, compute covariates from it
+        if target_series is not None and len(target_series) > 0:
+            import warnings
+            warnings.warn(
+                "target_series parameter is deprecated. Use covariates instead. "
+                "If you specify target_series, it will be converted to covariates automatically.",
+                DeprecationWarning,
+                stacklevel=2
+            )
+            # Compute covariates = all_columns - target_series
+            valid_targets = [t for t in target_series if t in all_columns]
+            missing_cols = [col for col in target_series if col not in all_columns]
             if missing_cols:
                 raise ValueError(
                     f"target_series columns {missing_cols} not found in data. "
-                    f"Available columns: {list(data.columns)}"
+                    f"Available columns: {all_columns}"
                 )
+            # Compute covariates from target_series
+            covariates = [col for col in all_columns if col not in valid_targets]
+        else:
+            covariates = covariates if covariates is not None else []
+        
+        # Compute target_series from covariates
+        # If covariates is empty, all series are targets
+        if covariates:
+            # Filter covariates to only those that exist in data
+            valid_covariates = [c for c in covariates if c in all_columns]
+            missing_covariates = [c for c in covariates if c not in all_columns]
+            if missing_covariates:
+                import warnings
+                warnings.warn(
+                    f"Some covariates not found in data: {missing_covariates}. "
+                    f"Available columns: {all_columns}",
+                    UserWarning
+                )
+            # target_series = all_columns - covariates
+            target_series_list = [col for col in all_columns if col not in valid_covariates]
+            self.covariates = valid_covariates
+        else:
+            # No covariates: all series are targets
+            target_series_list = all_columns
+            self.covariates = []
+        
+        # Store computed target_series for backward compatibility
         self.target_series = target_series_list
         self.data_original = data.copy()
-        self.target_scaler = target_scaler
+        # Use scaler parameter, fallback to target_scaler for backward compatibility
+        self.scaler = scaler if scaler is not None else target_scaler
         self.feature_scaler = feature_scaler
         
-        y = data[target_series_list]
-        X = data.drop(columns=target_series_list)
+        # Exclude time_idx from data when splitting X and y
+        # time_idx is metadata, not a data column
+        data_for_split = data.drop(columns=[time_idx]) if time_idx in data.columns else data
+        
+        y = data_for_split[target_series_list]
+        X = data_for_split.drop(columns=target_series_list)
         
         # Optionally scale feature series to avoid scale dominance in encoder input
         if feature_scaler is not None and not X.empty:
@@ -99,9 +148,9 @@ class DDFMDataset(Dataset):
                     self.feature_scaler.fit(X.values)
         
         # Check if data is already standardized (mean≈0, std≈1) to avoid double scaling
-        # If data is already scaled, we still need target_scaler for inverse transformation
+        # If data is already scaled, we still need scaler for inverse transformation
         # but we should NOT scale again
-        if target_scaler is not None:
+        if self.scaler is not None:
             # Check if target data appears already standardized
             y_mean = y.mean().abs().max()
             y_std = (y.std() - 1.0).abs().max()
@@ -109,10 +158,10 @@ class DDFMDataset(Dataset):
             
             # Check if scaler is already fitted
             scaler_is_fitted = (
-                hasattr(target_scaler, 'mean_') and 
-                target_scaler.mean_ is not None and
-                hasattr(target_scaler, 'scale_') and
-                target_scaler.scale_ is not None
+                hasattr(self.scaler, 'mean_') and 
+                self.scaler.mean_ is not None and
+                hasattr(self.scaler, 'scale_') and
+                self.scaler.scale_ is not None
             )
             
             if is_already_scaled:
@@ -130,7 +179,7 @@ class DDFMDataset(Dataset):
                         "Ensure scaler is fitted on original unscaled data before passing to DDFMDataset.",
                         UserWarning
                     )
-                    self.target_scaler.fit(y.values)
+                    self.scaler.fit(y.values)
                 # Don't transform - data is already scaled
             else:
                 # Data is not standardized - apply scaling
@@ -138,8 +187,8 @@ class DDFMDataset(Dataset):
                 if scaler_is_fitted:
                     # Check if scaler was fitted on data with similar scale
                     # If mean/scale are very different, warn user but keep scaler.
-                    scaler_mean = np.abs(target_scaler.mean_)
-                    scaler_scale = target_scaler.scale_
+                    scaler_mean = np.abs(self.scaler.mean_)
+                    scaler_scale = self.scaler.scale_
                     data_mean = np.abs(y.mean().values)
                     data_std = y.std().values
                     
@@ -157,9 +206,9 @@ class DDFMDataset(Dataset):
                         )
                 else:
                     # Scaler not fitted - fit on current data
-                    self.target_scaler.fit(y.values)
+                    self.scaler.fit(y.values)
                 
-                y_scaled = self.target_scaler.transform(y.values)
+                y_scaled = self.scaler.transform(y.values)
                 y = pd.DataFrame(y_scaled, index=y.index, columns=y.columns)
         
         self.data = pd.concat([X, y], axis=1)
@@ -168,6 +217,11 @@ class DDFMDataset(Dataset):
         self.missing_y = y.isna().values
         self.observed_y = ~self.missing_y
 
+    @property
+    def target_scaler(self):
+        """Backward compatibility: returns scaler."""
+        return self.scaler
+
     @property
     def target_nan_ratio(self) -> float:
         """Target interpolate ratio."""
@@ -249,13 +303,13 @@ class DDFMDataset(Dataset):
         Returns
         -------
         DDFMDataset
-            New dataset with same time_idx, target_series, target_scaler, feature_scaler.
+            New dataset with same time_idx, covariates, scaler, feature_scaler.
         """
         return cls(
             data=new_data,
             time_idx=dataset.time_idx,
-            target_series=dataset.target_series,
-            target_scaler=dataset.target_scaler,
+            covariates=getattr(dataset, 'covariates', None) or [],
+            scaler=getattr(dataset, 'scaler', None),
             feature_scaler=getattr(dataset, 'feature_scaler', None)
         )
     
@@ -404,8 +458,9 @@ class DDFMDataset(Dataset):
 class AutoencoderDataset:
     """Container for autoencoder training data with corrupted inputs and clean targets.
     
-    Stores pre-loaded tensors for efficient direct slicing (no DataLoader needed).
-    All tensors are expected to be on the correct device.
+    Stores pre-loaded tensors for efficient direct slicing. All tensors are expected 
+    to be on the correct device. Use direct tensor slicing (e.g., `dataset.full_input[i:j]`)
+    rather than indexing.
     
     Parameters
     ----------
@@ -440,20 +495,3 @@ class AutoencoderDataset:
     def __len__(self) -> int:
         """Return number of time steps."""
         return self.y_corrupted.shape[0]
-    
-    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]:
-        """Get item at index idx.
-        
-        Parameters
-        ----------
-        idx : int
-            Index of time step
-            
-        Returns
-        -------
-        full_input : torch.Tensor
-            Full input at time step idx (shape: (N_input,))
-        y_clean : torch.Tensor
-            Clean target at time step idx (shape: (N,))
-        """
-        return self._full_input[idx], self.y_clean[idx]