dfm-python 0.6.11__tar.gz → 0.6.13__tar.gz

{dfm_python-0.6.11 → dfm_python-0.6.13}/.gitignore

@@ -214,4 +214,5 @@ __marimo__/
 Nowcasting/
 agent/
 
-*.db
+*.db
+*.pkl

{dfm_python-0.6.11 → dfm_python-0.6.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.6.11
+Version: 0.6.13
 Summary: Dynamic Factor Model (DFM) estimation and forecasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme

{dfm_python-0.6.11 → dfm_python-0.6.13}/config/ddfm_exchange.yaml

@@ -20,6 +20,11 @@ tolerance: 0.0005  # Convergence tolerance (matching original tolerance=0.0005)
 disp: 10  # Display frequency for training progress (matching original disp=10)
 seed: 3  # Random seed (matching original seed=3)
 
+# Interpolation Parameters
+interpolation_method: linear  # Interpolation method for missing values
+interpolation_limit: 10  # Maximum consecutive NaNs to interpolate
+interpolation_limit_direction: both  # Direction to fill ('forward', 'backward', or 'both')
+
 # Data Handling (used by Dataset, not Model)
 clock: d  # Daily clock frequency (all series are daily)
 scaler: null  # No scaler - data should be pre-scaled before passing to Dataset

{dfm_python-0.6.11 → dfm_python-0.6.13}/config/ddfm_finance.yaml

@@ -18,6 +18,11 @@ tolerance: 0.0005  # Convergence tolerance
 disp: 10  # Display frequency for training progress
 seed: null  # Random seed (None uses default)
 
+# Interpolation Parameters
+interpolation_method: linear  # Interpolation method for missing values
+interpolation_limit: 10  # Maximum consecutive NaNs to interpolate
+interpolation_limit_direction: both  # Direction to fill ('forward', 'backward', or 'both')
+
 # Data Handling (used by Dataset, not Model)
 clock: m  # Monthly clock frequency
 scaler: standard  # Scaler for target series: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or null

{dfm_python-0.6.11 → dfm_python-0.6.13}/config/ddfm_macro.yaml

@@ -20,6 +20,11 @@ tolerance: 0.0005  # Convergence tolerance
 disp: 10  # Display frequency for training progress
 seed: null  # Random seed (None uses default)
 
+# Interpolation Parameters
+interpolation_method: linear  # Interpolation method for missing values
+interpolation_limit: 10  # Maximum consecutive NaNs to interpolate
+interpolation_limit_direction: both  # Direction to fill ('forward', 'backward', or 'both')
+
 # Data Handling (used by Dataset, not Model)
 clock: w  # Weekly clock frequency (base frequency for latent factors)
 scaler: standard  # Scaler for target series: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or null

{dfm_python-0.6.11 → dfm_python-0.6.13}/config/dfm_exchange.yaml

@@ -6,7 +6,7 @@
 # NOTE: Exchange tutorial is long daily data (T~7.6k). Full EM can take hours.
 # Keep tutorial runs short by capping iterations and loosening convergence.
 threshold: 1.0e-04  # EM convergence threshold (tutorial-friendly)
-max_iter: 50  # Maximum EM iterations (tutorial-friendly)
+max_iter: 10  # Maximum EM iterations (tutorial-friendly)
 
 # Data Handling
 clock: d  # Daily clock frequency (base frequency for latent factors)

{dfm_python-0.6.11 → dfm_python-0.6.13}/config/dfm_finance.yaml

@@ -7,7 +7,7 @@
 
 # EM Algorithm Parameters
 threshold: 1.0e-05  # EM convergence threshold
-max_iter: 12  # Maximum EM iterations
+max_iter: 10  # Maximum EM iterations
 
 # Data Handling
 clock: d  # Daily clock frequency (finance.csv uses daily index via date_id)

{dfm_python-0.6.11 → dfm_python-0.6.13}/config/dfm_macro.yaml

@@ -6,7 +6,7 @@
 
 # EM Algorithm Parameters
 threshold: 1.0e-05  # EM convergence threshold
-max_iter: 12  # Maximum EM iterations
+max_iter: 10  # Maximum EM iterations
 
 # Numerical Stability Parameters
 clip_ar_coefficients: true

{dfm_python-0.6.11 → dfm_python-0.6.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dfm-python"
-version = "0.6.11"
+version = "0.6.13"
 description = "Dynamic Factor Model (DFM) estimation and forecasting in Python"
 requires-python = ">=3.10"
 license = {text = "MIT"}

{dfm_python-0.6.11 → dfm_python-0.6.13}/src/dfm_python/__init__.py

@@ -63,7 +63,7 @@ Note: DFMConfig uses frequency dict to specify series (column names -> frequenci
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.5.60"
+__version__ = "0.6.13"
 
 # ============================================================================
 # PUBLIC API DEFINITION

{dfm_python-0.6.11 → dfm_python-0.6.13}/src/dfm_python/config/schema/params.py

@@ -163,11 +163,11 @@ class DFMModelState:
             )
 
 @dataclass
-class DDFMTrainingState:
-    """Training state for DDFM model.
+class DDFMModelState:
+    """DDFM model state including training state and state-space parameters.
     
-    Stores the current state of DDFM training, including convergence status,
-    loss, factors, and residuals. Used for checkpointing and resuming training.
+    Consolidates all DDFM model state for checkpointing: training state (convergence,
+    loss, factors, residuals) and fitted state-space parameters.
     
     Attributes
     ----------
@@ -184,15 +184,84 @@ class DDFMTrainingState:
     last_neurons : np.ndarray, optional
         Last layer neurons for MLP decoder (n_mc_samples x T x num_neurons) or (T x num_neurons).
         For linear decoder, this equals factors.
+    F : np.ndarray, optional
+        Transition matrix (m x m) - VAR(1) dynamics for factors (A in paper)
+    Q : np.ndarray, optional
+        Process noise covariance (m x m) - innovation covariance (W in original code)
+    mu_0 : np.ndarray, optional
+        Initial state mean (m,) - initial factor values
+    Sigma_0 : np.ndarray, optional
+        Initial state covariance (m x m) - initial uncertainty
+    H : np.ndarray, optional
+        Observation matrix (N x m) - decoder weights (measurement equation, theta in paper)
+    R : np.ndarray, optional
+        Observation noise covariance (N x N) - typically diagonal, small values
     """
+    # Training state
     num_iter: int = 0
     loss_now: Optional[float] = None
     converged: bool = False
     eps: Optional[np.ndarray] = None
     factors: Optional[np.ndarray] = None
     last_neurons: Optional[np.ndarray] = None
+    # State-space parameters (fitted during build_state_space)
+    F: Optional[np.ndarray] = None  # Transition matrix (m x m) - A in paper
+    Q: Optional[np.ndarray] = None  # Process noise covariance (m x m) - W in original code
+    mu_0: Optional[np.ndarray] = None  # Initial state mean (m,)
+    Sigma_0: Optional[np.ndarray] = None  # Initial state covariance (m x m)
+    H: Optional[np.ndarray] = None  # Observation matrix (N x m) - theta in paper
+    R: Optional[np.ndarray] = None  # Observation noise covariance (N x N)
     
-    def sync_from_model(self, model: Any) -> 'DDFMTrainingState':
+    @classmethod
+    def from_model(cls, model: Any) -> 'DDFMModelState':
+        """Create DDFMModelState from DDFM model instance.
+        
+        Parameters
+        ----------
+        model : DDFM
+            DDFM model instance
+            
+        Returns
+        -------
+        DDFMModelState
+            Model state dataclass
+        """
+        # Get training state from model attributes
+        factors = getattr(model, 'factors', None)
+        eps = getattr(model, 'eps', None)
+        last_neurons = getattr(model, 'last_neurons', None)
+        num_iter = getattr(model, '_num_iter', 0)
+        loss_now = getattr(model, 'loss_now', None)
+        converged = getattr(model, '_converged', False)
+        
+        # Get state-space parameters from training_state
+        F = Q = mu_0 = Sigma_0 = H = R = None
+        if hasattr(model, 'training_state') and model.training_state is not None:
+            ts = model.training_state
+            if hasattr(ts, 'F'):
+                F = ts.F
+                Q = ts.Q
+                mu_0 = ts.mu_0
+                Sigma_0 = ts.Sigma_0
+                H = ts.H
+                R = ts.R
+        
+        return cls(
+            num_iter=num_iter,
+            loss_now=loss_now,
+            converged=converged,
+            eps=eps,
+            factors=factors,
+            last_neurons=last_neurons,
+            F=F,
+            Q=Q,
+            mu_0=mu_0,
+            Sigma_0=Sigma_0,
+            H=H,
+            R=R
+        )
+    
+    def sync_from_model(self, model: Any) -> 'DDFMModelState':
         """Update this dataclass from model instance attributes.
         
         Parameters
@@ -202,7 +271,7 @@ class DDFMTrainingState:
             
         Returns
         -------
-        DDFMTrainingState
+        DDFMModelState
             Self (for chaining)
         """
         if hasattr(model, '_has_factors') and model._has_factors:
@@ -212,37 +281,52 @@ class DDFMTrainingState:
             self.num_iter = getattr(model, '_num_iter', 0)
             self.loss_now = getattr(model, 'loss_now', None)
             self.converged = getattr(model, '_converged', False)
+        
+        # Sync state-space parameters if available
+        if hasattr(model, 'training_state') and model.training_state is not None:
+            ts = model.training_state
+            if hasattr(ts, 'F'):
+                self.F = ts.F
+                self.Q = ts.Q
+                self.mu_0 = ts.mu_0
+                self.Sigma_0 = ts.Sigma_0
+                self.H = ts.H
+                self.R = ts.R
+        
         return self
-
-@dataclass
-class DDFMStateSpaceParams:
-    """State-space model parameters for DDFM (fitted during model training).
-    
-    These parameters are computed during DDFM.build_state_space() and represent
-    the fitted state-space model structure. Similar to DFMStateSpaceParams, this contains
-    the fitted state-space parameters but for DDFM. Naming follows original paper conventions.
     
-    Parameters
-    ----------
-    F : np.ndarray
-        Transition matrix (m x m) - VAR(1) dynamics for factors (A in paper)
-    Q : np.ndarray
-        Process noise covariance (m x m) - innovation covariance (W in original code)
-    mu_0 : np.ndarray
-        Initial state mean (m,) - initial factor values
-    Sigma_0 : np.ndarray
-        Initial state covariance (m x m) - initial uncertainty
-    H : np.ndarray
-        Observation matrix (N x m) - decoder weights (measurement equation, theta in paper)
-    R : np.ndarray
-        Observation noise covariance (N x N) - typically diagonal, small values
-    """
-    F: np.ndarray  # Transition matrix (m x m) - A in paper
-    Q: np.ndarray  # Process noise covariance (m x m) - W in original code
-    mu_0: np.ndarray  # Initial state mean (m,)
-    Sigma_0: np.ndarray  # Initial state covariance (m x m)
-    H: np.ndarray  # Observation matrix (N x m) - theta in paper
-    R: np.ndarray  # Observation noise covariance (N x N)
+    def apply_to_model(self, model: Any) -> None:
+        """Apply this state to DDFM model instance.
+        
+        Parameters
+        ----------
+        model : DDFM
+            DDFM model instance to update
+        """
+        # Apply training state
+        if self.factors is not None:
+            model.factors = self.factors
+        if self.eps is not None:
+            model.eps = self.eps
+        if self.last_neurons is not None:
+            model.last_neurons = self.last_neurons
+        model._num_iter = self.num_iter
+        model.loss_now = self.loss_now
+        model._converged = self.converged
+        
+        # Apply state-space parameters if available
+        if self.F is not None and self.Q is not None and self.mu_0 is not None:
+            if hasattr(model, 'training_state'):
+                model.training_state.F = self.F
+                model.training_state.Q = self.Q
+                model.training_state.mu_0 = self.mu_0
+                model.training_state.Sigma_0 = self.Sigma_0
+                model.training_state.H = self.H
+                model.training_state.R = self.R
+
+# Backward compatibility aliases
+DDFMTrainingState = DDFMModelState
+DDFMStateSpaceParams = DDFMModelState
 
 # Backward compatibility alias
 DFMStateSpaceParams = DFMModelState

{dfm_python-0.6.11 → dfm_python-0.6.13}/src/dfm_python/config/schema/results.py

@@ -13,7 +13,7 @@ from typing import Optional, List, Dict, Any, Tuple
 from datetime import datetime
 
 from .model import DFMConfig
-from .params import DDFMStateSpaceParams
+from .params import DDFMModelState
 
 
 
@@ -311,7 +311,7 @@ class DDFMResult(BaseResult):
         Architecture of the encoder network used.
     use_idiosyncratic : bool, optional
         Whether idiosyncratic components were modeled.
-    fit_params : DDFMStateSpaceParams, optional
+    fit_params : DDFMModelState, optional
         State-space model parameters created during fit (F, Q, mu_0, sigma_0, H, R).
         These are computed during build_state_space() and represent the fitted
         state-space model structure.
@@ -365,7 +365,7 @@ class DDFMResult(BaseResult):
     use_idiosyncratic: Optional[bool] = None  # Whether idio components were used
     
     # State-space parameters (fitted during training)
-    fit_params: Optional[DDFMStateSpaceParams] = None
+    fit_params: Optional[DDFMModelState] = None
     
     # Training hyperparameters
     training_max_iter: Optional[int] = None

dfm_python-0.6.13/src/dfm_python/dataset/ddfm_dataset.py

@@ -0,0 +1,228 @@
+"""PyTorch Dataset for Deep Dynamic Factor Model (DDFM)."""
+
+import torch
+from torch.utils.data import Dataset
+import numpy as np
+import pandas as pd
+try:
+    import polars as pl
+    _has_polars = True
+    PolarsDataFrame = pl.DataFrame
+except ImportError:
+    pl = None
+    _has_polars = False
+    PolarsDataFrame = type(None)  # Dummy type for type hints when polars not available
+from typing import Tuple, List, Optional, Union
+from sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler
+
+from ..config.constants import DEFAULT_TORCH_DTYPE
+
+
+class DDFMDataset(Dataset):
+    """Dataset for DDFM training.
+    
+    Scales target series if scaler provided.
+    
+    All columns are treated as targets by default. When covariates are specified,
+    they are excluded from targets (targets = all_columns - covariates).
+    
+    Parameters
+    ----------
+    data : pd.DataFrame | PolarsDataFrame
+        Input data. Target series will be scaled if scaler provided.
+    time_idx : str
+        Time index column name.
+    covariates : List[str], optional
+        Series to exclude from targets (used for factor extraction but not forecasted).
+        If None, all series are targets (default).
+    scaler : StandardScaler | RobustScaler | MinMaxScaler, optional
+        Scaler instance to scale target series. If None, no scaling.
+    """
+    
+    def __init__(
+        self,
+        data: Union[pd.DataFrame, PolarsDataFrame],
+        time_idx: str,
+        covariates: Optional[List[str]] = None,
+        scaler: Optional[Union[StandardScaler, RobustScaler, MinMaxScaler]] = None,
+    ):
+        if _has_polars and isinstance(data, pl.DataFrame):
+            data = data.to_pandas()
+        
+        data = data.copy()
+        data.sort_index(inplace=True)
+        
+        self.time_idx = time_idx
+        self.time_index = pd.Index(data[time_idx]) if time_idx in data.columns else data.index
+        self.data_original = data.copy()
+        
+        # Get variables (all columns excluding time_idx)
+        variables = [col for col in data.columns if col != time_idx] if time_idx in data.columns else list(data.columns)
+        
+        # Filter covariates and compute targets
+        covariates = [c for c in (covariates or []) if c in variables]
+        self.covariates = covariates
+        self.target_series = [col for col in variables if col not in covariates]
+        
+        # Split into covariates (X) and targets (y)
+        data_for_split = data.drop(columns=[time_idx]) if time_idx in data.columns else data
+        y = data_for_split[self.target_series]
+        X = data_for_split.drop(columns=self.target_series) if self.target_series else pd.DataFrame()
+        
+        # Scale covariates separately (temporary scaler, not stored)
+        if scaler is not None and not X.empty:
+            X = pd.DataFrame(type(scaler)().fit_transform(X.values), index=X.index, columns=X.columns)
+        
+        # Scale targets and store scaler for prediction
+        if scaler is not None:
+            y = pd.DataFrame(scaler.fit_transform(y.values), index=y.index, columns=y.columns)
+        
+        self.scaler = scaler
+        
+        # Store processed data
+        self.data = pd.concat([X, y], axis=1) if not X.empty else y
+        self.X = X.values if not X.empty else np.empty((len(y), 0))
+        self.y = y.values
+        self.missing_y = y.isna().values
+        self.observed_y = ~self.missing_y
+
+    @property
+    def target_shape(self) -> Tuple[int, int]:
+        """Target shape (T, num_target_series)."""
+        return self.y.shape
+
+    @property
+    def data_shape(self) -> Tuple[int, int]:
+        """Data shape (T, num_features + num_target_series)."""
+        return self.X.shape[0], self.X.shape[1] + self.y.shape[1]
+    
+    @property
+    def all_columns_are_targets(self) -> bool:
+        """Whether all columns are target series (no covariates)."""
+        return len(self.covariates) == 0
+    
+    def split_features_and_targets(self, data: pd.DataFrame) -> Tuple[Optional[pd.DataFrame], pd.DataFrame]:
+        """Split DataFrame into features (X) and targets (y)."""
+        if self.all_columns_are_targets:
+            return None, data
+        X = data.drop(columns=self.target_series)
+        y = data[self.target_series]
+        return X, y
+    
+    @property
+    def target_indices(self) -> np.ndarray:
+        """Target series column indices in processed data (self.data, excluding time_idx)."""
+        # target_series are column names in the processed data (self.data)
+        # self.data contains X + y (or just y), but not time_idx
+        return np.array([self.data.columns.get_loc(col) for col in self.target_series])
+    
+    @classmethod
+    def from_dataset(cls, new_data: Union[pd.DataFrame, PolarsDataFrame], dataset: 'DDFMDataset') -> 'DDFMDataset':
+        """Create new dataset with new data, preserving configuration."""
+        return cls(
+            data=new_data,
+            time_idx=dataset.time_idx,
+            covariates=dataset.covariates,
+            scaler=dataset.scaler
+        )
+    
+    def create_autoencoder_dataset(
+        self,
+        X: Optional[torch.Tensor],
+        y_tmp: torch.Tensor,
+        y_actual: torch.Tensor,
+        eps_draw: torch.Tensor
+    ) -> 'AutoencoderDataset':
+        """Create a single AutoencoderDataset with corrupted targets."""
+        return AutoencoderDataset(X=X, y_corrupted=y_tmp - eps_draw, y_clean=y_actual)
+    
+    def create_pretrain_dataset(
+        self,
+        data: pd.DataFrame,
+        device: Optional[torch.device] = None
+    ) -> 'AutoencoderDataset':
+        """Create AutoencoderDataset for pre-training (no corruption, clean data)."""
+        if device is None:
+            device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        
+        X_df, y_df = self.split_features_and_targets(data)
+        X = None if X_df is None else torch.from_numpy(X_df.values).to(dtype=DEFAULT_TORCH_DTYPE, device=device)
+        y = torch.from_numpy(y_df.values).to(dtype=DEFAULT_TORCH_DTYPE, device=device)
+        
+        return AutoencoderDataset(X=X, y_corrupted=y, y_clean=y)
+    
+    def create_autoencoder_datasets_list(
+        self,
+        n_mc_samples: int,
+        mu_eps: np.ndarray,
+        std_eps: np.ndarray,
+        X: Union[np.ndarray, pd.DataFrame],
+        y_tmp: Union[np.ndarray, pd.DataFrame],
+        y_actual: np.ndarray,
+        rng: np.random.RandomState,
+        device: Optional[torch.device] = None
+    ) -> List['AutoencoderDataset']:
+        """Create AutoencoderDataset instances with pre-sampled MC noise."""
+        if device is None:
+            device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        
+        X_array = X.values if isinstance(X, pd.DataFrame) else X
+        y_tmp_array = y_tmp.values if isinstance(y_tmp, pd.DataFrame) else y_tmp
+        T = y_tmp_array.shape[0]
+        
+        # Pre-sample all MC noise at once
+        eps_draws = rng.multivariate_normal(mu_eps, np.diag(std_eps), (n_mc_samples, T))
+        
+        # Convert to tensors once
+        X_tensor = torch.from_numpy(X_array).to(dtype=DEFAULT_TORCH_DTYPE, device=device) if X_array.size > 0 else None
+        y_tmp_tensor = torch.from_numpy(y_tmp_array).to(dtype=DEFAULT_TORCH_DTYPE, device=device)
+        y_actual_tensor = torch.from_numpy(y_actual).to(dtype=DEFAULT_TORCH_DTYPE, device=device)
+        eps_draws_tensor = torch.from_numpy(eps_draws).to(dtype=DEFAULT_TORCH_DTYPE, device=device)
+        
+        # Create datasets
+        return [
+            self.create_autoencoder_dataset(X_tensor, y_tmp_tensor, y_actual_tensor, eps_draws_tensor[i])
+            for i in range(n_mc_samples)
+        ]
+
+
+class AutoencoderDataset:
+    """Container for autoencoder training data with corrupted inputs and clean targets.
+    
+    Stores pre-loaded tensors for efficient direct slicing. All tensors are expected 
+    to be on the correct device. Use direct tensor slicing (e.g., `dataset.full_input[i:j]`)
+    rather than indexing.
+    
+    Parameters
+    ----------
+    X : torch.Tensor, optional
+        Features (T, N_features) - lags, dummies, etc. Not corrupted.
+    y_corrupted : torch.Tensor
+        Corrupted targets (T, num_target_series).
+    y_clean : torch.Tensor
+        Clean targets (T, num_target_series) for reconstruction.
+    """
+    
+    def __init__(
+        self,
+        X: Optional[torch.Tensor],
+        y_corrupted: torch.Tensor,
+        y_clean: torch.Tensor
+    ):
+        self.X = X
+        self.y_corrupted = y_corrupted
+        self.y_clean = y_clean
+        # Pre-compute full_input once (optimization: avoid torch.cat on every access)
+        if self.X is not None:
+            self._full_input = torch.cat([self.X, self.y_corrupted], dim=1)
+        else:
+            self._full_input = self.y_corrupted
+    
+    @property
+    def full_input(self) -> torch.Tensor:
+        """Full autoencoder input: clean X features + corrupted y targets."""
+        return self._full_input
+    
+    def __len__(self) -> int:
+        """Return number of time steps."""
+        return self.y_corrupted.shape[0]