dfm-python 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

dfm_python/__init__.py

@@ -65,7 +65,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"
 
 # ============================================================================
 # PUBLIC API DEFINITION

dfm_python/encoder/vae.py

@@ -82,13 +82,27 @@ if _has_torch:
             # Build layers
             prev_dim = input_dim
             for hidden_dim in hidden_dims:
-                self.layers.append(nn.Linear(prev_dim, hidden_dim))
+                layer = nn.Linear(prev_dim, hidden_dim)
+                # Initialize weights using Xavier/Kaiming initialization for better training stability
+                # Use Kaiming for ReLU, Xavier for tanh/sigmoid
+                if activation == 'relu':
+                    nn.init.kaiming_normal_(layer.weight, mode='fan_in', nonlinearity='relu')
+                else:
+                    nn.init.xavier_normal_(layer.weight, gain=1.0)
+                # Initialize bias to small values
+                if layer.bias is not None:
+                    nn.init.constant_(layer.bias, 0.0)
+                self.layers.append(layer)
                 if use_batch_norm:
                     self.batch_norms.append(nn.BatchNorm1d(hidden_dim))
                 prev_dim = hidden_dim
             
-            # Output layer (no activation, linear)
+            # Output layer (linear, no activation)
+            # Use smaller initialization for output layer to prevent large initial factors
             self.output_layer = nn.Linear(prev_dim, output_dim)
+            nn.init.xavier_normal_(self.output_layer.weight, gain=0.1)  # Smaller gain for output
+            if self.output_layer.bias is not None:
+                nn.init.constant_(self.output_layer.bias, 0.0)
         
         def forward(self, x: torch.Tensor) -> torch.Tensor:
             """Forward pass through encoder.

dfm_python/lightning/scaling.py

@@ -124,7 +124,10 @@ class NoScalingStrategy:
 # Scaler Factory
 # ============================================================================
 
-def _create_scaler_transformer(scaler_type: Optional[str]) -> Any:
+def _create_scaler_transformer(
+    scaler_type: Optional[str],
+    use_robust_as_default: bool = False
+) -> Any:
     """Create a raw sklearn scaler for unified scaling.
     
     Returns a raw sklearn scaler that can be used directly in TransformerPipeline.
@@ -160,10 +163,13 @@ def _create_scaler_transformer(scaler_type: Optional[str]) -> Any:
     )
     
     if scaler_type is None or scaler_type == 'none':
-        # Passthrough: no scaling
-        return FunctionTransformer(func=lambda x: x, inverse_func=lambda x: x)
-    
-    scaler_type_lower = scaler_type.lower()
+        if use_robust_as_default:
+            scaler_type_lower = 'robust'
+        else:
+            # Passthrough: no scaling
+            return FunctionTransformer(func=lambda x: x, inverse_func=lambda x: x)
+    else:
+        scaler_type_lower = scaler_type.lower()
     
     if scaler_type_lower == 'standard':
         return StandardScaler()

dfm_python/models/base.py

@@ -590,11 +590,14 @@ class BaseFactorModel(pl.LightningModule):
         """
         pass
     
+    @abstractmethod
     def update(
         self,
         X_std: np.ndarray,
         *,
-        kalman_filter: Optional[Any] = None
+        history: Optional[int] = None,
+        kalman_filter: Optional[Any] = None,
+        scaler: Optional[Any] = None
     ) -> 'BaseFactorModel':
         """Update factor state with standardized data.
         
@@ -608,8 +611,16 @@ class BaseFactorModel(pl.LightningModule):
             Standardized data array (T x N), where T is number of time periods
             and N is number of series. Data should already be standardized using
             result.Mx and result.Wx.
+        history : int, optional
+            Number of recent periods to use for factor state update. If None, uses
+            all provided data (default). If specified (e.g., 60), uses only the most
+            recent N periods. Initial state (Z_0, V_0) is always estimated from
+            full training data, but the update uses only recent history for efficiency.
         kalman_filter : Any, optional
             Kalman filter instance. If None, uses default or model's kalman filter.
+        scaler : Any, optional
+            If provided, attach/replace the model's scaler (e.g., refit on new regime).
+            When None, keep the existing scaler from training.
             
         Returns
         -------
@@ -620,48 +631,11 @@ class BaseFactorModel(pl.LightningModule):
         --------
         >>> # Update state with new data, then predict
         >>> model.update(X_std).predict(horizon=1)
-        >>> # Or update and predict separately
-        >>> model.update(X_std)
+        >>> # Or update with only recent 12 periods
+        >>> model.update(X_std, history=12)
         >>> forecast = model.predict(horizon=6)
         """
-        self._check_trained()
-        
-        result = self.result  # Use property which ensures non-None after _check_trained()
-        
-        # Validate input shape
-        if not isinstance(X_std, np.ndarray):
-            X_std = np.asarray(X_std)
-        if X_std.ndim != 2:
-            raise ValueError(
-                f"{self.__class__.__name__} update(): X_std must be 2D array (T x N), "
-                f"got shape {X_std.shape}"
-            )
-        
-        # Handle NaN/Inf values
-        X_std = np.where(np.isfinite(X_std), X_std, np.nan)
-        
-        # Update factor state (model-specific)
-        if hasattr(self, 'encoder') and self.encoder is not None:
-            # DDFM: Extract factors via encoder, then use Kalman filter
-            Z_last_updated = self._update_factor_state_ddfm(
-                X_std, result, kalman_filter
-            )
-        else:
-            # DFM: Use Kalman filter directly on standardized data
-            Z_last_updated = self._update_factor_state_dfm(
-                X_std, result, kalman_filter or getattr(self, 'kalman', None)
-            )
-        
-        # Update result.Z[-1, :] permanently
-        if Z_last_updated is not None:
-            result.Z[-1, :] = Z_last_updated
-        else:
-            _logger.warning(
-                f"{self.__class__.__name__} update(): Failed to update factor state, "
-                f"keeping current state"
-            )
-        
-        return self
+        pass
     
     def get_result(self) -> BaseResult:
         """Extract result from trained model.

dfm_python/models/ddfm.py

@@ -308,6 +308,10 @@ class DDFM(BaseFactorModel):
         decay_learning_rate: bool = True,
         min_obs_pretrain: int = 50,
         mult_epoch_pretrain: int = 1,
+        loss_function: str = 'mse',
+        huber_delta: float = 1.0,
+        weight_decay: float = 0.0,
+        grad_clip_val: float = 1.0,
         **kwargs
     ):
         """Initialize DDFM instance.
@@ -349,6 +353,19 @@ class DDFM(BaseFactorModel):
         mult_epoch_pretrain : int, default 1
             Multiplier for number of epochs during pre-training
             Display progress every 'disp' iterations
+        loss_function : str, default 'mse'
+            Loss function for training ('mse', 'huber'). 
+            'mse': Mean squared error (default, matches original DDFM)
+            'huber': Huber loss (more robust to outliers)
+        huber_delta : float, default 1.0
+            Delta parameter for Huber loss (only used if loss_function='huber').
+            Controls the transition point between quadratic and linear regions.
+        weight_decay : float, default 0.0
+            Weight decay (L2 regularization) for optimizer. Helps prevent overfitting to linear features.
+            Recommended: 1e-5 to 1e-3 for deeper encoders or when encoder collapses to linear behavior.
+        grad_clip_val : float, default 1.0
+            Maximum gradient norm for gradient clipping. Prevents training instability.
+            Set to 0.0 to disable gradient clipping.
         seed : int, optional
             Random seed for reproducibility
         **kwargs
@@ -383,6 +400,22 @@ class DDFM(BaseFactorModel):
         self.decay_learning_rate = decay_learning_rate
         self.min_obs_pretrain = min_obs_pretrain
         self.mult_epoch_pretrain = mult_epoch_pretrain
+        self.loss_function = loss_function.lower()
+        self.huber_delta = huber_delta
+        self.weight_decay = weight_decay
+        self.grad_clip_val = grad_clip_val
+        
+        # Validate loss function
+        if self.loss_function not in ['mse', 'huber']:
+            raise ValueError(
+                f"DDFM initialization failed: loss_function must be 'mse' or 'huber', got '{loss_function}'"
+            )
+        
+        # Validate gradient clipping value
+        if self.grad_clip_val < 0.0:
+            raise ValueError(
+                f"DDFM initialization failed: grad_clip_val must be >= 0.0, got {grad_clip_val}"
+            )
         
         # Determine number of factors
         # DDFM does not use block structure - num_factors is specified directly
@@ -586,7 +619,9 @@ class DDFM(BaseFactorModel):
         
         # Clip input data to prevent extreme values that cause NaN
         # Clip to reasonable range: -10 to 10 standard deviations
-        data_clipped = torch.clamp(data, min=-10.0, max=10.0)
+        # For deeper networks, use slightly tighter clipping to improve stability
+        clip_range = 8.0 if len(self.encoder_layers) > 2 else 10.0
+        data_clipped = torch.clamp(data, min=-clip_range, max=clip_range)
         
         # Forward pass
         reconstructed = self.forward(data_clipped)
@@ -603,8 +638,23 @@ class DDFM(BaseFactorModel):
         # Compute loss with missing data masking
         mask = torch.isfinite(target)
         target_clean = torch.where(mask, target, torch.zeros_like(target))
-        squared_diff = (target_clean - reconstructed) ** 2
-        loss = torch.sum(squared_diff * mask) / (torch.sum(mask) + 1e-8)
+        
+        # Use specified loss function
+        if self.loss_function == 'huber':
+            # Huber loss: more robust to outliers
+            # L_delta(a) = 0.5 * a^2 if |a| <= delta, else delta * (|a| - 0.5 * delta)
+            diff = target_clean - reconstructed
+            abs_diff = torch.abs(diff)
+            huber_loss = torch.where(
+                abs_diff <= self.huber_delta,
+                0.5 * diff ** 2,
+                self.huber_delta * (abs_diff - 0.5 * self.huber_delta)
+            )
+            loss = torch.sum(huber_loss * mask) / (torch.sum(mask) + 1e-8)
+        else:
+            # MSE loss (default)
+            squared_diff = (target_clean - reconstructed) ** 2
+            loss = torch.sum(squared_diff * mask) / (torch.sum(mask) + 1e-8)
         
         # Handle NaN/Inf in loss
         if not torch.isfinite(loss):
@@ -614,6 +664,9 @@ class DDFM(BaseFactorModel):
         # Log metrics
         self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True)
         
+        # Note: Gradient clipping is handled automatically by Lightning trainer if gradient_clip_val is set
+        # The grad_clip_val parameter is used in pre_train() and MCMC training for manual training loops
+        
         return loss
     
     
@@ -697,7 +750,8 @@ class DDFM(BaseFactorModel):
         
         optimizer = torch.optim.Adam(
             list(self.encoder.parameters()) + list(self.decoder.parameters()),
-            lr=self.learning_rate
+            lr=self.learning_rate,
+            weight_decay=self.weight_decay
         )
         
         if self.decay_learning_rate:
@@ -750,7 +804,8 @@ class DDFM(BaseFactorModel):
         
         optimizer = torch.optim.Adam(
             list(self.encoder.parameters()) + list(self.decoder.parameters()),
-            lr=lr
+            lr=lr,
+            weight_decay=self.weight_decay
         )
         
         return optimizer
@@ -842,7 +897,8 @@ class DDFM(BaseFactorModel):
         # Create optimizer for pre-training
         optimizer = torch.optim.Adam(
             list(self.encoder.parameters()) + list(self.decoder.parameters()),
-            lr=self.learning_rate
+            lr=self.learning_rate,
+            weight_decay=self.weight_decay
         )
         
         # Pre-train for epochs * mult_epoch_pretrain
@@ -889,10 +945,11 @@ class DDFM(BaseFactorModel):
                 loss.backward()
                 
                 # Gradient clipping for stability
-                torch.nn.utils.clip_grad_norm_(
-                    list(self.encoder.parameters()) + list(self.decoder.parameters()),
-                    max_norm=1.0
-                )
+                if self.grad_clip_val > 0.0:
+                    torch.nn.utils.clip_grad_norm_(
+                        list(self.encoder.parameters()) + list(self.decoder.parameters()),
+                        max_norm=self.grad_clip_val
+                    )
                 
                 optimizer.step()
                 
@@ -1388,12 +1445,120 @@ class DDFM(BaseFactorModel):
             Mx=Mx
         )
         
+        # Convert to numpy (handles torch inputs) and attempt scaler inverse_transform if available
+        X_forecast = np.asarray(
+            X_forecast.detach().cpu().numpy() if hasattr(X_forecast, "detach") else X_forecast
+        )
+        scaler = getattr(self, "scaler", None)
+        if scaler is not None and hasattr(scaler, "inverse_transform"):
+            try:
+                X_forecast = scaler.inverse_transform(X_forecast)
+            except Exception as e:
+                _logger.warning(
+                    f"DDFM prediction: scaler.inverse_transform failed, returning unstandardized values. "
+                    f"error={e}"
+                )
+        
         if return_series and return_factors:
             return X_forecast, Z_forecast
         if return_series:
             return X_forecast
         return Z_forecast
     
+    def update(
+        self,
+        X_std: np.ndarray,
+        *,
+        history: Optional[int] = None,
+        kalman_filter: Optional[Any] = None,
+        scaler: Optional[Any] = None
+    ) -> 'DDFM':
+        """Update factor state with standardized data.
+        
+        This method permanently updates the last factor state (result.Z[-1, :])
+        using the provided standardized data. Users should handle all preprocessing
+        (masking, imputation, standardization) before calling this method.
+        
+        Parameters
+        ----------
+        X_std : np.ndarray
+            Standardized data array (T x N), where T is number of time periods
+            and N is number of series. Data should already be standardized using
+            result.Mx and result.Wx.
+        history : int, optional
+            Number of recent periods to use for factor state update. If None, uses
+            all provided data (default). If specified (e.g., 60), uses only the most
+            recent N periods. Initial state (Z_0, V_0) is always estimated from
+            full training data, but the update uses only recent history for efficiency.
+        kalman_filter : Any, optional
+            Kalman filter instance. If None, uses default or model's kalman filter.
+            
+        Returns
+        -------
+        DDFM
+            Self for method chaining
+            
+        Examples
+        --------
+        >>> # Update state with new data, then predict
+        >>> model.update(X_std).predict(horizon=1)
+        >>> # Or update with only recent 12 periods
+        >>> model.update(X_std, history=12)
+        >>> forecast = model.predict(horizon=6)
+        """
+        self._check_trained()
+        
+        # Optionally replace scaler (e.g., if refit on new regime)
+        if scaler is not None:
+            self.scaler = scaler
+        
+        result = self.result  # Use property which ensures non-None after _check_trained()
+        
+        # Validate input shape
+        if not isinstance(X_std, np.ndarray):
+            X_std = np.asarray(X_std)
+        if X_std.ndim != 2:
+            raise ValueError(
+                f"DDFM update(): X_std must be 2D array (T x N), "
+                f"got shape {X_std.shape}"
+            )
+        
+        # Handle NaN/Inf values
+        X_std = np.where(np.isfinite(X_std), X_std, np.nan)
+        
+        # Filter to recent history if specified
+        # Note: Initial state (Z_0, V_0) from result is estimated from full training data,
+        # but we use only recent history for the update step
+        if history is not None and history > 0:
+            if X_std.shape[0] > history:
+                X_recent = X_std[-history:, :]
+                _logger.debug(
+                    f"DDFM update(): Using {history} most recent periods out of {X_std.shape[0]} total periods"
+                )
+            else:
+                X_recent = X_std
+                _logger.debug(
+                    f"DDFM update(): history={history} specified but data has only {X_std.shape[0]} periods, using all data"
+                )
+        else:
+            X_recent = X_std
+        
+        # Update factor state using encoder to extract factors, then Kalman filter
+        Z_last_updated = self._update_factor_state_ddfm(
+            X_recent, result, kalman_filter
+        )
+        
+        # Update result.Z[-1, :] permanently
+        if Z_last_updated is not None:
+            result.Z[-1, :] = Z_last_updated
+        else:
+            _logger.warning(
+                f"DDFM update(): Failed to update factor state, "
+                f"keeping current state"
+            )
+            
+        return self
+    
     @property
     def result(self) -> DDFMResult:
         """Get model result from training state.

dfm_python/models/dfm.py

@@ -23,11 +23,15 @@ from ..config import (
     MergedConfigSource,
 )
 from ..config.results import DFMResult, FitParams
+from ..config.utils import get_agg_structure, get_tent_weights, FREQUENCY_HIERARCHY, TENT_WEIGHTS_LOOKUP
 from ..logger import get_logger
 from ..ssm.em import EMAlgorithm, EMStepParams
 from ..ssm.kalman import KalmanFilter
 from .base import BaseFactorModel
 
+# Frequency to integer mapping for tensor conversion
+_FREQ_TO_INT = {'d': 1, 'w': 2, 'm': 3, 'q': 4, 'sa': 5, 'a': 6}
+
 if TYPE_CHECKING:
     from omegaconf import DictConfig
     from ..lightning import DFMDataModule
@@ -89,6 +93,8 @@ class DFM(BaseFactorModel):
         max_iter: int = 100,
         nan_method: int = 2,
         nan_k: int = 3,
+        tent_weights_dict: Optional[dict] = None,
+        mixed_freq: bool = False,
         **kwargs
     ):
         """Initialize DFM instance.
@@ -107,6 +113,12 @@ class DFM(BaseFactorModel):
             Missing data handling method
         nan_k : int, default 3
             Spline interpolation order
+        tent_weights_dict : dict, optional
+            [DEPRECATED] Optional tent weights to control aggregation. Use `mixed_freq` parameter instead.
+            Kept for backward compatibility only.
+        mixed_freq : bool, default False
+            If True, use tent kernels for mixed-frequency data. If False, treat all series as clock frequency.
+            When True, raises ValueError if any frequency pair is not in TENT_WEIGHTS_LOOKUP.
         **kwargs
             Additional arguments passed to BaseFactorModel
         """
@@ -119,6 +131,20 @@ class DFM(BaseFactorModel):
         self.max_iter = max_iter
         self.nan_method = nan_method
         self.nan_k = nan_k
+        self.mixed_freq = mixed_freq
+        
+        # Legacy tent_weights_dict parameter (deprecated - use mixed_freq instead)
+        # Only used when mixed_freq=False for backward compatibility
+        self._legacy_tent_weights_dict = tent_weights_dict
+        
+        # Mixed frequency parameters (set during initialize_from_data)
+        self._em_R_mat = None
+        self._em_q = None
+        self._em_nQ = 0
+        self._em_tent_weights_dict = None
+        self._em_frequencies = None
+        self._em_i_idio = None
+        self._em_idio_chain_lengths = None
         
         # Determine number of factors
         if num_factors is None:
@@ -178,6 +204,53 @@ class DFM(BaseFactorModel):
         # Parameters will be initialized during fit_em() or first training step
         pass
     
+    def _create_em_step_params(
+        self, 
+        y: torch.Tensor, 
+        device: torch.device, 
+        dtype: torch.dtype
+    ) -> EMStepParams:
+        """Create EM step parameters using stored mixed frequency parameters.
+        
+        Parameters
+        ----------
+        y : torch.Tensor
+            Data tensor (N x T)
+        device : torch.device
+            Device for tensors
+        dtype : torch.dtype
+            Data type for tensors
+            
+        Returns
+        -------
+        EMStepParams
+            EM step parameters
+        """
+        clock = getattr(self.config, 'clock', 'm')
+        N = y.shape[0]
+        
+        return EMStepParams(
+            y=y,
+            A=self.A,
+            C=self.C,
+            Q=self.Q,
+            R=self.R,
+            Z_0=self.Z_0,
+            V_0=self.V_0,
+            r=self.r.to(device),
+            p=self.p,
+            R_mat=self._em_R_mat,
+            q=self._em_q,
+            nQ=self._em_nQ,
+            i_idio=self._em_i_idio if self._em_i_idio is not None else torch.ones(N, device=device, dtype=dtype),
+            blocks=self.blocks.to(device),
+            tent_weights_dict=self._em_tent_weights_dict,
+            clock=clock,
+            frequencies=self._em_frequencies,
+            idio_chain_lengths=self._em_idio_chain_lengths if self._em_idio_chain_lengths is not None else torch.zeros(N, device=device, dtype=dtype),
+            config=self.config
+        )
+    
     def initialize_from_data(self, X: torch.Tensor) -> None:
         """Initialize parameters from data using PCA and OLS.
         
@@ -187,22 +260,100 @@ class DFM(BaseFactorModel):
             Standardized data (T x N)
         """
         opt_nan = {'method': self.nan_method, 'k': self.nan_k}
+        clock = getattr(self.config, 'clock', 'm')
         
-        # Use self.em.initialize_parameters() with direct tensor operations (no CPU transfers)
+        # Handle mixed_freq parameter
+        if self.mixed_freq:
+            # Use tent kernels for mixed-frequency data
+            agg_structure = get_agg_structure(self.config, clock=clock)
+            
+            # Validate that all required frequency pairs are in TENT_WEIGHTS_LOOKUP
+            frequencies_list = [s.frequency for s in self.config.series]
+            frequencies_set = set(frequencies_list)
+            clock_hierarchy = FREQUENCY_HIERARCHY.get(clock, 3)
+            
+            missing_pairs = []
+            for freq in frequencies_set:
+                freq_hierarchy = FREQUENCY_HIERARCHY.get(freq, 3)
+                if freq_hierarchy > clock_hierarchy:
+                    # This frequency is slower than clock, needs tent kernel
+                    tent_w = get_tent_weights(freq, clock)
+                    if tent_w is None:
+                        missing_pairs.append((freq, clock))
+            
+            if missing_pairs:
+                raise ValueError(
+                    f"mixed_freq=True but the following frequency pairs are not in TENT_WEIGHTS_LOOKUP: {missing_pairs}. "
+                    f"Available pairs: {list(TENT_WEIGHTS_LOOKUP.keys())}. "
+                    f"Either add the missing pairs to TENT_WEIGHTS_LOOKUP or set mixed_freq=False."
+                )
+            
+            # Convert tent_weights to torch tensors
+            tent_weights_dict = {k: torch.tensor(v, dtype=torch.float32, device=X.device) 
+                                for k, v in agg_structure['tent_weights'].items()}
+            
+            # Get R_mat and q from first structure (if any)
+            R_mat = None
+            q = None
+            if agg_structure['structures']:
+                first_structure = list(agg_structure['structures'].values())[0]
+                R_mat = torch.tensor(first_structure[0], dtype=torch.float32, device=X.device)
+                q = torch.tensor(first_structure[1], dtype=torch.float32, device=X.device)
+            
+            # Create frequencies array
+            frequencies_array = np.array(frequencies_list, dtype=object)
+            
+            # Count slower-frequency series
+            nQ = sum(1 for freq in frequencies_list 
+                    if FREQUENCY_HIERARCHY.get(freq, 3) > clock_hierarchy)
+            
+            # Compute i_idio (1 for clock frequency, 0 for slower frequencies)
+            i_idio = torch.tensor([1 if freq == clock else 0 for freq in frequencies_list], 
+                                 dtype=torch.float32, device=X.device)
+        else:
+            # Treat all series as clock frequency (unified frequency)
+            tent_weights_dict = self._legacy_tent_weights_dict
+            R_mat = None
+            q = None
+            frequencies_array = None
+            nQ = 0
+            i_idio = torch.ones(X.shape[1], dtype=torch.float32, device=X.device)
+        
+        # Convert frequencies to torch tensor
+        frequencies_tensor = None
+        if frequencies_array is not None:
+            frequencies_tensor = torch.tensor(
+                [_FREQ_TO_INT.get(f, 3) for f in frequencies_array], 
+                dtype=torch.int32, 
+                device=X.device
+            )
+        
+        # Store for reuse in EM steps
+        self._em_R_mat = R_mat
+        self._em_q = q
+        self._em_nQ = nQ
+        self._em_tent_weights_dict = tent_weights_dict
+        self._em_frequencies = frequencies_tensor
+        self._em_i_idio = i_idio
+        
+        # Idiosyncratic chain lengths (currently unused, set to zeros)
+        self._em_idio_chain_lengths = torch.zeros(X.shape[1], dtype=torch.int32, device=X.device)
+        
+        # Initialize parameters using EM algorithm
         A, C, Q, R, Z_0, V_0 = self.em.initialize_parameters(
             X,
             r=self.r.to(X.device),
             p=self.p,
             blocks=self.blocks.to(X.device),
             opt_nan=opt_nan,
-            R_mat=None,
-            q=None,
-            nQ=0,
-            i_idio=None,
-            clock=getattr(self.config, 'clock', 'm'),
-            tent_weights_dict=None,
-            frequencies=None,
-            idio_chain_lengths=None,
+            R_mat=R_mat,
+            q=q,
+            nQ=nQ,
+            i_idio=i_idio,
+            clock=clock,
+            tent_weights_dict=tent_weights_dict,
+            frequencies=frequencies_tensor,
+            idio_chain_lengths=self._em_idio_chain_lengths,
             config=self.config
         )
         
@@ -264,28 +415,8 @@ class DFM(BaseFactorModel):
         # EM expects y as (N x T), but data is (T x N)
         y = data.T  # (N x T)
         
-        # Create EM step parameters
-        em_params = EMStepParams(
-            y=y,
-            A=self.A,
-            C=self.C,
-            Q=self.Q,
-            R=self.R,
-            Z_0=self.Z_0,
-            V_0=self.V_0,
-            r=self.r.to(y.device),
-            p=self.p,
-            R_mat=None,
-            q=None,
-            nQ=0,
-            i_idio=torch.ones(y.shape[0], device=y.device, dtype=y.dtype),
-            blocks=self.blocks.to(y.device),
-            tent_weights_dict={},
-            clock=getattr(self.config, 'clock', 'm'),
-            frequencies=None,
-            idio_chain_lengths=torch.zeros(y.shape[0], device=y.device, dtype=y.dtype),
-            config=self.config
-        )
+        # Create EM step parameters using stored mixed frequency parameters
+        em_params = self._create_em_step_params(y, y.device, y.dtype)
         
         # Perform EM step - use self.em(...) instead of em_step(...)
         C_new, R_new, A_new, Q_new, Z_0_new, V_0_new, loglik = self.em(em_params)
@@ -387,28 +518,8 @@ class DFM(BaseFactorModel):
         
         # EM loop
         while num_iter < self.max_iter and not converged:
-            # Create EM step parameters
-            em_params = EMStepParams(
-                y=y,
-                A=self.A,
-                C=self.C,
-                Q=self.Q,
-                R=self.R,
-                Z_0=self.Z_0,
-                V_0=self.V_0,
-                r=self.r.to(device),
-                p=self.p,
-                R_mat=None,
-                q=None,
-                nQ=0,
-                i_idio=torch.ones(y.shape[0], device=device, dtype=dtype),
-                blocks=self.blocks.to(device),
-                tent_weights_dict={},
-                clock=getattr(self.config, 'clock', 'm'),
-                frequencies=None,
-                idio_chain_lengths=torch.zeros(y.shape[0], device=device, dtype=dtype),
-                config=self.config
-            )
+            # Create EM step parameters using stored mixed frequency parameters
+            em_params = self._create_em_step_params(y, device, dtype)
             
             # Perform EM step - use self.em(...) instead of em_step(...)
             C_new, R_new, A_new, Q_new, Z_0_new, V_0_new, loglik = self.em(em_params)
@@ -832,6 +943,21 @@ class DFM(BaseFactorModel):
                             f"(max deviation: {max_deviation:.1f} std devs). "
                             f"Possible numerical instability."
                         )
+        
+        # Convert to numpy (handles torch inputs) and attempt scaler inverse_transform if available
+        X_forecast = np.asarray(
+            X_forecast.detach().cpu().numpy() if hasattr(X_forecast, "detach") else X_forecast
+        )
+        scaler = getattr(self, "scaler", None)
+        if scaler is not None and hasattr(scaler, "inverse_transform"):
+            try:
+                X_forecast = scaler.inverse_transform(X_forecast)
+            except Exception as e:
+                _logger.warning(
+                    f"DFM prediction: scaler.inverse_transform failed, returning unstandardized values. "
+                    f"error={e}"
+                )
+        
         if return_factors and np.any(~np.isfinite(Z_forecast)):
             nan_count = np.sum(~np.isfinite(Z_forecast))
             raise ValueError(
@@ -846,6 +972,100 @@ class DFM(BaseFactorModel):
             return X_forecast
         return Z_forecast
     
+    def update(
+        self,
+        X_std: np.ndarray,
+        *,
+        history: Optional[int] = None,
+        kalman_filter: Optional[Any] = None,
+        scaler: Optional[Any] = None
+    ) -> 'DFM':
+        """Update factor state with standardized data.
+        
+        This method permanently updates the last factor state (result.Z[-1, :])
+        using the provided standardized data. Users should handle all preprocessing
+        (masking, imputation, standardization) before calling this method.
+        
+        Parameters
+        ----------
+        X_std : np.ndarray
+            Standardized data array (T x N), where T is number of time periods
+            and N is number of series. Data should already be standardized using
+            result.Mx and result.Wx.
+        history : int, optional
+            Number of recent periods to use for factor state update. If None, uses
+            all provided data (default). If specified (e.g., 60), uses only the most
+            recent N periods. Initial state (Z_0, V_0) is always estimated from
+            full training data, but the update uses only recent history for efficiency.
+        kalman_filter : Any, optional
+            Kalman filter instance. If None, uses default or model's kalman filter.
+            
+        Returns
+        -------
+        DFM
+            Self for method chaining
+            
+        Examples
+        --------
+        >>> # Update state with new data, then predict
+        >>> model.update(X_std).predict(horizon=1)
+        >>> # Or update with only recent 12 periods
+        >>> model.update(X_std, history=12)
+        >>> forecast = model.predict(horizon=6)
+        """
+        self._check_trained()
+        
+        # Optionally replace scaler (e.g., if refit on new regime)
+        if scaler is not None:
+            self.scaler = scaler
+        
+        result = self.result  # Use property which ensures non-None after _check_trained()
+        
+        # Validate input shape
+        if not isinstance(X_std, np.ndarray):
+            X_std = np.asarray(X_std)
+        if X_std.ndim != 2:
+            raise ValueError(
+                f"DFM update(): X_std must be 2D array (T x N), "
+                f"got shape {X_std.shape}"
+            )
+        
+        # Handle NaN/Inf values
+        X_std = np.where(np.isfinite(X_std), X_std, np.nan)
+        
+        # Filter to recent history if specified
+        # Note: Initial state (Z_0, V_0) from result is estimated from full training data,
+        # but we use only recent history for the update step
+        if history is not None and history > 0:
+            if X_std.shape[0] > history:
+                X_recent = X_std[-history:, :]
+                _logger.debug(
+                    f"DFM update(): Using {history} most recent periods out of {X_std.shape[0]} total periods"
+                )
+            else:
+                X_recent = X_std
+                _logger.debug(
+                    f"DFM update(): history={history} specified but data has only {X_std.shape[0]} periods, using all data"
+                )
+        else:
+            X_recent = X_std
+        
+        # Update factor state using Kalman filter directly on standardized data
+        Z_last_updated = self._update_factor_state_dfm(
+            X_recent, result, kalman_filter or getattr(self, 'kalman', None)
+        )
+        
+        # Update result.Z[-1, :] permanently
+        if Z_last_updated is not None:
+            result.Z[-1, :] = Z_last_updated
+        else:
+            _logger.warning(
+                f"DFM update(): Failed to update factor state, "
+                f"keeping current state"
+            )
+            
+        return self
+    
     @property
     def result(self) -> DFMResult:
         """Get model result from training state.

dfm_python/models/mcmc.py

@@ -347,10 +347,11 @@ class DDFMMCMCTrainer:
                     loss = torch.sum(squared_diff) / (torch.sum(mask) + 1e-8)
                     loss.backward()
                     # Gradient clipping to prevent NaN and improve stability
-                    torch.nn.utils.clip_grad_norm_(
-                        list(self.model.encoder.parameters()) + list(self.model.decoder.parameters()),
-                        max_norm=1.0
-                    )
+                    if self.model.grad_clip_val > 0.0:
+                        torch.nn.utils.clip_grad_norm_(
+                            list(self.model.encoder.parameters()) + list(self.model.decoder.parameters()),
+                            max_norm=self.model.grad_clip_val
+                        )
                     optimizer.step()
                 # Extract factors from this sample
                 x_sample_tensor = torch.tensor(x_sim_den[i, :, :], device=device, dtype=dtype)

dfm_python/ssm/em.py

@@ -570,7 +570,7 @@ class EMAlgorithm(nn.Module):
         resNaN = x_clean.clone()
         resNaN[indNaN] = torch.nan
         
-        # Determine tent kernel size (pC) for quarterly-monthly aggregation
+        # Determine tent kernel size (pC) for slower-frequency aggregation
         pC = 5  # Default: quarterly to monthly uses 5 periods [1,2,3,2,1]
         if R_mat is not None:
             pC = R_mat.shape[1]

{dfm_python-0.5.1.dist-info → dfm_python-0.5.3.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.5.1
+Version: 0.5.3
 Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme
 Project-URL: Repository, https://github.com/yourusername/dfm-python
 Project-URL: Issues, https://github.com/yourusername/dfm-python/issues
-Author: DFM Python Contributors
+Author: Minkey Chang
 License: MIT
 License-File: LICENSE
 Keywords: dfm,dynamic-factor-model,econometrics,forecasting,nowcasting,time-series
@@ -59,25 +59,25 @@ Description-Content-Type: text/markdown
 [![PyPI version](https://img.shields.io/pypi/v/dfm-python.svg)](https://pypi.org/project/dfm-python/)
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
 
-A comprehensive Python implementation of **Dynamic Factor Models (DFM)** and **Deep Dynamic Factor Models (DDFM)** for forecasting high-dimensional time series. Built with PyTorch Lightning for production-ready training and inference.
+A comprehensive Python implementation of **Dynamic Factor Models (DFM)** and **Deep Dynamic Factor Models (DDFM)** for forecasting high-dimensional time series. Built with PyTorch Lightning.
 
 ## Features
 
 ### Core Capabilities
-- ✅ **Mixed-frequency data**: Monthly, quarterly, semi-annual, annual series in one model
-- ✅ **Clock-based framework**: All factors evolve at a common clock frequency
-- ✅ **Block structure**: Flexible factor organization (global + sector-specific factors)
-- ✅ **Idiosyncratic components**: Per-series state augmentation for better fit
-- ✅ **Preprocessed data**: Users handle preprocessing (imputation, scaling) using sktime or other tools
-- ✅ **Forecasting**: Generate predictions for any horizon
-- ✅ **Nowcasting**: Estimate current period values using incomplete data
-- ✅ **Deep DFM (DDFM)**: Nonlinear encoder with PyTorch for capturing complex factor structures
+- **Mixed-frequency data**: Monthly, quarterly, semi-annual, annual series in one model
+- **Clock-based framework**: All factors evolve at a common clock frequency
+- **Block structure**: Flexible factor organization (global + sector-specific factors)
+- **Idiosyncratic components**: Per-series state augmentation for better fit
+- **Preprocessed data**: Users handle preprocessing (imputation, scaling) using sktime or other tools
+- **Forecasting**: Generate predictions for any horizon
+- **Nowcasting**: Estimate current period values using incomplete data
+- **Deep DFM (DDFM)**: Nonlinear encoder with PyTorch for capturing complex factor structures
 
 ### Technical Features
-- ✅ **PyTorch Lightning**: Standard training interface with DataModule and Trainer
-- ✅ **Multiple configuration methods**: YAML files, Python dictionaries, or Hydra
-- ✅ **Advanced numerical stability**: Adaptive regularization, spectral radius capping, variance floors
-- ✅ **Production-ready**: Comprehensive error handling, extensive testing, well-documented
+- **PyTorch Lightning**: Standard training interface with DataModule and Trainer
+- **Multiple configuration methods**: YAML files, Python dictionaries, or Hydra
+- **Advanced numerical stability**: Adaptive regularization, spectral radius capping, variance floors
+- **Error handling**: Comprehensive error handling, extensive testing, well-documented
 
 ## Installation
 
@@ -391,17 +391,17 @@ pytest src/test/test_trainer.py -v
 ## Project Status
 
 **Version**: 0.5.1  
-**Status**: Stable and production-ready  
-**Python**: 3.10+
+**Status**: Stable  
+**Python**: 3.10+  
 
 ### What's New in 0.5.1
 
-- ✨ **New `update()` method**: Replaces legacy `nowcast()` with flexible `update().predict()` pattern
-- 🔧 **Improved API**: Users now control all preprocessing (masking, imputation, standardization)
-- 🧹 **Code cleanup**: Removed legacy code and overengineering (~200+ lines removed)
-- 🐛 **Bug fixes**: Fixed DDFM `update()` method for single-factor models
-- 📚 **Updated tutorials**: All 4 tutorials rewritten with new pattern and VAR(1) configuration
-- ✅ **VAR(1) only**: Simplified to VAR(1) factor dynamics throughout  
+- **New `update()` method**: Replaces legacy `nowcast()` with flexible `update().predict()` pattern
+- **Improved API**: Users now control all preprocessing (masking, imputation, standardization)
+- **Code cleanup**: Removed legacy code and overengineering (~200+ lines removed)
+- **Bug fixes**: Fixed DDFM `update()` method for single-factor models
+- **Updated tutorials**: All 4 tutorials rewritten with new pattern and VAR(1) configuration
+- **VAR(1) only**: Simplified to VAR(1) factor dynamics throughout  
 
 ## License
 

{dfm_python-0.5.1.dist-info → dfm_python-0.5.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dfm_python/__init__.py,sha256=C_3NmIr6G7CyDoWB87Osp5b8XZR65LpHmflxMthue74,5683
+dfm_python/__init__.py,sha256=NU9ZxIsniiU9q1IVL8AuGVej9_ROM7nyIFD5dh4MfT4,5683
 dfm_python/config/__init__.py,sha256=NrTwjNzQRcYlk5Ht0hTF8o-6CCl5e4S_JxQAp1mNyCs,1507
 dfm_python/config/adapter.py,sha256=X-mjWlg6ebWp-VAxjAUssgHnmr8AXOvSxhTTVSHOIIk,27475
 dfm_python/config/results.py,sha256=hzXL6MwaIxzA4uzouh6Bc_57lbpedphN9ErTMgfkpeE,15967
@@ -11,22 +11,22 @@ dfm_python/data/utils.py,sha256=5aJFrQ09I3oda4KNut1dXVt-9GYMGouT5lDUpfUEb8Q,1330
 dfm_python/encoder/__init__.py,sha256=ljPTXq04cpOsGXNnma2EJ447kPFlytT6_mToLyKok7w,850
 dfm_python/encoder/base.py,sha256=MOSn_bsQP-GVonGprJymQq-_Q6tmCQex8VOWUJjyxNA,3029
 dfm_python/encoder/pca.py,sha256=FsvCqXD0AuASxymV2-5oAM2QVv_syhSyL3CmxVOXxgM,10584
-dfm_python/encoder/vae.py,sha256=nKgt327-oQ1KT6q3hdABI9z_1K26cKSLVXC1Criy0KA,21250
+dfm_python/encoder/vae.py,sha256=dmM243XrLedzwZ-4CdI3W8A9LH591JnlBjd72eHvXvs,22122
 dfm_python/lightning/__init__.py,sha256=_2DFZnMX3BOybuZZ76VJJ8gmMLZ7tFyO3yyN-A4_ECI,1247
 dfm_python/lightning/data_module.py,sha256=cZuBdGbk6meaeiw9ZPGkcsUW4v25ZP_km-u0Pkoh5bg,34815
-dfm_python/lightning/scaling.py,sha256=PrHCCNS7Z06xkq7_AiQv2bru2Cvra5mejQk6tAzr_hw,16844
+dfm_python/lightning/scaling.py,sha256=-8zYrBm3y4_Ke0ODh1-SJnC0pWef3ZJT6GRv8owo2o8,16997
 dfm_python/logger/__init__.py,sha256=2GwtfOG2ttbtJ5CrO6338R6GIao3YuY0HLiBeCoC0Ns,1066
 dfm_python/logger/inference_logger.py,sha256=TmXiWl703boJ3mIR-WgGpy6hfeF8a68OW9vrZppTTCE,10091
 dfm_python/logger/logger.py,sha256=1YzsUnvAnAK9CTcfttdSIMtgFvylO13LjQPMEsSs9xc,5187
 dfm_python/logger/train_logger.py,sha256=9VYhzwNYH1Rn7IQipKTY5wmVJcX1n6HDgnUZH0hChPY,17732
 dfm_python/models/__init__.py,sha256=3JsYZOqDGkl1eG8Zrc4-5gf7Sw47PzzDg2XBhsudB04,724
-dfm_python/models/base.py,sha256=-mKK2onDilvPLVRRrW3W7GYsMZxLMHjZWj6n6gL_xnw,30041
-dfm_python/models/ddfm.py,sha256=yZe7pxc81ZOsCwzoJ78UHXhEu5adRxkdTEvq-L17Sho,59415
-dfm_python/models/dfm.py,sha256=r4MnjhGv9bPB2spiluF6cWKw0cuwR_qq7pfw5oz8dlM,33675
-dfm_python/models/mcmc.py,sha256=QXBVxeBHREkc9K_xCGR1Y4sIFUkmX_RbRcgNktojuEc,23978
+dfm_python/models/base.py,sha256=WIx1d38bsaRkpfA4iYVG0_x7R1N8zVfi922LAlXqsEg,29261
+dfm_python/models/ddfm.py,sha256=oMHZTUMu7Dy1_8O1IK33prx_B15s0ex83Hq0lWP-91w,66657
+dfm_python/models/dfm.py,sha256=AK4gdwi5a_oEX8FqovyE19bCWN2GpL6SWiEx6-HsGqw,43318
+dfm_python/models/mcmc.py,sha256=IyQYyfNV5ou48Q3SfStPt3k07N74lSQE_u8W8QX93ro,24070
 dfm_python/models/utils.py,sha256=LAtvBqUYxnnl4dIaWl6MC-NHSuoyeNkHZppfWB1lCxQ,20193
 dfm_python/ssm/__init__.py,sha256=dFpt8_3jb_foISDNhma_JUAzch0VrKZcaeWP9zwpwoM,1161
-dfm_python/ssm/em.py,sha256=yRYA930YAqOn4B4oxsPER_-vzO8hDPzMYA_eGnn4Sec,63430
+dfm_python/ssm/em.py,sha256=sUiwuBVLTQBNHlBJ9YwemVY0wJYMTUqkuhyjPohENNw,63429
 dfm_python/ssm/kalman.py,sha256=KC6Q71VxM2NSdBVB5107QsNUm-_kE59MiyqrzImWz7E,45931
 dfm_python/ssm/utils.py,sha256=SYsu4E2zHAXXLfR2iQWNX-ywuzmwZj5kT4owHlyfvjU,13637
 dfm_python/trainer/__init__.py,sha256=gc8jRjM-y2XN911LldwYUYHAAuFEvzUJJvXhG6B3Sv4,25654
@@ -38,7 +38,7 @@ dfm_python/utils/diagnostics.py,sha256=UwrJfnn8VACATfQEkTcWw9eEJlTik-VCgTZ1dJABZ
 dfm_python/utils/helpers.py,sha256=iOVwkP5pti-q-HKOAkp6bUPAp7BPV7BBgh8jL76g9go,13049
 dfm_python/utils/statespace.py,sha256=P-GQiCP9HzI2d9PgYst4nmRuocEXr9gG1yzGYsYWkMg,44159
 dfm_python/utils/time.py,sha256=o6NRPuk4Ynk8kPaUnzR6xq3pPAO6aLUTR9N_3hn_N88,35651
-dfm_python-0.5.1.dist-info/METADATA,sha256=pf5lWV4TFMr4lKSNeUrtpHV_-ym3h_EaslIrAiioH8U,12159
-dfm_python-0.5.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dfm_python-0.5.1.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
-dfm_python-0.5.1.dist-info/RECORD,,
+dfm_python-0.5.3.dist-info/METADATA,sha256=6yDzvYrImsw5VC1FF2XyRGy7HLxlAs4c-jKU5cKMsw0,12007
+dfm_python-0.5.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dfm_python-0.5.3.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
+dfm_python-0.5.3.dist-info/RECORD,,