dfm-python 0.4.51__py3-none-any.whl → 0.5.1__py3-none-any.whl

dfm_python/__init__.py

@@ -6,7 +6,6 @@ This package implements a comprehensive Dynamic Factor Model framework with supp
 - Tent kernel aggregation for low-to-high frequency mapping
 - Expectation-Maximization (EM) algorithm for parameter estimation
 - Kalman filtering and smoothing for factor extraction
-- News decomposition for nowcasting
 - Deep Dynamic Factor Models (DDFM) with nonlinear encoders (requires PyTorch)
 
 The package implements a clock-based approach to mixed-frequency DFMs, where all latent 
@@ -23,7 +22,6 @@ Key Features:
     - Flexible block structure for factor modeling
     - Robust handling of missing data (internal spline interpolation)
     - Automatic standardization and data clipping
-    - News decomposition for forecast updates
 
 Example (Standard Lightning Pattern):
     >>> from dfm_python import DFM, DFMDataModule, DFMTrainer
@@ -67,7 +65,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.4.51"
+__version__ = "0.5.0"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -81,8 +79,7 @@ __version__ = "0.4.51"
 # 2. High-level API: DFM, DDFM, module-level convenience functions
 # 3. Core utilities: TimeIndex, diagnostics
 # 4. Models: BaseFactorModel, DDFM (low-level)
-# 5. Nowcasting: Nowcast, result classes, para_const
-# 6. Data & Results: DFMResult
+# 5. Data & Results: DFMResult
 # ============================================================================
 
 # Configuration (from config/ subpackage)
@@ -110,17 +107,6 @@ from .lightning import (
     EMAlgorithm,  # Module class
 )
 
-# Nowcasting (from nowcast/ subpackage)
-from .nowcast import (
-    Nowcast,
-    NowcastResult,
-)
-from .nowcast.helpers import (
-    para_const,
-    NewsDecompResult,
-    BacktestResult,
-)
-
 # Model implementations
 from .models.base import BaseFactorModel
 from .models.dfm import DFM
@@ -130,11 +116,9 @@ from .models.ddfm import DDFM, DDFMModel
 
 __all__ = [
     # Core classes
-    'DFM', 'Nowcast',
+    'DFM',
     # Model base and implementations
     'BaseFactorModel',
-    # Nowcast result classes
-    'NowcastResult', 'NewsDecompResult', 'BacktestResult',
     # Constants
     'DEFAULT_BLOCK_NAME',
     # Config sources
@@ -142,7 +126,6 @@ __all__ = [
     'MergedConfigSource', 'make_config_source',
     # Low-level API (functional interface - advanced usage)
     'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
-    'para_const',  # Internal utility for nowcasting
 ]
 
 # DDFM high-level API (PyTorch is mandatory)

dfm_python/config/adapter.py

@@ -214,8 +214,7 @@ class YamlSource:
                         frequency=series_data.get('frequency', 'm'),
                         transformation=series_data.get('transformation', 'lin'),
                         blocks=series_data.get('blocks', []),
-                        release_date=release_date,
-                        scaler=series_data.get('scaler', None)  # Optional, for per-series scaling
+                        release_date=release_date
                     ))
         
         # If no series loaded from separate files, try to get from main config

dfm_python/config/results.py

@@ -6,8 +6,9 @@ This module contains dataclasses for storing DFM estimation results and paramete
 import numpy as np
 import warnings
 from dataclasses import dataclass
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Dict, Any, Tuple
 from abc import ABC
+from datetime import datetime
 
 from .schema import DFMConfig
 
@@ -278,6 +279,59 @@ class DDFMResult(BaseResult):
     use_idiosyncratic: Optional[bool] = None  # Whether idio components were used
 
 
+@dataclass
+class NowcastResult:
+    """Result from a single nowcast calculation.
+    
+    This dataclass contains all information from a nowcast operation,
+    including the nowcast value, metadata about the data view, and
+    optional diagnostic information.
+    
+    Attributes
+    ----------
+    target_series : str
+        Target series ID that was nowcasted.
+    target_period : datetime
+        Target period for the nowcast (the period being estimated).
+    view_date : datetime
+        View date (when data is available). This determines which
+        data points are masked/unmasked in the nowcast calculation.
+    nowcast_value : float
+        The calculated nowcast value for the target series.
+    confidence_interval : Tuple[float, float], optional
+        Confidence interval (lower, upper) for the nowcast, if available.
+    factors_at_view : np.ndarray, optional
+        Factor values at the view_date (m,). These are the updated
+        factor states after applying the data view masking.
+    dfm_result : BaseResult, optional
+        Full DFM/DDFM result for this view. Can be used for further
+        analysis or diagnostics.
+    data_availability : Dict[str, int], optional
+        Dictionary with keys 'n_available' and 'n_missing' indicating
+        how many data points were available vs missing in the data view.
+    
+    Examples
+    --------
+    >>> from dfm_python import DFM
+    >>> import numpy as np
+    >>> model = DFM()
+    >>> trainer.fit(model, data_module)
+    >>> # Update state with new data, then predict
+    >>> X_std = np.random.randn(10, 5)  # Standardized data
+    >>> model.update(X_std)
+    >>> forecast = model.predict(horizon=1)
+    >>> print(f"Forecast: {forecast[0, 0]}")
+    """
+    target_series: str
+    target_period: datetime
+    view_date: datetime
+    nowcast_value: float
+    confidence_interval: Optional[Tuple[float, float]] = None  # (lower, upper)
+    factors_at_view: Optional[np.ndarray] = None  # Factor values at view_date
+    dfm_result: Optional[BaseResult] = None  # Full DFM/DDFM result for this view
+    data_availability: Optional[Dict[str, int]] = None  # n_available, n_missing
+
+
 @dataclass
 class FitParams:
     """Parameter overrides for DFM model fitting.

dfm_python/config/schema.py

@@ -200,11 +200,6 @@ class SeriesConfig:
         - Positive value (1-31): Day of month when data is released
         - Negative value: Days before end of previous month when data is released
         Example: 25 = released on 25th of each month, -5 = released 5 days before end of previous month
-    scaler : str, optional
-        Scaler type for this series: 'standard', 'robust', 'minmax', 'maxabs', 
-        'quantile', or None/null (no scaling). Used by create_scaling_transformer_from_config()
-        to automatically create per-series scaling transformers. If None, uses default
-        scaling strategy (typically 'standard').
     """
     # Required fields (no defaults)
     frequency: str
@@ -215,7 +210,6 @@ class SeriesConfig:
     series_name: Optional[str] = None  # Optional metadata for display
     units: Optional[str] = None  # Optional metadata for display only (used in news.py output)
     release_date: Optional[int] = None  # Release date for pseudo real-time nowcasting
-    scaler: Optional[str] = None  # Scaler type: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or None
     
     def __post_init__(self):
         """Validate fields after initialization."""
@@ -308,6 +302,7 @@ class BaseModelConfig:
     nan_method: int = 2  # Missing data handling method (1-5). Preprocessing step before Kalman Filter-based handling
     nan_k: int = 3  # Spline parameter for NaN interpolation (cubic spline)
     clock: str = 'm'  # Base frequency for nowcasting (global clock): 'd', 'w', 'm', 'q', 'sa', 'a' (defaults to 'm' for monthly)
+    scaler: Optional[str] = 'standard'  # Unified scaler type for all series: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or None (no scaling). Default: 'standard' for unified scaling.
     
     # ========================================================================
     # Internal cache (not user-configurable)
@@ -403,6 +398,11 @@ class BaseModelConfig:
                     f"Block '{block_name}' validation failed: AR lag must be at least 1, got {ar_lag}. "
                     f"Please set ar_lag >= 1 for block '{block_name}'."
                 )
+            if ar_lag > 2:
+                raise ValueError(
+                    f"Block '{block_name}' validation failed: AR lag must be at most 2 (maximum supported VAR order is VAR(2)), got {ar_lag}. "
+                    f"Please set ar_lag to 1 (VAR(1)) or 2 (VAR(2)) for block '{block_name}'."
+                )
         
         # Auto-generate series_id if not provided and convert blocks to indices
         n_blocks = len(self.block_names)
@@ -613,7 +613,7 @@ class DFMConfig(BaseModelConfig):
     # ========================================================================
     # EM Algorithm Parameters (HOW - controls the algorithm)
     # ========================================================================
-    ar_lag: int = 1  # Number of lags in AR transition equation (lookback window)
+    ar_lag: int = 1  # Number of lags in AR transition equation (lookback window). Must be 1 or 2 (maximum supported order is VAR(2))
     threshold: float = 1e-5  # EM convergence threshold
     max_iter: int = 5000  # Maximum EM iterations
     
@@ -660,6 +660,7 @@ class DFMConfig(BaseModelConfig):
             'nan_method': data.get('nan_method', 2),
             'nan_k': data.get('nan_k', 3),
             'clock': data.get('clock', 'm'),
+            'scaler': data.get('scaler', 'standard'),  # Unified scaler for all series (default: 'standard')
         }
     
     @classmethod
@@ -716,12 +717,12 @@ class DDFMConfig(BaseModelConfig):
     # ========================================================================
     encoder_layers: Optional[List[int]] = None  # Hidden layer dimensions for encoder (default: [64, 32])
     num_factors: Optional[int] = None  # Number of factors (inferred from config if None)
-    activation: str = 'tanh'  # Activation function ('tanh', 'relu', 'sigmoid', default: 'tanh')
+    activation: str = 'relu'  # Activation function ('tanh', 'relu', 'sigmoid', default: 'relu' to match original DDFM)
     use_batch_norm: bool = True  # Use batch normalization in encoder (default: True)
     learning_rate: float = 0.001  # Learning rate for Adam optimizer (default: 0.001)
     epochs: int = 100  # Number of training epochs (default: 100)
-    batch_size: int = 32  # Batch size for training (default: 32)
-    factor_order: int = 1  # VAR lag order for factor dynamics (1 or 2, default: 1)
+    batch_size: int = 100  # Batch size for training (default: 100 to match original DDFM)
+    factor_order: int = 1  # VAR lag order for factor dynamics. Must be 1 or 2 (maximum supported order is VAR(2), default: 1)
     use_idiosyncratic: bool = True  # Model idio components with AR(1) dynamics (default: True)
     min_obs_idio: int = 5  # Minimum observations for idio AR(1) estimation (default: 5)
     
@@ -750,15 +751,15 @@ class DDFMConfig(BaseModelConfig):
     def _extract_ddfm(cls, data: Dict[str, Any]) -> Dict[str, Any]:
         """Extract DDFM-specific parameters from config dict."""
         base_params = cls._extract_base(data)
-        # Handle both direct keys and ddfm_ prefix format (for backward compatibility)
+        # Handle both direct keys and ddfm_ prefix format
         base_params.update({
             'encoder_layers': data.get('encoder_layers') or data.get('ddfm_encoder_layers', None),
             'num_factors': data.get('num_factors') or data.get('ddfm_num_factors', None),
-            'activation': data.get('activation') or data.get('ddfm_activation', 'tanh'),
+            'activation': data.get('activation') or data.get('ddfm_activation', 'relu'),
             'use_batch_norm': data.get('use_batch_norm', data.get('ddfm_use_batch_norm', True)),
             'learning_rate': data.get('learning_rate', data.get('ddfm_learning_rate', 0.001)),
             'epochs': data.get('epochs', data.get('ddfm_epochs', 100)),
-            'batch_size': data.get('batch_size', data.get('ddfm_batch_size', 32)),
+            'batch_size': data.get('batch_size', data.get('ddfm_batch_size', 100)),
             'factor_order': data.get('factor_order', data.get('ddfm_factor_order', 1)),
             'use_idiosyncratic': data.get('use_idiosyncratic', data.get('ddfm_use_idiosyncratic', True)),
             'min_obs_idio': data.get('min_obs_idio', data.get('ddfm_min_obs_idio', 5)),
@@ -832,8 +833,7 @@ class DDFMConfig(BaseModelConfig):
                     transformation=series_cfg.get('transformation', 'lin'),
                     blocks=series_blocks,
                     units=series_cfg.get('units', None),  # Optional, for display only
-                    release_date=series_cfg.get('release_date', None),  # Optional, for nowcasting
-                    scaler=series_cfg.get('scaler', None)  # Optional, for per-series scaling
+                    release_date=series_cfg.get('release_date', None)  # Optional, for nowcasting
                 ))
         
         # Convert blocks_dict to dict of block properties

dfm_python/data/__init__.py

@@ -4,6 +4,7 @@ This module provides:
 - Dataset classes: DFMDataset (linear DFM), DDFMDataset (deep DDFM)
 - DataLoader factories: create_dfm_dataloader, create_ddfm_dataloader
 - Data reading: read_data, load_data
+- Nowcast data classes moved to src.nowcasting in main project
 
 Note: Transformation functions are application-specific and should be provided
 by users in their preprocessing pipelines. The package does not include
@@ -13,6 +14,7 @@ transformation utilities to remain generic.
 from .dataset import DFMDataset, DDFMDataset
 from .dataloader import create_dfm_dataloader, create_ddfm_dataloader
 from .utils import read_data, load_data
+# Nowcast data classes moved to src.nowcasting in main project
 
 __all__ = [
     # Datasets

dfm_python/data/dataloader.py

@@ -101,7 +101,7 @@ def create_dfm_dataloader(
 
 def create_ddfm_dataloader(
     dataset: DDFMDataset,
-    batch_size: int = 32,
+    batch_size: int = 100,
     shuffle: bool = True,
     num_workers: int = 0,
     pin_memory: bool = True
@@ -115,8 +115,8 @@ def create_ddfm_dataloader(
     ----------
     dataset : DDFMDataset
         DDFM dataset instance with windowed sequences
-    batch_size : int, default 32
-        Batch size for training
+    batch_size : int, default 100
+        Batch size for training (matches original DDFM)
     shuffle : bool, default True
         Whether to shuffle samples (typically True for training)
     num_workers : int, default 0

dfm_python/data/utils.py

@@ -17,7 +17,6 @@ from ..config import DFMConfig
 from ..config.utils import FREQUENCY_HIERARCHY
 from ..utils.time import TimeIndex, parse_timestamp
 from ..utils.helpers import (
-    safe_get_attr,
     get_frequencies,
     get_series_ids,
 )
@@ -209,7 +208,7 @@ def load_data(
     Time : TimeIndex
         Time index for the data (aligned to clock frequency)
     Z : np.ndarray
-        Original untransformed data (T x N), same as X (for backward compatibility)
+        Original untransformed data (T x N), same as X
         
     Raises
     ------
@@ -267,7 +266,7 @@ def load_data(
     _logger.info(f"Loaded data: {X.shape[0]} time periods, {X.shape[1]} series (raw, not transformed)")
     
     # Validate data quality
-    clock = safe_get_attr(config, 'clock', 'm')
+    clock = getattr(config, 'clock', 'm')
     clock_hierarchy = FREQUENCY_HIERARCHY.get(clock, 3)
     
     frequencies = get_frequencies(config)

dfm_python/encoder/vae.py

@@ -454,6 +454,21 @@ def extract_decoder_params(decoder) -> Tuple[np.ndarray, np.ndarray]:
     # Decoder weight is already (N x m), so no transpose needed
     C = weight
     
+    # Check for NaN in extracted C matrix (indicates numerical instability during training)
+    if np.any(np.isnan(C)):
+        nan_count = np.sum(np.isnan(C))
+        nan_ratio = nan_count / C.size
+        from ..logger import get_logger
+        _logger = get_logger(__name__)
+        _logger.warning(
+            f"extract_decoder_params: C matrix contains {nan_count}/{C.size} NaN values ({nan_ratio:.1%}). "
+            f"This indicates the decoder weights contain NaN, likely due to numerical instability during training. "
+            f"Possible causes: (1) learning rate too high, (2) gradient explosion, (3) unstable training."
+        )
+        # Replace NaN with zeros to prevent further issues
+        C = np.nan_to_num(C, nan=0.0)
+        _logger.warning("Replaced NaN values in C matrix with zeros.")
+    
     return C, bias
 
 

dfm_python/lightning/__init__.py

@@ -28,15 +28,14 @@ from .scaling import (
 )
 
 # DFMTrainingState is defined in models.dfm and exported here for convenience
-# Note: dfm_module.py has been removed - DFM class implements EM directly
 from ..models.dfm import DFMTrainingState
 
 __all__ = [
     # Kalman filter
-    'KalmanFilter',  # New module class
+    'KalmanFilter',
     'KalmanFilterState',
     # EM algorithm
-    'EMAlgorithm',  # New module class
+    'EMAlgorithm',
     'EMStepParams',
     # Data handling
     'DFMDataModule',
@@ -50,7 +49,5 @@ __all__ = [
     'NoScalingStrategy',
     # Training state (defined in models.dfm)
     'DFMTrainingState',
-    # DDFMLightningModule removed: deprecated and unused (DDFM uses DDFMModel directly)
-    # DDFMTrainingState removed: duplicate (also defined in models.ddfm)
 ]
 

dfm_python/lightning/data_module.py

@@ -43,13 +43,11 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
     
     This function depends on sktime's pipeline structure to traverse
     TransformerPipeline and sklearn transformers (StandardScaler, etc.).
-    Note: TabularToSeriesAdaptor support is for backward compatibility only.
     
     Parameters
     ----------
     transformer : Any
-        Transformer to search (can be any scaler, StandardScaler directly, or TabularToSeriesAdaptor for backward compatibility). 
-        TransformerPipeline, sklearn Pipeline, etc.)
+        Transformer to search (StandardScaler, TransformerPipeline, sklearn Pipeline, etc.)
         
     Returns
     -------
@@ -60,9 +58,7 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
     if transformer is None:
         return None
     
-    # Check if transformer is TabularToSeriesAdaptor (backward compatibility only)
-    # Note: New code should use StandardScaler() directly per sktime docs
-    # This check is kept for backward compatibility with old code that may still use TabularToSeriesAdaptor
+    # Check if transformer is wrapped (TabularToSeriesAdaptor or similar)
     if hasattr(transformer, 'transformer'):
         # Recursively search the wrapped transformer
         wrapped = transformer.transformer
@@ -78,8 +74,7 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
             if scaler is not None:
                 return scaler
     
-    # Check if transformer is ColumnTransformer (backward compatibility only - not recommended for new code)
-    # Unified scaling with StandardScaler is preferred per sktime docs
+    # Check if transformer is ColumnEnsembleTransformer (or ColumnTransformer)
     if hasattr(transformer, 'transformers'):
         for name, trans, cols in transformer.transformers:
             scaler = _get_scaler(trans)
@@ -187,34 +182,76 @@ def _get_scale(scaler: Any, data: np.ndarray) -> Optional[np.ndarray]:
 def _normalize_wx(wx: np.ndarray) -> np.ndarray:
     """Normalize Wx to avoid division by zero.
     
+    This function replaces zero or NaN values in Wx with 1.0 to prevent
+    division by zero during standardization/unstandardization.
+    
     Parameters
     ----------
     wx : np.ndarray
-        Scale values (N,)
+        Scale values (N,), may contain zeros or NaN
         
     Returns
     -------
     np.ndarray
-        Normalized scale values with zeros replaced by 1.0
+        Normalized scale values with zeros and NaN replaced by 1.0
     """
-    return np.where(wx == 0, 1.0, wx)
+    # Replace both zero and NaN with 1.0
+    return np.where((wx == 0) | np.isnan(wx), 1.0, wx)
 
 
 def _compute_mx_wx(data: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
     """Compute Mx and Wx from data as fallback.
     
+    This function computes mean (Mx) and standard deviation (Wx) from data,
+    handling missing values (NaN) by using nan-aware functions. If NaN values
+    are detected, a warning is issued recommending imputation.
+    
     Parameters
     ----------
     data : np.ndarray
-        Processed data array (T x N)
+        Processed data array (T x N), may contain NaN values
         
     Returns
     -------
     Tuple[np.ndarray, np.ndarray]
-        (Mx, Wx) tuple where Mx is mean and Wx is normalized std
+        (Mx, Wx) tuple where Mx is mean and Wx is normalized std.
+        NaN values are handled using nanmean and nanstd.
+        
+    Notes
+    -----
+    If data contains NaN values, this function will:
+    1. Use np.nanmean() and np.nanstd() to compute statistics ignoring NaN
+    2. Issue a warning recommending data imputation for better results
+    3. Continue processing with available data
     """
-    mx = np.mean(data, axis=0)
-    wx = np.std(data, axis=0)
+    # Check for NaN values
+    has_nan = np.any(np.isnan(data))
+    if has_nan:
+        nan_count = np.sum(np.isnan(data))
+        nan_pct = 100.0 * nan_count / data.size
+        _logger.warning(
+            f"Data contains {nan_count} NaN values ({nan_pct:.1f}%). "
+            f"Using nanmean/nanstd to compute standardization parameters. "
+            f"For better results, consider imputing missing values before creating DataModule. "
+            f"Suggested approaches: forward-fill, backward-fill, or interpolation."
+        )
+    
+    # Use nan-aware functions to handle missing values
+    mx = np.nanmean(data, axis=0)
+    wx = np.nanstd(data, axis=0)
+    
+    # Check if any series have all NaN values (would result in NaN std)
+    nan_std_mask = np.isnan(wx)
+    if np.any(nan_std_mask):
+        n_nan_std = np.sum(nan_std_mask)
+        _logger.warning(
+            f"{n_nan_std} series have all NaN values, resulting in NaN standard deviation. "
+            f"These will be normalized to 1.0 to avoid division by zero. "
+            f"Consider imputing or removing these series."
+        )
+        # Replace NaN std with 1.0 (will be normalized by _normalize_wx anyway)
+        wx = np.where(nan_std_mask, 1.0, wx)
+    
     wx = _normalize_wx(wx)
     return mx, wx
 
@@ -261,15 +298,17 @@ class DFMDataModule(lightning_pl.LightningDataModule):
     
     This DataModule handles data loading for DFM/DDFM models.
     
-    **Important**: This package expects preprocessed data. All preprocessing (imputation, 
-    scaling, feature engineering, etc.) must be done by the user before passing data to 
-    this DataModule. Users are responsible for creating their own preprocessing pipelines 
-    using sktime or other tools.
+    **Important**: DFM and DDFM can handle missing data (NaN values) implicitly:
+    - **DFM**: Uses Kalman filter's `handle_missing_data()` method to skip NaN observations
+    - **DDFM**: Uses state-space model and MCMC procedure to handle missing data through
+      idiosyncratic component estimation
     
     **Usage Pattern**:
-    - Provide already-preprocessed data (standardized, no missing values, feature-engineered)
+    - Data can contain NaN values - models will handle them implicitly
     - If `pipeline=None`, a passthrough transformer is used by default (no-op)
     - Users can optionally provide their preprocessing pipeline to extract statistics (Mx/Wx)
+    - For better performance, users can preprocess data (imputation, scaling) before passing,
+      but it's not required - models will handle missing data automatically
     
     For linear DFM, this uses DFMDataset which returns full sequences.
     For DDFM, use DDFMDataModule which uses DDFMDataset with windowing.
@@ -311,9 +350,10 @@ class DFMDataModule(lightning_pl.LightningDataModule):
     data_path : str or Path, optional
         Path to data file (CSV). If None, data must be provided.
     data : np.ndarray or pd.DataFrame, optional
-        Preprocessed data array or DataFrame. Data must be:
-        - Standardized/scaled (mean=0, std=1 recommended for DFM/DDFM)
-        - No missing values (NaN-free)
+        Data array or DataFrame. Can contain NaN values - DFM/DDFM will handle them:
+        - DFM: Uses Kalman filter to implicitly handle missing data
+        - DDFM: Uses state-space model and MCMC to handle missing data
+        - Standardized/scaled data (mean=0, std=1) is recommended for better performance
         - Feature-engineered if needed
         If None, data_path must be provided.
     time_index : TimeIndex, optional
@@ -417,20 +457,22 @@ class DFMDataModule(lightning_pl.LightningDataModule):
         This method is called by Lightning to set up the data module.
         It loads preprocessed data and extracts statistics from the pipeline for forecasting/nowcasting.
         
-        **Important**: Data must be fully preprocessed by the user before passing to this DataModule.
-        All preprocessing (imputation, scaling, feature engineering, etc.) is the user's responsibility.
+        **Important**: DFM and DDFM can handle missing data (NaN values) implicitly:
+        - **DFM**: Uses Kalman filter's `handle_missing_data()` method
+        - **DDFM**: Uses state-space model and MCMC procedure
         
         **Pipeline Purpose**:
         - The pipeline is used to extract statistics (e.g., Mx/Wx from StandardScaler) needed for
           forecasting and nowcasting operations, not to preprocess the data
-        - Data is already preprocessed - the pipeline is fitted to extract standardization parameters
         - The pipeline is fitted on the **entire dataset** to extract statistics
         - If `val_split` is provided, it's used for validation during training (e.g., DDFM neural network)
         
         **Usage**:
-        - Provide preprocessed data (standardized, no missing values)
+        - Data can contain NaN values - models will handle them automatically
         - Optionally provide pipeline (e.g., TransformerPipeline with StandardScaler) to extract Mx/Wx for predictions
         - If pipeline=None, uses passthrough (no statistics extracted, Mx/Wx computed from data)
+        - For better performance, users can preprocess data (imputation, scaling) before passing,
+          but it's not required
         """
         # Load data if not already provided
         if self.data is None:
@@ -451,6 +493,9 @@ class DFMDataModule(lightning_pl.LightningDataModule):
             self.time_index = Time
         
         # Convert to pandas DataFrame if needed
+        # Note: DFM/DDFM can handle NaN values implicitly via Kalman filter
+        # (DFM uses handle_missing_data() in Kalman filter, DDFM uses state-space model)
+        # So we don't validate NaN here - let the models handle it
         if isinstance(self.data, np.ndarray):
             series_ids = self.config.get_series_ids()
             X_df = pd.DataFrame(self.data, columns=series_ids)
@@ -505,7 +550,10 @@ class DFMDataModule(lightning_pl.LightningDataModule):
                 f"automatically extracted from the pipeline."
             ) from e
         
-        # Ensure output is pandas DataFrame
+        # Note: DFM/DDFM can handle NaN values implicitly via Kalman filter
+        # So we don't validate NaN here - let the models handle it
+        
+        # Ensure output is pandas DataFrame with compatible index
         if not isinstance(X_transformed, pd.DataFrame):
             # If transformer returned numpy array, convert to pandas
             if isinstance(X_transformed, np.ndarray):
@@ -532,6 +580,45 @@ class DFMDataModule(lightning_pl.LightningDataModule):
                         # Create generic column names
                         X_transformed = pd.DataFrame(X_transformed, 
                             columns=[f'feature_{i}' for i in range(n_cols)])
+                
+                # Ensure index is compatible (DatetimeIndex, PeriodIndex, or RangeIndex)
+                # Use input DataFrame index if length matches
+                if len(X_transformed) == len(X_df):
+                    if isinstance(X_df.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
+                        X_transformed.index = X_df.index
+                    else:
+                        # Try to convert to DatetimeIndex
+                        try:
+                            X_transformed.index = pd.to_datetime(X_df.index)
+                        except (ValueError, TypeError):
+                            # Fallback to RangeIndex
+                            X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
+                else:
+                    # Length doesn't match - use RangeIndex
+                    X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
+        
+        # Ensure DataFrame index is compatible (DatetimeIndex, PeriodIndex, or RangeIndex)
+        # This handles cases where pipeline output has incompatible index
+        if isinstance(X_transformed, pd.DataFrame):
+            if not isinstance(X_transformed.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
+                # Try to use input DataFrame index if length matches
+                if len(X_transformed) == len(X_df):
+                    if isinstance(X_df.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
+                        X_transformed.index = X_df.index
+                    else:
+                        # Try to convert to DatetimeIndex
+                        try:
+                            X_transformed.index = pd.to_datetime(X_df.index)
+                        except (ValueError, TypeError):
+                            # Fallback to RangeIndex
+                            X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
+                else:
+                    # Try to convert existing index
+                    try:
+                        X_transformed.index = pd.to_datetime(X_transformed.index)
+                    except (ValueError, TypeError):
+                        # Fallback to RangeIndex
+                        X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
             elif hasattr(X_transformed, 'to_numpy'):
                 # Has to_numpy method (pandas DataFrame) - convert to numpy then back to DataFrame
                 # Try to preserve column names if available
@@ -572,9 +659,7 @@ class DFMDataModule(lightning_pl.LightningDataModule):
         # This is optional - some pipelines may not have standardization
         # Mx and Wx are already initialized in __init__
         try:
-            # Extract scaler from pipeline (works with TransformerPipeline, StandardScaler directly, or TabularToSeriesAdaptor for backward compatibility)
-            # Supports any scaler type: StandardScaler, MinMaxScaler, RobustScaler, etc.
-            # Note: New code should use StandardScaler() directly per sktime docs (unified scaling)
+            # Extract scaler from pipeline (supports TransformerPipeline, StandardScaler, etc.)
             scaler = _get_scaler(pipeline_to_use)
             
             if scaler is not None: