dfm-python 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

dfm_python/__init__.py

@@ -67,7 +67,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.4.3"
+__version__ = "0.4.5"
 
 # ============================================================================
 # PUBLIC API DEFINITION

dfm_python/lightning/__init__.py

@@ -31,13 +31,6 @@ from .scaling import (
 # Note: dfm_module.py has been removed - DFM class implements EM directly
 from ..models.dfm import DFMTrainingState
 
-# DDFMLightningModule is deprecated and unused - DDFM class uses DDFMModel directly.
-# Keeping ddfm_module.py file for reference only, but not exporting it.
-# from .ddfm_module import (
-#     DDFMLightningModule,  # Deprecated: not used (DDFM uses DDFMModel)
-#     DDFMTrainingState,  # Duplicate: also defined in models.ddfm
-# )
-
 __all__ = [
     # Kalman filter
     'KalmanFilter',  # New module class

dfm_python/models/base.py

@@ -499,43 +499,6 @@ class BaseFactorModel(pl.LightningModule):
             self.training_state = None
         return self
     
-    def plot(self, **kwargs) -> 'BaseFactorModel':
-        """Plot common visualizations (not yet implemented).
-        
-        .. note::
-            This method is a placeholder for future visualization features.
-            For now, use external plotting libraries (matplotlib, plotly) with
-            model results: ``result = model.get_result()`` then plot ``result.Z``
-            (factors) or ``result.X_sm`` (smoothed data).
-        
-        Parameters
-        ----------
-        **kwargs
-            Additional keyword arguments (reserved for future use)
-            
-        Returns
-        -------
-        BaseFactorModel
-            Self for method chaining
-            
-        Raises
-        ------
-        ValueError
-            If model has not been trained yet
-        """
-        if self.training_state is None:
-            error_msg = self._format_error_message(
-                operation="plotting",
-                reason="model has not been trained yet",
-                guidance="Please call trainer.fit(model, data_module) first"
-            )
-            raise ValueError(error_msg)
-        _logger.warning(
-            "Plot functionality not yet implemented. Use external plotting libraries "
-            "with model results: result = model.get_result(); plot(result.Z)"
-        )
-        return self
-    
     def load_pickle(self, path: Union[str, Path], **kwargs) -> 'BaseFactorModel':
         """Load a saved model from pickle file.
         

dfm_python/nowcast/helpers.py

@@ -236,8 +236,6 @@ from .utils import (
     extract_news,
 )
 
-# Backward compatibility aliases removed - use direct imports from nowcast.utils
-
 # ============================================================================
 # Backtest result classes (merged from backtest.py)
 # ============================================================================

dfm_python/nowcast/splitters.py

@@ -4,19 +4,19 @@ This module provides custom splitter implementations for nowcasting evaluation
 that respect publication lags and backward-looking nature of nowcasting.
 """
 
-from typing import Iterator, Tuple, List, Optional, Union
+from typing import Iterator, Tuple, List, Optional, Union, Any
 from datetime import datetime, timedelta
 import numpy as np
 import pandas as pd
 
-from sktime.split import BaseSplitter
+from sktime.split.base import BaseSplitter
 from sktime.forecasting.base import BaseForecaster
 
 from ..config import DFMConfig
 from ..utils.time import TimeIndex, parse_timestamp, to_python_datetime
 from ..utils.data import create_data_view, calculate_release_date
-from ..utils.helpers import get_series_ids
-from .utils import calc_backward_date, get_higher_frequency, get_clock_frequency
+from ..utils.helpers import get_series_ids, get_clock_frequency
+from .utils import calc_backward_date, get_higher_frequency
 
 
 class NowcastingSplitter(BaseSplitter):

dfm_python/nowcast/utils.py

@@ -213,10 +213,6 @@ def check_config(saved_config: Any, current_config: DFMConfig) -> None:
         # If comparison fails, continue anyway
 
 
-# transform_data and _transform_series removed - use DataModule with custom transformers instead
-# transform_series alias removed
-
-
 def extract_news(
     singlenews: np.ndarray,
     weight: np.ndarray,

{dfm_python-0.4.3.dist-info → dfm_python-0.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.4.3
+Version: 0.4.5
 Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme

{dfm_python-0.4.3.dist-info → dfm_python-0.4.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dfm_python/__init__.py,sha256=aQ4O9IEetFvKSTmgyTOccHPiSKWcvTD5b-g5kGaXhZM,6159
+dfm_python/__init__.py,sha256=wSJzWhUQkHZKVcDGx1W36QzTj6zZ-OgZ82qiTMWf0qg,6159
 dfm_python/config/__init__.py,sha256=NrTwjNzQRcYlk5Ht0hTF8o-6CCl5e4S_JxQAp1mNyCs,1507
 dfm_python/config/adapter.py,sha256=KS9C35A1FMRjpU-yJ19sYciurIy0KWtrcJxosoIzOCE,27575
 dfm_python/config/results.py,sha256=k_WY8v_rJyA8e5zr6Pq6KwNLcq1CbCpavndgBYyaqLs,13778
@@ -12,27 +12,24 @@ dfm_python/encoder/__init__.py,sha256=ljPTXq04cpOsGXNnma2EJ447kPFlytT6_mToLyKok7
 dfm_python/encoder/base.py,sha256=MOSn_bsQP-GVonGprJymQq-_Q6tmCQex8VOWUJjyxNA,3029
 dfm_python/encoder/pca.py,sha256=FsvCqXD0AuASxymV2-5oAM2QVv_syhSyL3CmxVOXxgM,10584
 dfm_python/encoder/vae.py,sha256=a5LYl9bb2j2kcZL-irPMTlwHTotCAMOCNA6BmG0xqw4,20431
-dfm_python/lightning/__init__.py,sha256=czIV9GYj2z2BxK30Wpbtp7grc6gmRycwzu6NbYgG1uc,1853
-dfm_python/lightning/callbacks.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dfm_python/lightning/__init__.py,sha256=S73pzk72n7Mk8to6urty_PULKyo83Y4FiXNWSScI2-4,1522
 dfm_python/lightning/data_module.py,sha256=GKXNUgTb4Hivs1NFrj05YVH7UEwIt8o4sa_DEOuoYSI,30478
 dfm_python/lightning/scaling.py,sha256=o6DlimsZjf0Z3zosezURin6NBTOE9kAfdcdXKHbZdz4,16359
-dfm_python/lightning/utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dfm_python/logger/__init__.py,sha256=2GwtfOG2ttbtJ5CrO6338R6GIao3YuY0HLiBeCoC0Ns,1066
 dfm_python/logger/inference_logger.py,sha256=EeFfux4UxXhIosg8dEjbkkdm38_4LKZdGS_6p_xpzWI,10118
 dfm_python/logger/logger.py,sha256=cledaY-VvZx_HzsV4Ukuw5tP5sEvkxiAqOKJlwbDWXA,3768
 dfm_python/logger/train_logger.py,sha256=7ufZqDxexRmTI_kokoOyAGYQBmipRb0dALLXawJqq4E,17759
 dfm_python/models/__init__.py,sha256=mrs2N5OoFeNWCjS6luPjmwzFosNDkpdMiBoA24NMrE4,640
-dfm_python/models/base.py,sha256=0bIMEWbPJGgp5vC_uU_qt3ZtT1XkDQ2rsBiTUiNjwKM,21052
+dfm_python/models/base.py,sha256=QG3Mc2c2ccZx_Zmr1CnNO8B3Ek6yCm86EoKQccycsBA,19718
 dfm_python/models/ddfm.py,sha256=SbVva4gwYM8dN3ibf725Svqv4jyeTZEFOja87ZHeJqk,107381
 dfm_python/models/dfm.py,sha256=pfBMI5-vxLPzYkG7xa6yWY1TkDqpBdtGAdv0rRazKpA,25382
-dfm_python/models/estimation.py,sha256=hc10Dw4E7reBF2dXDjqyY2Z9_RDLNM4QBZIYSFJMdds,14005
 dfm_python/nowcast/__init__.py,sha256=AI51eoxdpKh-7EksDHdGcQple6bN_Gferp8VRCNmpYU,954
 dfm_python/nowcast/dataview.py,sha256=KpLZHVu9QPFDmTWWxZdlTTQ8Z1Q-676NJ47TpWKr2-I,4439
-dfm_python/nowcast/helpers.py,sha256=9YgTQbReDFnw0otmsAvO0xA0IfwR8nhb9K0Ru82uVjU,13849
+dfm_python/nowcast/helpers.py,sha256=pUWVrMMrc2N8MwD6qQw_a-kXUAw_t8CKe4Mjnf-z96U,13767
 dfm_python/nowcast/nowcast.py,sha256=bXlYUVZ-Lx05u2eqD42OtFhGpYAecx2GFQ2c9xpuueg,50746
-dfm_python/nowcast/splitters.py,sha256=xaqvcQlWwGl7itlVi7KgWfLGDnNYtBQcxG4lHDWWR5A,15768
+dfm_python/nowcast/splitters.py,sha256=3RWhRxInoLVW6Oh34Shu88IT3WBM9TK3Wpig2cpQ_oI,15778
 dfm_python/nowcast/transformers.py,sha256=Kf_Wd3_xeCKqpddsNnoPOhip1sWF7AhuzcsKSzaHMQo,10645
-dfm_python/nowcast/utils.py,sha256=_bl9_GsEZZP9I8z2qqFnzpjzzYjMeJP_W7U0QiosUJw,9362
+dfm_python/nowcast/utils.py,sha256=aFbXTWbBAYebhznfa15X8kL8drAe7i-yECMtnjMDRvc,9230
 dfm_python/ssm/__init__.py,sha256=dFpt8_3jb_foISDNhma_JUAzch0VrKZcaeWP9zwpwoM,1161
 dfm_python/ssm/em.py,sha256=N1cGTpsE1A6phLWDHYB6mYm8RYd09FZSwOweGjoSa8Y,34760
 dfm_python/ssm/kalman.py,sha256=USWXDJ5nj1sk6FXgG2zXe-VYGdKDYj-kGruIvy8cfHU,24501
@@ -46,7 +43,7 @@ dfm_python/utils/diagnostics.py,sha256=UwrJfnn8VACATfQEkTcWw9eEJlTik-VCgTZ1dJABZ
 dfm_python/utils/helpers.py,sha256=t-hJNgmPbk5KV5nGP7z_MF0sVh-VJMdQyoDvRBzwTV8,13020
 dfm_python/utils/statespace.py,sha256=pyKy1D8bLMmd7sDvAVSb2n74U0IfH2YuYreIfyg3MAA,43823
 dfm_python/utils/time.py,sha256=l9xu1nwLquWpRgWayRELPhTymV9P47s3s4Q8b-gl0IM,35745
-dfm_python-0.4.3.dist-info/METADATA,sha256=OLTnar9fKHbI8jSefyJz-jShnp1bSUqZqXxRR8B1PMw,48890
-dfm_python-0.4.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dfm_python-0.4.3.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
-dfm_python-0.4.3.dist-info/RECORD,,
+dfm_python-0.4.5.dist-info/METADATA,sha256=0Av0uXKYn-qwlaLTxBPo-K5_Evv56v2j1nkzyXxk2pY,48890
+dfm_python-0.4.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dfm_python-0.4.5.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
+dfm_python-0.4.5.dist-info/RECORD,,

dfm_python/models/estimation.py

@@ -1,371 +0,0 @@
-"""Dynamic Factor Model (DFM) estimation using PyTorch Lightning.
-
-This module provides the core DFM estimation function (_dfm_core) which
-uses PyTorch Lightning modules for training.
-
-The implementation uses a clock-based approach, where all latent factors
-evolve at a common clock frequency, with lower-frequency observations
-mapped to higher-frequency latent states via deterministic tent kernels.
-
-Note: This module now acts as a wrapper around Lightning modules. The actual
-estimation logic is in dfm_python.lightning modules.
-"""
-
-import numpy as np
-from typing import Tuple, Optional, Any, Dict, Union
-import warnings
-import logging
-import pandas as pd
-from ..logger import get_logger
-
-from ..config import DFMConfig
-from ..utils.time import calculate_rmse
-from ..utils.diagnostics import (
-    _display_dfm_tables,
-)
-from ..utils.helpers import (
-    safe_get_method, safe_get_attr, resolve_param,
-    get_series_names, get_frequencies, ParameterResolver
-)
-
-from ..config.utils import (
-    get_agg_structure,
-    FREQUENCY_HIERARCHY,
-)
-
-from ..config.results import DFMResult, FitParams
-
-_logger = get_logger(__name__)
-
-
-def _prepare_data(
-    X: np.ndarray,
-    config: DFMConfig,
-    params: Optional[FitParams] = None,
-) -> Tuple[np.ndarray, np.ndarray, Dict[str, Any]]:
-    """Prepare data and resolve all parameters from config and overrides.
-    
-    Parameters
-    ----------
-    X : np.ndarray
-        Input data matrix (T x N)
-    config : DFMConfig
-        Configuration object
-    params : FitParams, optional
-        Parameter overrides. If None, all values from config are used.
-    
-    Returns
-    -------
-    X_clean : np.ndarray
-        Cleaned input data (Inf replaced with NaN)
-    blocks : np.ndarray
-        Block structure array (N x n_blocks)
-    params_dict : dict
-        Dictionary of resolved parameters
-    """
-    # Clean input data
-    inf_mask = np.isinf(X)
-    if np.any(inf_mask):
-        X = np.where(inf_mask, np.nan, X)
-        warnings.warn("Data contains Inf values, replaced with NaN", UserWarning)
-    
-    blocks = config.get_blocks_array()
-    T, N = X.shape
-    
-    # Initialize params if not provided
-    if params is None:
-        params = FitParams()
-    
-    # Use ParameterResolver for consistent parameter resolution
-    resolver = ParameterResolver(config, params)
-    params_dict = resolver.resolve_estimation_params()
-    
-    # Add model structure parameters
-    params_dict['r'] = (np.array(config.factors_per_block) 
-                        if config.factors_per_block is not None 
-                        else np.ones(blocks.shape[1]))
-    params_dict['T'] = T
-    params_dict['N'] = N
-    
-    # Display blocks structure if debug logging enabled
-    if _logger.isEnabledFor(logging.DEBUG):
-        try:
-            series_names = get_series_names(config)
-            block_names = (config.block_names if len(config.block_names) == blocks.shape[1] 
-                          else [f'Block_{i+1}' for i in range(blocks.shape[1])])
-            # Create pandas DataFrame
-            df_dict = {block_names[i]: blocks[:, i].tolist() for i in range(blocks.shape[1])}
-            df_dict['series'] = [name.replace(' ', '_') for name in series_names]
-            df = pd.DataFrame(df_dict)
-            _logger.debug('Block Loading Structure')
-            _logger.debug(f'\n{df}')
-            _logger.debug(f'Blocks shape: {blocks.shape}')
-        except Exception as e:
-            _logger.debug(f'Error displaying block structure: {e}')
-    
-    return X, blocks, params_dict
-
-
-def _prepare_agg_structure(
-    config: DFMConfig,
-    clock: str
-) -> Tuple[Dict[str, np.ndarray], Optional[np.ndarray], Optional[np.ndarray], Optional[np.ndarray], np.ndarray, int, np.ndarray]:
-    """Prepare aggregation structure for mixed-frequency handling.
-    
-    Returns
-    -------
-    tent_weights_dict : dict
-        Dictionary mapping frequency pairs to tent weights
-    R_mat : np.ndarray or None
-        Constraint matrix for tent kernel aggregation
-    q : np.ndarray or None
-        Constraint vector for tent kernel aggregation
-    frequencies : np.ndarray or None
-        Array of frequencies for each series
-    i_idio : np.ndarray
-        Indicator array (1 for clock frequency, 0 for slower frequencies)
-    nQ : int
-        Number of slower-frequency series
-    idio_chain_lengths : np.ndarray
-        Array of idiosyncratic chain lengths per series (0, 1, or tent length)
-    """
-    from ..config.utils import compute_idio_lengths
-    
-    agg_info = get_agg_structure(config, clock=clock)
-    tent_weights_dict = agg_info.get('tent_weights', {})
-    frequencies = np.array(get_frequencies(config)) if config.series else None
-    
-    # Find R_mat and q for tent kernel constraints
-    R_mat = None
-    q = None
-    if agg_info['structures']:
-        max_periods = 0
-        for (slower_freq, clock_freq), (R, q_vec) in agg_info['structures'].items():
-            if R is not None:
-                n_periods = R.shape[1]
-                if n_periods > max_periods:
-                    max_periods = n_periods
-                    R_mat = R
-                    q = q_vec
-    
-    # Compute i_idio and nQ
-    if frequencies is not None:
-        clock_hierarchy = FREQUENCY_HIERARCHY.get(clock, 3)
-        N = len(frequencies)
-        i_idio = np.array([
-            1 if j >= len(frequencies) or FREQUENCY_HIERARCHY.get(frequencies[j], 3) <= clock_hierarchy
-            else 0
-            for j in range(N)
-        ])
-        nQ = N - np.sum(i_idio)
-    else:
-        i_idio = np.ones(config.get_blocks_array().shape[0])
-        nQ = 0
-    
-    # Compute idio chain lengths
-    idio_chain_lengths = compute_idio_lengths(config, clock, tent_weights_dict)
-    
-    return tent_weights_dict, R_mat, q, frequencies, i_idio, nQ, idio_chain_lengths
-
-
-
-
-def _dfm_core(
-    X: np.ndarray,
-    config: DFMConfig,
-    params: Optional[FitParams] = None,
-    Mx: Optional[np.ndarray] = None,
-    Wx: Optional[np.ndarray] = None,
-    **kwargs
-) -> DFMResult:
-    """Estimate dynamic factor model using EM algorithm.
-    
-    This is the main function for estimating a Dynamic Factor Model (DFM). It implements
-    the complete EM algorithm workflow:
-    
-    1. **Initialization**: Compute initial parameter estimates via PCA and OLS
-    2. **EM Iterations**: Iteratively update parameters until convergence
-    3. **Final Smoothing**: Run Kalman smoother with final parameters to obtain
-       smoothed factors and data
-    
-    The DFM models observed time series as:
-    
-    .. math::
-        y_t = C Z_t + e_t,   e_t \\sim N(0, R)
-        Z_t = A Z_{t-1} + v_t,   v_t \\sim N(0, Q)
-    
-    where:
-    - y_t is the n x 1 vector of observed series at time t
-    - Z_t is the m x 1 vector of unobserved factors
-    - C is the n x m loading matrix
-    - A is the m x m transition matrix
-    - R and Q are covariance matrices
-    
-    Parameters
-    ----------
-    X : np.ndarray
-        Data matrix (T x N), where T is time periods and N is number of series.
-        Data should already be transformed and standardized (use DFMDataModule with custom transformer).
-        Data can contain missing values (NaN), which are handled via spline interpolation.
-        Missing values are allowed but excessive missing data (>50%) will trigger warnings.
-    config : DFMConfig
-        Unified DFM configuration object containing:
-        - Model structure: Blocks (N x n_blocks), Frequency (per series), 
-          Transformation (per series), factors_per_block
-        - Estimation parameters: ar_lag, threshold, max_iter, nan_method, nan_k
-        Typically obtained from `load_config()`.
-    params : FitParams, optional
-        Parameter overrides. If None, all values from config are used.
-        All parameters in FitParams are optional and override corresponding config values.
-    Mx : np.ndarray, optional
-        Mean values used for standardization (N,). Required if data is already standardized.
-    Wx : np.ndarray, optional
-        Standard deviation values used for standardization (N,). Required if data is already standardized.
-    **kwargs
-        Additional parameter overrides (merged into params if provided).
-        Valid parameter names: threshold, max_iter, ar_lag, nan_method, nan_k,
-        clock, clip_ar_coefficients, ar_clip_min, ar_clip_max, clip_data_values,
-        data_clip_threshold, use_regularization, regularization_scale,
-        min_eigenvalue, max_eigenvalue, use_damped_updates, damping_factor.
-    
-    Returns
-    -------
-    DFMResult
-        Dataclass containing all estimation results:
-        - x_sm, X_sm: Smoothed data (standardized and unstandardized)
-        - Z: Smoothed factor estimates
-        - C, A, Q, R: Estimated parameters
-        - Mx, Wx: Standardization parameters
-        - Z_0, V_0: Initial state and covariance
-        - r, p: Model structure parameters
-        
-    Raises
-    ------
-    ValueError
-        If inputs are invalid (dimensions, data quality, parameters).
-        Also raised during EM iterations if numerical issues occur (e.g., NaN/Inf).
-    TypeError
-        If input types are incorrect (e.g., X is not numpy array).
-        
-    Notes
-    -----
-    - The function expects data to already be transformed and standardized.
-      Use DFMDataModule with custom sktime transformer to apply transformations and standardization.
-    - Initial conditions are computed via `init_conditions()`
-    - EM iterations continue until convergence or max_iter=5000
-    - Missing data is handled by the Kalman filter during estimation
-    - Convergence messages and progress are printed during execution
-    
-    Examples
-    --------
-    >>> from dfm_python import DFM
-    >>> from dfm_python.config.adapter import YamlSource
-    >>> from dfm_python import DFMDataModule
-    >>> from dfm_python.data.utils import load_data
-    >>> from datetime import datetime
-    >>> # Load configuration from YAML or create DFMConfig directly
-    >>> config = YamlSource('config.yaml').load()
-    >>> # Load data from file
-    >>> X, Time, Z = load_data('data.csv', config, sample_start=datetime(2000, 1, 1))
-    >>> # Estimate DFM
-    >>> model = DFM()
-    >>> Res = model.fit(X, config, threshold=1e-4)
-    >>> # Access results
-    >>> factors = Res.Z  # (T x m) factor estimates
-    >>> loadings = Res.C  # (N x m) factor loadings
-    >>> smoothed_data = Res.X_sm  # (T x N) smoothed data
-    >>> # Compute common factor (first factor)
-    >>> common_factor = Res.Z[:, 0]
-    >>> # Project factor onto a series
-    >>> series_idx = 0
-    >>> series_factor = Res.Z @ Res.C[series_idx, :].T
-    """
-    _logger.info('Estimating the dynamic factor model (DFM)')
-    
-    # Merge kwargs into params if provided
-    if kwargs:
-        if params is None:
-            params = FitParams.from_kwargs(**kwargs)
-        else:
-            # Update params with kwargs (only valid parameter names)
-            valid_params = {
-                'threshold', 'max_iter', 'ar_lag', 'nan_method', 'nan_k',
-                'clock', 'clip_ar_coefficients', 'ar_clip_min', 'ar_clip_max',
-                'clip_data_values', 'data_clip_threshold', 'use_regularization',
-                'regularization_scale', 'min_eigenvalue', 'max_eigenvalue',
-                'use_damped_updates', 'damping_factor'
-            }
-            for k, v in kwargs.items():
-                if k in valid_params and hasattr(params, k):
-                    setattr(params, k, v)
-    
-    # Step 1: Prepare data and resolve parameters
-    X, blocks, params_dict = _prepare_data(X, config, params)
-    
-    # Extract key parameters
-    r = params_dict['r']
-    nan_method = params_dict['nan_method']
-    nan_k = params_dict['nan_k']
-    threshold = params_dict['threshold']
-    max_iter = params_dict['max_iter']
-    clock = params_dict['clock']
-    
-    # Step 2: Prepare aggregation structure (for diagnostics only)
-    _, _, _, _, _, nQ, _ = _prepare_agg_structure(config, clock)
-    
-    # Step 3: Validate preprocessing
-    if Mx is None or Wx is None:
-        raise ValueError(
-            "Mx and Wx must be provided. Data should be preprocessed using DFMDataModule with custom transformer. "
-            "before calling _dfm_core()."
-        )
-    
-    # Step 3.5: Handle missing data implicitly
-    # Missing data (NaN) is handled automatically by the Kalman filter during estimation.
-    # The Kalman filter's handle_missing_data() method removes NaN observations at each time step,
-    # allowing the EM algorithm to work with incomplete data. This is the standard approach
-    # in state-space models (see Harvey 1989, Mariano & Murasawa 2003).
-    # Preprocessing (rem_nans_spline) handles leading/trailing NaNs, but remaining NaNs
-    # are handled implicitly during Kalman filtering.
-    if np.any(np.isnan(X)):
-        nan_ratio = np.sum(np.isnan(X)) / X.size
-        if nan_ratio > 0.5:
-            _logger.warning(
-                f"High proportion of missing data ({nan_ratio:.1%}). "
-                "Results may be unstable. Consider preprocessing data more thoroughly."
-            )
-        else:
-            _logger.debug(
-                f"Data contains {nan_ratio:.1%} missing values. "
-                "These will be handled implicitly by the Kalman filter during estimation."
-            )
-    
-    # Step 4-8: Use Lightning module for estimation
-    import torch
-    
-    # Convert to torch tensor
-    X_torch = torch.tensor(X, dtype=torch.float32)
-    
-    # Create DFM model (Lightning module)
-    from ..models.dfm import DFM
-    model = DFM(
-        config=config,
-        num_factors=int(np.sum(r)),
-        threshold=threshold,
-        max_iter=max_iter,
-        nan_method=nan_method,
-        nan_k=nan_k
-    )
-    
-    # Run EM algorithm
-    model.fit_em(X_torch, Mx=Mx, Wx=Wx)
-    
-    # Extract results
-    Res = model.get_result()
-    
-    # Display diagnostic tables if debug logging is enabled
-    if _logger.isEnabledFor(logging.DEBUG):
-        _display_dfm_tables(Res, config, nQ)
-    
-    return Res
-