dfm-python 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

dfm_python/__init__.py

@@ -26,23 +26,39 @@ Key Features:
 
 Example (High-level API - Recommended):
     >>> import dfm_python as dfm
+    >>> from dfm_python.lightning import DFMDataModule
+    >>> from sktime.transformations.compose import ColumnTransformer
+    >>> 
     >>> # Linear DFM
     >>> model = dfm.DFM()
     >>> model.load_config('config/default.yaml')
-    >>> model.load_data('data/sample_data.csv')
-    >>> model.train(max_iter=100)
+    >>> 
+    >>> # Create transformer (user must provide)
+    >>> transformer = ColumnTransformer([...])  # User-defined sktime transformer
+    >>> 
+    >>> # Create DataModule
+    >>> data_module = DFMDataModule(config=model.config, transformer=transformer, data_path='data/sample_data.csv')
+    >>> data_module.setup()
+    >>> 
+    >>> # Train
+    >>> model.train(data_module, max_iter=100)
     >>> Xf, Zf = model.predict(horizon=6)
     >>> 
     >>> # Or use DDFM (separate class)
     >>> ddfm_model = dfm.DDFM(encoder_layers=[64, 32], num_factors=2)
     >>> ddfm_model.load_config('config/default.yaml')
-    >>> ddfm_model.load_data('data/sample_data.csv')
-    >>> ddfm_model.train(epochs=100)
+    >>> 
+    >>> # Create DataModule for DDFM
+    >>> data_module_ddfm = DFMDataModule(config=ddfm_model.config, transformer=transformer, data_path='data/sample_data.csv')
+    >>> data_module_ddfm.setup()
+    >>> 
+    >>> # Train
+    >>> ddfm_model.train(data_module_ddfm, epochs=100)
     >>> Xf, Zf = ddfm_model.predict(horizon=6)
     
 Example (Low-level API - For advanced usage):
     >>> from dfm_python import DFM, DFMConfig, SeriesConfig
-    >>> from dfm_python.dataloader import load_data  # Preferred import
+    >>> from dfm_python.transformations.utils import read_data  # Preferred import
     >>> # Option 1: Load from YAML
     >>> config = load_config('config.yaml')
     >>> # Option 2: Create directly
@@ -50,15 +66,25 @@ Example (Low-level API - For advanced usage):
     ...     series=[SeriesConfig(frequency='m', transformation='lin', blocks=[1], series_id='series1')],
     ...     block_names=['Global']
     ... )
-    >>> X, Time, _ = load_data('data.csv', config)  # or use database adapter
+    >>> from dfm_python.lightning import DFMDataModule
+    >>> from sktime.transformations.compose import ColumnTransformer
+    >>> 
+    >>> # Create transformer (user must provide)
+    >>> transformer = ColumnTransformer([...])  # User-defined
+    >>> 
+    >>> # Create DataModule
+    >>> data_module = DFMDataModule(config=config, transformer=transformer, data_path='data.csv')
+    >>> data_module.setup()
+    >>> 
+    >>> # Fit model
     >>> model = DFM()
-    >>> result = model.fit(X, config)
+    >>> result = model.fit(data_module, config)
     >>> factors = result.Z  # Extract estimated factors
 
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.3.1"
+__version__ = "0.4.0"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -84,45 +110,61 @@ from .config import (
 )
 
 # Data utilities
-from .dataloader import transform_data
+# Note: transform_data and DFMScaler have been removed - users must provide their own sktime transformers to DFMDataModule
 
 # Results
-from .core import DFMResult
+from .config.results import DFMResult, DDFMResult, BaseResult
 
-# Core utilities (from core/ subpackage)
-from .core import calculate_rmse, diagnose_series, print_series_diagnosis
-from .core.state_space import run_kf, skf, fis, miss_data
-# DFMCore is now an alias for DFMLinear (backward compatibility)
+# Utilities (from utils/ subpackage)
+from .utils.diagnostics import diagnose_series, print_series_diagnosis
+from .utils.time import calculate_rmse
+# DFMCore is an alias for DFMLinear (backward compatibility)
 from .models.dfm import DFMLinear
 DFMCore = DFMLinear
 
-# Nowcasting (already in nowcasting/ subpackage)
-from .nowcasting import (
+# PyTorch Lightning modules (recommended)
+try:
+    from .lightning import (
+        DFMLightningModule,
+        DDFMLightningModule,
+        DFMDataModule,
+        KalmanFilter,  # New module class
+        EMAlgorithm,  # New module class
+    )
+    _has_lightning = True
+except ImportError:
+    _has_lightning = False
+    DFMLightningModule = None  # type: ignore
+    DDFMLightningModule = None  # type: ignore
+    DFMDataModule = None  # type: ignore
+    KalmanFilter = None  # type: ignore
+    EMAlgorithm = None  # type: ignore
+
+# Nowcasting (from nowcast/ subpackage)
+from .nowcast import (
     Nowcast,
-    para_const,
     NowcastResult,
+)
+from .nowcast.helpers import (
+    para_const,
     NewsDecompResult,
     BacktestResult,
 )
 
-# High-level API (from api/ subpackage)
-from .api import (
-    DFM, _dfm_instance, from_yaml, from_spec, from_spec_df, from_dict,
-    load_config, load_data, load_pickle, train, predict, plot, reset, create_model
-)
-
 # Model implementations
 from .models.base import BaseFactorModel
-from .models.dfm import DFMLinear
+from .models.dfm import DFMLinear, DFM
+from .models.dfm import (
+    from_yaml, from_spec, from_spec_df, from_dict,
+    load_config, load_pickle, train, predict, plot, reset, create_model
+)
+# Note: load_data has been removed - use DFMDataModule instead
 
 # DDFM high-level API and low-level model (both optional, requires PyTorch)
 try:
-    from .api import DDFM as DDFMAPI, _ddfm_instance
-    from .api import load_config_ddfm, load_data_ddfm, train_ddfm, predict_ddfm, plot_ddfm, reset_ddfm
-    from .models.ddfm import DDFM as DDFMModel
+    from .models.ddfm import DDFM, DDFMModel
+    from .models.ddfm import load_config_ddfm, load_data_ddfm, train_ddfm, predict_ddfm, plot_ddfm, reset_ddfm
     _has_ddfm = True
-    # Export high-level API as DDFM
-    DDFM = DDFMAPI
 except ImportError:
     _has_ddfm = False
     DDFM = None  # type: ignore
@@ -133,30 +175,7 @@ except ImportError:
     predict_ddfm = None  # type: ignore
     plot_ddfm = None  # type: ignore
     reset_ddfm = None  # type: ignore
-
-# Expose properties as module-level attributes
-# Use property-like access via functions or direct attribute access
-# Since 'config' conflicts with the config module, we'll use a different approach
-# Users can access: dfm.get_config(), dfm.get_data(), dfm.get_result()
-def get_config():
-    """Get current configuration."""
-    return _dfm_instance.config
-
-def get_data():
-    """Get current data matrix."""
-    return _dfm_instance.data
-
-def get_time():
-    """Get current time index."""
-    return _dfm_instance.time
-
-def get_result():
-    """Get training result."""
-    return _dfm_instance.result
-
-def get_original_data():
-    """Get original (untransformed) data matrix."""
-    return _dfm_instance.original_data
+    # DDFM module-level functions removed - create DDFM() instance directly
 
 __all__ = [
     # Core classes
@@ -172,22 +191,46 @@ __all__ = [
     'MergedConfigSource', 'make_config_source',
     # High-level API (module-level - recommended)
     'load_config',
-    'load_data', 'load_pickle', 'train', 'predict', 'plot', 'reset', 'create_model',
-    'get_config', 'get_data', 'get_time', 'get_result', 'get_original_data',
+    'load_pickle', 'train', 'predict', 'plot', 'reset', 'create_model',
     # Convenience constructors (cleaner API)
     'from_yaml', 'from_spec', 'from_spec_df', 'from_dict',
     # Low-level API (functional interface - advanced usage)
-    'transform_data',
-    'DFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
-    'run_kf', 'skf', 'fis', 'miss_data',
-    'para_const',  # Internal utility, kept for backward compatibility
+    'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
+    'para_const',  # Internal utility for nowcasting
 ]
 
 # Add DDFM high-level API and convenience functions if available
 if _has_ddfm:
     __all__.extend([
         'DDFM',  # High-level API class
-        'load_config_ddfm', 'load_data_ddfm', 'train_ddfm', 
+        'DDFMModel',  # Low-level implementation
+        'load_config_ddfm', 'train_ddfm', 
         'predict_ddfm', 'plot_ddfm', 'reset_ddfm'
+        # Note: load_data_ddfm has been removed - use DFMDataModule instead
+    ])
+
+# Add Lightning modules if available
+if _has_lightning:
+    __all__.extend([
+        'DFMLightningModule',
+        'DDFMLightningModule',
+        'DFMDataModule',
+        'KalmanFilter',  # New module class
+        'EMAlgorithm',  # New module class
+    ])
+
+# Add Trainer classes if available
+try:
+    from .trainer import DFMTrainer, DDFMTrainer
+    _has_trainers = True
+except ImportError:
+    _has_trainers = False
+    DFMTrainer = None  # type: ignore
+    DDFMTrainer = None  # type: ignore
+
+if _has_trainers:
+    __all__.extend([
+        'DFMTrainer',
+        'DDFMTrainer',
     ])
 

dfm_python/config/__init__.py

@@ -6,10 +6,12 @@ This subpackage provides:
 """
 
 from .schema import (
-    DFMConfig, SeriesConfig, BlockConfig, Params,
+    BaseModelConfig, DFMConfig, DDFMConfig, SeriesConfig, BlockConfig,
     DEFAULT_GLOBAL_BLOCK_NAME,
-    validate_frequency, validate_transformation,
 )
+from .params import Params, FitParams
+from .results import BaseResult, DFMResult, DDFMResult, DFMParams
+from .utils import validate_frequency, validate_transformation
 from .io import (
     ConfigSource,
     YamlSource,
@@ -18,18 +20,46 @@ from .io import (
     MergedConfigSource,
     make_config_source,
     from_spec,
-    _load_config_from_dataframe,
-    _write_series_blocks_yaml,
+    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
+    # are available from io module but not exported as public API
+)
+from .structure import (
+    FREQUENCY_HIERARCHY,
+    PERIODS_PER_YEAR,
+    get_periods_per_year,
+    compute_idio_chain_lengths,
+    get_tent_weights_for_pair,
+    generate_tent_weights,
+    generate_R_mat,
+    get_aggregation_structure,
+    group_series_by_frequency,
 )
 
 __all__ = [
     # Schema
-    'DFMConfig', 'SeriesConfig', 'BlockConfig', 'Params',
+    'BaseModelConfig', 'DFMConfig', 'DDFMConfig', 'SeriesConfig', 'BlockConfig',
     'DEFAULT_GLOBAL_BLOCK_NAME',
+    # Parameters
+    'Params', 'FitParams',
+    # Results
+    'BaseResult', 'DFMResult', 'DDFMResult', 'DFMParams',
+    # Utilities
     'validate_frequency', 'validate_transformation',
     # IO
     'ConfigSource', 'YamlSource', 'DictSource',
     'HydraSource', 'MergedConfigSource', 'make_config_source',
-    'from_spec', '_load_config_from_dataframe', '_write_series_blocks_yaml',
+    'from_spec',
+    # Structure utilities
+    'FREQUENCY_HIERARCHY',
+    'PERIODS_PER_YEAR',
+    'get_periods_per_year',
+    'compute_idio_chain_lengths',
+    'get_tent_weights_for_pair',
+    'generate_tent_weights',
+    'generate_R_mat',
+    'get_aggregation_structure',
+    'group_series_by_frequency',
+    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
+    # are available from io module but not exported as public API
 ]
 

dfm_python/config/params.py

@@ -0,0 +1,202 @@
+"""Parameter dataclasses for DFM models.
+
+This module contains parameter configuration classes:
+- Params: Global estimation parameters with defaults
+- FitParams: Optional parameter overrides for model fitting
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+from .utils import validate_frequency
+
+
+@dataclass
+class Params:
+    """Global estimation parameters for DFM (main settings only).
+    
+    This dataclass contains only the global knobs that control the EM algorithm
+    and numerical stability. It does NOT include series or block definitions.
+    Use this with spec CSV files to provide main settings while the spec CSV
+    defines all series and their block memberships.
+    
+    All parameters have defaults matching the standard YAML configuration.
+    
+    Attributes
+    ----------
+    ar_lag : int
+        AR lag for factor transition equation (typically 1)
+    threshold : float
+        EM convergence threshold (default: 1e-5)
+    max_iter : int
+        Maximum EM iterations (default: 5000)
+    nan_method : int
+        Missing data handling method (1-5, default: 2 = spline interpolation).
+        This preprocessing step is followed by Kalman Filter-based missing data
+        handling during DFM estimation, following standard practice in state-space
+        models (Mariano & Murasawa 2003, Harvey 1989).
+    nan_k : int
+        Spline parameter for NaN interpolation (default: 3 for cubic spline)
+    clock : str
+        Base frequency for all latent factors (default: 'm' for monthly)
+    clip_ar_coefficients : bool
+        Enable AR coefficient clipping for stationarity (default: True)
+    ar_clip_min : float
+        Minimum AR coefficient (default: -0.99)
+    ar_clip_max : float
+        Maximum AR coefficient (default: 0.99)
+    warn_on_ar_clip : bool
+        Warn when AR coefficients are clipped (default: True)
+    clip_data_values : bool
+        Enable clipping of extreme data values (default: True)
+    data_clip_threshold : float
+        Clip values beyond this many standard deviations (default: 100.0)
+    warn_on_data_clip : bool
+        Warn when data values are clipped (default: True)
+    use_regularization : bool
+        Enable regularization for numerical stability (default: True)
+    regularization_scale : float
+        Scale factor for ridge regularization (default: 1e-5)
+    min_eigenvalue : float
+        Minimum eigenvalue for positive definite matrices (default: 1e-8)
+    max_eigenvalue : float
+        Maximum eigenvalue cap (default: 1e6)
+    warn_on_regularization : bool
+        Warn when regularization is applied (default: True)
+    use_damped_updates : bool
+        Enable damped updates when likelihood decreases (default: True)
+    damping_factor : float
+        Damping factor: 0.8 = 80% new, 20% old (default: 0.8)
+    warn_on_damped_update : bool
+        Warn when damped updates are used (default: True)
+    augment_idio : bool
+        Enable state augmentation with idiosyncratic components (default: True)
+    augment_idio_slow : bool
+        Enable tent-length chains for slower-frequency series (default: True)
+    idio_rho0 : float
+        Initial AR coefficient for idiosyncratic components (default: 0.1)
+    idio_min_var : float
+        Minimum variance for idiosyncratic innovation covariance (default: 1e-8)
+    
+    Examples
+    --------
+    >>> from dfm_python.config import Params
+    >>> params = Params(max_iter=100, threshold=1e-4)
+    >>> dfm.from_spec('data/spec.csv', params=params)
+    """
+    # Estimation parameters
+    ar_lag: int = 1
+    threshold: float = 1e-5
+    max_iter: int = 5000
+    nan_method: int = 2
+    nan_k: int = 3
+    clock: str = 'm'
+    
+    # Numerical stability - AR clipping
+    clip_ar_coefficients: bool = True
+    ar_clip_min: float = -0.99
+    ar_clip_max: float = 0.99
+    warn_on_ar_clip: bool = True
+    
+    # Numerical stability - Data clipping
+    clip_data_values: bool = True
+    data_clip_threshold: float = 100.0
+    warn_on_data_clip: bool = True
+    
+    # Numerical stability - Regularization
+    use_regularization: bool = True
+    regularization_scale: float = 1e-5
+    min_eigenvalue: float = 1e-8
+    max_eigenvalue: float = 1e6
+    warn_on_regularization: bool = True
+    
+    # Numerical stability - Damped updates
+    use_damped_updates: bool = True
+    damping_factor: float = 0.8
+    warn_on_damped_update: bool = True
+    
+    # Idiosyncratic component augmentation
+    augment_idio: bool = True  # Enable state augmentation with idiosyncratic components (default: True)
+    augment_idio_slow: bool = True  # Enable tent-length chains for slower-frequency series (default: True)
+    idio_rho0: float = 0.1  # Initial AR coefficient for idiosyncratic components (default: 0.1)
+    idio_min_var: float = 1e-8  # Minimum variance for idiosyncratic innovation covariance (default: 1e-8)
+    
+    def __post_init__(self):
+        """Validate parameters."""
+        self.clock = validate_frequency(self.clock)
+        if self.threshold <= 0:
+            raise ValueError(f"threshold must be positive, got {self.threshold}")
+        if self.max_iter < 1:
+            raise ValueError(f"max_iter must be at least 1, got {self.max_iter}")
+        if self.ar_clip_min >= self.ar_clip_max:
+            raise ValueError(f"ar_clip_min ({self.ar_clip_min}) must be < ar_clip_max ({self.ar_clip_max})")
+        if not (-1 < self.ar_clip_min < 1):
+            raise ValueError(f"ar_clip_min must be in (-1, 1), got {self.ar_clip_min}")
+        if not (-1 < self.ar_clip_max < 1):
+            raise ValueError(f"ar_clip_max must be in (-1, 1), got {self.ar_clip_max}")
+
+
+@dataclass
+class FitParams:
+    """Parameters for fitting DFM models.
+    
+    This dataclass groups all optional parameters that can override
+    config values during model fitting. This reduces method signature
+    complexity and improves code readability.
+    
+    All parameters are optional. If None, the corresponding value
+    from DFMConfig will be used during parameter resolution.
+    """
+    # Convergence parameters
+    threshold: Optional[float] = None
+    max_iter: Optional[int] = None
+    
+    # Model structure
+    ar_lag: Optional[int] = None
+    num_factors: Optional[int] = None
+    
+    # Missing data handling
+    nan_method: Optional[int] = None
+    nan_k: Optional[int] = None
+    
+    # AR coefficient clipping
+    clip_ar_coefficients: Optional[bool] = None
+    ar_clip_min: Optional[float] = None
+    ar_clip_max: Optional[float] = None
+    
+    # Data clipping
+    clip_data_values: Optional[bool] = None
+    data_clip_threshold: Optional[float] = None
+    
+    # Regularization
+    use_regularization: Optional[bool] = None
+    regularization_scale: Optional[float] = None
+    min_eigenvalue: Optional[float] = None
+    max_eigenvalue: Optional[float] = None
+    
+    # Damping
+    use_damped_updates: Optional[bool] = None
+    damping_factor: Optional[float] = None
+    
+    def to_dict(self) -> dict:
+        """Convert to dictionary, excluding None values."""
+        return {k: v for k, v in self.__dict__.items() if v is not None}
+    
+    @classmethod
+    def from_kwargs(cls, **kwargs) -> 'FitParams':
+        """Create FitParams from keyword arguments.
+        
+        Filters kwargs to only include valid parameter names,
+        ignoring any extra arguments.
+        """
+        valid_params = {
+            'threshold', 'max_iter', 'ar_lag', 'num_factors',
+            'nan_method', 'nan_k',
+            'clip_ar_coefficients', 'ar_clip_min', 'ar_clip_max',
+            'clip_data_values', 'data_clip_threshold',
+            'use_regularization', 'regularization_scale',
+            'min_eigenvalue', 'max_eigenvalue',
+            'use_damped_updates', 'damping_factor'
+        }
+        filtered = {k: v for k, v in kwargs.items() if k in valid_params}
+        return cls(**filtered)
+