dfm-python 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

dfm_python/__init__.py

@@ -7,6 +7,7 @@ This package implements a comprehensive Dynamic Factor Model framework with supp
 - Expectation-Maximization (EM) algorithm for parameter estimation
 - Kalman filtering and smoothing for factor extraction
 - News decomposition for nowcasting
+- Deep Dynamic Factor Models (DDFM) with nonlinear encoders (optional, requires PyTorch)
 
 The package implements a clock-based approach to mixed-frequency DFMs, where all latent 
 factors (global and block-level) are synchronized to a common "clock" frequency, typically 
@@ -25,16 +26,39 @@ Key Features:
 
 Example (High-level API - Recommended):
     >>> import dfm_python as dfm
-    >>> dfm.load_config('config/default.yaml')
-    >>> dfm.load_data('data/sample_data.csv')
-    >>> dfm.train(max_iter=1)
-    >>> Xf, Zf = dfm.predict(horizon=6)
-    >>> dfm.plot(kind='factor', factor_index=0, forecast_horizon=6, save_path='outputs/factor_forecast.png')
-    >>> factors = dfm.get_result().Z  # Access factors via result
+    >>> from dfm_python.lightning import DFMDataModule
+    >>> from sktime.transformations.compose import ColumnTransformer
+    >>> 
+    >>> # Linear DFM
+    >>> model = dfm.DFM()
+    >>> model.load_config('config/default.yaml')
+    >>> 
+    >>> # Create transformer (user must provide)
+    >>> transformer = ColumnTransformer([...])  # User-defined sktime transformer
+    >>> 
+    >>> # Create DataModule
+    >>> data_module = DFMDataModule(config=model.config, transformer=transformer, data_path='data/sample_data.csv')
+    >>> data_module.setup()
+    >>> 
+    >>> # Train
+    >>> model.train(data_module, max_iter=100)
+    >>> Xf, Zf = model.predict(horizon=6)
+    >>> 
+    >>> # Or use DDFM (separate class)
+    >>> ddfm_model = dfm.DDFM(encoder_layers=[64, 32], num_factors=2)
+    >>> ddfm_model.load_config('config/default.yaml')
+    >>> 
+    >>> # Create DataModule for DDFM
+    >>> data_module_ddfm = DFMDataModule(config=ddfm_model.config, transformer=transformer, data_path='data/sample_data.csv')
+    >>> data_module_ddfm.setup()
+    >>> 
+    >>> # Train
+    >>> ddfm_model.train(data_module_ddfm, epochs=100)
+    >>> Xf, Zf = ddfm_model.predict(horizon=6)
     
 Example (Low-level API - For advanced usage):
     >>> from dfm_python import DFM, DFMConfig, SeriesConfig
-    >>> from dfm_python.data import load_data  # Preferred import
+    >>> from dfm_python.transformations.utils import read_data  # Preferred import
     >>> # Option 1: Load from YAML
     >>> config = load_config('config.yaml')
     >>> # Option 2: Create directly
@@ -42,62 +66,122 @@ Example (Low-level API - For advanced usage):
     ...     series=[SeriesConfig(frequency='m', transformation='lin', blocks=[1], series_id='series1')],
     ...     block_names=['Global']
     ... )
-    >>> X, Time, _ = load_data('data.csv', config)  # or use database adapter
+    >>> from dfm_python.lightning import DFMDataModule
+    >>> from sktime.transformations.compose import ColumnTransformer
+    >>> 
+    >>> # Create transformer (user must provide)
+    >>> transformer = ColumnTransformer([...])  # User-defined
+    >>> 
+    >>> # Create DataModule
+    >>> data_module = DFMDataModule(config=config, transformer=transformer, data_path='data.csv')
+    >>> data_module.setup()
+    >>> 
+    >>> # Fit model
     >>> model = DFM()
-    >>> result = model.fit(X, config)
+    >>> result = model.fit(data_module, config)
     >>> factors = result.Z  # Extract estimated factors
 
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.3.0"
-
+__version__ = "0.4.0"
+
+# ============================================================================
+# PUBLIC API DEFINITION
+# ============================================================================
+# This __init__.py is the single source of truth for the public API.
+# All symbols exported here are considered stable public API.
+# Internal reorganization should not break these imports.
+#
+# Public API categories:
+# 1. Configuration: DFMConfig, SeriesConfig, BlockConfig, Params, config sources
+# 2. High-level API: DFM, DDFM, module-level convenience functions
+# 3. Core utilities: DFMCore, run_kf, TimeIndex, diagnostics
+# 4. Models: BaseFactorModel, DFMLinear, DDFM (low-level)
+# 5. Nowcasting: Nowcast, result classes, para_const
+# 6. Data & Results: DFMResult, transform_data
+# ============================================================================
+
+# Configuration (from config/ subpackage)
 from .config import (
     DFMConfig, SeriesConfig, BlockConfig, Params, DEFAULT_GLOBAL_BLOCK_NAME,
     ConfigSource, YamlSource, DictSource, HydraSource,
     MergedConfigSource, make_config_source,
 )
-from .data import transform_data
-from .dfm import DFMResult
-from .core import calculate_rmse, diagnose_series, print_series_diagnosis
-from .dfm import DFMCore  # Core DFM class from dfm.py
-from .kalman import run_kf, skf, fis, miss_data
-from .nowcast import Nowcast, para_const, NowcastResult, NewsDecompResult, BacktestResult
-
-# Import high-level API (extends core DFM with convenience methods)
-# Import module-level convenience functions directly from api.py to avoid duplication
-from .api import (
-    DFM, _dfm_instance, from_yaml, from_spec, from_spec_df, from_dict,
-    load_config, load_data, load_pickle, train, predict, plot, reset
-)
-
-# Expose properties as module-level attributes
-# Use property-like access via functions or direct attribute access
-# Since 'config' conflicts with the config module, we'll use a different approach
-# Users can access: dfm.get_config(), dfm.get_data(), dfm.get_result()
-def get_config():
-    """Get current configuration."""
-    return _dfm_instance.config
-
-def get_data():
-    """Get current data matrix."""
-    return _dfm_instance.data
-
-def get_time():
-    """Get current time index."""
-    return _dfm_instance.time
 
-def get_result():
-    """Get training result."""
-    return _dfm_instance.result
+# Data utilities
+# Note: transform_data and DFMScaler have been removed - users must provide their own sktime transformers to DFMDataModule
+
+# Results
+from .config.results import DFMResult, DDFMResult, BaseResult
+
+# Utilities (from utils/ subpackage)
+from .utils.diagnostics import diagnose_series, print_series_diagnosis
+from .utils.time import calculate_rmse
+# DFMCore is an alias for DFMLinear (backward compatibility)
+from .models.dfm import DFMLinear
+DFMCore = DFMLinear
+
+# PyTorch Lightning modules (recommended)
+try:
+    from .lightning import (
+        DFMLightningModule,
+        DDFMLightningModule,
+        DFMDataModule,
+        KalmanFilter,  # New module class
+        EMAlgorithm,  # New module class
+    )
+    _has_lightning = True
+except ImportError:
+    _has_lightning = False
+    DFMLightningModule = None  # type: ignore
+    DDFMLightningModule = None  # type: ignore
+    DFMDataModule = None  # type: ignore
+    KalmanFilter = None  # type: ignore
+    EMAlgorithm = None  # type: ignore
+
+# Nowcasting (from nowcast/ subpackage)
+from .nowcast import (
+    Nowcast,
+    NowcastResult,
+)
+from .nowcast.helpers import (
+    para_const,
+    NewsDecompResult,
+    BacktestResult,
+)
 
-def get_original_data():
-    """Get original (untransformed) data matrix."""
-    return _dfm_instance.original_data
+# Model implementations
+from .models.base import BaseFactorModel
+from .models.dfm import DFMLinear, DFM
+from .models.dfm import (
+    from_yaml, from_spec, from_spec_df, from_dict,
+    load_config, load_pickle, train, predict, plot, reset, create_model
+)
+# Note: load_data has been removed - use DFMDataModule instead
+
+# DDFM high-level API and low-level model (both optional, requires PyTorch)
+try:
+    from .models.ddfm import DDFM, DDFMModel
+    from .models.ddfm import load_config_ddfm, load_data_ddfm, train_ddfm, predict_ddfm, plot_ddfm, reset_ddfm
+    _has_ddfm = True
+except ImportError:
+    _has_ddfm = False
+    DDFM = None  # type: ignore
+    DDFMModel = None  # type: ignore
+    load_config_ddfm = None  # type: ignore
+    load_data_ddfm = None  # type: ignore
+    train_ddfm = None  # type: ignore
+    predict_ddfm = None  # type: ignore
+    plot_ddfm = None  # type: ignore
+    reset_ddfm = None  # type: ignore
+    # DDFM module-level functions removed - create DDFM() instance directly
 
 __all__ = [
     # Core classes
     'DFMConfig', 'SeriesConfig', 'BlockConfig', 'Params', 'DFM', 'DFMCore', 'Nowcast',
+    # Model base and implementations
+    'BaseFactorModel', 'DFMLinear',
     # Nowcast result classes
     'NowcastResult', 'NewsDecompResult', 'BacktestResult',
     # Constants
@@ -107,14 +191,46 @@ __all__ = [
     'MergedConfigSource', 'make_config_source',
     # High-level API (module-level - recommended)
     'load_config',
-    'load_data', 'load_pickle', 'train', 'predict', 'plot', 'reset',
-    'get_config', 'get_data', 'get_time', 'get_result', 'get_original_data',
+    'load_pickle', 'train', 'predict', 'plot', 'reset', 'create_model',
     # Convenience constructors (cleaner API)
     'from_yaml', 'from_spec', 'from_spec_df', 'from_dict',
     # Low-level API (functional interface - advanced usage)
-    'transform_data',
-    'DFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
-    'run_kf', 'skf', 'fis', 'miss_data',
-    'para_const',  # Internal utility, kept for backward compatibility
+    'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
+    'para_const',  # Internal utility for nowcasting
 ]
 
+# Add DDFM high-level API and convenience functions if available
+if _has_ddfm:
+    __all__.extend([
+        'DDFM',  # High-level API class
+        'DDFMModel',  # Low-level implementation
+        'load_config_ddfm', 'train_ddfm', 
+        'predict_ddfm', 'plot_ddfm', 'reset_ddfm'
+        # Note: load_data_ddfm has been removed - use DFMDataModule instead
+    ])
+
+# Add Lightning modules if available
+if _has_lightning:
+    __all__.extend([
+        'DFMLightningModule',
+        'DDFMLightningModule',
+        'DFMDataModule',
+        'KalmanFilter',  # New module class
+        'EMAlgorithm',  # New module class
+    ])
+
+# Add Trainer classes if available
+try:
+    from .trainer import DFMTrainer, DDFMTrainer
+    _has_trainers = True
+except ImportError:
+    _has_trainers = False
+    DFMTrainer = None  # type: ignore
+    DDFMTrainer = None  # type: ignore
+
+if _has_trainers:
+    __all__.extend([
+        'DFMTrainer',
+        'DDFMTrainer',
+    ])
+

dfm_python/config/__init__.py

@@ -0,0 +1,65 @@
+"""Configuration subpackage for DFM.
+
+This subpackage provides:
+- Schema (DFMConfig, SeriesConfig, BlockConfig, Params) in schema.py
+- IO (ConfigSource, YamlSource, etc.) in io.py
+"""
+
+from .schema import (
+    BaseModelConfig, DFMConfig, DDFMConfig, SeriesConfig, BlockConfig,
+    DEFAULT_GLOBAL_BLOCK_NAME,
+)
+from .params import Params, FitParams
+from .results import BaseResult, DFMResult, DDFMResult, DFMParams
+from .utils import validate_frequency, validate_transformation
+from .io import (
+    ConfigSource,
+    YamlSource,
+    DictSource,
+    HydraSource,
+    MergedConfigSource,
+    make_config_source,
+    from_spec,
+    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
+    # are available from io module but not exported as public API
+)
+from .structure import (
+    FREQUENCY_HIERARCHY,
+    PERIODS_PER_YEAR,
+    get_periods_per_year,
+    compute_idio_chain_lengths,
+    get_tent_weights_for_pair,
+    generate_tent_weights,
+    generate_R_mat,
+    get_aggregation_structure,
+    group_series_by_frequency,
+)
+
+__all__ = [
+    # Schema
+    'BaseModelConfig', 'DFMConfig', 'DDFMConfig', 'SeriesConfig', 'BlockConfig',
+    'DEFAULT_GLOBAL_BLOCK_NAME',
+    # Parameters
+    'Params', 'FitParams',
+    # Results
+    'BaseResult', 'DFMResult', 'DDFMResult', 'DFMParams',
+    # Utilities
+    'validate_frequency', 'validate_transformation',
+    # IO
+    'ConfigSource', 'YamlSource', 'DictSource',
+    'HydraSource', 'MergedConfigSource', 'make_config_source',
+    'from_spec',
+    # Structure utilities
+    'FREQUENCY_HIERARCHY',
+    'PERIODS_PER_YEAR',
+    'get_periods_per_year',
+    'compute_idio_chain_lengths',
+    'get_tent_weights_for_pair',
+    'generate_tent_weights',
+    'generate_R_mat',
+    'get_aggregation_structure',
+    'group_series_by_frequency',
+    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
+    # are available from io module but not exported as public API
+]
+

dfm_python/{config_sources.py → config/io.py}

@@ -15,7 +15,7 @@ from typing import Protocol, Optional, Dict, Any, Union, Tuple
 from pathlib import Path
 from dataclasses import is_dataclass, asdict
 
-from .config import DFMConfig, SeriesConfig, BlockConfig, DEFAULT_GLOBAL_BLOCK_NAME
+from .schema import DFMConfig, SeriesConfig, BlockConfig, DEFAULT_GLOBAL_BLOCK_NAME
 import polars as pl
 
 
@@ -378,7 +378,7 @@ def _write_series_blocks_yaml(
             else:
                 blocks_names = list(series.blocks)
         else:
-            blocks_names = ['Block_Global']
+            blocks_names = [DEFAULT_GLOBAL_BLOCK_NAME]
         
         series_entry = {
             'series_name': series.series_name,
@@ -482,7 +482,7 @@ def from_spec(
                 # Blocks are already names
                 blocks_names = list(series.blocks)
         else:
-            blocks_names = ['Block_Global']  # Default
+            blocks_names = [DEFAULT_GLOBAL_BLOCK_NAME]  # Default
         
         series_entry = {
             'series_name': series.series_name,
@@ -764,7 +764,7 @@ def _load_config_from_dataframe(df: pl.DataFrame) -> DFMConfig:
         blocks = []
         
         if block_cols:
-            # CSV format: Block_Global, Block_Consumption, etc. columns with 1/0 values
+            # CSV format: global block, block_1, etc. columns with 1/0 values
             for block_col in block_cols:
                 if block_col in df_dict:
                     block_value = df_dict[block_col][i]
@@ -789,13 +789,13 @@ def _load_config_from_dataframe(df: pl.DataFrame) -> DFMConfig:
                 elif isinstance(blocks_str, list):
                     blocks = blocks_str
                 else:
-                    blocks = ['Block_Global']
+                    blocks = [DEFAULT_GLOBAL_BLOCK_NAME]
             else:
-                blocks = ['Block_Global']
+                blocks = [DEFAULT_GLOBAL_BLOCK_NAME]
         
-        # Ensure at least Block_Global
+        # Ensure at least global block
         if not blocks:
-            blocks = ['Block_Global']
+            blocks = [DEFAULT_GLOBAL_BLOCK_NAME]
         
         # Track block names
         for block in blocks:
@@ -824,7 +824,7 @@ def _load_config_from_dataframe(df: pl.DataFrame) -> DFMConfig:
         ))
     
     # Create default blocks if none specified
-    block_names = sorted(block_names_set) if block_names_set else ['Block_Global']
+    block_names = sorted(block_names_set) if block_names_set else [DEFAULT_GLOBAL_BLOCK_NAME]
     blocks = {}
     for block_name in block_names:
         blocks[block_name] = BlockConfig(factors=1, ar_lag=1, clock='m')

dfm_python/config/params.py

@@ -0,0 +1,202 @@
+"""Parameter dataclasses for DFM models.
+
+This module contains parameter configuration classes:
+- Params: Global estimation parameters with defaults
+- FitParams: Optional parameter overrides for model fitting
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+from .utils import validate_frequency
+
+
+@dataclass
+class Params:
+    """Global estimation parameters for DFM (main settings only).
+    
+    This dataclass contains only the global knobs that control the EM algorithm
+    and numerical stability. It does NOT include series or block definitions.
+    Use this with spec CSV files to provide main settings while the spec CSV
+    defines all series and their block memberships.
+    
+    All parameters have defaults matching the standard YAML configuration.
+    
+    Attributes
+    ----------
+    ar_lag : int
+        AR lag for factor transition equation (typically 1)
+    threshold : float
+        EM convergence threshold (default: 1e-5)
+    max_iter : int
+        Maximum EM iterations (default: 5000)
+    nan_method : int
+        Missing data handling method (1-5, default: 2 = spline interpolation).
+        This preprocessing step is followed by Kalman Filter-based missing data
+        handling during DFM estimation, following standard practice in state-space
+        models (Mariano & Murasawa 2003, Harvey 1989).
+    nan_k : int
+        Spline parameter for NaN interpolation (default: 3 for cubic spline)
+    clock : str
+        Base frequency for all latent factors (default: 'm' for monthly)
+    clip_ar_coefficients : bool
+        Enable AR coefficient clipping for stationarity (default: True)
+    ar_clip_min : float
+        Minimum AR coefficient (default: -0.99)
+    ar_clip_max : float
+        Maximum AR coefficient (default: 0.99)
+    warn_on_ar_clip : bool
+        Warn when AR coefficients are clipped (default: True)
+    clip_data_values : bool
+        Enable clipping of extreme data values (default: True)
+    data_clip_threshold : float
+        Clip values beyond this many standard deviations (default: 100.0)
+    warn_on_data_clip : bool
+        Warn when data values are clipped (default: True)
+    use_regularization : bool
+        Enable regularization for numerical stability (default: True)
+    regularization_scale : float
+        Scale factor for ridge regularization (default: 1e-5)
+    min_eigenvalue : float
+        Minimum eigenvalue for positive definite matrices (default: 1e-8)
+    max_eigenvalue : float
+        Maximum eigenvalue cap (default: 1e6)
+    warn_on_regularization : bool
+        Warn when regularization is applied (default: True)
+    use_damped_updates : bool
+        Enable damped updates when likelihood decreases (default: True)
+    damping_factor : float
+        Damping factor: 0.8 = 80% new, 20% old (default: 0.8)
+    warn_on_damped_update : bool
+        Warn when damped updates are used (default: True)
+    augment_idio : bool
+        Enable state augmentation with idiosyncratic components (default: True)
+    augment_idio_slow : bool
+        Enable tent-length chains for slower-frequency series (default: True)
+    idio_rho0 : float
+        Initial AR coefficient for idiosyncratic components (default: 0.1)
+    idio_min_var : float
+        Minimum variance for idiosyncratic innovation covariance (default: 1e-8)
+    
+    Examples
+    --------
+    >>> from dfm_python.config import Params
+    >>> params = Params(max_iter=100, threshold=1e-4)
+    >>> dfm.from_spec('data/spec.csv', params=params)
+    """
+    # Estimation parameters
+    ar_lag: int = 1
+    threshold: float = 1e-5
+    max_iter: int = 5000
+    nan_method: int = 2
+    nan_k: int = 3
+    clock: str = 'm'
+    
+    # Numerical stability - AR clipping
+    clip_ar_coefficients: bool = True
+    ar_clip_min: float = -0.99
+    ar_clip_max: float = 0.99
+    warn_on_ar_clip: bool = True
+    
+    # Numerical stability - Data clipping
+    clip_data_values: bool = True
+    data_clip_threshold: float = 100.0
+    warn_on_data_clip: bool = True
+    
+    # Numerical stability - Regularization
+    use_regularization: bool = True
+    regularization_scale: float = 1e-5
+    min_eigenvalue: float = 1e-8
+    max_eigenvalue: float = 1e6
+    warn_on_regularization: bool = True
+    
+    # Numerical stability - Damped updates
+    use_damped_updates: bool = True
+    damping_factor: float = 0.8
+    warn_on_damped_update: bool = True
+    
+    # Idiosyncratic component augmentation
+    augment_idio: bool = True  # Enable state augmentation with idiosyncratic components (default: True)
+    augment_idio_slow: bool = True  # Enable tent-length chains for slower-frequency series (default: True)
+    idio_rho0: float = 0.1  # Initial AR coefficient for idiosyncratic components (default: 0.1)
+    idio_min_var: float = 1e-8  # Minimum variance for idiosyncratic innovation covariance (default: 1e-8)
+    
+    def __post_init__(self):
+        """Validate parameters."""
+        self.clock = validate_frequency(self.clock)
+        if self.threshold <= 0:
+            raise ValueError(f"threshold must be positive, got {self.threshold}")
+        if self.max_iter < 1:
+            raise ValueError(f"max_iter must be at least 1, got {self.max_iter}")
+        if self.ar_clip_min >= self.ar_clip_max:
+            raise ValueError(f"ar_clip_min ({self.ar_clip_min}) must be < ar_clip_max ({self.ar_clip_max})")
+        if not (-1 < self.ar_clip_min < 1):
+            raise ValueError(f"ar_clip_min must be in (-1, 1), got {self.ar_clip_min}")
+        if not (-1 < self.ar_clip_max < 1):
+            raise ValueError(f"ar_clip_max must be in (-1, 1), got {self.ar_clip_max}")
+
+
+@dataclass
+class FitParams:
+    """Parameters for fitting DFM models.
+    
+    This dataclass groups all optional parameters that can override
+    config values during model fitting. This reduces method signature
+    complexity and improves code readability.
+    
+    All parameters are optional. If None, the corresponding value
+    from DFMConfig will be used during parameter resolution.
+    """
+    # Convergence parameters
+    threshold: Optional[float] = None
+    max_iter: Optional[int] = None
+    
+    # Model structure
+    ar_lag: Optional[int] = None
+    num_factors: Optional[int] = None
+    
+    # Missing data handling
+    nan_method: Optional[int] = None
+    nan_k: Optional[int] = None
+    
+    # AR coefficient clipping
+    clip_ar_coefficients: Optional[bool] = None
+    ar_clip_min: Optional[float] = None
+    ar_clip_max: Optional[float] = None
+    
+    # Data clipping
+    clip_data_values: Optional[bool] = None
+    data_clip_threshold: Optional[float] = None
+    
+    # Regularization
+    use_regularization: Optional[bool] = None
+    regularization_scale: Optional[float] = None
+    min_eigenvalue: Optional[float] = None
+    max_eigenvalue: Optional[float] = None
+    
+    # Damping
+    use_damped_updates: Optional[bool] = None
+    damping_factor: Optional[float] = None
+    
+    def to_dict(self) -> dict:
+        """Convert to dictionary, excluding None values."""
+        return {k: v for k, v in self.__dict__.items() if v is not None}
+    
+    @classmethod
+    def from_kwargs(cls, **kwargs) -> 'FitParams':
+        """Create FitParams from keyword arguments.
+        
+        Filters kwargs to only include valid parameter names,
+        ignoring any extra arguments.
+        """
+        valid_params = {
+            'threshold', 'max_iter', 'ar_lag', 'num_factors',
+            'nan_method', 'nan_k',
+            'clip_ar_coefficients', 'ar_clip_min', 'ar_clip_max',
+            'clip_data_values', 'data_clip_threshold',
+            'use_regularization', 'regularization_scale',
+            'min_eigenvalue', 'max_eigenvalue',
+            'use_damped_updates', 'damping_factor'
+        }
+        filtered = {k: v for k, v in kwargs.items() if k in valid_params}
+        return cls(**filtered)
+