dfm-python 0.4.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

dfm_python/__init__.py

@@ -7,7 +7,7 @@ This package implements a comprehensive Dynamic Factor Model framework with supp
 - Expectation-Maximization (EM) algorithm for parameter estimation
 - Kalman filtering and smoothing for factor extraction
 - News decomposition for nowcasting
-- Deep Dynamic Factor Models (DDFM) with nonlinear encoders (optional, requires PyTorch)
+- Deep Dynamic Factor Models (DDFM) with nonlinear encoders (requires PyTorch)
 
 The package implements a clock-based approach to mixed-frequency DFMs, where all latent 
 factors (global and block-level) are synchronized to a common "clock" frequency, typically 
@@ -18,73 +18,56 @@ Note: Higher frequencies (daily, weekly) than the clock are not supported. If an
 has a frequency faster than the clock, a ValueError will be raised.
 
 Key Features:
-    - Unified configuration system (YAML with Hydra/OmegaConf, or direct DFMConfig objects)
+    - Hydra-based configuration (YAML files) - primary configuration method
+    - Expects preprocessed data from users - users handle all preprocessing, package extracts statistics from pipeline
     - Flexible block structure for factor modeling
-    - Robust handling of missing data
-    - Comprehensive transformation support
+    - Robust handling of missing data (internal spline interpolation)
+    - Automatic standardization and data clipping
     - News decomposition for forecast updates
 
-Example (High-level API - Recommended):
-    >>> import dfm_python as dfm
-    >>> from dfm_python.lightning import DFMDataModule
-    >>> from sktime.transformations.compose import ColumnTransformer
+Example (Standard Lightning Pattern):
+    >>> from dfm_python import DFM, DFMDataModule, DFMTrainer
+    >>> import pandas as pd
     >>> 
-    >>> # Linear DFM
-    >>> model = dfm.DFM()
-    >>> model.load_config('config/default.yaml')
+    >>> # Step 1: Load and preprocess data
+    >>> df = pd.read_csv('data/sample_data.csv')
+    >>> df_processed = df[[col for col in df.columns if col != 'date']]
     >>> 
-    >>> # Create transformer (user must provide)
-    >>> transformer = ColumnTransformer([...])  # User-defined sktime transformer
+    >>> # Step 2: Create DataModule
+    >>> dm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
+    >>> dm.setup()
     >>> 
-    >>> # Create DataModule
-    >>> data_module = DFMDataModule(config=model.config, transformer=transformer, data_path='data/sample_data.csv')
-    >>> data_module.setup()
+    >>> # Step 3: Create model and load config
+    >>> model = DFM()
+    >>> model.load_config('config/default.yaml')
     >>> 
-    >>> # Train
-    >>> model.train(data_module, max_iter=100)
+    >>> # Step 4: Create trainer and fit (standard Lightning pattern)
+    >>> trainer = DFMTrainer(max_epochs=100)
+    >>> trainer.fit(model, dm)
+    >>> 
+    >>> # Step 5: Predict
     >>> Xf, Zf = model.predict(horizon=6)
     >>> 
-    >>> # Or use DDFM (separate class)
-    >>> ddfm_model = dfm.DDFM(encoder_layers=[64, 32], num_factors=2)
-    >>> ddfm_model.load_config('config/default.yaml')
+    >>> # Or use DDFM (same pattern)
+    >>> from dfm_python import DDFM, DDFMTrainer
     >>> 
-    >>> # Create DataModule for DDFM
-    >>> data_module_ddfm = DFMDataModule(config=ddfm_model.config, transformer=transformer, data_path='data/sample_data.csv')
-    >>> data_module_ddfm.setup()
+    >>> dm_ddfm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
+    >>> dm_ddfm.setup()
     >>> 
-    >>> # Train
-    >>> ddfm_model.train(data_module_ddfm, epochs=100)
+    >>> ddfm_model = DDFM(encoder_layers=[64, 32], num_factors=2)
+    >>> ddfm_model.load_config('config/default.yaml')
+    >>> 
+    >>> trainer_ddfm = DDFMTrainer(max_epochs=100)
+    >>> trainer_ddfm.fit(ddfm_model, dm_ddfm)
     >>> Xf, Zf = ddfm_model.predict(horizon=6)
     
-Example (Low-level API - For advanced usage):
-    >>> from dfm_python import DFM, DFMConfig, SeriesConfig
-    >>> from dfm_python.transformations.utils import read_data  # Preferred import
-    >>> # Option 1: Load from YAML
-    >>> config = load_config('config.yaml')
-    >>> # Option 2: Create directly
-    >>> config = DFMConfig(
-    ...     series=[SeriesConfig(frequency='m', transformation='lin', blocks=[1], series_id='series1')],
-    ...     block_names=['Global']
-    ... )
-    >>> from dfm_python.lightning import DFMDataModule
-    >>> from sktime.transformations.compose import ColumnTransformer
-    >>> 
-    >>> # Create transformer (user must provide)
-    >>> transformer = ColumnTransformer([...])  # User-defined
-    >>> 
-    >>> # Create DataModule
-    >>> data_module = DFMDataModule(config=config, transformer=transformer, data_path='data.csv')
-    >>> data_module.setup()
-    >>> 
-    >>> # Fit model
-    >>> model = DFM()
-    >>> result = model.fit(data_module, config)
-    >>> factors = result.Z  # Extract estimated factors
+Note: DFMConfig and SeriesConfig are internal implementation details.
+    Users should use Hydra YAML configuration files instead.
 
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.4.0"
+__version__ = "0.4.3"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -94,23 +77,22 @@ __version__ = "0.4.0"
 # Internal reorganization should not break these imports.
 #
 # Public API categories:
-# 1. Configuration: DFMConfig, SeriesConfig, BlockConfig, Params, config sources
+# 1. Configuration: DFMConfig, SeriesConfig, config sources
 # 2. High-level API: DFM, DDFM, module-level convenience functions
-# 3. Core utilities: DFMCore, run_kf, TimeIndex, diagnostics
-# 4. Models: BaseFactorModel, DFMLinear, DDFM (low-level)
+# 3. Core utilities: TimeIndex, diagnostics
+# 4. Models: BaseFactorModel, DDFM (low-level)
 # 5. Nowcasting: Nowcast, result classes, para_const
-# 6. Data & Results: DFMResult, transform_data
+# 6. Data & Results: DFMResult
 # ============================================================================
 
 # Configuration (from config/ subpackage)
 from .config import (
-    DFMConfig, SeriesConfig, BlockConfig, Params, DEFAULT_GLOBAL_BLOCK_NAME,
+    DEFAULT_BLOCK_NAME,
     ConfigSource, YamlSource, DictSource, HydraSource,
     MergedConfigSource, make_config_source,
 )
-
-# Data utilities
-# Note: transform_data and DFMScaler have been removed - users must provide their own sktime transformers to DFMDataModule
+# Internal imports (for backward compatibility, but not recommended)
+from .config import DFMConfig, SeriesConfig  # Internal use only
 
 # Results
 from .config.results import DFMResult, DDFMResult, BaseResult
@@ -118,27 +100,15 @@ from .config.results import DFMResult, DDFMResult, BaseResult
 # Utilities (from utils/ subpackage)
 from .utils.diagnostics import diagnose_series, print_series_diagnosis
 from .utils.time import calculate_rmse
-# DFMCore is an alias for DFMLinear (backward compatibility)
-from .models.dfm import DFMLinear
-DFMCore = DFMLinear
-
-# PyTorch Lightning modules (recommended)
-try:
-    from .lightning import (
-        DFMLightningModule,
-        DDFMLightningModule,
-        DFMDataModule,
-        KalmanFilter,  # New module class
-        EMAlgorithm,  # New module class
-    )
-    _has_lightning = True
-except ImportError:
-    _has_lightning = False
-    DFMLightningModule = None  # type: ignore
-    DDFMLightningModule = None  # type: ignore
-    DFMDataModule = None  # type: ignore
-    KalmanFilter = None  # type: ignore
-    EMAlgorithm = None  # type: ignore
+
+# PyTorch Lightning modules (mandatory dependency)
+# Users can import these directly from dfm_python
+from .lightning import (
+    DFMDataModule,
+    DFMDataset,  # Dataset class (usually not needed directly)
+    KalmanFilter,  # Module class
+    EMAlgorithm,  # Module class
+)
 
 # Nowcasting (from nowcast/ subpackage)
 from .nowcast import (
@@ -153,84 +123,48 @@ from .nowcast.helpers import (
 
 # Model implementations
 from .models.base import BaseFactorModel
-from .models.dfm import DFMLinear, DFM
-from .models.dfm import (
-    from_yaml, from_spec, from_spec_df, from_dict,
-    load_config, load_pickle, train, predict, plot, reset, create_model
-)
-# Note: load_data has been removed - use DFMDataModule instead
-
-# DDFM high-level API and low-level model (both optional, requires PyTorch)
-try:
-    from .models.ddfm import DDFM, DDFMModel
-    from .models.ddfm import load_config_ddfm, load_data_ddfm, train_ddfm, predict_ddfm, plot_ddfm, reset_ddfm
-    _has_ddfm = True
-except ImportError:
-    _has_ddfm = False
-    DDFM = None  # type: ignore
-    DDFMModel = None  # type: ignore
-    load_config_ddfm = None  # type: ignore
-    load_data_ddfm = None  # type: ignore
-    train_ddfm = None  # type: ignore
-    predict_ddfm = None  # type: ignore
-    plot_ddfm = None  # type: ignore
-    reset_ddfm = None  # type: ignore
-    # DDFM module-level functions removed - create DDFM() instance directly
+from .models.dfm import DFM
+
+# DDFM high-level API and low-level model (PyTorch is mandatory)
+from .models.ddfm import DDFM, DDFMModel
 
 __all__ = [
     # Core classes
-    'DFMConfig', 'SeriesConfig', 'BlockConfig', 'Params', 'DFM', 'DFMCore', 'Nowcast',
+    'DFM', 'Nowcast',
     # Model base and implementations
-    'BaseFactorModel', 'DFMLinear',
+    'BaseFactorModel',
     # Nowcast result classes
     'NowcastResult', 'NewsDecompResult', 'BacktestResult',
     # Constants
-    'DEFAULT_GLOBAL_BLOCK_NAME',
+    'DEFAULT_BLOCK_NAME',
     # Config sources
     'ConfigSource', 'YamlSource', 'DictSource', 'HydraSource',
     'MergedConfigSource', 'make_config_source',
-    # High-level API (module-level - recommended)
-    'load_config',
-    'load_pickle', 'train', 'predict', 'plot', 'reset', 'create_model',
-    # Convenience constructors (cleaner API)
-    'from_yaml', 'from_spec', 'from_spec_df', 'from_dict',
     # Low-level API (functional interface - advanced usage)
     'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
     'para_const',  # Internal utility for nowcasting
 ]
 
-# Add DDFM high-level API and convenience functions if available
-if _has_ddfm:
-    __all__.extend([
-        'DDFM',  # High-level API class
-        'DDFMModel',  # Low-level implementation
-        'load_config_ddfm', 'train_ddfm', 
-        'predict_ddfm', 'plot_ddfm', 'reset_ddfm'
-        # Note: load_data_ddfm has been removed - use DFMDataModule instead
-    ])
-
-# Add Lightning modules if available
-if _has_lightning:
-    __all__.extend([
-        'DFMLightningModule',
-        'DDFMLightningModule',
-        'DFMDataModule',
-        'KalmanFilter',  # New module class
-        'EMAlgorithm',  # New module class
-    ])
-
-# Add Trainer classes if available
-try:
-    from .trainer import DFMTrainer, DDFMTrainer
-    _has_trainers = True
-except ImportError:
-    _has_trainers = False
-    DFMTrainer = None  # type: ignore
-    DDFMTrainer = None  # type: ignore
-
-if _has_trainers:
-    __all__.extend([
-        'DFMTrainer',
-        'DDFMTrainer',
-    ])
+# DDFM high-level API (PyTorch is mandatory)
+__all__.extend([
+    'DDFM',  # High-level API class
+    'DDFMModel',  # Low-level implementation (used internally)
+])
+
+# Lightning modules (mandatory dependency)
+__all__.extend([
+    'DFMDataModule',
+    'DFMDataset',  # Dataset class (usually not needed directly)
+    'KalmanFilter',  # Module class
+    'EMAlgorithm',  # Module class
+])
+
+# Trainer classes (mandatory dependency)
+from .trainer import DFMTrainer, DDFMTrainer
+
+__all__.extend([
+    'DFMTrainer',
+    'DDFMTrainer',
+])
+
 

dfm_python/config/__init__.py

@@ -1,65 +1,60 @@
 """Configuration subpackage for DFM.
 
 This subpackage provides:
-- Schema (DFMConfig, SeriesConfig, BlockConfig, Params) in schema.py
-- IO (ConfigSource, YamlSource, etc.) in io.py
+- Schema (DFMConfig, SeriesConfig) in schema.py
+- IO (ConfigSource, YamlSource, etc.) in adapter.py
 """
 
 from .schema import (
-    BaseModelConfig, DFMConfig, DDFMConfig, SeriesConfig, BlockConfig,
-    DEFAULT_GLOBAL_BLOCK_NAME,
+    BaseModelConfig, DFMConfig, DDFMConfig, SeriesConfig,
+    DEFAULT_BLOCK_NAME,
 )
-from .params import Params, FitParams
-from .results import BaseResult, DFMResult, DDFMResult, DFMParams
+from .results import BaseResult, DFMResult, DDFMResult, FitParams
 from .utils import validate_frequency, validate_transformation
-from .io import (
+from .adapter import (
     ConfigSource,
     YamlSource,
     DictSource,
     HydraSource,
     MergedConfigSource,
     make_config_source,
-    from_spec,
-    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
-    # are available from io module but not exported as public API
 )
-from .structure import (
+from .utils import (
     FREQUENCY_HIERARCHY,
     PERIODS_PER_YEAR,
     get_periods_per_year,
-    compute_idio_chain_lengths,
-    get_tent_weights_for_pair,
+    get_annual_factor,
+    compute_idio_lengths,
+    get_tent_weights,
     generate_tent_weights,
     generate_R_mat,
-    get_aggregation_structure,
-    group_series_by_frequency,
+    get_agg_structure,
+    group_by_freq,
 )
 
 __all__ = [
     # Schema
-    'BaseModelConfig', 'DFMConfig', 'DDFMConfig', 'SeriesConfig', 'BlockConfig',
-    'DEFAULT_GLOBAL_BLOCK_NAME',
-    # Parameters
-    'Params', 'FitParams',
+    'BaseModelConfig', 'DFMConfig', 'DDFMConfig', 'SeriesConfig',
+    'DEFAULT_BLOCK_NAME',
+    # Parameter overrides
+    'FitParams',
     # Results
-    'BaseResult', 'DFMResult', 'DDFMResult', 'DFMParams',
+    'BaseResult', 'DFMResult', 'DDFMResult',
     # Utilities
     'validate_frequency', 'validate_transformation',
     # IO
     'ConfigSource', 'YamlSource', 'DictSource',
     'HydraSource', 'MergedConfigSource', 'make_config_source',
-    'from_spec',
-    # Structure utilities
+    # Frequency and aggregation utilities
     'FREQUENCY_HIERARCHY',
     'PERIODS_PER_YEAR',
     'get_periods_per_year',
-    'compute_idio_chain_lengths',
-    'get_tent_weights_for_pair',
+    'get_annual_factor',
+    'compute_idio_lengths',
+    'get_tent_weights',
     'generate_tent_weights',
     'generate_R_mat',
-    'get_aggregation_structure',
-    'group_series_by_frequency',
-    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
-    # are available from io module but not exported as public API
+    'get_agg_structure',
+    'group_by_freq',
 ]