dfm-python 0.3.1__py3-none-any.whl → 0.4.2__py3-none-any.whl

dfm_python/__init__.py

@@ -7,7 +7,7 @@ This package implements a comprehensive Dynamic Factor Model framework with supp
 - Expectation-Maximization (EM) algorithm for parameter estimation
 - Kalman filtering and smoothing for factor extraction
 - News decomposition for nowcasting
-- Deep Dynamic Factor Models (DDFM) with nonlinear encoders (optional, requires PyTorch)
+- Deep Dynamic Factor Models (DDFM) with nonlinear encoders (requires PyTorch)
 
 The package implements a clock-based approach to mixed-frequency DFMs, where all latent 
 factors (global and block-level) are synchronized to a common "clock" frequency, typically 
@@ -18,47 +18,56 @@ Note: Higher frequencies (daily, weekly) than the clock are not supported. If an
 has a frequency faster than the clock, a ValueError will be raised.
 
 Key Features:
-    - Unified configuration system (YAML with Hydra/OmegaConf, or direct DFMConfig objects)
+    - Hydra-based configuration (YAML files) - primary configuration method
+    - Expects preprocessed data from users - package applies internal transformations automatically
     - Flexible block structure for factor modeling
-    - Robust handling of missing data
-    - Comprehensive transformation support
+    - Robust handling of missing data (internal spline interpolation)
+    - Automatic standardization and data clipping
     - News decomposition for forecast updates
 
-Example (High-level API - Recommended):
-    >>> import dfm_python as dfm
-    >>> # Linear DFM
-    >>> model = dfm.DFM()
+Example (Standard Lightning Pattern):
+    >>> from dfm_python import DFM, DFMDataModule, DFMTrainer
+    >>> import polars as pl
+    >>> 
+    >>> # Step 1: Load and preprocess data
+    >>> df = pl.read_csv('data/sample_data.csv')
+    >>> df_processed = df.select([col for col in df.columns if col != 'date'])
+    >>> 
+    >>> # Step 2: Create DataModule
+    >>> dm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
+    >>> dm.setup()
+    >>> 
+    >>> # Step 3: Create model and load config
+    >>> model = DFM()
     >>> model.load_config('config/default.yaml')
-    >>> model.load_data('data/sample_data.csv')
-    >>> model.train(max_iter=100)
+    >>> 
+    >>> # Step 4: Create trainer and fit (standard Lightning pattern)
+    >>> trainer = DFMTrainer(max_epochs=100)
+    >>> trainer.fit(model, dm)
+    >>> 
+    >>> # Step 5: Predict
     >>> Xf, Zf = model.predict(horizon=6)
     >>> 
-    >>> # Or use DDFM (separate class)
-    >>> ddfm_model = dfm.DDFM(encoder_layers=[64, 32], num_factors=2)
+    >>> # Or use DDFM (same pattern)
+    >>> from dfm_python import DDFM, DDFMTrainer
+    >>> 
+    >>> dm_ddfm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
+    >>> dm_ddfm.setup()
+    >>> 
+    >>> ddfm_model = DDFM(encoder_layers=[64, 32], num_factors=2)
     >>> ddfm_model.load_config('config/default.yaml')
-    >>> ddfm_model.load_data('data/sample_data.csv')
-    >>> ddfm_model.train(epochs=100)
+    >>> 
+    >>> trainer_ddfm = DDFMTrainer(max_epochs=100)
+    >>> trainer_ddfm.fit(ddfm_model, dm_ddfm)
     >>> Xf, Zf = ddfm_model.predict(horizon=6)
     
-Example (Low-level API - For advanced usage):
-    >>> from dfm_python import DFM, DFMConfig, SeriesConfig
-    >>> from dfm_python.dataloader import load_data  # Preferred import
-    >>> # Option 1: Load from YAML
-    >>> config = load_config('config.yaml')
-    >>> # Option 2: Create directly
-    >>> config = DFMConfig(
-    ...     series=[SeriesConfig(frequency='m', transformation='lin', blocks=[1], series_id='series1')],
-    ...     block_names=['Global']
-    ... )
-    >>> X, Time, _ = load_data('data.csv', config)  # or use database adapter
-    >>> model = DFM()
-    >>> result = model.fit(X, config)
-    >>> factors = result.Z  # Extract estimated factors
+Note: DFMConfig and SeriesConfig are internal implementation details.
+    Users should use Hydra YAML configuration files instead.
 
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.3.1"
+__version__ = "0.4.2"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -68,126 +77,96 @@ __version__ = "0.3.1"
 # Internal reorganization should not break these imports.
 #
 # Public API categories:
-# 1. Configuration: DFMConfig, SeriesConfig, BlockConfig, Params, config sources
+# 1. Configuration: DFMConfig, SeriesConfig, config sources
 # 2. High-level API: DFM, DDFM, module-level convenience functions
-# 3. Core utilities: DFMCore, run_kf, TimeIndex, diagnostics
+# 3. Core utilities: DFMLinear, TimeIndex, diagnostics
 # 4. Models: BaseFactorModel, DFMLinear, DDFM (low-level)
 # 5. Nowcasting: Nowcast, result classes, para_const
-# 6. Data & Results: DFMResult, transform_data
+# 6. Data & Results: DFMResult
 # ============================================================================
 
 # Configuration (from config/ subpackage)
 from .config import (
-    DFMConfig, SeriesConfig, BlockConfig, Params, DEFAULT_GLOBAL_BLOCK_NAME,
+    DEFAULT_BLOCK_NAME,
     ConfigSource, YamlSource, DictSource, HydraSource,
     MergedConfigSource, make_config_source,
 )
+# Internal imports (for backward compatibility, but not recommended)
+from .config import DFMConfig, SeriesConfig  # Internal use only
 
 # Data utilities
-from .dataloader import transform_data
 
 # Results
-from .core import DFMResult
-
-# Core utilities (from core/ subpackage)
-from .core import calculate_rmse, diagnose_series, print_series_diagnosis
-from .core.state_space import run_kf, skf, fis, miss_data
-# DFMCore is now an alias for DFMLinear (backward compatibility)
-from .models.dfm import DFMLinear
-DFMCore = DFMLinear
+from .config.results import DFMResult, DDFMResult, BaseResult
+
+# Utilities (from utils/ subpackage)
+from .utils.diagnostics import diagnose_series, print_series_diagnosis
+from .utils.time import calculate_rmse
+
+# PyTorch Lightning modules (mandatory dependency)
+# Users can import these directly from dfm_python
+from .lightning import (
+    DFMDataModule,
+    DFMDataset,  # Dataset class (usually not needed directly)
+    KalmanFilter,  # Module class
+    EMAlgorithm,  # Module class
+)
 
-# Nowcasting (already in nowcasting/ subpackage)
-from .nowcasting import (
+# Nowcasting (from nowcast/ subpackage)
+from .nowcast import (
     Nowcast,
-    para_const,
     NowcastResult,
+)
+from .nowcast.helpers import (
+    para_const,
     NewsDecompResult,
     BacktestResult,
 )
 
-# High-level API (from api/ subpackage)
-from .api import (
-    DFM, _dfm_instance, from_yaml, from_spec, from_spec_df, from_dict,
-    load_config, load_data, load_pickle, train, predict, plot, reset, create_model
-)
-
 # Model implementations
 from .models.base import BaseFactorModel
-from .models.dfm import DFMLinear
-
-# DDFM high-level API and low-level model (both optional, requires PyTorch)
-try:
-    from .api import DDFM as DDFMAPI, _ddfm_instance
-    from .api import load_config_ddfm, load_data_ddfm, train_ddfm, predict_ddfm, plot_ddfm, reset_ddfm
-    from .models.ddfm import DDFM as DDFMModel
-    _has_ddfm = True
-    # Export high-level API as DDFM
-    DDFM = DDFMAPI
-except ImportError:
-    _has_ddfm = False
-    DDFM = None  # type: ignore
-    DDFMModel = None  # type: ignore
-    load_config_ddfm = None  # type: ignore
-    load_data_ddfm = None  # type: ignore
-    train_ddfm = None  # type: ignore
-    predict_ddfm = None  # type: ignore
-    plot_ddfm = None  # type: ignore
-    reset_ddfm = None  # type: ignore
-
-# Expose properties as module-level attributes
-# Use property-like access via functions or direct attribute access
-# Since 'config' conflicts with the config module, we'll use a different approach
-# Users can access: dfm.get_config(), dfm.get_data(), dfm.get_result()
-def get_config():
-    """Get current configuration."""
-    return _dfm_instance.config
-
-def get_data():
-    """Get current data matrix."""
-    return _dfm_instance.data
-
-def get_time():
-    """Get current time index."""
-    return _dfm_instance.time
-
-def get_result():
-    """Get training result."""
-    return _dfm_instance.result
-
-def get_original_data():
-    """Get original (untransformed) data matrix."""
-    return _dfm_instance.original_data
+from .models.dfm import DFMLinear, DFM
+
+# DDFM high-level API and low-level model (PyTorch is mandatory)
+from .models.ddfm import DDFM, DDFMModel
 
 __all__ = [
     # Core classes
-    'DFMConfig', 'SeriesConfig', 'BlockConfig', 'Params', 'DFM', 'DFMCore', 'Nowcast',
+    'DFM', 'DFMLinear', 'Nowcast',
     # Model base and implementations
-    'BaseFactorModel', 'DFMLinear',
+    'BaseFactorModel',
     # Nowcast result classes
     'NowcastResult', 'NewsDecompResult', 'BacktestResult',
     # Constants
-    'DEFAULT_GLOBAL_BLOCK_NAME',
+    'DEFAULT_BLOCK_NAME',
     # Config sources
     'ConfigSource', 'YamlSource', 'DictSource', 'HydraSource',
     'MergedConfigSource', 'make_config_source',
-    # High-level API (module-level - recommended)
-    'load_config',
-    'load_data', 'load_pickle', 'train', 'predict', 'plot', 'reset', 'create_model',
-    'get_config', 'get_data', 'get_time', 'get_result', 'get_original_data',
-    # Convenience constructors (cleaner API)
-    'from_yaml', 'from_spec', 'from_spec_df', 'from_dict',
     # Low-level API (functional interface - advanced usage)
-    'transform_data',
-    'DFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
-    'run_kf', 'skf', 'fis', 'miss_data',
-    'para_const',  # Internal utility, kept for backward compatibility
+    'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
+    'para_const',  # Internal utility for nowcasting
 ]
 
-# Add DDFM high-level API and convenience functions if available
-if _has_ddfm:
-    __all__.extend([
-        'DDFM',  # High-level API class
-        'load_config_ddfm', 'load_data_ddfm', 'train_ddfm', 
-        'predict_ddfm', 'plot_ddfm', 'reset_ddfm'
-    ])
+# DDFM high-level API (PyTorch is mandatory)
+__all__.extend([
+    'DDFM',  # High-level API class
+    'DDFMModel',  # Low-level implementation (used internally)
+])
+
+# Lightning modules (mandatory dependency)
+__all__.extend([
+    'DFMDataModule',
+    'DFMDataset',  # Dataset class (usually not needed directly)
+    'KalmanFilter',  # Module class
+    'EMAlgorithm',  # Module class
+])
+
+# Trainer classes (mandatory dependency)
+from .trainer import DFMTrainer, DDFMTrainer
+
+__all__.extend([
+    'DFMTrainer',
+    'DDFMTrainer',
+])
+
 

dfm_python/config/__init__.py

@@ -1,35 +1,65 @@
 """Configuration subpackage for DFM.
 
 This subpackage provides:
-- Schema (DFMConfig, SeriesConfig, BlockConfig, Params) in schema.py
-- IO (ConfigSource, YamlSource, etc.) in io.py
+- Schema (DFMConfig, SeriesConfig) in schema.py
+- IO (ConfigSource, YamlSource, etc.) in adapter.py
 """
 
 from .schema import (
-    DFMConfig, SeriesConfig, BlockConfig, Params,
-    DEFAULT_GLOBAL_BLOCK_NAME,
-    validate_frequency, validate_transformation,
+    BaseModelConfig, DFMConfig, DDFMConfig, SeriesConfig,
+    DEFAULT_BLOCK_NAME,
 )
-from .io import (
+from .results import BaseResult, DFMResult, DDFMResult, FitParams
+from .utils import validate_frequency, validate_transformation
+from .adapter import (
     ConfigSource,
     YamlSource,
     DictSource,
     HydraSource,
     MergedConfigSource,
     make_config_source,
-    from_spec,
-    _load_config_from_dataframe,
-    _write_series_blocks_yaml,
+    # Note: from_spec is in models/dfm.py, not in adapter
+    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
+    # are available from adapter module but not exported as public API
+)
+from .utils import (
+    FREQUENCY_HIERARCHY,
+    PERIODS_PER_YEAR,
+    get_periods_per_year,
+    get_annual_factor,
+    compute_idio_chain_lengths,
+    get_tent_weights_for_pair,
+    generate_tent_weights,
+    generate_R_mat,
+    get_aggregation_structure,
+    group_series_by_frequency,
 )
 
 __all__ = [
     # Schema
-    'DFMConfig', 'SeriesConfig', 'BlockConfig', 'Params',
-    'DEFAULT_GLOBAL_BLOCK_NAME',
+    'BaseModelConfig', 'DFMConfig', 'DDFMConfig', 'SeriesConfig',
+    'DEFAULT_BLOCK_NAME',
+    # Parameter overrides
+    'FitParams',
+    # Results
+    'BaseResult', 'DFMResult', 'DDFMResult',
+    # Utilities
     'validate_frequency', 'validate_transformation',
     # IO
     'ConfigSource', 'YamlSource', 'DictSource',
     'HydraSource', 'MergedConfigSource', 'make_config_source',
-    'from_spec', '_load_config_from_dataframe', '_write_series_blocks_yaml',
+    # Frequency and aggregation utilities
+    'FREQUENCY_HIERARCHY',
+    'PERIODS_PER_YEAR',
+    'get_periods_per_year',
+    'get_annual_factor',
+    'compute_idio_chain_lengths',
+    'get_tent_weights_for_pair',
+    'generate_tent_weights',
+    'generate_R_mat',
+    'get_aggregation_structure',
+    'group_series_by_frequency',
+    # Note: Private functions (_load_config_from_dataframe, _write_series_blocks_yaml)
+    # are available from adapter module but not exported as public API
 ]