dfm-python 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

dfm_python/__init__.py

@@ -50,7 +50,7 @@ Example (Low-level API - For advanced usage):
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 
 from .config import DFMConfig, SeriesConfig, BlockConfig, Params, DEFAULT_GLOBAL_BLOCK_NAME
 from .config import (
@@ -69,7 +69,7 @@ from .kalman import run_kf, skf, fis, miss_data
 from .news import update_nowcast, news_dfm, para_const
 
 # Import high-level API (extends core DFM with convenience methods)
-from .dfm_api import DFM, _dfm_instance, from_yaml, from_spec, from_spec_df, from_dict
+from .api import DFM, _dfm_instance, from_yaml, from_spec, from_spec_df, from_dict
 
 # Module-level API: expose singleton instance methods and properties
 # This allows: import dfm_python as dfm; dfm.load_config(...); dfm.result

dfm_python/{dfm_api.py → api.py}

@@ -1,7 +1,8 @@
-"""High-level API for Dynamic Factor Model.
+"""High-level API for Dynamic Factor Model (convenience layer).
 
-This module provides an object-oriented interface for DFM estimation.
-Use this for a simpler, more intuitive API compared to the functional interface.
+This module provides an object-oriented interface for DFM estimation
+on top of the core `DFM` in `dfm.py`. Use this for a simpler, more
+intuitive workflow:
 
 Example:
     >>> import dfm_python as dfm
@@ -16,7 +17,7 @@ Example:
     >>> factors = result.Z
 """
 
-from typing import Optional, Union, Dict, Any
+from typing import Optional, Union, Dict, Any, Tuple
 from pathlib import Path
 import numpy as np
 import pandas as pd
@@ -29,6 +30,7 @@ from .config import (
     MergedConfigSource,
 )
 from .dfm import DFM as _DFMCore, DFMResult
+from .core.helpers import safe_get_method, safe_get_attr
 
 
 class DFM(_DFMCore):
@@ -84,53 +86,8 @@ class DFM(_DFMCore):
     ) -> 'DFM':
         """Load configuration from various sources.
         
-        This method provides a unified interface for loading configurations
-        from YAML files, dictionaries, spec CSV files, or Hydra configs.
-        It also supports merging configurations (e.g., YAML base + Spec CSV series).
-        
-        Parameters
-        ----------
-        source : str, Path, dict, DFMConfig, or ConfigSource, optional
-            Configuration source. If DFMConfig, used directly.
-            If str/Path, treated as YAML file path.
-            If dict, treated as dictionary config.
-            If ConfigSource, used directly.
-        yaml : str or Path, optional
-            Explicit YAML file path
-        mapping : dict, optional
-            Explicit dictionary config
-        spec : str or Path, optional
-            Explicit spec CSV file path (series definitions only)
-        hydra : DictConfig or dict, optional
-            Explicit Hydra config
-        base : str, Path, dict, or ConfigSource, optional
-            Base configuration (for merging). Used with `override`.
-        override : str, Path, dict, or ConfigSource, optional
-            Override configuration (for merging). Used with `base`.
-            Series from override replace base series.
-            
-        Returns
-        -------
-        DFM
-            Self for method chaining.
-            
-        Examples
-        --------
-        >>> # From YAML file
-        >>> dfm.load_config('config/default.yaml')
-        >>> 
-        >>> # From dictionary
-        >>> dfm.load_config({'series': [...], 'clock': 'm'})
-        >>> 
-        >>> # Explicit keyword arguments
-        >>> dfm.load_config(yaml='config/default.yaml')
-        >>> dfm.load_config(spec='data/spec.csv')
-        >>> 
-        >>> # Merge YAML base + Spec CSV series
-        >>> dfm.load_config(base='config/default.yaml', override='data/spec.csv')
-        >>> 
-        >>> # Direct DFMConfig object
-        >>> dfm.load_config(DFMConfig(...))
+        Unified interface for YAML files, dictionaries, spec CSV, or Hydra configs.
+        Supports merging configurations (e.g., YAML base + Spec CSV series).
         """
         # Handle direct DFMConfig
         if isinstance(source, DFMConfig):
@@ -171,29 +128,7 @@ class DFM(_DFMCore):
                   data_path: Optional[Union[str, Path]] = None,
                   data: Optional[np.ndarray] = None,
                   **kwargs) -> 'DFM':
-        """Load data from file or use provided array.
-        
-        Parameters
-        ----------
-        data_path : str or Path, optional
-            Path to data file (CSV). If None, must provide `data`.
-        data : np.ndarray, optional
-            Data matrix directly (T x N). If None, must provide `data_path`.
-        **kwargs
-            Additional arguments passed to `load_data()`:
-            - sample_start: Start date for data sampling
-            - sample_end: End date for data sampling
-            
-        Returns
-        -------
-        DFM
-            Self for method chaining.
-            
-        Example:
-            >>> dfm.load_data('data/sample_data.csv')
-            >>> # or
-            >>> dfm.load_data(data=X, sample_start=pd.Timestamp('2000-01-01'))
-        """
+        """Load data from file or use provided array."""
         if self._config is None:
             raise ValueError("Configuration must be loaded before loading data. "
                            "Call load_config() first.")
@@ -203,14 +138,12 @@ class DFM(_DFMCore):
                 data_path, self._config, **kwargs
             )
         elif data is not None:
-            # If data is provided directly, we still need to transform it
-            # For now, assume it's already in the right format
-            # In the future, we might want to add transform_data() call here
+            # If data is provided directly, we still need a time index
             self._data = data
             if 'time' in kwargs:
                 self._time = kwargs['time']
             else:
-                # Generate default time index
+                # Generate default monthly-end index
                 self._time = pd.date_range(
                     start='2000-01-01', 
                     periods=len(data), 
@@ -226,25 +159,7 @@ class DFM(_DFMCore):
               threshold: Optional[float] = None,
               max_iter: Optional[int] = None,
               **kwargs) -> 'DFM':
-        """Train the DFM model.
-        
-        Parameters
-        ----------
-        threshold : float, optional
-            EM convergence threshold. If None, uses config.threshold.
-        max_iter : int, optional
-            Maximum EM iterations. If None, uses config.max_iter.
-        **kwargs
-            Additional arguments (for future extensibility).
-            
-        Returns
-        -------
-        DFM
-            Self for method chaining.
-            
-        Example:
-            >>> dfm.train(threshold=1e-4, max_iter=100)
-        """
+        """Train the DFM model using EM algorithm."""
         if self._config is None:
             raise ValueError("Configuration must be loaded before training. "
                            "Call load_config() first.")
@@ -262,23 +177,16 @@ class DFM(_DFMCore):
         # Attach metadata for convenient OOP access
         if self._time is not None:
             self._result.time_index = self._time
-        if hasattr(self._config, 'get_series_ids'):
-            self._result.series_ids = self._config.get_series_ids()
-        if hasattr(self._config, 'block_names'):
-            self._result.block_names = self._config.block_names
+        series_ids = safe_get_method(self._config, 'get_series_ids')
+        if series_ids is not None:
+            self._result.series_ids = series_ids
+        block_names = safe_get_attr(self._config, 'block_names')
+        if block_names is not None:
+            self._result.block_names = block_names
         return self
     
     def reset(self) -> 'DFM':
-        """Reset all state (config, data, results).
-        
-        Returns
-        -------
-        DFM
-            Self for method chaining.
-            
-        Example:
-            >>> dfm.reset()  # Clear everything
-        """
+        """Reset all state (config, data, results)."""
         super().__init__()
         self._time = None
         self._original_data = None
@@ -287,24 +195,29 @@ class DFM(_DFMCore):
     # ---------------------------------------------------------------------
     # Inference API
     # ---------------------------------------------------------------------
-    def predict(self, horizon: int = 12, *, return_series: bool = True, return_factors: bool = True):
+    def predict(self, horizon: int = 12, *, return_series: bool = True, return_factors: bool = True) -> Union[Tuple[np.ndarray, np.ndarray], np.ndarray]:
         """Forecast series and/or factors for a given horizon using the trained model.
         
         Parameters
         ----------
         horizon : int
-            Number of periods to forecast.
+            Number of periods ahead to forecast (default: 12)
         return_series : bool
-            If True, return forecasted observed series (denormalized).
+            Whether to return forecasted series (default: True)
         return_factors : bool
-            If True, return forecasted latent factors.
-        
+            Whether to return forecasted factors (default: True)
+            
         Returns
         -------
-        tuple | np.ndarray
-            - If both True: (X_forecast, Z_forecast)
-            - If only factors: Z_forecast
-            - If only series: X_forecast
+        If both return_series and return_factors are True:
+            Tuple[np.ndarray, np.ndarray]
+                (X_forecast, Z_forecast) where X_forecast is (horizon x N) and Z_forecast is (horizon x m)
+        If only return_series is True:
+            np.ndarray
+                X_forecast (horizon x N)
+        If only return_factors is True:
+            np.ndarray
+                Z_forecast (horizon x m)
         """
         if self._result is None:
             raise ValueError("Model must be trained before calling predict(). Call train() first.")
@@ -335,26 +248,7 @@ class DFM(_DFMCore):
     
     def plot(self, *, kind: str = 'factor', factor_index: int = 0, forecast_horizon: Optional[int] = None,
              save_path: Optional[Union[str, Path]] = None, show: bool = False):
-        """Quick plotting utility for common visualizations.
-        
-        Parameters
-        ----------
-        kind : str
-            What to plot. Supported: 'factor' (single factor time series).
-        factor_index : int
-            Which factor index to plot (default: 0 = first/global).
-        forecast_horizon : int, optional
-            If provided, overlay deterministic factor forecast.
-        save_path : str or Path, optional
-            If provided, save the figure to this path.
-        show : bool
-            If True, display the figure (useful in notebooks).
-        
-        Returns
-        -------
-        Path | None
-            Save path if saved, else None.
-        """
+        """Quick plotting utility for common visualizations."""
         if self._result is None:
             raise ValueError("No results to plot. Train the model first.")
         try:
@@ -379,7 +273,7 @@ class DFM(_DFMCore):
             Z_fore = self.predict(forecast_horizon, return_series=False, return_factors=True)
             # Build forecast dates
             last_date = time_hist[-1]
-            # Use same monthly spacing; if freq missing, fallback to monthly
+            # Use same monthly-end spacing; if freq missing, fallback to monthly-end
             try:
                 forecast_dates = pd.date_range(start=last_date + pd.tseries.frequencies.to_offset('ME'),
                                                periods=forecast_horizon, freq='ME')
@@ -391,7 +285,6 @@ class DFM(_DFMCore):
         
         title_block = None
         if hasattr(self._result, 'block_names') and self._result.block_names:
-            # factor grouping is block-wise; use index 0 as 'Global' heuristic
             title_block = self._result.block_names[0] if factor_index == 0 else None
         plt.title(f"Factor {factor_index}" + (f" ({title_block})" if title_block else ""))
         plt.xlabel("Date")
@@ -428,13 +321,7 @@ def load_config(
     base: Optional[Union[str, Path, Dict[str, Any], ConfigSource]] = None,
     override: Optional[Union[str, Path, Dict[str, Any], ConfigSource]] = None,
 ) -> DFM:
-    """Load configuration (module-level convenience function).
-    
-    This is a convenience function that uses the global DFM instance.
-    For multiple models, create separate DFM() instances.
-    
-    See DFM.load_config() for parameter documentation.
-    """
+    """Load configuration (module-level convenience function)."""
     return _dfm_instance.load_config(
         source=source,
         yaml=yaml,
@@ -477,58 +364,12 @@ def reset() -> DFM:
 
 # Convenience constructors for cleaner API
 def from_yaml(yaml_path: Union[str, Path]) -> DFM:
-    """Load configuration from YAML file (convenience constructor).
-    
-    This is a cleaner alternative to `load_config(yaml=...)`.
-    
-    Parameters
-    ----------
-    yaml_path : str or Path
-        Path to YAML configuration file
-        
-    Returns
-    -------
-    DFM
-        DFM instance for method chaining
-        
-    Examples
-    --------
-    >>> import dfm_python as dfm
-    >>> dfm.from_yaml('config/default.yaml')
-    >>> dfm.load_data('data/sample_data.csv')
-    >>> dfm.train(max_iter=1)
-    """
+    """Load configuration from YAML file (convenience constructor)."""
     return _dfm_instance.load_config(yaml=yaml_path)
 
 
 def from_spec(spec_path: Union[str, Path], params: Optional[Params] = None) -> DFM:
-    """Load configuration from spec CSV file (convenience constructor).
-    
-    This is a cleaner alternative to `load_config(config=params, spec=...)`.
-    The spec CSV defines all series and their block memberships.
-    The params object (optional) provides main settings (threshold, max_iter, etc.).
-    
-    Parameters
-    ----------
-    spec_path : str or Path
-        Path to spec CSV file (series definitions)
-    params : Params, optional
-        Global estimation parameters. If None, uses defaults from Params().
-        
-    Returns
-    -------
-    DFM
-        DFM instance for method chaining
-        
-    Examples
-    --------
-    >>> from dfm_python.config import Params
-    >>> import dfm_python as dfm
-    >>> params = Params(max_iter=100, threshold=1e-4)
-    >>> dfm.from_spec('data/spec.csv', params=params)
-    >>> dfm.load_data('data/sample_data.csv')
-    >>> dfm.train(max_iter=params.max_iter)
-    """
+    """Load configuration from spec CSV file (convenience constructor)."""
     from .config import SpecCSVSource
     
     if params is None:
@@ -548,33 +389,7 @@ def from_spec(spec_path: Union[str, Path], params: Optional[Params] = None) -> D
 
 
 def from_spec_df(spec_df: pd.DataFrame, params: Optional[Params] = None) -> DFM:
-    """Load configuration from spec DataFrame (convenience constructor).
-    
-    Same as `from_spec()` but accepts a pandas DataFrame instead of a CSV file path.
-    Useful when you already have the spec data in memory.
-    
-    Parameters
-    ----------
-    spec_df : pd.DataFrame
-        DataFrame containing series definitions (same format as spec CSV)
-    params : Params, optional
-        Global estimation parameters. If None, uses defaults from Params().
-        
-    Returns
-    -------
-    DFM
-        DFM instance for method chaining
-        
-    Examples
-    --------
-    >>> import pandas as pd
-    >>> from dfm_python.config import Params
-    >>> import dfm_python as dfm
-    >>> spec_df = pd.read_csv('data/spec.csv')
-    >>> params = Params(max_iter=100)
-    >>> dfm.from_spec_df(spec_df, params=params)
-    >>> dfm.load_data('data/sample_data.csv')
-    """
+    """Load configuration from spec DataFrame (convenience constructor)."""
     from .data_loader import _load_config_from_dataframe
     
     if params is None:
@@ -593,33 +408,7 @@ def from_spec_df(spec_df: pd.DataFrame, params: Optional[Params] = None) -> DFM:
 
 
 def from_dict(mapping: Dict[str, Any]) -> DFM:
-    """Load configuration from dictionary (convenience constructor).
-    
-    This is a cleaner alternative to `load_config(mapping=...)`.
-    Useful for programmatic configuration without YAML or CSV files.
-    
-    Parameters
-    ----------
-    mapping : dict
-        Dictionary containing full DFM configuration (series, blocks, params)
-        
-    Returns
-    -------
-    DFM
-        DFM instance for method chaining
-        
-    Examples
-    --------
-    >>> import dfm_python as dfm
-    >>> config_dict = {
-    ...     'clock': 'm',
-    ...     'max_iter': 100,
-    ...     'blocks': {'Block_Global': {'factors': 1, 'clock': 'm'}},
-    ...     'series': [{'series_id': 's1', 'frequency': 'm', 'transformation': 'lin', 'blocks': [0]}]
-    ... }
-    >>> dfm.from_dict(config_dict)
-    >>> dfm.load_data('data/sample_data.csv')
-    """
+    """Load configuration from dictionary (convenience constructor)."""
     return _dfm_instance.load_config(mapping=mapping)
 
 
@@ -628,3 +417,4 @@ def from_dict(mapping: Dict[str, Any]) -> DFM:
 __all__ = ['DFM', 'load_config', 'load_data', 'train', 'reset', 
            'from_yaml', 'from_spec', 'from_spec_df', 'from_dict']
 
+

dfm_python/config.py

@@ -1050,7 +1050,28 @@ class DictSource:
         self.mapping = mapping
     
     def load(self) -> DFMConfig:
-        """Load configuration from dictionary."""
+        """Load configuration from dictionary.
+        
+        If the dictionary is partial (e.g., only max_iter, threshold),
+        it will be merged with a minimal default config.
+        """
+        # Check if this is a partial config (missing series or blocks)
+        has_series = 'series' in self.mapping and self.mapping['series']
+        has_blocks = 'blocks' in self.mapping and self.mapping['blocks']
+        
+        if not has_series or not has_blocks:
+            # This is a partial config - create a minimal default and merge
+            minimal_default = {
+                'series': [],
+                'blocks': {},
+                'clock': 'm',
+                'max_iter': 5000,
+                'threshold': 1e-5
+            }
+            # Merge: mapping takes precedence
+            merged = {**minimal_default, **self.mapping}
+            return DFMConfig.from_dict(merged)
+        
         return DFMConfig.from_dict(self.mapping)
 
 
@@ -1142,10 +1163,36 @@ class MergedConfigSource:
         from dataclasses import fields
         
         base_cfg = self.base.load()
-        override_cfg = self.override.load()
+        
+        # Check if override is a partial config (DictSource with partial dict)
+        override_is_partial = False
+        if isinstance(self.override, DictSource):
+            has_series = 'series' in self.override.mapping and self.override.mapping['series']
+            has_blocks = 'blocks' in self.override.mapping and self.override.mapping['blocks']
+            override_is_partial = not (has_series and has_blocks)
+        
+        if override_is_partial:
+            # Handle partial override: merge fields directly without loading full DFMConfig
+            override_dict = self.override.mapping
+            override_cfg = base_cfg  # Use base as template
+        else:
+            override_cfg = self.override.load()
         
         # Merge blocks: override takes precedence
-        merged_blocks = {**base_cfg.blocks, **override_cfg.blocks}
+        if override_is_partial and 'blocks' in override_dict:
+            # Merge block dicts
+            merged_blocks = {**base_cfg.blocks, **override_dict['blocks']}
+        else:
+            merged_blocks = {**base_cfg.blocks, **override_cfg.blocks}
+        
+        # Use override's series if provided and non-empty, otherwise use base's series
+        if override_is_partial:
+            if 'series' in override_dict and override_dict['series']:
+                merged_series = override_dict['series']
+            else:
+                merged_series = base_cfg.series
+        else:
+            merged_series = override_cfg.series if (override_cfg.series and len(override_cfg.series) > 0) else base_cfg.series
         
         # Get all config fields (excluding derived/computed fields)
         excluded_fields = {'series', 'blocks', 'block_names', 'factors_per_block', '_cached_blocks'}
@@ -1155,11 +1202,29 @@ class MergedConfigSource:
             if field.name not in excluded_fields
         }
         
-        # Create merged config: base settings + override series + merged blocks
+        # Override settings from override_cfg or override_dict
+        if override_is_partial:
+            override_settings = {
+                field.name: override_dict.get(field.name, getattr(base_cfg, field.name))
+                for field in fields(DFMConfig)
+                if field.name not in excluded_fields
+            }
+        else:
+            override_settings = {
+                field.name: getattr(override_cfg, field.name)
+                for field in fields(DFMConfig)
+                if field.name not in excluded_fields
+                and hasattr(override_cfg, field.name)
+            }
+        
+        # Merge settings: base + override (override takes precedence)
+        merged_settings = {**base_settings, **override_settings}
+        
+        # Create merged config: merged settings + merged series + merged blocks
         return DFMConfig(
-            series=override_cfg.series,  # Override series
-            blocks=merged_blocks,  # Merged blocks
-            **base_settings  # All other settings from base
+            series=merged_series,
+            blocks=merged_blocks,
+            **merged_settings
         )
 
 

dfm_python/core/__init__.py

@@ -0,0 +1,59 @@
+"""Core utilities for DFM estimation.
+
+This package contains core functionality organized by domain:
+- em.py: EM algorithm core (init_conditions, em_step, em_converged)
+- numeric.py: Numerical utilities (matrix operations, regularization, clipping)
+- diagnostics.py: Diagnostic functions and output formatting
+- results.py: Result metrics (RMSE calculation)
+- grouping.py: Frequency grouping utilities
+- helpers.py: Common helper functions for code patterns
+"""
+
+from .em import init_conditions, em_step, em_converged
+from .numeric import (
+    _ensure_symmetric,
+    _compute_principal_components,
+    _clean_matrix,
+    _ensure_positive_definite,
+    _compute_regularization_param,
+    _clip_ar_coefficients,
+    _apply_ar_clipping,
+    _cap_max_eigenvalue,
+    _estimate_ar_coefficient,
+    _safe_divide,
+    _check_finite,
+    _ensure_square_matrix,
+)
+from .results import calculate_rmse
+from .grouping import group_series_by_frequency
+from .diagnostics import (
+    _display_dfm_tables,
+    diagnose_series,
+    print_series_diagnosis,
+)
+from .helpers import safe_get_method, safe_get_attr
+
+__all__ = [
+    'init_conditions',
+    'em_step',
+    'em_converged',
+    '_ensure_symmetric',
+    '_compute_principal_components',
+    '_clean_matrix',
+    '_ensure_positive_definite',
+    '_compute_regularization_param',
+    '_clip_ar_coefficients',
+    '_apply_ar_clipping',
+    '_cap_max_eigenvalue',
+    '_estimate_ar_coefficient',
+    '_safe_divide',
+    '_check_finite',
+    '_ensure_square_matrix',
+    'calculate_rmse',
+    'group_series_by_frequency',
+    '_display_dfm_tables',
+    'diagnose_series',
+    'print_series_diagnosis',
+    'safe_get_method',
+    'safe_get_attr',
+]