dfm-python 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

dfm_python/__init__.py

@@ -1,6 +1,38 @@
-"""Core nowcasting modules for DFM estimation and forecasting."""
+"""Dynamic Factor Model (DFM) package for Python.
 
-from .config import DFMConfig, AppConfig
+This package implements a comprehensive Dynamic Factor Model framework with support for:
+- Mixed-frequency time series data (daily, weekly, monthly, quarterly, semi-annual, annual)
+- Clock-based synchronization of latent factors
+- Tent kernel aggregation for low-to-high frequency mapping
+- Expectation-Maximization (EM) algorithm for parameter estimation
+- Kalman filtering and smoothing for factor extraction
+- News decomposition for nowcasting
+
+The package follows the FRBNY (Federal Reserve Bank of New York) approach to mixed-frequency
+DFMs, where all latent factors (global and block-level) are synchronized to a common "clock"
+frequency, typically monthly. Lower-frequency observed variables are mapped to higher-frequency
+latent states using deterministic tent kernels in the observation equation.
+
+Key Features:
+    - Unified configuration system (YAML/CSV)
+    - Flexible block structure for factor modeling
+    - Robust handling of missing data
+    - Comprehensive transformation support
+    - News decomposition for forecast updates
+
+Example:
+    >>> from dfm_python import load_config, load_data, dfm
+    >>> config = load_config('config.yaml')
+    >>> X, Time, _ = load_data('data.csv', config)
+    >>> result = dfm(X, config)
+    >>> factors = result.Z  # Extract estimated factors
+
+For detailed documentation, see the README.md file.
+"""
+
+__version__ = "0.1.3"
+
+from .config import DFMConfig
 from .data_loader import (
     load_config, load_config_from_yaml, load_config_from_csv, load_data, transform_data
 )
@@ -14,7 +46,7 @@ ModelSpec = DFMConfig  # Deprecated: use DFMConfig
 load_spec = load_config  # Deprecated: use load_config
 
 __all__ = [
-    'DFMConfig', 'AppConfig',
+    'DFMConfig',
     # Backward compatibility
     'ModelConfig',  # Alias for DFMConfig
     'load_config', 'load_config_from_yaml', 'load_config_from_csv',

dfm_python/config.py

@@ -1,4 +1,15 @@
-"""Configuration models for DFM nowcasting using OmegaConf and Hydra."""
+"""Configuration models for DFM nowcasting using OmegaConf and Hydra.
+
+This module defines the configuration structure for Dynamic Factor Models,
+providing a unified interface for specifying:
+- Series definitions (frequency, transformation, block loadings)
+- Block structure and factor dimensions
+- Estimation parameters (EM algorithm settings, missing data handling)
+- Clock frequency for mixed-frequency synchronization
+
+The configuration system supports both YAML and CSV formats, with validation
+and type checking to ensure model specifications are correct before estimation.
+"""
 
 import numpy as np
 from typing import List, Optional, Dict, Any
@@ -106,6 +117,7 @@ class DFMConfig:
     max_iter: int = 5000  # Maximum EM iterations
     nan_method: int = 2  # Missing data handling method (1-5)
     nan_k: int = 3  # Spline parameter for NaN interpolation
+    clock: str = 'm'  # Base frequency for nowcasting (global clock): 'd', 'w', 'm', 'q', 'sa', 'a' (defaults to 'm' for monthly)
     
     # ========================================================================
     # Internal cache (not user-configurable)
@@ -113,9 +125,26 @@ class DFMConfig:
     _cached_blocks: Optional[np.ndarray] = field(default=None, init=False, repr=False)
     
     def __post_init__(self):
-        """Validate blocks structure and consistency."""
+        """Validate blocks structure and consistency.
+        
+        This method performs comprehensive validation of the DFM configuration:
+        - Ensures at least one series is specified
+        - Validates block structure consistency across all series
+        - Ensures all series load on the global block
+        - Validates factor dimensions match block structure
+        - Validates clock frequency
+        
+        Raises
+        ------
+        ValueError
+            If any validation check fails, with a descriptive error message
+            indicating what needs to be fixed.
+        """
         if not self.series:
-            raise ValueError("At least one series must be specified")
+            raise ValueError(
+                "DFM configuration must contain at least one series. "
+                "Please add series definitions to your configuration."
+            )
         
         # Auto-generate series_id if not provided
         for i, s in enumerate(self.series):
@@ -132,16 +161,19 @@ class DFMConfig:
         for i, s in enumerate(self.series):
             if len(s.blocks) != n_blocks:
                 raise ValueError(
-                    f"Series {i} ({s.series_id}) has {len(s.blocks)} blocks, "
-                    f"but expected {n_blocks} (from block_names)"
+                    f"Series {i} ('{s.series_id}') has {len(s.blocks)} block loadings, "
+                    f"but expected {n_blocks} (from block_names: {self.block_names}). "
+                    f"Each series must specify a loading (0 or 1) for each block."
                 )
         
         # Check first column (global block) is all 1s
         for i, s in enumerate(self.series):
             if s.blocks[0] != 1:
                 raise ValueError(
-                    f"Series {i} ({s.series_id}) must load on global block "
-                    f"(first block must be 1)"
+                    f"Series {i} ('{s.series_id}') must load on the global block "
+                    f"(first block '{self.block_names[0]}'). "
+                    f"All series must have blocks[0] = 1. "
+                    f"Current value: {s.blocks[0]}"
                 )
         
         # Validate factors_per_block if provided
@@ -149,10 +181,21 @@ class DFMConfig:
             if len(self.factors_per_block) != n_blocks:
                 raise ValueError(
                     f"factors_per_block length ({len(self.factors_per_block)}) must match "
-                    f"number of blocks ({n_blocks})"
+                    f"number of blocks ({n_blocks}). "
+                    f"Block names: {self.block_names}. "
+                    f"Please provide one factor count per block."
                 )
             if any(f < 1 for f in self.factors_per_block):
-                raise ValueError("factors_per_block must contain positive integers (>= 1)")
+                invalid_blocks = [i for i, f in enumerate(self.factors_per_block) if f < 1]
+                raise ValueError(
+                    f"factors_per_block must contain positive integers (>= 1). "
+                    f"Invalid values found at block indices {invalid_blocks}: "
+                    f"{[self.factors_per_block[i] for i in invalid_blocks]}. "
+                    f"Each block must have at least one factor."
+                )
+        
+        # Validate clock
+        self.clock = validate_frequency(self.clock)
     
     # Convenience properties for backward compatibility
     @property
@@ -250,18 +293,19 @@ class DFMConfig:
                 category=str(cat_val) if cat_val is not None else ''
             ))
         
-        # Extract estimation parameters if provided
-        return cls(
-            series=series_list,
-            block_names=data.get('BlockNames', data.get('block_names', [])),
-            factors_per_block=data.get('factors_per_block', None),
-            # Estimation parameters
-            ar_lag=data.get('ar_lag', 1),
-            threshold=data.get('threshold', 1e-5),
-            max_iter=data.get('max_iter', 5000),
-            nan_method=data.get('nan_method', 2),
-            nan_k=data.get('nan_k', 3)
-        )
+            # Extract estimation parameters if provided
+            return cls(
+                series=series_list,
+                block_names=data.get('BlockNames', data.get('block_names', [])),
+                factors_per_block=data.get('factors_per_block', None),
+                # Estimation parameters
+                ar_lag=data.get('ar_lag', 1),
+                threshold=data.get('threshold', 1e-5),
+                max_iter=data.get('max_iter', 5000),
+                nan_method=data.get('nan_method', 2),
+                nan_k=data.get('nan_k', 3),
+                clock=data.get('clock', 'm')
+            )
     
     @classmethod
     def from_dict(cls, data: Dict[str, Any]) -> 'DFMConfig':
@@ -331,7 +375,8 @@ class DFMConfig:
                 threshold=data.get('threshold', 1e-5),
                 max_iter=data.get('max_iter', 5000),
                 nan_method=data.get('nan_method', 2),
-                nan_k=data.get('nan_k', 3)
+                nan_k=data.get('nan_k', 3),
+                clock=data.get('clock', 'm')
             )
         
         # New format with series list
@@ -350,7 +395,8 @@ class DFMConfig:
                 threshold=data.get('threshold', 1e-5),
                 max_iter=data.get('max_iter', 5000),
                 nan_method=data.get('nan_method', 2),
-                nan_k=data.get('nan_k', 3)
+                nan_k=data.get('nan_k', 3),
+                clock=data.get('clock', 'm')
             )
         
         # Direct instantiation (shouldn't happen often, but handle it)
@@ -415,6 +461,7 @@ if HYDRA_AVAILABLE and ConfigStore is not None:
             max_iter: int = 5000
             nan_method: int = 2
             nan_k: int = 3
+            clock: str = 'm'
         
         # Register schemas in config groups (following Hydra docs pattern)
         # These can be referenced in YAML defaults lists for validation

dfm_python/data_loader.py

@@ -1,4 +1,14 @@
-"""Data loading, specification parsing, and transformation functions."""
+"""Data loading and transformation utilities for DFM estimation.
+
+This module provides comprehensive data handling for Dynamic Factor Models:
+- Configuration loading from YAML and CSV files
+- Data loading from CSV with automatic date parsing
+- Time series transformations (differences, percent changes, etc.)
+- Data sorting and alignment with configuration
+
+The module supports flexible configuration formats and handles common data
+issues such as missing dates, inconsistent formats, and transformation errors.
+"""
 
 import numpy as np
 import pandas as pd
@@ -9,7 +19,8 @@ import warnings
 import logging
 
 from .config import DFMConfig
-# Backward compatibility
+
+# Backward compatibility alias (deprecated - use DFMConfig)
 ModelConfig = DFMConfig
 
 logger = logging.getLogger(__name__)
@@ -53,16 +64,34 @@ def load_config_from_yaml(configfile: Union[str, Path]) -> DFMConfig:
     cfg = OmegaConf.load(configfile)
     cfg_dict = OmegaConf.to_container(cfg, resolve=True)
     
+    # Extract model config and dfm parameters
+    model_dict = None
+    dfm_params = {}
+    
     # Handle nested structure (if config has @package model: directive)
     if 'series' in cfg_dict and 'block_names' in cfg_dict:
         # Direct model config structure
-        return DFMConfig.from_dict(cfg_dict)
+        model_dict = cfg_dict
     elif 'model' in cfg_dict:
         # Nested under 'model' key (from @package model:)
-        return DFMConfig.from_dict(cfg_dict['model'])
+        model_dict = cfg_dict['model']
     else:
         # Try to construct from top-level keys
-        return DFMConfig.from_dict(cfg_dict)
+        model_dict = cfg_dict
+    
+    # Extract dfm parameters (estimation parameters like clock)
+    if 'dfm' in cfg_dict:
+        dfm_params = cfg_dict['dfm']
+    
+    # Merge dfm parameters into model config
+    if dfm_params:
+        if isinstance(model_dict, dict):
+            model_dict = {**model_dict, **dfm_params}
+        else:
+            # If model_dict is not a dict, create new dict
+            model_dict = {'series': model_dict, **dfm_params} if model_dict else dfm_params
+    
+    return DFMConfig.from_dict(model_dict)
 
 
 def load_config_from_csv(configfile: Union[str, Path]) -> DFMConfig:
@@ -291,16 +320,36 @@ def _transform_series(Z: np.ndarray, formula: str, freq: str, step: int) -> np.n
 
 
 def transform_data(Z: np.ndarray, Time: pd.DatetimeIndex, config: DFMConfig) -> Tuple[np.ndarray, pd.DatetimeIndex, np.ndarray]:
-    """Transform each data series based on specification."""
+    """Transform each data series based on specification.
+    
+    Handles all frequencies: daily (d), weekly (w), monthly (m), 
+    quarterly (q), semi-annual (sa), annual (a).
+    """
     T, N = Z.shape
     X = np.full((T, N), np.nan)
     
+    # Frequency to step mapping (step = number of base periods per observation)
+    # Base frequency is monthly, so step is months per observation
+    freq_to_step = {
+        'd': 1,   # Daily: 1 month per observation (aggregated to monthly)
+        'w': 1,   # Weekly: 1 month per observation (aggregated to monthly)
+        'm': 1,   # Monthly: 1 month per observation
+        'q': 3,   # Quarterly: 3 months per observation
+        'sa': 6,  # Semi-annual: 6 months per observation
+        'a': 12,  # Annual: 12 months per observation
+    }
+    
     for i in range(N):
-        step = 3 if config.Frequency[i] == 'q' else 1
-        X[:, i] = _transform_series(Z[:, i], config.Transformation[i], config.Frequency[i], step)
+        freq = config.Frequency[i]
+        step = freq_to_step.get(freq, 1)  # Default to 1 if unknown frequency
+        X[:, i] = _transform_series(Z[:, i], config.Transformation[i], freq, step)
+    
+    # Drop initial observations based on highest frequency in data
+    # Find maximum step (longest period) to determine drop period
+    max_step = max([freq_to_step.get(f, 1) for f in config.Frequency])
+    # Drop period should be at least max_step + 1 to handle transformations
+    drop = max(4, max_step + 1)  # At least 4 for quarterly, more for annual
     
-    # Drop first quarter of observations (4 months) since transformations cause missing values
-    drop = 4
     if T > drop:
         return X[drop:], Time[drop:], Z[drop:]
     return X, Time, Z