dfm-python 0.4.7__py3-none-any.whl → 0.4.9__py3-none-any.whl

dfm_python/__init__.py

@@ -65,7 +65,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.4.7"
+__version__ = "0.4.9"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -80,7 +80,6 @@ __version__ = "0.4.7"
 # 3. Core utilities: TimeIndex, diagnostics
 # 4. Models: BaseFactorModel, DDFM (low-level)
 # 5. Data & Results: DFMResult
-# 6. Data & Results: DFMResult
 # ============================================================================
 
 # Configuration (from config/ subpackage)

dfm_python/logger/logger.py

@@ -37,20 +37,29 @@ def get_logger(name: str) -> logging.Logger:
     # Configure package logger if not already configured
     # Use a flag to avoid re-configuring if already done
     if not hasattr(package_logger, '_dfm_configured'):
-        # Configure root logger for dfm_python package
-        handler = logging.StreamHandler(sys.stdout)
-        handler.setFormatter(
-            logging.Formatter(
-                '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-                datefmt='%Y-%m-%d %H:%M:%S'
+        # CRITICAL: Check if handlers already exist to prevent duplicates
+        # This can happen if configure_logging() was called before get_logger()
+        if not package_logger.handlers:
+            # Configure handler for dfm_python package logger
+            handler = logging.StreamHandler(sys.stdout)
+            handler.setFormatter(
+                logging.Formatter(
+                    '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S'
+                )
             )
-        )
-        package_logger.addHandler(handler)
+            package_logger.addHandler(handler)
+        
         package_logger.setLevel(logging.INFO)
+        # CRITICAL: Disable propagation to root logger to prevent duplicate messages
+        # Root logger may have handlers from other code (e.g., basicConfig)
+        # By setting propagate=False, we ensure messages only go through our handler
+        package_logger.propagate = False
         # Mark as configured to avoid duplicate handlers
         package_logger._dfm_configured = True
     
-    # Ensure child logger propagates to parent (default, but explicit for clarity)
+    # Child logger should propagate to package logger (not root)
+    # Since package logger has propagate=False, messages won't go to root
     logger.propagate = True
     
     return logger
@@ -110,6 +119,10 @@ def configure_logging(
     if hasattr(logger, '_dfm_configured'):
         delattr(logger, '_dfm_configured')
     
+    # CRITICAL: Disable propagation to root logger to prevent duplicate messages
+    # If root logger also has handlers, messages would be logged twice
+    logger.propagate = False
+    
     # Add console handler
     console_handler = logging.StreamHandler(sys.stdout)
     console_handler.setLevel(level)

dfm_python/models/__init__.py

@@ -17,8 +17,10 @@ __all__ = [
 
 # DDFM implementation
 from .ddfm import DDFM, DDFMModel
+from .mcmc import DDFMMCMCTrainer
 __all__.extend([
     'DDFM',  # High-level API
     'DDFMModel',  # Low-level implementation
+    'DDFMMCMCTrainer',  # MCMC training procedure
 ])
 

dfm_python/models/base.py

@@ -11,6 +11,7 @@ from abc import abstractmethod
 from typing import Optional, Union, Tuple, Any, Dict, TYPE_CHECKING
 from pathlib import Path
 import numpy as np
+import torch
 import pytorch_lightning as pl
 
 if TYPE_CHECKING:
@@ -349,6 +350,394 @@ class BaseFactorModel(pl.LightningModule):
         
         return X_torch, Mx, Wx
     
+    def _compute_default_horizon(self) -> int:
+        """Compute default horizon based on clock frequency.
+        
+        Returns
+        -------
+        int
+            Default horizon (1 year of periods based on clock frequency)
+        """
+        from ..config.utils import get_periods_per_year
+        from ..utils.helpers import get_clock_frequency
+        clock = get_clock_frequency(self.config, 'm')
+        return get_periods_per_year(clock)
+    
+    def _validate_horizon(self, horizon: int) -> None:
+        """Validate horizon parameter.
+        
+        Parameters
+        ----------
+        horizon : int
+            Horizon value to validate
+            
+        Raises
+        ------
+        ValueError
+            If horizon is not a positive integer
+        """
+        if horizon <= 0:
+            raise ValueError(
+                self._format_error_message(
+                    operation="prediction",
+                    reason=f"horizon must be a positive integer, got {horizon}",
+                    guidance="Please provide a positive integer value for the forecast horizon"
+                )
+            )
+    
+    def _forecast_var_factors(
+        self,
+        Z_last: np.ndarray,
+        A: np.ndarray,
+        p: int,
+        horizon: int,
+        Z_prev: Optional[np.ndarray] = None
+    ) -> np.ndarray:
+        """Forecast factors using VAR dynamics.
+        
+        Supports VAR(1) and VAR(2) factor dynamics.
+        
+        Parameters
+        ----------
+        Z_last : np.ndarray
+            Last factor state (m,)
+        A : np.ndarray
+            Transition matrix. For VAR(1): (m x m), for VAR(2): (m x 2m)
+        p : int
+            VAR order (1 or 2)
+        horizon : int
+            Number of periods to forecast
+        Z_prev : np.ndarray, optional
+            Previous factor state for VAR(2) (m,). Required if p == 2.
+            
+        Returns
+        -------
+        np.ndarray
+            Forecasted factors (horizon x m)
+        """
+        if p == 1:
+            # VAR(1): f_t = A @ f_{t-1}
+            Z_forecast = np.zeros((horizon, Z_last.shape[0]))
+            Z_forecast[0, :] = A @ Z_last
+            for h in range(1, horizon):
+                Z_forecast[h, :] = A @ Z_forecast[h - 1, :]
+        elif p == 2:
+            # VAR(2): f_t = A1 @ f_{t-1} + A2 @ f_{t-2}
+            if Z_prev is None:
+                # Fallback to VAR(1) if not enough history
+                A1 = A[:, :Z_last.shape[0]]
+                Z_forecast = np.zeros((horizon, Z_last.shape[0]))
+                Z_forecast[0, :] = A1 @ Z_last
+                for h in range(1, horizon):
+                    Z_forecast[h, :] = A1 @ Z_forecast[h - 1, :]
+            else:
+                A1 = A[:, :Z_last.shape[0]]
+                A2 = A[:, Z_last.shape[0]:]
+                Z_forecast = np.zeros((horizon, Z_last.shape[0]))
+                Z_forecast[0, :] = A1 @ Z_last + A2 @ Z_prev
+                if horizon > 1:
+                    Z_forecast[1, :] = A1 @ Z_forecast[0, :] + A2 @ Z_last
+                for h in range(2, horizon):
+                    Z_forecast[h, :] = A1 @ Z_forecast[h - 1, :] + A2 @ Z_forecast[h - 2, :]
+        else:
+            raise ValueError(
+                self._format_error_message(
+                    operation="prediction",
+                    reason=f"unsupported VAR order {p}",
+                    guidance="Only VAR(1) and VAR(2) are supported. Please use factor_order=1 or factor_order=2"
+                )
+            )
+        return Z_forecast
+    
+    def _transform_factors_to_observations(
+        self,
+        Z_forecast: np.ndarray,
+        C: np.ndarray,
+        Wx: np.ndarray,
+        Mx: np.ndarray
+    ) -> np.ndarray:
+        """Transform forecasted factors to observed series.
+        
+        Parameters
+        ----------
+        Z_forecast : np.ndarray
+            Forecasted factors (horizon x m)
+        C : np.ndarray
+            Loading matrix (N x m)
+        Wx : np.ndarray
+            Standard deviation values for unstandardization (N,)
+        Mx : np.ndarray
+            Mean values for unstandardization (N,)
+            
+        Returns
+        -------
+        np.ndarray
+            Forecasted observations (horizon x N)
+        """
+        X_forecast_std = Z_forecast @ C.T
+        X_forecast = X_forecast_std * Wx + Mx
+        return X_forecast
+    
+    def _update_factor_state_with_history(
+        self,
+        history: int,
+        result: 'BaseResult',
+        kalman_filter: Optional[Any] = None
+    ) -> Optional[np.ndarray]:
+        """Update factor state using recent N periods of data.
+        
+        This method re-runs the Kalman filter (for DFM) or extracts factors
+        via encoder and runs Kalman filter (for DDFM) using only the most
+        recent N periods of data for efficiency.
+        
+        Parameters
+        ----------
+        history : int
+            Number of recent periods to use for state update
+        result : BaseResult
+            Model result containing parameters (A, C, Q, R, Z_0, V_0)
+        kalman_filter : Any, optional
+            Kalman filter instance. If None, creates a new one.
+            
+        Returns
+        -------
+        np.ndarray or None
+            Updated last factor state (m,), or None if update failed
+        """
+        # Get data from DataModule
+        if not hasattr(self, '_data_module') or self._data_module is None:
+            # Try to get from trainer
+            if hasattr(self, 'trainer') and self.trainer is not None:
+                if hasattr(self.trainer, 'datamodule') and self.trainer.datamodule is not None:
+                    self._data_module = self.trainer.datamodule
+        
+        if not hasattr(self, '_data_module') or self._data_module is None:
+            _logger.warning(
+                f"{self.__class__.__name__} predict(): DataModule not available for history={history}, "
+                f"using training state instead"
+            )
+            return None
+        
+        try:
+            # Get processed data (T x N)
+            X_torch = self._data_module.get_processed_data()
+            
+            # Convert to numpy if needed
+            if isinstance(X_torch, torch.Tensor):
+                X_data = X_torch.cpu().numpy()
+            else:
+                X_data = np.asarray(X_torch)
+            
+            # Slice to recent N periods
+            if X_data.shape[0] > history:
+                X_recent = X_data[-history:, :]
+                _logger.debug(
+                    f"{self.__class__.__name__} predict(): Using recent {history} periods "
+                    f"(out of {X_data.shape[0]}) for state update"
+                )
+            else:
+                X_recent = X_data
+                _logger.debug(
+                    f"{self.__class__.__name__} predict(): Using all {X_data.shape[0]} periods "
+                    f"(less than history={history}) for state update"
+                )
+            
+            # Standardize data using result's Mx and Wx
+            Mx = result.Mx
+            Wx = result.Wx
+            if Mx is not None and Wx is not None:
+                # Check if data is already standardized (mean ~0, std ~1)
+                if np.abs(np.nanmean(X_recent)) > 0.1 or np.nanstd(X_recent) > 2.0:
+                    # Data not standardized, standardize it
+                    X_recent_std = (X_recent - Mx) / np.where(Wx != 0, Wx, 1.0)
+                else:
+                    # Data already standardized
+                    X_recent_std = X_recent
+            else:
+                X_recent_std = X_recent
+            
+            # Handle NaN values (missing data)
+            X_recent_std = np.where(np.isfinite(X_recent_std), X_recent_std, np.nan)
+            
+            # Model-specific state update
+            # Use __class__.__name__ to avoid circular imports
+            # Check if model has encoder attribute (DDFM-specific)
+            if hasattr(self, 'encoder') and self.encoder is not None:
+                # DDFM: Extract factors via encoder, then use Kalman filter
+                return self._update_factor_state_ddfm(
+                    X_recent_std, result, kalman_filter
+                )
+            else:
+                # DFM: Use Kalman filter directly on standardized data
+                return self._update_factor_state_dfm(
+                    X_recent_std, result, kalman_filter
+                )
+        except Exception as e:
+            _logger.warning(
+                f"{self.__class__.__name__} predict(): Failed to update factor state with history={history}, "
+                f"using training state instead. Error: {type(e).__name__}: {str(e)}"
+            )
+            return None
+    
+    def _update_factor_state_dfm(
+        self,
+        X_recent_std: np.ndarray,
+        result: 'BaseResult',
+        kalman_filter: Optional[Any] = None
+    ) -> np.ndarray:
+        """Update factor state for DFM using Kalman filter.
+        
+        Parameters
+        ----------
+        X_recent_std : np.ndarray
+            Standardized recent data (T x N)
+        result : BaseResult
+            Model result containing parameters
+        kalman_filter : Any, optional
+            Kalman filter instance
+            
+        Returns
+        -------
+        np.ndarray
+            Updated last factor state (m,)
+        """
+        import torch
+        
+        # Convert to torch tensor: (N x T) format for Kalman filter
+        Y = torch.tensor(X_recent_std.T, dtype=torch.float32)  # (N x T)
+        
+        # Extract parameters
+        A = result.A
+        C = result.C
+        Q = result.Q
+        R = result.R
+        Z_0 = result.Z_0
+        V_0 = result.V_0
+        
+        # Convert parameters to torch
+        A_torch = torch.tensor(A, dtype=torch.float32)
+        C_torch = torch.tensor(C, dtype=torch.float32)
+        Q_torch = torch.tensor(Q, dtype=torch.float32)
+        R_torch = torch.tensor(R, dtype=torch.float32)
+        Z_0_torch = torch.tensor(Z_0, dtype=torch.float32)
+        V_0_torch = torch.tensor(V_0, dtype=torch.float32)
+        
+        # Re-run Kalman filter with recent data
+        if kalman_filter is None:
+            if hasattr(self, 'kalman') and self.kalman is not None:
+                kalman_filter = self.kalman
+            else:
+                from ..ssm.kalman import KalmanFilter
+                kalman_filter = KalmanFilter(
+                    min_eigenval=1e-8,
+                    inv_regularization=1e-6,
+                    cholesky_regularization=1e-8
+                )
+        
+        # Run Kalman smoother
+        zsmooth, Vsmooth, _, _ = kalman_filter(
+            Y, A_torch, C_torch, Q_torch, R_torch, Z_0_torch, V_0_torch
+        )
+        
+        # zsmooth is (m x (T+1)), transpose to ((T+1) x m)
+        Zsmooth = zsmooth.T  # ((T+1) x m)
+        
+        # Get last factor state (skip initial state at index 0)
+        Z_last = Zsmooth[-1, :].cpu().numpy()  # (m,)
+        
+        return Z_last
+    
+    def _update_factor_state_ddfm(
+        self,
+        X_recent_std: np.ndarray,
+        result: 'BaseResult',
+        kalman_filter: Optional[Any] = None
+    ) -> np.ndarray:
+        """Update factor state for DDFM using encoder and Kalman filter.
+        
+        For DDFM, we first extract factors using the encoder, then run
+        Kalman filter on those factors to get smoothed state.
+        
+        Parameters
+        ----------
+        X_recent_std : np.ndarray
+            Standardized recent data (T x N)
+        result : BaseResult
+            Model result containing parameters
+        kalman_filter : Any, optional
+            Kalman filter instance
+            
+        Returns
+        -------
+        np.ndarray
+            Updated last factor state (m,)
+        """
+        import torch
+        
+        # Extract factors using encoder
+        # Note: encoder is a DDFM-specific attribute, accessed via model instance
+        if not hasattr(self, 'encoder') or self.encoder is None:
+            _logger.warning(
+                f"{self.__class__.__name__} predict(): Encoder not available for history update, "
+                "using training state instead"
+            )
+            return None
+        
+        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.encoder.eval()
+        
+        # Convert to torch and extract factors
+        X_tensor = torch.tensor(X_recent_std, device=device, dtype=torch.float32)
+        with torch.no_grad():
+            factors_raw = self.encoder(X_tensor).cpu().numpy()  # (T x m)
+        
+        # Extract parameters for Kalman filter
+        A = result.A
+        Q = result.Q
+        Z_0 = result.Z_0
+        V_0 = result.V_0
+        p = result.p  # VAR order
+        
+        # For DDFM, the measurement equation is identity (factors are directly observed)
+        # So C = I (identity matrix), R = small noise
+        m = factors_raw.shape[1]
+        C = np.eye(m)  # Identity matrix (factors are directly observed)
+        R = np.eye(m) * 1e-8  # Small noise for numerical stability
+        
+        # Convert to torch tensor: (m x T) format for Kalman filter
+        # Note: For DDFM, we filter factors directly, so Y = factors.T
+        Y = torch.tensor(factors_raw.T, dtype=torch.float32)  # (m x T)
+        
+        # Convert parameters to torch
+        A_torch = torch.tensor(A, dtype=torch.float32)
+        C_torch = torch.tensor(C, dtype=torch.float32)
+        Q_torch = torch.tensor(Q, dtype=torch.float32)
+        R_torch = torch.tensor(R, dtype=torch.float32)
+        Z_0_torch = torch.tensor(Z_0, dtype=torch.float32)
+        V_0_torch = torch.tensor(V_0, dtype=torch.float32)
+        
+        # Create or use Kalman filter
+        if kalman_filter is None:
+            from ..ssm.kalman import KalmanFilter
+            kalman_filter = KalmanFilter(
+                min_eigenval=1e-8,
+                inv_regularization=1e-6,
+                cholesky_regularization=1e-8
+            )
+        
+        # Run Kalman smoother on factors
+        zsmooth, Vsmooth, _, _ = kalman_filter(
+            Y, A_torch, C_torch, Q_torch, R_torch, Z_0_torch, V_0_torch
+        )
+        
+        # zsmooth is (m x (T+1)), transpose to ((T+1) x m)
+        Zsmooth = zsmooth.T  # ((T+1) x m)
+        
+        # Get last factor state (skip initial state at index 0)
+        Z_last = Zsmooth[-1, :].cpu().numpy()  # (m,)
+        
+        return Z_last
+    
     @property
     def result(self) -> BaseResult:
         """Get last fit result.
@@ -366,6 +755,7 @@ class BaseFactorModel(pl.LightningModule):
         self,
         horizon: Optional[int] = None,
         *,
+        history: Optional[int] = None,
         return_series: bool = True,
         return_factors: bool = True
     ) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
@@ -375,6 +765,11 @@ class BaseFactorModel(pl.LightningModule):
         ----------
         horizon : int, optional
             Number of periods ahead to forecast. If None, uses default based on clock frequency.
+        history : int, optional
+            Number of historical periods to use for Kalman filter update before prediction.
+            If None, uses full history (default). If specified (e.g., 60), uses only the most
+            recent N periods for efficiency. Initial state (Z_0, V_0) is always estimated from
+            full history (including any new data beyond training period).
         return_series : bool, default True
             Whether to return forecasted series.
         return_factors : bool, default True
@@ -387,6 +782,13 @@ class BaseFactorModel(pl.LightningModule):
             - If both return_series and return_factors are True: (X_forecast, Z_forecast)
             - If only return_series is True: X_forecast
             - If only return_factors is True: Z_forecast
+            
+        Notes
+        -----
+        When history is specified, the method uses only the most recent N periods for
+        Kalman filter update, improving computational efficiency. The initial state
+        (Z_0, V_0) is always estimated from full history (including any new data beyond
+        training period), ensuring accuracy while maintaining efficiency.
         """
         pass
     
@@ -403,6 +805,65 @@ class BaseFactorModel(pl.LightningModule):
         """
         raise NotImplementedError("Subclasses must implement get_result()")
     
+    @property
+    def nowcast(self) -> 'Nowcast':
+        """Get nowcast manager for generating nowcasts.
+        
+        Returns
+        -------
+        Nowcast
+            Nowcast manager object that provides methods for:
+            - Generating nowcasts (forecasts for current period)
+            - Computing revision impacts
+            - Analyzing data releases
+            - News decomposition
+            
+        Raises
+        ------
+        ValueError
+            If model has not been trained yet. Call trainer.fit(model, data_module) first.
+            
+        Examples
+        --------
+        >>> model = DFM()
+        >>> model.load_config('config.yaml')
+        >>> trainer.fit(model, data_module)
+        >>> nowcast = model.nowcast
+        >>> value = nowcast('gdp', view_date='2024-01-15', target_period='2024Q1')
+        """
+        self._check_trained()
+        
+        # Import here to avoid circular dependency
+        try:
+            from ..nowcast import Nowcast
+        except ImportError:
+            # Fallback: try importing from src.nowcasting if dfm_python.nowcast doesn't exist
+            try:
+                import sys
+                from pathlib import Path
+                # Add src to path if not already there
+                src_path = Path(__file__).parent.parent.parent.parent / 'src'
+                if str(src_path) not in sys.path:
+                    sys.path.insert(0, str(src_path))
+                from nowcasting import Nowcast
+            except ImportError:
+                error_msg = self._format_error_message(
+                    operation="nowcast access",
+                    reason="Nowcast class not available",
+                    guidance="Please ensure nowcasting module is available"
+                )
+                raise ValueError(error_msg)
+        
+        # Get DataModule from model
+        data_module = getattr(self, '_data_module', None)
+        if data_module is None:
+            # Try to get from trainer if available
+            if hasattr(self, 'trainer') and self.trainer is not None:
+                if hasattr(self.trainer, 'datamodule') and self.trainer.datamodule is not None:
+                    data_module = self.trainer.datamodule
+        
+        return Nowcast(self, data_module=data_module)
+    
     def _load_config_common(
         self,
         source: Optional[Union[str, Path, Dict[str, Any], DFMConfig, ConfigSource]] = None,