dfm-python 0.4.6__py3-none-any.whl → 0.4.7__py3-none-any.whl

dfm_python/__init__.py

@@ -6,7 +6,6 @@ This package implements a comprehensive Dynamic Factor Model framework with supp
 - Tent kernel aggregation for low-to-high frequency mapping
 - Expectation-Maximization (EM) algorithm for parameter estimation
 - Kalman filtering and smoothing for factor extraction
-- News decomposition for nowcasting
 - Deep Dynamic Factor Models (DDFM) with nonlinear encoders (requires PyTorch)
 
 The package implements a clock-based approach to mixed-frequency DFMs, where all latent 
@@ -23,7 +22,6 @@ Key Features:
     - Flexible block structure for factor modeling
     - Robust handling of missing data (internal spline interpolation)
     - Automatic standardization and data clipping
-    - News decomposition for forecast updates
 
 Example (Standard Lightning Pattern):
     >>> from dfm_python import DFM, DFMDataModule, DFMTrainer
@@ -67,7 +65,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.4.6"
+__version__ = "0.4.7"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -81,7 +79,7 @@ __version__ = "0.4.6"
 # 2. High-level API: DFM, DDFM, module-level convenience functions
 # 3. Core utilities: TimeIndex, diagnostics
 # 4. Models: BaseFactorModel, DDFM (low-level)
-# 5. Nowcasting: Nowcast, result classes, para_const
+# 5. Data & Results: DFMResult
 # 6. Data & Results: DFMResult
 # ============================================================================
 
@@ -90,8 +88,9 @@ from .config import (
     DEFAULT_BLOCK_NAME,
     ConfigSource, YamlSource, DictSource, HydraSource,
     MergedConfigSource, make_config_source,
-    DFMConfig, SeriesConfig,  # Internal use - prefer Hydra YAML configs
 )
+# Internal imports (for backward compatibility, but not recommended)
+from .config import DFMConfig, SeriesConfig  # Internal use only
 
 # Results
 from .config.results import DFMResult, DDFMResult, BaseResult
@@ -101,23 +100,15 @@ from .utils.diagnostics import diagnose_series, print_series_diagnosis
 from .utils.time import calculate_rmse
 
 # PyTorch Lightning modules (mandatory dependency)
+# Users can import these directly from dfm_python
 from .lightning import (
     DFMDataModule,
-    DFMDataset,
-    KalmanFilter,
-    EMAlgorithm,
+    DFMDataset,  # Dataset class (usually not needed directly)
+    KalmanFilter,  # Module class
+    EMAlgorithm,  # Module class
 )
 
-# Nowcasting (from nowcast/ subpackage)
-from .nowcast import (
-    Nowcast,
-    NowcastResult,
-)
-from .nowcast.helpers import (
-    para_const,
-    NewsDecompResult,
-    BacktestResult,
-)
+# Nowcasting removed - use src.nowcast instead
 
 # Model implementations
 from .models.base import BaseFactorModel
@@ -128,11 +119,9 @@ from .models.ddfm import DDFM, DDFMModel
 
 __all__ = [
     # Core classes
-    'DFM', 'Nowcast',
+    'DFM',
     # Model base and implementations
     'BaseFactorModel',
-    # Nowcast result classes
-    'NowcastResult', 'NewsDecompResult', 'BacktestResult',
     # Constants
     'DEFAULT_BLOCK_NAME',
     # Config sources
@@ -140,7 +129,6 @@ __all__ = [
     'MergedConfigSource', 'make_config_source',
     # Low-level API (functional interface - advanced usage)
     'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
-    'para_const',
 ]
 
 # DDFM high-level API (PyTorch is mandatory)
@@ -152,9 +140,9 @@ __all__.extend([
 # Lightning modules (mandatory dependency)
 __all__.extend([
     'DFMDataModule',
-    'DFMDataset',
-    'KalmanFilter',
-    'EMAlgorithm',
+    'DFMDataset',  # Dataset class (usually not needed directly)
+    'KalmanFilter',  # Module class
+    'EMAlgorithm',  # Module class
 ])
 
 # Trainer classes (mandatory dependency)

dfm_python/models/base.py

@@ -91,7 +91,6 @@ class BaseFactorModel(pl.LightningModule):
         self._result: Optional[BaseResult] = None
         self.training_state: Optional[Any] = None
         self._data_module: Optional[Any] = None
-        self._nowcast: Optional[Any] = None
     
     @property
     def config(self) -> DFMConfig:
@@ -494,7 +493,6 @@ class BaseFactorModel(pl.LightningModule):
         self._config = None
         self._data_module = None
         self._result = None
-        self._nowcast = None
         if hasattr(self, 'training_state'):
             self.training_state = None
         return self

dfm_python/models/ddfm.py

@@ -31,9 +31,8 @@ from ..config.results import DDFMResult
 from ..config.utils import get_periods_per_year
 from ..encoder.vae import Decoder, Encoder, extract_decoder_params
 from ..logger import get_logger
-from ..nowcast.dataview import DataView
-from ..nowcast.nowcast import Nowcast
-from ..utils.data import create_data_view, rem_nans_spline
+# Nowcasting removed - use src.nowcast instead
+from ..utils.data import rem_nans_spline
 from ..utils.helpers import (
     find_series_index,
     get_clock_frequency,
@@ -291,131 +290,8 @@ class DDFMModel:
             return X_forecast
         return Z_forecast
     
-    def generate_dataset(
-        self,
-        target_series: str,
-        periods: List[datetime],
-        backward: int = 0,
-        forward: int = 0,
-        dataview: Optional['DataView'] = None
-    ) -> Dict[str, Any]:
-        """Generate dataset for DFM evaluation.
-        
-        Note: Requires data to be stored during fit(). Data is automatically
-        stored from DataModule during fit().
-        """
-        if not hasattr(self, '_data') or self._data is None:
-            error_msg = format_error_message(
-                model_type="DDFM",
-                operation="generate_dataset",
-                reason="model has not been fitted with DataModule yet",
-                guidance="Please call trainer.fit(model, data_module) first to store data"
-            )
-            raise ValueError(error_msg)
-        
-        i_series = find_series_index(self._config, target_series)
-        X_features, y_baseline, y_actual, metadata, backward_results = [], [], [], [], []
-        
-        if dataview is not None:
-            dataview_factory = dataview
-        else:
-            # Convert data to numpy if needed
-            if hasattr(self._data, 'to_numpy'):
-                X_data = self._data.to_numpy()
-            else:
-                X_data = np.asarray(self._data)
-            
-            dataview_factory = DataView.from_arrays(
-                X=X_data, Time=self._time,
-                Z=self._original_data, config=self._config,
-                X_frame=self._data_frame
-            )
-        if dataview_factory.config is None:
-            dataview_factory.config = self._config
-        
-        for period in periods:
-            view_obj = dataview_factory.with_view_date(period)
-            X_view, Time_view, _ = view_obj.materialize()
-            
-            if backward > 0:
-                nowcasts, data_view_dates = [], []
-                for weeks_back in range(backward, -1, -1):
-                    data_view_date = period - timedelta(weeks=weeks_back)
-                    view_past = dataview_factory.with_view_date(data_view_date)
-                    X_view_past, Time_view_past, _ = view_past.materialize()
-                    # Access nowcast through _nowcast_ref (set by high-level DDFM class)
-                    nowcast_obj = getattr(self, '_nowcast_ref', None)
-                    if nowcast_obj is None:
-                        error_msg = format_error_message(
-                            model_type="DDFM",
-                            operation="nowcast",
-                            reason="requires high-level DDFM instance",
-                            guidance="Please call nowcast() from DDFM class, not DDFMModel"
-                        )
-                        raise ValueError(error_msg)
-                    nowcast_val = nowcast_obj(
-                        target_series=target_series,
-                        view_date=view_past.view_date or data_view_date,
-                        target_period=period
-                    )
-                    nowcasts.append(nowcast_val)
-                    data_view_dates.append(view_past.view_date or data_view_date)
-                baseline_nowcast = nowcasts[-1]
-                backward_results.append({
-                    'nowcasts': np.array(nowcasts),
-                    'data_view_dates': data_view_dates,
-                    'target_date': period
-                })
-            else:
-                # Access nowcast through _nowcast_ref (set by high-level DDFM class)
-                nowcast_obj = getattr(self, '_nowcast_ref', None)
-                if nowcast_obj is None:
-                    error_msg = format_error_message(
-                        model_type="DDFM",
-                        operation="nowcast",
-                        reason="requires high-level DDFM instance",
-                        guidance="Please call nowcast() from DDFM class, not DDFMModel"
-                    )
-                    raise ValueError(error_msg)
-                baseline_nowcast = nowcast_obj(
-                    target_series=target_series,
-                    view_date=view_obj.view_date or period,
-                    target_period=period
-                )
-            
-            y_baseline.append(baseline_nowcast)
-            t_idx = find_time_index(self._time, period)
-            actual_val = np.nan
-            # Convert data to numpy for indexing
-            if hasattr(self._data, 'to_numpy'):
-                data_array = self._data.to_numpy()
-            else:
-                data_array = np.asarray(self._data)
-            if t_idx is not None and t_idx < data_array.shape[0] and i_series < data_array.shape[1]:
-                actual_val = data_array[t_idx, i_series]
-            y_actual.append(actual_val)
-            
-            # Extract features
-            if self._result is not None and hasattr(self._result, 'Z'):
-                latest_factors = self._result.Z[-1, :] if self._result.Z.shape[0] > 0 else np.zeros(self._result.Z.shape[1])
-            else:
-                latest_factors = np.array([])
-            if X_view.shape[0] > 0:
-                mean_residual = np.nanmean(X_view[-1, :]) if X_view.shape[0] > 0 else 0.0
-            else:
-                mean_residual = 0.0
-            features = np.concatenate([latest_factors, [mean_residual]])
-            X_features.append(features)
-            metadata.append({'period': period, 'target_series': target_series})
-        
-        return {
-            'X': np.array(X_features),
-            'y_baseline': np.array(y_baseline),
-            'y_actual': np.array(y_actual),
-            'y_target': np.array(y_actual) - np.array(y_baseline),
-            'metadata': metadata,
-            'backward_results': backward_results if backward > 0 else []
-        }
+    # generate_dataset() method removed - application-specific functionality
+    # Use src.nowcast for nowcasting-related dataset generation
     
     def get_state(
         self,
@@ -450,22 +326,12 @@ class DDFMModel:
         else:
             X_data = np.asarray(self._data)
         
-        X_view, Time_view, _ = create_data_view(
-            X=X_data, Time=self._time,
-            Z=self._original_data, config=self._config, view_date=t
-        )
+        # create_data_view removed - use src.nowcast for data view functionality
+        # For get_state, we use the raw data directly
+        X_view = X_data
         
-        # Access nowcast through _nowcast_ref (set by high-level DDFM class)
-        nowcast_obj = getattr(self, '_nowcast_ref', None)
-        if nowcast_obj is None:
-            error_msg = format_error_message(
-                model_type="DDFM",
-                operation="nowcast",
-                reason="requires high-level DDFM instance",
-                guidance="Please call nowcast() from DDFM class, not DDFMModel"
-            )
-            raise ValueError(error_msg)
-        baseline_nowcast = nowcast_obj(target_series=target_series, view_date=t, target_period=None)
+        # Nowcast removed - use src.nowcast for nowcasting functionality
+        baseline_nowcast = np.nan  # Placeholder - use src.nowcast for actual nowcast values
         
         baseline_forecast, actual_history, residuals, factors_history = [], [], [], []
         t_idx = find_time_index(self._time, t)
@@ -480,7 +346,11 @@ class DDFMModel:
         
         for i in range(max(0, t_idx - lookback + 1), t_idx + 1):
             if i < X_data.shape[0]:
-                forecast_val = baseline_nowcast
+                # Use smoothed value from result if available, otherwise NaN
+                if self._result is not None and hasattr(self._result, 'X_sm') and i < self._result.X_sm.shape[0]:
+                    forecast_val = self._result.X_sm[i, i_series] if i_series < self._result.X_sm.shape[1] else np.nan
+                else:
+                    forecast_val = np.nan
                 baseline_forecast.append(forecast_val)
                 actual_val = X_data[i, i_series] if i_series < X_data.shape[1] else np.nan
                 actual_history.append(actual_val)
@@ -505,7 +375,6 @@ class DDFMModel:
 # ============================================================================
 
 if TYPE_CHECKING:
-    from ..nowcast import Nowcast
     from ..lightning import DFMDataModule
 
 class DDFM(BaseFactorModel):
@@ -874,9 +743,11 @@ class DDFM(BaseFactorModel):
                 f"Possible causes: (1) learning rate too high, (2) gradient explosion, "
                 f"(3) extreme input values, (4) unstable activation functions."
             )
-            # Return a large but finite loss to signal the issue, but don't update weights
-            # This prevents NaN from propagating to decoder weights
-            loss = torch.tensor(1e6, device=data.device, dtype=data.dtype, requires_grad=False)
+            # Return a large but finite loss to signal the issue
+            # Must have requires_grad=True for Lightning backward() to work
+            # Create loss from model parameters to ensure it's in the computation graph
+            dummy_loss = sum(p.sum() * 0.0 for p in self.parameters() if p.requires_grad)
+            loss = dummy_loss + torch.tensor(1e6, device=data.device, dtype=data.dtype, requires_grad=True)
             self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True)
             return loss
         
@@ -2604,113 +2475,8 @@ class DDFM(BaseFactorModel):
         return self
     
     
-    def generate_dataset(
-        self,
-        target_series: str,
-        periods: List[datetime],
-        backward: int = 0,
-        forward: int = 0,
-        dataview: Optional['DataView'] = None
-    ) -> Dict[str, Any]:
-        """Generate dataset for DFM evaluation.
-        
-        Note: Requires data to be stored during fit(). Data is automatically
-        stored from DataModule during fit().
-        """
-        if not hasattr(self, '_data') or self._data is None:
-            error_msg = format_error_message(
-                model_type="DDFM",
-                operation="generate_dataset",
-                reason="model has not been fitted with DataModule yet",
-                guidance="Please call trainer.fit(model, data_module) first to store data"
-            )
-            raise ValueError(error_msg)
-        
-        i_series = find_series_index(self._config, target_series)
-        X_features, y_baseline, y_actual, metadata, backward_results = [], [], [], [], []
-        
-        if dataview is not None:
-            dataview_factory = dataview
-        else:
-            # Convert data to numpy if needed
-            if hasattr(self._data, 'to_numpy'):
-                X_data = self._data.to_numpy()
-            else:
-                X_data = np.asarray(self._data)
-            
-            dataview_factory = DataView.from_arrays(
-                X=X_data, Time=self._time,
-                Z=self._original_data, config=self._config,
-                X_frame=self._data_frame
-            )
-        if dataview_factory.config is None:
-            dataview_factory.config = self._config
-        
-        for period in periods:
-            view_obj = dataview_factory.with_view_date(period)
-            X_view, Time_view, _ = view_obj.materialize()
-            
-            if backward > 0:
-                nowcasts, data_view_dates = [], []
-                for weeks_back in range(backward, -1, -1):
-                    data_view_date = period - timedelta(weeks=weeks_back)
-                    view_past = dataview_factory.with_view_date(data_view_date)
-                    X_view_past, Time_view_past, _ = view_past.materialize()
-                    # Use nowcast property directly
-                    nowcast_val = self.nowcast(
-                        target_series=target_series,
-                        view_date=view_past.view_date or data_view_date,
-                        target_period=period
-                    )
-                    nowcasts.append(nowcast_val)
-                    data_view_dates.append(view_past.view_date or data_view_date)
-                baseline_nowcast = nowcasts[-1]
-                backward_results.append({
-                    'nowcasts': np.array(nowcasts),
-                    'data_view_dates': data_view_dates,
-                    'target_date': period
-                })
-            else:
-                # Use nowcast property directly
-                baseline_nowcast = self.nowcast(
-                    target_series=target_series,
-                    view_date=view_obj.view_date or period,
-                    target_period=period
-                )
-            
-            y_baseline.append(baseline_nowcast)
-            t_idx = find_time_index(self._time, period)
-            actual_val = np.nan
-            # Convert data to numpy for indexing
-            if hasattr(self._data, 'to_numpy'):
-                data_array = self._data.to_numpy()
-            else:
-                data_array = np.asarray(self._data)
-            if t_idx is not None and t_idx < data_array.shape[0] and i_series < data_array.shape[1]:
-                actual_val = data_array[t_idx, i_series]
-            y_actual.append(actual_val)
-            
-            # Extract features
-            if self._result is not None and hasattr(self._result, 'Z'):
-                latest_factors = self._result.Z[-1, :] if self._result.Z.shape[0] > 0 else np.zeros(self._result.Z.shape[1])
-            else:
-                latest_factors = np.array([])
-            if X_view.shape[0] > 0:
-                mean_residual = np.nanmean(X_view[-1, :]) if X_view.shape[0] > 0 else 0.0
-            else:
-                mean_residual = 0.0
-            features = np.concatenate([latest_factors, [mean_residual]])
-            X_features.append(features)
-            metadata.append({'period': period, 'target_series': target_series})
-        
-        return {
-            'X': np.array(X_features),
-            'y_baseline': np.array(y_baseline),
-            'y_actual': np.array(y_actual),
-            'y_target': np.array(y_actual) - np.array(y_baseline),
-            'metadata': metadata,
-            'backward_results': backward_results if backward > 0 else []
-        }
+    # generate_dataset() method removed - application-specific functionality
+    # Use src.nowcast for nowcasting-related dataset generation
     
     def get_state(
         self,
@@ -2720,7 +2486,7 @@ class DDFM(BaseFactorModel):
     ) -> Dict[str, Any]:
         """Get DFM state at time t.
         
-        Returns state information including baseline nowcast, actual history,
+        Returns state information including baseline forecast, actual history,
         residuals, and factor history for the specified time period.
         """
         if not hasattr(self, '_data') or self._data is None:
@@ -2743,8 +2509,8 @@ class DDFM(BaseFactorModel):
         else:
             X_data = np.asarray(self._data)
         
-        # Use nowcast property directly
-        baseline_nowcast = self.nowcast(target_series=target_series, view_date=t, target_period=None)
+        # Nowcast removed - use src.nowcast for nowcasting functionality
+        baseline_nowcast = np.nan  # Placeholder - use src.nowcast for actual nowcast values
         
         baseline_forecast, actual_history, residuals, factors_history = [], [], [], []
         t_idx = find_time_index(self._time, t)
@@ -2759,7 +2525,11 @@ class DDFM(BaseFactorModel):
         
         for i in range(max(0, t_idx - lookback + 1), t_idx + 1):
             if i < X_data.shape[0]:
-                forecast_val = baseline_nowcast
+                # Use smoothed value from result if available, otherwise NaN
+                if self._result is not None and hasattr(self._result, 'X_sm') and i < self._result.X_sm.shape[0]:
+                    forecast_val = self._result.X_sm[i, i_series] if i_series < self._result.X_sm.shape[1] else np.nan
+                else:
+                    forecast_val = np.nan
                 baseline_forecast.append(forecast_val)
                 actual_val = X_data[i, i_series] if i_series < X_data.shape[1] else np.nan
                 actual_history.append(actual_val)

dfm_python/models/dfm.py

@@ -219,6 +219,10 @@ class DFM(BaseFactorModel):
         For DFM, each "step" is actually one EM iteration. The batch contains
         the full time series data.
         
+        **Important**: If `fit_em()` has already completed training (in `on_train_start()`),
+        this method skips additional EM iterations to prevent double training and log-likelihood
+        deterioration. The model will use the results from `fit_em()` instead.
+        
         Parameters
         ----------
         batch : torch.Tensor or tuple
@@ -231,6 +235,19 @@ class DFM(BaseFactorModel):
         loss : torch.Tensor
             Negative log-likelihood (to minimize)
         """
+        # CRITICAL FIX: Skip additional EM iterations if fit_em() already completed training
+        # This prevents double training that causes log-likelihood deterioration.
+        # fit_em() runs in on_train_start() and completes the full EM algorithm.
+        # Lightning's training loop should not run additional EM iterations after that.
+        if self.training_state is not None:
+            # fit_em() has already run - use its results instead of doing more EM iterations
+            loglik = self.training_state.loglik
+            self.log('loglik', loglik, on_step=True, on_epoch=True, prog_bar=True)
+            self.log('em_iteration', float(self.training_state.num_iter), on_step=True, on_epoch=True)
+            # Return negative log-likelihood as loss (consistent with normal training_step)
+            device = next(self.parameters()).device if len(list(self.parameters())) > 0 else torch.device('cpu')
+            return -torch.tensor(loglik, device=device, dtype=torch.float32)
+        
         # Handle both tuple and single tensor batches
         if isinstance(batch, tuple):
             data, _ = batch
@@ -306,7 +323,10 @@ class DFM(BaseFactorModel):
         
         if converged:
             self.training_state.converged = True
-            _logger.info(f"EM algorithm converged at iteration {self.current_epoch}")
+            # Only log convergence once, not on every training_step call
+            if not hasattr(self, '_convergence_logged'):
+                _logger.info(f"EM algorithm converged at iteration {self.training_state.num_iter}")
+                self._convergence_logged = True
     
     def fit_em(
         self,
@@ -363,6 +383,8 @@ class DFM(BaseFactorModel):
         
         # Initialize state
         previous_loglik = float('-inf')
+        best_loglik = float('-inf')
+        best_params = None
         num_iter = 0
         converged = False
         
@@ -422,6 +444,41 @@ class DFM(BaseFactorModel):
                 self.Z_0.data = Z_0_new
                 self.V_0.data = V_0_new
             
+            # Track best log-likelihood and parameters (for early stopping on divergence)
+            if loglik > best_loglik:
+                best_loglik = loglik
+                # Store best parameters in case we need to revert
+                best_params = {
+                    'A': self.A.data.clone(),
+                    'C': self.C.data.clone(),
+                    'Q': self.Q.data.clone(),
+                    'R': self.R.data.clone(),
+                    'Z_0': self.Z_0.data.clone(),
+                    'V_0': self.V_0.data.clone()
+                }
+            
+            # Check for significant log-likelihood deterioration (divergence detection)
+            # Stop early if log-likelihood worsens by more than 1000 from best (indicates divergence)
+            if num_iter > 10 and best_loglik != float('-inf') and loglik < best_loglik - 1000:
+                _logger.warning(
+                    f"EM algorithm: Log-likelihood deteriorated significantly from best value. "
+                    f"Best: {best_loglik:.4f}, Current: {loglik:.4f}, Deterioration: {best_loglik - loglik:.4f}. "
+                    f"Stopping early at iteration {num_iter + 1} to prevent further divergence. "
+                    f"Reverting to best parameters from iteration {num_iter}."
+                )
+                # Revert to best parameters
+                if best_params is not None:
+                    with torch.no_grad():
+                        self.A.data = best_params['A']
+                        self.C.data = best_params['C']
+                        self.Q.data = best_params['Q']
+                        self.R.data = best_params['R']
+                        self.Z_0.data = best_params['Z_0']
+                        self.V_0.data = best_params['V_0']
+                # Use best log-likelihood for final state
+                loglik = best_loglik
+                break
+            
             # Check convergence - use self.em.check_convergence() instead of em_converged()
             if num_iter > 2:
                 converged, change = self.em.check_convergence(