dfm-python 0.5.5__py3-none-any.whl → 0.5.51__py3-none-any.whl

dfm_python/__init__.py

@@ -113,8 +113,8 @@ from .lightning import (
 from .models.base import BaseFactorModel
 from .models.dfm import DFM
 
-# DDFM high-level API and low-level model (PyTorch is mandatory)
-from .models.ddfm import DDFM, DDFMModel
+# DDFM high-level API (PyTorch is mandatory)
+from .models.ddfm import DDFM
 
 __all__ = [
     # Core classes
@@ -133,7 +133,6 @@ __all__ = [
 # DDFM high-level API (PyTorch is mandatory)
 __all__.extend([
     'DDFM',  # High-level API class
-    'DDFMModel',  # Low-level implementation (used internally)
 ])
 
 # Lightning modules (mandatory dependency)

dfm_python/config/adapter.py

@@ -208,13 +208,16 @@ class YamlSource:
                         except (ValueError, TypeError):
                             release_date = None
                     
+                    # Remove transformation and blocks if present (not in SeriesConfig)
+                    series_data_clean = {k: v for k, v in series_data.items() 
+                                       if k not in ['transformation', 'blocks']}
                     series_list.append(SeriesConfig(
                         series_id=series_id,
-                        series_name=series_data.get('series_name', series_id),
-                        frequency=series_data.get('frequency', 'm'),
-                        transformation=series_data.get('transformation', 'lin'),
-                        blocks=series_data.get('blocks', []),
-                        release_date=release_date
+                        series_name=series_data_clean.get('series_name', series_id),
+                        frequency=series_data_clean.get('frequency', 'm'),
+                        # transformation removed - handled by preprocessing pipeline
+                        # blocks removed - defined in DFMConfig
+                        release_date=series_data_clean.get('release_date', release_date)
                     ))
         
         # If no series loaded from separate files, try to get from main config
@@ -224,17 +227,23 @@ class YamlSource:
                 for series_item in series_data:
                     if isinstance(series_item, str):
                         # Series is just a string ID - create minimal config
-                        series_list.append(SeriesConfig(series_id=series_item, frequency='m', transformation='lin', blocks=[1]))
+                        series_list.append(SeriesConfig(series_id=series_item, frequency='m'))
                     elif isinstance(series_item, dict):
-                        series_list.append(SeriesConfig(**series_item))
+                        # Remove transformation and blocks if present (not in SeriesConfig)
+                        series_item_clean = {k: v for k, v in series_item.items() 
+                                           if k not in ['transformation', 'blocks']}
+                        series_list.append(SeriesConfig(**series_item_clean))
             elif isinstance(series_data, dict):
                 for series_id, series_item in series_data.items():
                     if isinstance(series_item, dict):
                         series_item['series_id'] = series_id
-                        series_list.append(SeriesConfig(**series_item))
+                        # Remove transformation and blocks if present (not in SeriesConfig)
+                        series_item_clean = {k: v for k, v in series_item.items() 
+                                           if k not in ['transformation', 'blocks']}
+                        series_list.append(SeriesConfig(**series_item_clean))
                     elif isinstance(series_item, str):
                         # Series value is just a string - create minimal config
-                        series_list.append(SeriesConfig(series_id=series_id, frequency='m', transformation='lin', blocks=[1]))
+                        series_list.append(SeriesConfig(series_id=series_id, frequency='m'))
         
         # Load block properties from model config (factors, ar_lag, clock)
         # Model config defines all blocks and their properties
@@ -250,28 +259,8 @@ class YamlSource:
         if blocks_dict_raw is not None:
             blocks_properties_from_config.update(blocks_dict_raw)
         
-        # Derive blocks from series configs (new pattern: blocks discovered from series)
-        # Collect all unique block names from series
-        block_names_from_series = set()
-        for series in series_list:
-            if hasattr(series, 'blocks') and series.blocks:
-                # Handle both list of strings and list of ints
-                if isinstance(series.blocks, list):
-                    for block in series.blocks:
-                        if isinstance(block, str):
-                            block_names_from_series.add(block)
-        
-        # If no blocks specified in series, use default single block
-        # Use first block from model config if available, otherwise create default
-        if not block_names_from_series:
-            # Check if model config defines blocks
-            if blocks_properties_from_config:
-                # Use first block from model config
-                first_block_name = list(blocks_properties_from_config.keys())[0]
-                block_names_from_series.add(first_block_name)
-            else:
-                # Create default generic block
-                block_names_from_series.add(DEFAULT_BLOCK_NAME)
+        # Note: SeriesConfig no longer contains blocks information.
+        # Blocks are defined in DFMConfig, not in SeriesConfig.
         
         # Create block dicts for each block
         # Priority: 1) Model config properties, 2) Defaults from main_settings
@@ -296,24 +285,13 @@ class YamlSource:
                     'clock': default_clock
                 }
         
-        # Then, ensure all blocks referenced by series exist
-        for block_name in block_names_from_series:
-            if block_name not in blocks_dict:
-                # Create block with defaults if not in model config
-                blocks_dict[block_name] = {
-                    'factors': 1,
-                    'ar_lag': default_ar_lag,
-                    'clock': default_clock
-                }
-        
-        # Convert block names to list for from_dict() if series use integer block indices
-        # Check if series use integer indices instead of block names
-        block_names_list = None
-        if series_list:
-            first_series_blocks = series_list[0].blocks if hasattr(series_list[0], 'blocks') else []
-            if first_series_blocks and isinstance(first_series_blocks[0], int):
-                # Series use integer indices, need block_names for from_dict()
-                block_names_list = list(blocks_dict.keys())
+        # If no blocks specified in config, use default single block
+        if not blocks_dict:
+            blocks_dict[DEFAULT_BLOCK_NAME] = {
+                'factors': 1,
+                'ar_lag': default_ar_lag,
+                'clock': default_clock
+            }
         
         # Build dictionary for from_dict() which handles type detection
         # Include all main_settings (which may contain DDFM parameters from model config)
@@ -323,10 +301,6 @@ class YamlSource:
             **main_settings
         }
         
-        # Add block_names if series use integer indices
-        if block_names_list is not None:
-            config_dict['block_names'] = block_names_list
-        
         # Use from_dict() which now correctly detects DDFM configs
         # The detection bug has been fixed in schema.py
         return DFMConfig.from_dict(config_dict)
@@ -644,12 +618,16 @@ if HYDRA_AVAILABLE and ConfigStore is not None:
             
             @schema_dataclass
             class SeriesConfigSchema:
-                """Schema for SeriesConfig validation in Hydra."""
+                """Schema for SeriesConfig validation in Hydra.
+                
+                Note: transformation is handled by preprocessing pipeline, not in SeriesConfig.
+                Note: blocks are defined in DFMConfig, not in SeriesConfig.
+                """
                 series_id: str
                 series_name: str
                 frequency: str
-                transformation: str
-                blocks: ListType[int]
+                # transformation removed - handled by preprocessing pipeline
+                # blocks removed - defined in DFMConfig
                 units: Optional[str] = None  # Optional, for display only
             
             @schema_dataclass

dfm_python/config/schema.py

@@ -36,7 +36,7 @@ except ImportError:
 DEFAULT_BLOCK_NAME = 'Block_0'
 
 # Import validation functions from utils
-from .utils import validate_frequency, validate_transformation
+from .utils import validate_frequency
 
 
 # ============================================================================
@@ -93,14 +93,17 @@ def _infer_blocks(
     series_list: List['SeriesConfig'],
     data: Dict[str, Any]
 ) -> Dict[str, Dict[str, Any]]:
-    """Infer blocks from series when blocks not explicitly provided.
+    """Infer blocks from configuration data when blocks not explicitly provided.
+    
+    Note: SeriesConfig no longer contains blocks information.
+    Blocks are defined in DFMConfig, not in SeriesConfig.
     
     Parameters
     ----------
     series_list : List[SeriesConfig]
-        List of series configurations
+        List of series configurations (blocks information not used)
     data : Dict[str, Any]
-        Configuration data (for clock default)
+        Configuration data (for clock default and block_names)
         
     Returns
     -------
@@ -108,31 +111,17 @@ def _infer_blocks(
         Dictionary mapping block names to block config dicts
     """
     blocks_dict = {}
-    block_names_set = set()
-    
-    # Collect all unique block names from series
-    for s in series_list:
-        if isinstance(s, dict):
-            blocks = s.get('blocks', [])
-        else:
-            blocks = s.blocks
-        
-        if isinstance(blocks, list) and len(blocks) > 0:
-            if isinstance(blocks[0], str):
-                # Block names as strings
-                block_names_set.update(blocks)
-            else:
-                # Block indices - need block_names to map
-                if 'block_names' in data:
-                    block_names_list = data['block_names']
-                    for idx in blocks:
-                        if isinstance(idx, int) and idx < len(block_names_list):
-                            block_names_set.add(block_names_list[idx])
     
-    # Create default blocks for each unique block name
-    clock = data.get('clock', 'm')
-    for block_name in sorted(block_names_set):
-        blocks_dict[block_name] = {'factors': 1, 'ar_lag': 1, 'clock': clock}
+    # Try to get block_names from data
+    if 'block_names' in data:
+        block_names_list = data['block_names']
+        clock = data.get('clock', 'm')
+        for block_name in block_names_list:
+            blocks_dict[block_name] = {'factors': 1, 'ar_lag': 1, 'clock': clock}
+    else:
+        # Default: create default block if no blocks specified
+        clock = data.get('clock', 'm')
+        blocks_dict[DEFAULT_BLOCK_NAME] = {'factors': 1, 'ar_lag': 1, 'clock': clock}
     
     return blocks_dict
 
@@ -179,15 +168,13 @@ class SeriesConfig:
     This is a generic DFM configuration - no API or database-specific fields.
     For API/database integration, implement adapters in your application layer.
     
+    Note: Transformation is handled by preprocessing pipeline, not in SeriesConfig.
+    Note: Blocks are defined in DFMConfig, not in SeriesConfig.
+    
     Attributes
     ----------
     frequency : str
         Series frequency: 'm' (monthly), 'q' (quarterly), 'sa' (semi-annual), 'a' (annual)
-    transformation : str
-        Transformation code: 'lin', 'pch', 'pca', etc.
-    blocks : List[str] or List[int]
-        Block names (strings) or block indices (ints) this series loads on.
-        Must include the global block (first block).
     series_id : str, optional
         Unique identifier (auto-generated if None)
     series_name : str, optional
@@ -203,8 +190,6 @@ class SeriesConfig:
     """
     # Required fields (no defaults)
     frequency: str
-    transformation: str
-    blocks: Union[List[str], List[int]]  # Can be block names (str) or indices (int)
     # Optional fields (with defaults - must come after required fields)
     series_id: Optional[str] = None  # Auto-generated if None: "series_0", "series_1", etc.
     series_name: Optional[str] = None  # Optional metadata for display
@@ -214,65 +199,9 @@ class SeriesConfig:
     def __post_init__(self):
         """Validate fields after initialization."""
         self.frequency = validate_frequency(self.frequency)
-        self.transformation = validate_transformation(self.transformation)
         # Auto-generate series_name if not provided
         if self.series_name is None and self.series_id:
             self.series_name = self.series_id
-    
-    def to_block_indices(self, block_names: List[str]) -> List[int]:
-        """Convert block names to indices.
-        
-        Parameters
-        ----------
-        block_names : List[str]
-            List of block names in order
-            
-        Returns
-        -------
-        List[int]
-            Block indices (0 or 1) for each block
-        """
-        if not self.blocks:
-            raise ValueError(
-                f"Series '{self.series_id}' validation failed: no blocks specified. "
-                f"Please specify blocks for series '{self.series_id}'."
-            )
-        
-        # If already integers, validate and return
-        if isinstance(self.blocks[0], int):
-            if len(self.blocks) != len(block_names):
-                raise ValueError(
-                    f"Series '{self.series_id}' validation failed: has {len(self.blocks)} block indices "
-                    f"but {len(block_names)} blocks defined. "
-                    f"Please ensure series '{self.series_id}' has exactly {len(block_names)} block indices."
-                )
-            return list(self.blocks)
-        
-        # Convert block names to indices
-        # Normalize block names (handle both underscore and hyphen variants)
-        block_names_normalized = {name.replace('-', '_'): name for name in block_names}
-        blocks_normalized = {name.replace('-', '_'): name for name in self.blocks}
-        
-        block_indices = [0] * len(block_names)
-        for block_name in self.blocks:
-            block_name_norm = block_name.replace('-', '_')
-            if block_name_norm not in block_names_normalized:
-                # Try exact match first
-                if block_name in block_names:
-                    block_indices[block_names.index(block_name)] = 1
-                else:
-                    raise ValueError(
-                        f"Series '{self.series_id}' validation failed: references block '{block_name}' "
-                        f"which is not in block_names: {block_names}. "
-                        f"Block names are case-sensitive and must match exactly. "
-                        f"Please use one of the defined block names: {block_names}."
-                    )
-            else:
-                # Use normalized name to find index
-                actual_block_name = block_names_normalized[block_name_norm]
-                block_indices[block_names.index(actual_block_name)] = 1
-        
-        return block_indices
 
 
 @dataclass
@@ -538,63 +467,37 @@ class DFMConfig(BaseModelConfig):
                     f"Please set ar_lag to 1 (VAR(1)) or 2 (VAR(2)) for block '{block_name}'."
                 )
         
-        # Auto-generate series_id if not provided and convert blocks to indices
-        n_blocks = len(self.block_names)
-        for i, s in enumerate(self.series):
-            # Convert block names to indices if needed
-            if isinstance(s.blocks, list) and len(s.blocks) > 0:
-                if isinstance(s.blocks[0], str):
-                    # Convert block names to indices
-                    block_indices = s.to_block_indices(self.block_names)
-                    object.__setattr__(s, 'blocks', block_indices)
-        
-        # Validate all series have correct number of blocks
-        for i, s in enumerate(self.series):
-            if len(s.blocks) != n_blocks:
-                raise ValueError(
-                    f"Series {i} ('{s.series_id}') validation failed: has {len(s.blocks)} block loadings, "
-                    f"but expected {n_blocks} (from block_names: {self.block_names}). "
-                    f"Each series must specify a loading (0 or 1) for each block. "
-                    f"Please ensure series '{s.series_id}' has exactly {n_blocks} block loadings."
-                )
-        
-        # Check first column (global block) is all 1s
+        # Auto-generate series_id if not provided
+        # Note: Blocks are defined in DFMConfig, not in SeriesConfig.
+        # Series-to-block mapping is handled separately (e.g., via series_to_blocks dict or default to global block).
         for i, s in enumerate(self.series):
-            if s.blocks[0] != 1:
-                raise ValueError(
-                    f"Series {i} ('{s.series_id}') validation failed: must load on the global block "
-                    f"(first block '{self.block_names[0]}'). "
-                    f"All series must have blocks[0] = 1. "
-                    f"Current value: {s.blocks[0]}. "
-                    f"Please set blocks[0] = 1 for series '{s.series_id}'."
-                )
+            if s.series_id is None:
+                object.__setattr__(s, 'series_id', f"series_{i}")
         
         # Validate block clock constraints: series frequency <= block clock
+        # All series load on all blocks by default (can be customized via series_to_blocks mapping)
         for i, s in enumerate(self.series):
             series_freq_hierarchy = FREQUENCY_HIERARCHY.get(s.frequency, 3)
             
-            # Find which blocks this series loads on
-            for block_idx, loads_on_block in enumerate(s.blocks):
-                if loads_on_block == 1:
-                    block_name = self.block_names[block_idx]
-                    block_cfg = self.blocks[block_name]
-                    block_clock = block_cfg.get('clock', self.clock)
-                    block_clock_hierarchy = FREQUENCY_HIERARCHY.get(block_clock, 3)
-                    
-                    # Series frequency must be <= block clock (slower or equal)
-                    if series_freq_hierarchy < block_clock_hierarchy:
-                        # Suggest valid frequencies for the series
-                        valid_freqs = [freq for freq, hier in FREQUENCY_HIERARCHY.items() 
-                                      if hier >= block_clock_hierarchy]
-                        valid_freqs_str = ', '.join(sorted(valid_freqs))
-                        raise ValueError(
-                            f"Series '{s.series_id}' has frequency '{s.frequency}' which is faster than "
-                            f"block '{block_name}' clock '{block_clock}'. "
-                            f"Series in a block must have frequency <= block clock. "
-                            f"Suggested fix: change series frequency to one of [{valid_freqs_str}] "
-                            f"(slower or equal to block clock '{block_clock}'), "
-                            f"or set block clock to '{s.frequency}' or faster."
-                        )
+            # Check against all blocks (series can load on multiple blocks)
+            for block_name, block_cfg in self.blocks.items():
+                block_clock = block_cfg.get('clock', self.clock)
+                block_clock_hierarchy = FREQUENCY_HIERARCHY.get(block_clock, 3)
+                
+                # Series frequency must be <= block clock (slower or equal)
+                if series_freq_hierarchy < block_clock_hierarchy:
+                    # Suggest valid frequencies for the series
+                    valid_freqs = [freq for freq, hier in FREQUENCY_HIERARCHY.items() 
+                                  if hier >= block_clock_hierarchy]
+                    valid_freqs_str = ', '.join(sorted(valid_freqs))
+                    raise ValueError(
+                        f"Series '{s.series_id}' has frequency '{s.frequency}' which is faster than "
+                        f"block '{block_name}' clock '{block_clock}'. "
+                        f"Series in a block must have frequency <= block clock. "
+                        f"Suggested fix: change series frequency to one of [{valid_freqs_str}] "
+                        f"(slower or equal to block clock '{block_clock}'), "
+                        f"or set block clock to '{s.frequency}' or faster."
+                    )
         
         # Validate factors_per_block
         if any(f < 1 for f in self.factors_per_block):
@@ -607,10 +510,26 @@ class DFMConfig(BaseModelConfig):
             )
     
     def get_blocks_array(self) -> np.ndarray:
-        """Get blocks as numpy array (snake_case - recommended, cached)."""
+        """Get blocks as numpy array (snake_case - recommended, cached).
+        
+        Note: SeriesConfig no longer contains blocks information.
+        By default, all series load on all blocks (all 1s).
+        Custom series-to-block mapping can be provided via series_to_blocks attribute.
+        """
         if self._cached_blocks is None:
-            blocks_list = [s.blocks for s in self.series]
-            self._cached_blocks = np.array(blocks_list, dtype=int)
+            n_series = len(self.series)
+            n_blocks = len(self.block_names)
+            # Default: all series load on all blocks (all 1s)
+            # Custom mapping can be provided via series_to_blocks attribute if needed
+            if hasattr(self, 'series_to_blocks') and self.series_to_blocks is not None:
+                blocks_list = []
+                for s in self.series:
+                    series_blocks = self.series_to_blocks.get(s.series_id, [1] * n_blocks)
+                    blocks_list.append(series_blocks)
+                self._cached_blocks = np.array(blocks_list, dtype=int)
+            else:
+                # Default: all series load on all blocks
+                self._cached_blocks = np.ones((n_series, n_blocks), dtype=int)
         return self._cached_blocks
     
     # ========================================================================
@@ -747,53 +666,26 @@ class DDFMConfig(BaseModelConfig):
             block_names = data.get('block_names', [])
             factors_per_block = data.get('factors_per_block', None)
         
-        # If block_names is still empty, try to infer from series blocks
-        if not block_names and 'series' in data and isinstance(data['series'], dict):
-            # Collect all unique block names from series
-            all_blocks = set()
-            for series_cfg in data['series'].values():
-                if isinstance(series_cfg, dict):
-                    series_blocks = series_cfg.get('blocks', [])
-                    if isinstance(series_blocks, list):
-                        all_blocks.update(series_blocks)
-            if all_blocks:
-                # Ensure first block is used as default
-                block_names = []
-                if all_blocks:
-                    # Use first block as global
-                    first_block = sorted(all_blocks)[0]
-                    block_names.append(first_block)
-                    all_blocks.remove(first_block)
-                block_names.extend(sorted(all_blocks))
-                factors_per_block = [1] * len(block_names)
-                # Create default blocks dict for later use
-                blocks_dict = {name: {'factors': 1, 'clock': 'm'} for name in block_names}
-        
-        # Parse series dict: {series_id: {frequency: ..., blocks: [block_names], ...}}
+        # Note: SeriesConfig no longer contains blocks information.
+        # Blocks must be defined in DFMConfig, not inferred from series.
+        # If block_names is still empty, create default block
+        if not block_names:
+            block_names = [DEFAULT_BLOCK_NAME]
+            factors_per_block = [1]
+            blocks_dict = {DEFAULT_BLOCK_NAME: {'factors': 1, 'ar_lag': 1, 'clock': data.get('clock', 'm')}}
+        
+        # Parse series dict: {series_id: {frequency: ..., ...}}
+        # Note: transformation is handled by preprocessing pipeline, not in SeriesConfig
+        # Note: blocks are defined in DFMConfig, not in SeriesConfig
         series_list = []
         for series_id, series_cfg in data['series'].items():
             if isinstance(series_cfg, dict):
-                # Convert block names to binary array (0/1 for each block)
-                series_blocks_names = series_cfg.get('blocks', [])
-                if isinstance(series_blocks_names, list) and series_blocks_names:
-                    # Create binary array: 1 if series loads on block, 0 otherwise
-                    # First block (Global) must always be 1
-                    series_blocks = [0] * len(block_names)
-                    series_blocks[0] = 1  # Global block is always 1
-                    for block_name in series_blocks_names:
-                        if block_name in block_names:
-                            block_idx = block_names.index(block_name)
-                            series_blocks[block_idx] = 1
-                else:
-                    # Default: only global block
-                    series_blocks = [1] + [0] * (len(block_names) - 1)
-                
                 series_list.append(SeriesConfig(
                     series_id=series_id,
                     series_name=series_cfg.get('series_name', series_id),
                     frequency=series_cfg.get('frequency', 'm'),
-                    transformation=series_cfg.get('transformation', 'lin'),
-                    blocks=series_blocks,
+                    # transformation removed - handled by preprocessing pipeline
+                    # blocks removed - defined in DFMConfig
                     units=series_cfg.get('units', None),  # Optional, for display only
                     release_date=series_cfg.get('release_date', None)  # Optional, for nowcasting
                 ))

dfm_python/decoder/__init__.py

@@ -0,0 +1,14 @@
+"""Decoder modules for DDFM.
+
+This package provides decoder implementations for reconstructing observations
+from latent factors in the Deep Dynamic Factor Model (DDFM).
+"""
+
+from .linear import Decoder
+from .mlp import MLPDecoder
+
+__all__ = [
+    'Decoder',
+    'MLPDecoder',
+]
+

dfm_python/decoder/linear.py

@@ -0,0 +1,68 @@
+"""Linear decoder network for DDFM.
+
+Maps latent factors f_t back to observed variables X_t using a linear transformation.
+This preserves interpretability and allows Kalman filtering.
+"""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import torch
+    import torch.nn as nn
+else:
+    torch = None
+    nn = None
+
+try:
+    import torch
+    import torch.nn as nn
+    _has_torch = True
+except ImportError:
+    _has_torch = False
+    torch = None
+    nn = None
+
+
+if _has_torch:
+    class Decoder(nn.Module):
+        """Linear decoder network for DDFM.
+        
+        Maps latent factors f_t back to observed variables X_t using a linear transformation.
+        This preserves interpretability and allows Kalman filtering.
+        """
+        
+        def __init__(self, input_dim: int, output_dim: int, use_bias: bool = True):
+            """Initialize linear decoder.
+            
+            Parameters
+            ----------
+            input_dim : int
+                Number of factors (input dimension)
+            output_dim : int
+                Number of series (output dimension)
+            use_bias : bool
+                Whether to use bias term
+            """
+            super().__init__()
+            self.decoder = nn.Linear(input_dim, output_dim, bias=use_bias)
+        
+        def forward(self, f: "torch.Tensor") -> "torch.Tensor":
+            """Forward pass through decoder.
+            
+            Parameters
+            ----------
+            f : torch.Tensor
+                Factors (batch_size x input_dim)
+                
+            Returns
+            -------
+            torch.Tensor
+                Reconstructed observations (batch_size x output_dim)
+            """
+            return self.decoder(f)
+else:
+    # Placeholder class when PyTorch is not available
+    class Decoder:
+        """Placeholder Decoder class when PyTorch is not available."""
+        def __init__(self, *args, **kwargs):
+            raise ImportError("PyTorch is required for DDFM. Install with: pip install dfm-python[deep]")