dfm-python 0.1.4__py3-none-any.whl

dfm_python/utils/__init__.py

@@ -0,0 +1,27 @@
+"""Utility functions for data preprocessing and summary statistics."""
+
+from .data_utils import rem_nans_spline, summarize
+from .aggregation import (
+    generate_tent_weights,
+    generate_R_mat,
+    get_tent_weights_for_pair,
+    get_aggregation_structure,
+    FREQUENCY_HIERARCHY,
+    TENT_WEIGHTS_LOOKUP,
+    MAX_TENT_SIZE,
+)
+
+__all__ = [
+    'rem_nans_spline',
+    'summarize',
+    'generate_tent_weights',
+    'generate_R_mat',
+    'get_tent_weights_for_pair',
+    'get_aggregation_structure',
+    'FREQUENCY_HIERARCHY',
+    'TENT_WEIGHTS_LOOKUP',
+    'MAX_TENT_SIZE',
+]
+
+# Note: Direct import of data_utils module available for backward compatibility
+

dfm_python/utils/aggregation.py

@@ -0,0 +1,258 @@
+"""Aggregation utilities for mixed-frequency data in Dynamic Factor Models.
+
+This module implements the clock-based aggregation framework for handling mixed-frequency
+time series in Dynamic Factor Models. The key innovation is the use of deterministic
+tent kernels to map lower-frequency observed variables to higher-frequency latent states
+within the observation equation.
+
+The module provides:
+- Frequency hierarchy definitions
+- Tent weight generation and lookup
+- Constraint matrix (R_mat) generation from tent weights
+- Aggregation structure computation based on global clock
+
+The tent kernel approach follows the FRBNY methodology, where quarterly (or other
+lower-frequency) series are connected to monthly latent factors through weighted
+aggregation constraints. This allows all factors to evolve at the same clock frequency
+while properly handling mixed-frequency observations.
+"""
+
+import numpy as np
+from typing import Tuple, Optional, Dict, Any
+from ..config import DFMConfig
+
+# ============================================================================
+# Frequency Hierarchy
+# ============================================================================
+
+# Frequency hierarchy (from highest to lowest frequency)
+# Used to determine which frequencies are slower/faster than the clock
+FREQUENCY_HIERARCHY: Dict[str, int] = {
+    'd': 1,   # Daily (highest frequency)
+    'w': 2,   # Weekly
+    'm': 3,   # Monthly
+    'q': 4,   # Quarterly
+    'sa': 5,  # Semi-annual
+    'a': 6    # Annual (lowest frequency)
+}
+
+# ============================================================================
+# Tent Kernel Configuration
+# ============================================================================
+
+# Maximum tent kernel size (number of periods)
+# For frequency gaps larger than this, the missing data approach is used instead
+# This prevents excessively large constraint matrices that would be computationally
+# expensive and potentially numerically unstable
+MAX_TENT_SIZE: int = 12
+
+# Deterministic tent weights lookup for supported frequency pairs
+# Format: (slower_freq, faster_freq) -> tent_weights_array
+# 
+# These weights define how lower-frequency observations aggregate to higher-frequency
+# latent states. The tent shape (symmetric, peaking in the middle) follows FRBNY's
+# approach and ensures smooth aggregation while giving more weight to the central
+# periods of the aggregation window.
+#
+# Example: ('q', 'm'): [1, 2, 3, 2, 1] means a quarterly observation aggregates
+# 5 monthly latent states with weights 1, 2, 3, 2, 1 (peaking at the middle month).
+TENT_WEIGHTS_LOOKUP: Dict[Tuple[str, str], np.ndarray] = {
+    ('q', 'm'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: quarterly -> monthly
+    ('sa', 'm'): np.array([1, 2, 3, 4, 3, 2, 1]),             # 7 periods: semi-annual -> monthly
+    ('a', 'm'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: annual -> monthly
+    ('m', 'w'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: monthly -> weekly
+    ('q', 'w'): np.array([1, 2, 3, 4, 5, 4, 3, 2, 1]),       # 9 periods: quarterly -> weekly
+    ('sa', 'q'): np.array([1, 2, 1]),                         # 3 periods: semi-annual -> quarterly
+    ('a', 'q'): np.array([1, 2, 3, 2, 1]),                    # 5 periods: annual -> quarterly
+    ('a', 'sa'): np.array([1, 2, 1]),                         # 3 periods: annual -> semi-annual
+}
+
+
+def generate_tent_weights(n_periods: int, tent_type: str = 'symmetric') -> np.ndarray:
+    """Generate tent-shaped weights for aggregation.
+    
+    Parameters:
+    -----------
+    n_periods : int
+        Number of base periods to aggregate (e.g., 5 for monthly->quarterly)
+    tent_type : str
+        Type of tent: 'symmetric' (default), 'linear', 'exponential'
+        
+    Returns:
+    --------
+    weights : np.ndarray
+        Array of weights that sum to a convenient number
+        
+    Examples:
+    --------
+    >>> generate_tent_weights(5, 'symmetric')
+    array([1, 2, 3, 2, 1])  # Classic tent for monthly->quarterly
+    
+    >>> generate_tent_weights(7, 'symmetric')
+    array([1, 2, 3, 4, 3, 2, 1])  # Weekly aggregation
+    """
+    if tent_type == 'symmetric':
+        if n_periods % 2 == 1:
+            # Odd number: symmetric around middle
+            half = n_periods // 2
+            weights = np.concatenate([
+                np.arange(1, half + 2),      # [1, 2, ..., peak]
+                np.arange(half, 0, -1)       # [peak-1, ..., 2, 1]
+            ])
+        else:
+            # Even number: symmetric with two peaks
+            half = n_periods // 2
+            weights = np.concatenate([
+                np.arange(1, half + 1),     # [1, 2, ..., half]
+                np.arange(half, 0, -1)       # [half, ..., 2, 1]
+            ])
+    elif tent_type == 'linear':
+        # Linear weights (simple average)
+        weights = np.ones(n_periods)
+    elif tent_type == 'exponential':
+        # Exponential decay from center
+        center = n_periods / 2
+        weights = np.exp(-np.abs(np.arange(n_periods) - center) / (n_periods / 4))
+        weights = weights / weights.sum() * n_periods  # Normalize
+    else:
+        raise ValueError(f"Unknown tent_type: {tent_type}. Must be 'symmetric', 'linear', or 'exponential'")
+    
+    return weights.astype(int)
+
+
+def generate_R_mat(tent_weights: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+    """Generate constraint matrix R_mat from tent weights.
+    
+    Parameters:
+    -----------
+    tent_weights : np.ndarray
+        Tent weights array, e.g., [1, 2, 3, 2, 1] for monthly->quarterly
+        
+    Returns:
+    --------
+    R_mat : np.ndarray
+        Constraint matrix of shape (n-1) × n
+    q : np.ndarray
+        Constraint vector of zeros, shape (n-1,)
+        
+    Examples:
+    --------
+    >>> weights = np.array([1, 2, 3, 2, 1])
+    >>> R_mat, q = generate_R_mat(weights)
+    >>> # Returns the classic monthly->quarterly R_mat
+    """
+    n = len(tent_weights)
+    w1 = tent_weights[0]  # First weight (reference)
+    
+    # Create constraint matrix: (n-1) rows × n columns
+    R_mat = np.zeros((n - 1, n))
+    q = np.zeros(n - 1)
+    
+    # Row i: relates w1*c1 to w(i+1)*c(i+1)
+    # Constraint: w1*c1 - w(i+1)*c(i+1) = 0
+    for i in range(n - 1):
+        R_mat[i, 0] = w1              # Coefficient for c1
+        R_mat[i, i + 1] = -tent_weights[i + 1]  # Coefficient for c(i+1)
+        # All other columns remain 0
+    
+    return R_mat, q
+
+
+def get_tent_weights_for_pair(slower_freq: str, faster_freq: str) -> Optional[np.ndarray]:
+    """Get deterministic tent weights for a frequency pair.
+    
+    Parameters:
+    -----------
+    slower_freq : str
+        Slower frequency (e.g., 'q' for quarterly)
+    faster_freq : str
+        Faster frequency (e.g., 'm' for monthly) - this is the clock
+    
+    Returns:
+    --------
+    tent_weights : np.ndarray or None
+        Tent weights array if pair is supported, None otherwise
+        
+    Examples:
+    --------
+    >>> get_tent_weights_for_pair('q', 'm')
+    array([1, 2, 3, 2, 1])  # Quarterly -> monthly
+    
+    >>> get_tent_weights_for_pair('m', 'd')
+    None  # Not supported (too large gap)
+    """
+    return TENT_WEIGHTS_LOOKUP.get((slower_freq, faster_freq))
+
+
+def get_aggregation_structure(
+    config: DFMConfig, 
+    clock: str = 'm'
+) -> Dict[str, Any]:
+    """Get aggregation structure for all frequency combinations in config based on clock.
+    
+    This function determines which series need tent kernels (those with frequencies
+    slower than the clock) and generates the corresponding constraint matrices (R_mat)
+    and constraint vectors (q) for use in constrained least squares estimation.
+    
+    The aggregation structure follows the clock-based approach:
+    - All latent factors evolve at the clock frequency
+    - Series with frequencies slower than the clock use tent kernels
+    - Series with frequencies faster than the clock use missing data approach
+    - If tent kernel size exceeds MAX_TENT_SIZE, missing data approach is used
+    
+    Parameters
+    ----------
+    config : DFMConfig
+        Model configuration containing series frequencies and structure
+    clock : str, optional
+        Base frequency (global clock) for nowcasting, by default 'm' (monthly).
+        All latent factors will evolve at this frequency.
+        
+    Returns
+    -------
+    aggregation_info : Dict[str, Any]
+        Dictionary containing:
+        - 'structures': Dict[Tuple[str, str], Tuple[np.ndarray, np.ndarray]]
+            Maps (slower_freq, clock) tuples to (R_mat, q) constraint pairs
+        - 'tent_weights': Dict[str, np.ndarray]
+            Maps frequency strings to their tent weight arrays
+        - 'n_periods': Dict[str, int]
+            Maps frequency strings to tent kernel sizes
+        - 'clock': str
+            The clock frequency used
+            
+    Examples
+    --------
+    >>> from dfm_python import load_config
+    >>> config = load_config('config.yaml')
+    >>> agg_info = get_aggregation_structure(config, clock='m')
+    >>> # Check which frequencies need tent kernels
+    >>> print(agg_info['tent_weights'])
+    {'q': array([1, 2, 3, 2, 1]), 'sa': array([1, 2, 3, 4, 3, 2, 1])}
+    """
+    frequencies = set(config.Frequency)
+    structures = {}
+    tent_weights = {}
+    n_periods_map = {}
+    
+    # Find series with frequencies slower than clock (need tent kernels)
+    for freq in frequencies:
+        if FREQUENCY_HIERARCHY.get(freq, 999) > FREQUENCY_HIERARCHY.get(clock, 0):
+            # This frequency is slower than clock, check if tent kernel is available
+            tent_w = get_tent_weights_for_pair(freq, clock)
+            if tent_w is not None and len(tent_w) <= MAX_TENT_SIZE:
+                # Tent kernel available and within size limit
+                tent_weights[freq] = tent_w
+                n_periods_map[freq] = len(tent_w)
+                # Generate R_mat from tent weights
+                R_mat, q = generate_R_mat(tent_w)
+                structures[(freq, clock)] = (R_mat, q)
+            # If tent kernel not available or too large, use missing data approach (no structure needed)
+    
+    return {
+        'structures': structures,
+        'tent_weights': tent_weights,
+        'n_periods': n_periods_map,
+        'clock': clock
+    }
+

dfm_python/utils/data_utils.py

@@ -0,0 +1,165 @@
+"""Utility functions for data preprocessing and summary statistics.
+
+This module provides essential data handling utilities for Dynamic Factor Models:
+- Missing value treatment via spline interpolation
+- Data summary and visualization
+- Robust handling of edge cases (all-NaN series, constant series, etc.)
+
+The spline interpolation methods follow standard econometric practice for handling
+missing data in time series, with options for different treatment strategies
+depending on data characteristics.
+"""
+
+import numpy as np
+from scipy.interpolate import CubicSpline
+from scipy.signal import lfilter
+from typing import Tuple, Optional
+import logging
+
+# Set up logger
+_logger = logging.getLogger(__name__)
+
+
+def rem_nans_spline(X: np.ndarray, method: int = 2, k: int = 3) -> Tuple[np.ndarray, np.ndarray]:
+    """Treat NaNs in dataset for DFM estimation."""
+    T, N = X.shape
+    indNaN = np.isnan(X)
+    
+    def _remove_leading_trailing(threshold: float):
+        """Remove rows with NaN count above threshold."""
+        rem = np.sum(indNaN, axis=1) > (N * threshold if threshold < 1 else threshold)
+        nan_lead = np.cumsum(rem) == np.arange(1, T + 1)
+        nan_end = np.cumsum(rem[::-1]) == np.arange(1, T + 1)[::-1]
+        return ~(nan_lead | nan_end)
+    
+    def _fill_missing(x: np.ndarray, mask: np.ndarray):
+        """Fill missing values using spline interpolation and moving average."""
+        # Ensure mask matches x length
+        if len(mask) != len(x):
+            mask = mask[:len(x)]
+        
+        non_nan = np.where(~mask)[0]
+        if len(non_nan) < 2:
+            return x
+        
+        x_filled = x.copy()
+        if non_nan[-1] >= len(x):
+            non_nan = non_nan[non_nan < len(x)]
+        if len(non_nan) < 2:
+            return x
+        
+        x_filled[non_nan[0]:non_nan[-1]+1] = CubicSpline(non_nan, x[non_nan])(np.arange(non_nan[0], min(non_nan[-1]+1, len(x))))
+        x_filled[mask[:len(x_filled)]] = np.nanmedian(x_filled)
+        
+        # Moving average filter
+        pad = np.concatenate([np.full(k, x_filled[0]), x_filled, np.full(k, x_filled[-1])])
+        ma = lfilter(np.ones(2*k+1)/(2*k+1), 1, pad)[2*k+1:]
+        if len(ma) == len(x_filled):
+            x_filled[mask[:len(x_filled)]] = ma[mask[:len(x_filled)]]
+        return x_filled
+    
+    if method == 1:
+        # Replace all missing values
+        for i in range(N):
+            mask = indNaN[:, i]
+            x = X[:, i].copy()
+            x[mask] = np.nanmedian(x)
+            pad = np.concatenate([np.full(k, x[0]), x, np.full(k, x[-1])])
+            ma = lfilter(np.ones(2*k+1)/(2*k+1), 1, pad)[2*k+1:]
+            x[mask] = ma[mask]
+            X[:, i] = x
+    
+    elif method == 2:
+        # Remove >80% NaN rows, then fill
+        mask = _remove_leading_trailing(0.8)
+        X = X[mask]
+        indNaN = np.isnan(X)
+        for i in range(N):
+            X[:, i] = _fill_missing(X[:, i], indNaN[:, i])
+    
+    elif method == 3:
+        # Only remove all-NaN rows
+        mask = _remove_leading_trailing(N)
+        X = X[mask]
+        indNaN = np.isnan(X)
+    
+    elif method == 4:
+        # Remove all-NaN rows, then fill
+        mask = _remove_leading_trailing(N)
+        X = X[mask]
+        indNaN = np.isnan(X)
+        for i in range(N):
+            X[:, i] = _fill_missing(X[:, i], indNaN[:, i])
+    
+    elif method == 5:
+        # Fill missing values
+        for i in range(N):
+            X[:, i] = _fill_missing(X[:, i], indNaN[:, i])
+    
+    return X, indNaN
+
+
+def summarize(X: np.ndarray, Time, config, vintage: Optional[str] = None) -> None:
+    """Display data summary table."""
+    T, N = X.shape
+    
+    print("\n\n\nTable 2: Data Summary")
+    print(f"N = {N:4d} data series")
+    
+    try:
+        time_start = Time[0].strftime('%Y-%m-%d') if hasattr(Time[0], 'strftime') else str(Time[0])
+        time_end = Time[-1].strftime('%Y-%m-%d') if hasattr(Time[-1], 'strftime') else str(Time[-1])
+    except (AttributeError, ValueError, TypeError) as e:
+        _logger.debug(f"summarize: Failed to format time range: {type(e).__name__}: {str(e)}")
+        time_start = time_end = "N/A"
+    
+    print(f"T = {T:4d} observations from {time_start:10s} to {time_end:10s}\n")
+    print(f"{'Data Series':30s} | {'Observations':17s}    {'Units':12s}    "
+          f"{'Frequency':10s}    {'Mean':8s}    {'Std. Dev.':8s}    {'Min':8s}    {'Max':8s}")
+    print("-" * 130)
+    
+    for i in range(N):
+        t_obs = ~np.isnan(X[:, i])
+        obs_idx = np.where(t_obs)[0]
+        
+        name = config.SeriesName[i][:27] + "..." if len(config.SeriesName[i]) > 30 else config.SeriesName[i]
+        sid = f"[{config.SeriesID[i][:25]}...]" if len(config.SeriesID[i]) > 28 else f"[{config.SeriesID[i]}]"
+        
+        freq = 'Monthly' if config.Frequency[i] == 'm' else 'Quarterly'
+        trans = config.Transformation[i]
+        units_t = 'MoM%' if trans == 'pch' and config.Frequency[i] == 'm' else \
+                  'QoQ% AR' if trans == 'pca' and config.Frequency[i] == 'q' else \
+                  config.UnitsTransformed[i]
+        
+        date_range = "N/A"
+        if len(obs_idx) > 0:
+            try:
+                fmt = '%b %Y' if config.Frequency[i] == 'm' else '%Y-%m'
+                # Use iloc for positional indexing
+                if hasattr(Time, 'iloc'):
+                    date_range = f"{Time.iloc[obs_idx[0]].strftime(fmt)}-{Time.iloc[obs_idx[-1]].strftime(fmt)}"
+                else:
+                    date_range = f"{Time[obs_idx[0]].strftime(fmt)}-{Time[obs_idx[-1]].strftime(fmt)}"
+            except (AttributeError, ValueError, TypeError) as e:
+                # Fallback to positional indexing
+                _logger.debug(f"summarize: Failed to format date range with strftime: {type(e).__name__}: {str(e)}")
+                try:
+                    if hasattr(Time, 'iloc'):
+                        date_range = f"{Time.iloc[obs_idx[0]]}-{Time.iloc[obs_idx[-1]]}"
+                    else:
+                        date_range = f"{Time[obs_idx[0]]}-{Time[obs_idx[-1]]}"
+                except (AttributeError, ValueError, TypeError, IndexError) as e2:
+                    _logger.debug(f"summarize: Failed to format date range fallback: {type(e2).__name__}: {str(e2)}")
+                    date_range = "N/A"
+        
+        y = X[t_obs, i]
+        if len(y) == 0:
+            stats = (np.nan, np.nan, np.nan, np.nan)
+        else:
+            stats = np.nanmean(y), np.nanstd(y), np.nanmin(y) if len(y) > 0 else np.nan, np.nanmax(y) if len(y) > 0 else np.nan
+        
+        print(f"{name:30s} | {len(obs_idx):17d}    {units_t:12s}    {freq:10s}    "
+              f"{stats[0]:8.1f}    {stats[1]:8.1f}    {stats[2]:8.1f}    {stats[3]:8.1f}")
+        print(f"{sid:30s} | {date_range:17s}\n")
+    
+    print("\n\n\n")