dfm-python 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

dfm_python/__init__.py

@@ -36,7 +36,7 @@ from .config import DFMConfig
 from .data_loader import (
     load_config, load_config_from_yaml, load_config_from_csv, load_data, transform_data
 )
-from .dfm import DFMResult, dfm
+from .dfm import DFMResult, dfm, calculate_rmse
 from .kalman import run_kf, skf, fis, miss_data
 from .news import update_nowcast, news_dfm, para_const
 
@@ -51,7 +51,7 @@ __all__ = [
     'ModelConfig',  # Alias for DFMConfig
     'load_config', 'load_config_from_yaml', 'load_config_from_csv',
     'load_data', 'transform_data',
-    'DFMResult', 'dfm',
+    'DFMResult', 'dfm', 'calculate_rmse',
     'run_kf', 'skf', 'fis', 'miss_data',
     'update_nowcast', 'news_dfm', 'para_const',
     # Deprecated aliases

dfm_python/config.py

@@ -119,6 +119,32 @@ class DFMConfig:
     nan_k: int = 3  # Spline parameter for NaN interpolation
     clock: str = 'm'  # Base frequency for nowcasting (global clock): 'd', 'w', 'm', 'q', 'sa', 'a' (defaults to 'm' for monthly)
     
+    # ========================================================================
+    # Numerical Stability Parameters (transparent and configurable)
+    # ========================================================================
+    # AR Coefficient Clipping
+    clip_ar_coefficients: bool = True  # Enable AR coefficient clipping for stationarity
+    ar_clip_min: float = -0.99  # Minimum AR coefficient (must be > -1 for stationarity)
+    ar_clip_max: float = 0.99   # Maximum AR coefficient (must be < 1 for stationarity)
+    warn_on_ar_clip: bool = True  # Warn when AR coefficients are clipped (indicates near-unit root)
+    
+    # Data Value Clipping
+    clip_data_values: bool = True  # Enable clipping of extreme data values
+    data_clip_threshold: float = 100.0  # Clip values beyond this many standard deviations
+    warn_on_data_clip: bool = True  # Warn when data values are clipped (indicates outliers)
+    
+    # Regularization
+    use_regularization: bool = True  # Enable regularization for numerical stability
+    regularization_scale: float = 1e-6  # Scale factor for ridge regularization (relative to trace)
+    min_eigenvalue: float = 1e-8  # Minimum eigenvalue for positive definite matrices
+    max_eigenvalue: float = 1e6   # Maximum eigenvalue cap to prevent explosion
+    warn_on_regularization: bool = True  # Warn when regularization is applied
+    
+    # Damped Updates
+    use_damped_updates: bool = True  # Enable damped updates when likelihood decreases
+    damping_factor: float = 0.8  # Damping factor (0.8 = 80% new, 20% old)
+    warn_on_damped_update: bool = True  # Warn when damped updates are used
+    
     # ========================================================================
     # Internal cache (not user-configurable)
     # ========================================================================
@@ -293,8 +319,8 @@ class DFMConfig:
                 category=str(cat_val) if cat_val is not None else ''
             ))
         
-            # Extract estimation parameters if provided
-            return cls(
+        # Extract estimation parameters if provided
+        return cls(
                 series=series_list,
                 block_names=data.get('BlockNames', data.get('block_names', [])),
                 factors_per_block=data.get('factors_per_block', None),
@@ -304,7 +330,23 @@ class DFMConfig:
                 max_iter=data.get('max_iter', 5000),
                 nan_method=data.get('nan_method', 2),
                 nan_k=data.get('nan_k', 3),
-                clock=data.get('clock', 'm')
+                clock=data.get('clock', 'm'),
+                # Numerical stability parameters
+                clip_ar_coefficients=data.get('clip_ar_coefficients', True),
+                ar_clip_min=data.get('ar_clip_min', -0.99),
+                ar_clip_max=data.get('ar_clip_max', 0.99),
+                warn_on_ar_clip=data.get('warn_on_ar_clip', True),
+                clip_data_values=data.get('clip_data_values', True),
+                data_clip_threshold=data.get('data_clip_threshold', 100.0),
+                warn_on_data_clip=data.get('warn_on_data_clip', True),
+                use_regularization=data.get('use_regularization', True),
+                regularization_scale=data.get('regularization_scale', 1e-6),
+                min_eigenvalue=data.get('min_eigenvalue', 1e-8),
+                max_eigenvalue=data.get('max_eigenvalue', 1e6),
+                warn_on_regularization=data.get('warn_on_regularization', True),
+                use_damped_updates=data.get('use_damped_updates', True),
+                damping_factor=data.get('damping_factor', 0.8),
+                warn_on_damped_update=data.get('warn_on_damped_update', True)
             )
     
     @classmethod
@@ -376,7 +418,23 @@ class DFMConfig:
                 max_iter=data.get('max_iter', 5000),
                 nan_method=data.get('nan_method', 2),
                 nan_k=data.get('nan_k', 3),
-                clock=data.get('clock', 'm')
+                clock=data.get('clock', 'm'),
+                # Numerical stability parameters
+                clip_ar_coefficients=data.get('clip_ar_coefficients', True),
+                ar_clip_min=data.get('ar_clip_min', -0.99),
+                ar_clip_max=data.get('ar_clip_max', 0.99),
+                warn_on_ar_clip=data.get('warn_on_ar_clip', True),
+                clip_data_values=data.get('clip_data_values', True),
+                data_clip_threshold=data.get('data_clip_threshold', 100.0),
+                warn_on_data_clip=data.get('warn_on_data_clip', True),
+                use_regularization=data.get('use_regularization', True),
+                regularization_scale=data.get('regularization_scale', 1e-6),
+                min_eigenvalue=data.get('min_eigenvalue', 1e-8),
+                max_eigenvalue=data.get('max_eigenvalue', 1e6),
+                warn_on_regularization=data.get('warn_on_regularization', True),
+                use_damped_updates=data.get('use_damped_updates', True),
+                damping_factor=data.get('damping_factor', 0.8),
+                warn_on_damped_update=data.get('warn_on_damped_update', True)
             )
         
         # New format with series list
@@ -396,7 +454,23 @@ class DFMConfig:
                 max_iter=data.get('max_iter', 5000),
                 nan_method=data.get('nan_method', 2),
                 nan_k=data.get('nan_k', 3),
-                clock=data.get('clock', 'm')
+                clock=data.get('clock', 'm'),
+                # Numerical stability parameters
+                clip_ar_coefficients=data.get('clip_ar_coefficients', True),
+                ar_clip_min=data.get('ar_clip_min', -0.99),
+                ar_clip_max=data.get('ar_clip_max', 0.99),
+                warn_on_ar_clip=data.get('warn_on_ar_clip', True),
+                clip_data_values=data.get('clip_data_values', True),
+                data_clip_threshold=data.get('data_clip_threshold', 100.0),
+                warn_on_data_clip=data.get('warn_on_data_clip', True),
+                use_regularization=data.get('use_regularization', True),
+                regularization_scale=data.get('regularization_scale', 1e-6),
+                min_eigenvalue=data.get('min_eigenvalue', 1e-8),
+                max_eigenvalue=data.get('max_eigenvalue', 1e6),
+                warn_on_regularization=data.get('warn_on_regularization', True),
+                use_damped_updates=data.get('use_damped_updates', True),
+                damping_factor=data.get('damping_factor', 0.8),
+                warn_on_damped_update=data.get('warn_on_damped_update', True)
             )
         
         # Direct instantiation (shouldn't happen often, but handle it)

dfm_python/data_loader.py

@@ -74,7 +74,13 @@ def load_config_from_yaml(configfile: Union[str, Path]) -> DFMConfig:
         model_dict = cfg_dict
     elif 'model' in cfg_dict:
         # Nested under 'model' key (from @package model:)
-        model_dict = cfg_dict['model']
+        model_dict = cfg_dict['model'].copy() if isinstance(cfg_dict['model'], dict) else {}
+        # If series is at top level, merge it into model_dict
+        if 'series' in cfg_dict and 'series' not in model_dict:
+            model_dict['series'] = cfg_dict['series']
+    elif 'series' in cfg_dict:
+        # Series at top level, construct model_dict from top-level keys
+        model_dict = {k: v for k, v in cfg_dict.items() if k not in ['dfm', 'data', 'experiment_name', 'output_dir', 'defaults']}
     else:
         # Try to construct from top-level keys
         model_dict = cfg_dict
@@ -362,6 +368,10 @@ def read_data(datafile: Union[str, Path]) -> Tuple[np.ndarray, pd.DatetimeIndex,
     - First column: Date (YYYY-MM-DD format or pandas parseable)
     - Subsequent columns: Series data (one column per series)
     - Header row: Series IDs (matching config.SeriesID)
+    
+    Alternatively, CSV can have metadata columns first:
+    - Metadata columns: series_id, series_name, frequency, transformation, category, units
+    - Then date columns starting from the first date column
     """
     datafile = Path(datafile)
     if not datafile.exists():
@@ -369,13 +379,94 @@ def read_data(datafile: Union[str, Path]) -> Tuple[np.ndarray, pd.DatetimeIndex,
     
     # Read CSV file
     try:
-        df = pd.read_csv(datafile, index_col=0, parse_dates=True)
+        df = pd.read_csv(datafile)
     except Exception as e:
         raise ValueError(f"Failed to read CSV file {datafile}: {e}")
     
+    # Check if first column is a date column or metadata
+    first_col = df.columns[0]
+    
+    # Try to parse first column as date
+    try:
+        pd.to_datetime(df[first_col].iloc[0])
+        is_date_first = True
+    except (ValueError, TypeError):
+        is_date_first = False
+    
+    # If first column is not a date, check if CSV is in "long" format (one row per series)
+    if not is_date_first:
+        # Check if first column contains series IDs (string values)
+        first_col_values = df[first_col].astype(str)
+        # If first column looks like series IDs and we have date columns, transpose
+        if 'series_id' in first_col.lower() or first_col_values.str.match(r'^[A-Z0-9_]+$').any():
+            # Long format: transpose so series are columns and dates are rows
+            # Find first date column
+            date_col_idx = None
+            for i, col in enumerate(df.columns):
+                try:
+                    # Try to parse first value as date
+                    pd.to_datetime(df[col].iloc[0])
+                    date_col_idx = i
+                    break
+                except (ValueError, TypeError):
+                    continue
+            
+            if date_col_idx is None:
+                raise ValueError(f"Could not find date column in CSV file {datafile}")
+            
+            # Set series_id as index, then transpose
+            series_id_col = df.columns[0]
+            df = df.set_index(series_id_col)
+            
+            # Get date columns (from date_col_idx onwards)
+            date_cols = df.columns[date_col_idx:]
+            df_data = df[date_cols].T  # Transpose: dates become rows, series become columns
+            
+            # Convert date column names to datetime index
+            df_data.index = pd.to_datetime(df_data.index)
+            
+            mnemonics = df_data.columns.tolist()
+            Time = df_data.index
+            Z = df_data.apply(pd.to_numeric, errors='coerce').values.astype(float)
+            
+            return Z, Time, mnemonics
+        else:
+            # Find first column that looks like a date
+            date_col_idx = None
+            for i, col in enumerate(df.columns):
+                try:
+                    # Try to parse first value as date
+                    pd.to_datetime(df[col].iloc[0])
+                    date_col_idx = i
+                    break
+                except (ValueError, TypeError):
+                    continue
+            
+            if date_col_idx is None:
+                raise ValueError(f"Could not find date column in CSV file {datafile}")
+            
+            # Use first date column as index, drop metadata columns
+            date_col = df.columns[date_col_idx]
+            df = df.set_index(date_col)
+            df.index = pd.to_datetime(df.index)
+            # Drop any remaining non-numeric columns (metadata)
+            numeric_cols = []
+            for col in df.columns:
+                try:
+                    pd.to_numeric(df[col], errors='coerce')
+                    numeric_cols.append(col)
+                except (ValueError, TypeError):
+                    continue
+            df = df[numeric_cols]
+    else:
+        # Standard format: first column is date
+        df = df.set_index(first_col)
+        df.index = pd.to_datetime(df.index)
+    
     mnemonics = df.columns.tolist()
     Time = df.index
-    Z = df.values.astype(float)
+    # Convert to float, handling any remaining non-numeric values
+    Z = df.apply(pd.to_numeric, errors='coerce').values.astype(float)
     
     return Z, Time, mnemonics