dfm-python 0.4.2__tar.gz → 0.4.5__tar.gz

{dfm_python-0.4.2 → dfm_python-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dfm-python
-Version: 0.4.2
+Version: 0.4.5
 Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
 Project-URL: Homepage, https://github.com/yourusername/dfm-python
 Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme
@@ -22,7 +22,7 @@ Requires-Python: >=3.10
 Requires-Dist: hydra-core>=1.3.2
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: numpy>=1.24.0
-Requires-Dist: polars>=0.20.0
+Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyarrow>=22.0.0
 Requires-Dist: pytorch-lightning>=2.0.0
 Requires-Dist: scikit-learn>=1.7.2
@@ -45,6 +45,8 @@ Provides-Extra: dev
 Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
 Requires-Dist: pytest>=7.0.0; extra == 'dev'
 Requires-Dist: scikit-learn>=1.0.0; extra == 'dev'
+Provides-Extra: forecasting
+Requires-Dist: sktime[forecasting]>=0.40.1; extra == 'forecasting'
 Provides-Extra: hydra
 Requires-Dist: hydra-core>=1.3.0; extra == 'hydra'
 Requires-Dist: omegaconf>=2.3.0; extra == 'hydra'
@@ -78,7 +80,7 @@ A Dynamic Factor Model is a powerful statistical framework that:
 - ✅ **Clock-based framework**: All factors evolve at a common clock frequency (configurable: daily, weekly, monthly, quarterly, semi-annual, annual)
 - ✅ **Block structure**: Flexible factor organization (global common factor + sector-specific factors)
 - ✅ **Idiosyncratic components**: Per-series state augmentation for better fit (AR(1) for clock-frequency, tent-length chains for slower frequencies)
-- ✅ **Robust missing data**: Spline interpolation preprocessing + Kalman filter handling during estimation
+- ✅ **Preprocessed data**: Package expects preprocessed data from users - users handle all preprocessing (imputation, scaling, feature engineering) using sktime or other tools
 - ✅ **News decomposition**: Attribute forecast changes to specific data releases
 - ✅ **Nowcasting & Forecasting**: Generate predictions for any horizon
 - ✅ **Deep Dynamic Factor Models (DDFM)**: Nonlinear encoder with PyTorch for capturing complex factor structures (optional, requires `dfm-python[deep]`)
@@ -103,7 +105,7 @@ pip install dfm-python
 **Requirements**: 
 - Python >= 3.10
 - numpy >= 1.24.0
-- polars >= 0.20.0
+- pandas >= 2.0.0
 - scipy >= 1.10.0
 
 Optional dependencies:
@@ -119,7 +121,7 @@ Optional dependencies:
 import dfm_python as dfm
 import numpy as np
 from datetime import datetime
-import polars as pl
+import pandas as pd
 
 # Generate or load your data (T × N: time periods × series)
 # Data should be standardized or will be standardized automatically
@@ -215,36 +217,25 @@ dfm.train(max_iter=5000, threshold=1e-5)
 X_forecast, Z_forecast = dfm.predict(horizon=None)  # Or specify: horizon=12
 ```
 
-### Example 3: Using Spec CSV (Recommended for CSV Workflows)
+### Example 3: Using YAML Configuration (Recommended)
 
 ```python
 import dfm_python as dfm
-from dfm_python.config import Params
-import polars as pl
-
-# 1. Create spec CSV (see Data Format section)
-# 2. Load configuration
-params = Params(
-    max_iter=5000,
-    threshold=1e-5,
-    clock='m',
-    augment_idio=True,  # Enable idio components
-    idio_rho0=0.1       # Initial AR coefficient for idio
-)
-spec_df = pl.read_csv('data/spec.csv')
-dfm.from_spec_df(spec_df, params=params)
 
-# 3. Load data
+# 1. Load configuration from YAML file
+dfm.from_yaml('config/default.yaml')
+
+# 2. Load data
 dfm.load_data('data/data.csv', sample_start='2020-01-01', sample_end='2023-12-31')
 
-# 4. Train
+# 3. Train
 dfm.train()
 
-# 5. Forecast and visualize (horizon=None defaults to 1 year of periods)
+# 4. Forecast and visualize (horizon=None defaults to 1 year of periods)
 X_forecast, Z_forecast = dfm.predict(horizon=None)  # Or specify: horizon=12
 dfm.plot(kind='factor', factor_index=0, forecast_horizon=None, save_path='factor_forecast.png')
 
-# 6. Access results
+# 5. Access results
 result = dfm.get_result()
 print(f"Converged: {result.converged}, Iterations: {result.num_iter}")
 print(f"Log-likelihood: {result.loglik:.2f}")
@@ -297,28 +288,7 @@ decomposition abilities (which require the data view).
 
 The package supports multiple ways to configure your model:
 
-#### 1. Spec CSV + Params (Recommended for CSV workflows)
-
-**Best for**: Users with existing CSV files and Excel-based workflows
-
-```python
-from dfm_python.config import Params
-import polars as pl
-
-params = Params(
-    max_iter=5000,
-    threshold=1e-5,
-    clock='m',
-    augment_idio=True,
-    augment_idio_slow=True,
-    idio_rho0=0.1,
-    idio_min_var=1e-8
-)
-spec_df = pl.read_csv('data/spec.csv')
-dfm.from_spec_df(spec_df, params=params)
-```
-
-#### 2. YAML Configuration
+#### 1. YAML Configuration
 
 **Best for**: Users who prefer declarative configuration files
 
@@ -331,7 +301,7 @@ dfm.load_data('data/sample_data.csv')
 dfm.train()
 ```
 
-#### 3. Hydra Decorator (Advanced)
+#### 2. Hydra Decorator (Advanced)
 
 **Best for**: Users already using Hydra for configuration management
 
@@ -356,7 +326,7 @@ if __name__ == "__main__":
 
 **CLI Overrides**: `python script.py max_iter=10 threshold=1e-4 blocks.Block_Global.factors=2`
 
-#### 4. Direct Python Objects
+#### 3. Direct Python Objects
 
 **Best for**: Programmatic configuration and integration with other Python code
 
@@ -459,7 +429,7 @@ Date,gdp_real,consumption,investment,employment,retail_sales
 **Notes**:
 - Quarterly series (e.g., `gdp_real`) only have values at quarter-end months
 - Monthly series (e.g., `employment`, `retail_sales`) have values every month
-- Missing values are handled automatically via spline interpolation and Kalman filtering
+- **Data must be preprocessed**: Users must handle missing values, scaling, and feature engineering before passing data to the package. Use sktime or other tools for preprocessing.
 
 ### Supported Frequencies
 
@@ -856,11 +826,9 @@ config = DFMConfig(series=series, blocks=blocks)
 
 ```python
 import dfm_python as dfm
-import polars as pl
+import pandas as pd
 
 # Configuration
-spec_df = pl.read_csv(spec_path)
-dfm.from_spec_df(spec_df, params=None)     # Convert spec DataFrame to YAML + load
 dfm.from_yaml(yaml_path)                    # Load from YAML file
 dfm.from_dict(config_dict)                  # Load from dictionary
 dfm.load_config(hydra=cfg)                  # Load from Hydra DictConfig
@@ -1141,13 +1109,6 @@ python tutorial/basic_tutorial.py \
   --config-name default \
   data_path=data/sample_data.csv
 
-# Convert spec CSV to YAML first, then use
-python -c "import dfm_python as dfm; dfm.from_spec('data/sample_spec.csv')"
-python tutorial/basic_tutorial.py \
-  --config-path config \
-  --config-name sample_spec \
-  data_path=data/sample_data.csv
-
 # With CLI overrides
 python tutorial/basic_tutorial.py \
   --config-path config \
@@ -1160,7 +1121,6 @@ python tutorial/basic_tutorial.py \
 
 Comprehensive tutorial covering:
 - **Hydra-based configuration**: All configs managed through YAML files
-- **Spec CSV conversion**: Convert CSV to YAML using `dfm.from_spec()`, then load via Hydra
 - **CLI parameter overrides**: Override any parameter via command line
 - **Data preparation and transformation**
 - **Model training and convergence**
@@ -1223,14 +1183,14 @@ DATA_VIEW_SOURCE=file  # or database
 
 ### Recent Improvements (v0.3.1)
 
-- ✅ **Polars-backed Data Views + Kalman Cache**  
-  `create_data_view()` now applies release-date masks via Polars, avoiding per-view NumPy copies. `Nowcast.backtest()` reuses cached Kalman filter/smoother states across successive data views, giving large speed-ups for 12-step pseudo real-time tests.
+- ✅ **Pandas-backed Data Views + Kalman Cache**  
+  `create_data_view()` now applies release-date masks via pandas, avoiding per-view NumPy copies. `Nowcast.backtest()` reuses cached Kalman filter/smoother states across successive data views, giving large speed-ups for 12-step pseudo real-time tests.
 - ✅ **Backtest Trajectory Plot**  
   `BacktestResult.plot_trajectory()` draws 이전 vs. 업데이트 nowcast curves plus monthly actuals along a relative week axis (`-12주 … 현재`). The tutorial simply calls this helper—no more hand-built plotting code.
 - ✅ **Higher-Frequency Backsteps**  
   `Nowcast.backtest(..., backward_steps=12, higher_freq=True)` now emits weekly snapshots even when the model clock is monthly, so you can inspect 12주 worth of “pseudo releases” immediately.
 - ✅ **Reload & Reuse**  
-  `DFM.load_pickle()` restores config, time index, and cached Polars data views so you can call `predict()`/`nowcast()` right after loading without re-reading CSVs.
+  `DFM.load_pickle()` restores config, time index, and cached pandas data views so you can call `predict()`/`nowcast()` right after loading without re-reading CSVs.
 - ✅ **API + Docs Refresh**  
   README/tutorial sections now highlight the faster backtests, new trajectory plots, and guidelines for interpreting flattened Q-matrix values or “strange” news decomposition outputs.
 
@@ -1244,7 +1204,7 @@ DATA_VIEW_SOURCE=file  # or database
 - ✅ **Model Result Storage**: File-based and SQLite adapters for saving/loading model results
 - ✅ **Data View Management**: Time-point specific data views for pseudo real-time evaluation
 - ✅ **Helper Function Consolidation**: Reduced code duplication with consolidated helper functions
-  - `to_python_datetime()`: Generic datetime conversion (handles polars datetime, strings, etc.)
+  - `to_python_datetime()`: Generic datetime conversion (handles pandas Timestamp, strings, etc.)
   - `extract_last_date()`: Generic time index extraction
   - `clock_to_datetime_freq()`: Shared clock-to-datetime frequency mapping
   - `get_periods_per_year()`: Generic periods-per-year calculation

{dfm_python-0.4.2 → dfm_python-0.4.5}/README.md

@@ -24,7 +24,7 @@ A Dynamic Factor Model is a powerful statistical framework that:
 - ✅ **Clock-based framework**: All factors evolve at a common clock frequency (configurable: daily, weekly, monthly, quarterly, semi-annual, annual)
 - ✅ **Block structure**: Flexible factor organization (global common factor + sector-specific factors)
 - ✅ **Idiosyncratic components**: Per-series state augmentation for better fit (AR(1) for clock-frequency, tent-length chains for slower frequencies)
-- ✅ **Robust missing data**: Spline interpolation preprocessing + Kalman filter handling during estimation
+- ✅ **Preprocessed data**: Package expects preprocessed data from users - users handle all preprocessing (imputation, scaling, feature engineering) using sktime or other tools
 - ✅ **News decomposition**: Attribute forecast changes to specific data releases
 - ✅ **Nowcasting & Forecasting**: Generate predictions for any horizon
 - ✅ **Deep Dynamic Factor Models (DDFM)**: Nonlinear encoder with PyTorch for capturing complex factor structures (optional, requires `dfm-python[deep]`)
@@ -49,7 +49,7 @@ pip install dfm-python
 **Requirements**: 
 - Python >= 3.10
 - numpy >= 1.24.0
-- polars >= 0.20.0
+- pandas >= 2.0.0
 - scipy >= 1.10.0
 
 Optional dependencies:
@@ -65,7 +65,7 @@ Optional dependencies:
 import dfm_python as dfm
 import numpy as np
 from datetime import datetime
-import polars as pl
+import pandas as pd
 
 # Generate or load your data (T × N: time periods × series)
 # Data should be standardized or will be standardized automatically
@@ -161,36 +161,25 @@ dfm.train(max_iter=5000, threshold=1e-5)
 X_forecast, Z_forecast = dfm.predict(horizon=None)  # Or specify: horizon=12
 ```
 
-### Example 3: Using Spec CSV (Recommended for CSV Workflows)
+### Example 3: Using YAML Configuration (Recommended)
 
 ```python
 import dfm_python as dfm
-from dfm_python.config import Params
-import polars as pl
-
-# 1. Create spec CSV (see Data Format section)
-# 2. Load configuration
-params = Params(
-    max_iter=5000,
-    threshold=1e-5,
-    clock='m',
-    augment_idio=True,  # Enable idio components
-    idio_rho0=0.1       # Initial AR coefficient for idio
-)
-spec_df = pl.read_csv('data/spec.csv')
-dfm.from_spec_df(spec_df, params=params)
 
-# 3. Load data
+# 1. Load configuration from YAML file
+dfm.from_yaml('config/default.yaml')
+
+# 2. Load data
 dfm.load_data('data/data.csv', sample_start='2020-01-01', sample_end='2023-12-31')
 
-# 4. Train
+# 3. Train
 dfm.train()
 
-# 5. Forecast and visualize (horizon=None defaults to 1 year of periods)
+# 4. Forecast and visualize (horizon=None defaults to 1 year of periods)
 X_forecast, Z_forecast = dfm.predict(horizon=None)  # Or specify: horizon=12
 dfm.plot(kind='factor', factor_index=0, forecast_horizon=None, save_path='factor_forecast.png')
 
-# 6. Access results
+# 5. Access results
 result = dfm.get_result()
 print(f"Converged: {result.converged}, Iterations: {result.num_iter}")
 print(f"Log-likelihood: {result.loglik:.2f}")
@@ -243,28 +232,7 @@ decomposition abilities (which require the data view).
 
 The package supports multiple ways to configure your model:
 
-#### 1. Spec CSV + Params (Recommended for CSV workflows)
-
-**Best for**: Users with existing CSV files and Excel-based workflows
-
-```python
-from dfm_python.config import Params
-import polars as pl
-
-params = Params(
-    max_iter=5000,
-    threshold=1e-5,
-    clock='m',
-    augment_idio=True,
-    augment_idio_slow=True,
-    idio_rho0=0.1,
-    idio_min_var=1e-8
-)
-spec_df = pl.read_csv('data/spec.csv')
-dfm.from_spec_df(spec_df, params=params)
-```
-
-#### 2. YAML Configuration
+#### 1. YAML Configuration
 
 **Best for**: Users who prefer declarative configuration files
 
@@ -277,7 +245,7 @@ dfm.load_data('data/sample_data.csv')
 dfm.train()
 ```
 
-#### 3. Hydra Decorator (Advanced)
+#### 2. Hydra Decorator (Advanced)
 
 **Best for**: Users already using Hydra for configuration management
 
@@ -302,7 +270,7 @@ if __name__ == "__main__":
 
 **CLI Overrides**: `python script.py max_iter=10 threshold=1e-4 blocks.Block_Global.factors=2`
 
-#### 4. Direct Python Objects
+#### 3. Direct Python Objects
 
 **Best for**: Programmatic configuration and integration with other Python code
 
@@ -405,7 +373,7 @@ Date,gdp_real,consumption,investment,employment,retail_sales
 **Notes**:
 - Quarterly series (e.g., `gdp_real`) only have values at quarter-end months
 - Monthly series (e.g., `employment`, `retail_sales`) have values every month
-- Missing values are handled automatically via spline interpolation and Kalman filtering
+- **Data must be preprocessed**: Users must handle missing values, scaling, and feature engineering before passing data to the package. Use sktime or other tools for preprocessing.
 
 ### Supported Frequencies
 
@@ -802,11 +770,9 @@ config = DFMConfig(series=series, blocks=blocks)
 
 ```python
 import dfm_python as dfm
-import polars as pl
+import pandas as pd
 
 # Configuration
-spec_df = pl.read_csv(spec_path)
-dfm.from_spec_df(spec_df, params=None)     # Convert spec DataFrame to YAML + load
 dfm.from_yaml(yaml_path)                    # Load from YAML file
 dfm.from_dict(config_dict)                  # Load from dictionary
 dfm.load_config(hydra=cfg)                  # Load from Hydra DictConfig
@@ -1087,13 +1053,6 @@ python tutorial/basic_tutorial.py \
   --config-name default \
   data_path=data/sample_data.csv
 
-# Convert spec CSV to YAML first, then use
-python -c "import dfm_python as dfm; dfm.from_spec('data/sample_spec.csv')"
-python tutorial/basic_tutorial.py \
-  --config-path config \
-  --config-name sample_spec \
-  data_path=data/sample_data.csv
-
 # With CLI overrides
 python tutorial/basic_tutorial.py \
   --config-path config \
@@ -1106,7 +1065,6 @@ python tutorial/basic_tutorial.py \
 
 Comprehensive tutorial covering:
 - **Hydra-based configuration**: All configs managed through YAML files
-- **Spec CSV conversion**: Convert CSV to YAML using `dfm.from_spec()`, then load via Hydra
 - **CLI parameter overrides**: Override any parameter via command line
 - **Data preparation and transformation**
 - **Model training and convergence**
@@ -1169,14 +1127,14 @@ DATA_VIEW_SOURCE=file  # or database
 
 ### Recent Improvements (v0.3.1)
 
-- ✅ **Polars-backed Data Views + Kalman Cache**  
-  `create_data_view()` now applies release-date masks via Polars, avoiding per-view NumPy copies. `Nowcast.backtest()` reuses cached Kalman filter/smoother states across successive data views, giving large speed-ups for 12-step pseudo real-time tests.
+- ✅ **Pandas-backed Data Views + Kalman Cache**  
+  `create_data_view()` now applies release-date masks via pandas, avoiding per-view NumPy copies. `Nowcast.backtest()` reuses cached Kalman filter/smoother states across successive data views, giving large speed-ups for 12-step pseudo real-time tests.
 - ✅ **Backtest Trajectory Plot**  
   `BacktestResult.plot_trajectory()` draws 이전 vs. 업데이트 nowcast curves plus monthly actuals along a relative week axis (`-12주 … 현재`). The tutorial simply calls this helper—no more hand-built plotting code.
 - ✅ **Higher-Frequency Backsteps**  
   `Nowcast.backtest(..., backward_steps=12, higher_freq=True)` now emits weekly snapshots even when the model clock is monthly, so you can inspect 12주 worth of “pseudo releases” immediately.
 - ✅ **Reload & Reuse**  
-  `DFM.load_pickle()` restores config, time index, and cached Polars data views so you can call `predict()`/`nowcast()` right after loading without re-reading CSVs.
+  `DFM.load_pickle()` restores config, time index, and cached pandas data views so you can call `predict()`/`nowcast()` right after loading without re-reading CSVs.
 - ✅ **API + Docs Refresh**  
   README/tutorial sections now highlight the faster backtests, new trajectory plots, and guidelines for interpreting flattened Q-matrix values or “strange” news decomposition outputs.
 
@@ -1190,7 +1148,7 @@ DATA_VIEW_SOURCE=file  # or database
 - ✅ **Model Result Storage**: File-based and SQLite adapters for saving/loading model results
 - ✅ **Data View Management**: Time-point specific data views for pseudo real-time evaluation
 - ✅ **Helper Function Consolidation**: Reduced code duplication with consolidated helper functions
-  - `to_python_datetime()`: Generic datetime conversion (handles polars datetime, strings, etc.)
+  - `to_python_datetime()`: Generic datetime conversion (handles pandas Timestamp, strings, etc.)
   - `extract_last_date()`: Generic time index extraction
   - `clock_to_datetime_freq()`: Shared clock-to-datetime frequency mapping
   - `get_periods_per_year()`: Generic periods-per-year calculation

dfm_python-0.4.5/config/README.md

@@ -0,0 +1,174 @@
+# Configuration Structure
+
+This directory contains all configuration files for the Nowcasting system. The configuration follows a **unified structure** where experiment configs contain all necessary settings including preprocessing and series configurations.
+
+## Directory Structure
+
+```
+config/
+├── experiment/     # Experiment configurations (unified structure)
+├── model/         # Model-specific hyperparameters
+├── preprocess/    # Preprocessing templates (for reference)
+└── series/        # Series configurations (legacy, now embedded in experiments)
+```
+
+## Unified Configuration Structure
+
+### Experiment Configs (`config/experiment/`)
+
+Experiment configs now contain **all** configuration in a single file:
+
+```yaml
+# Experiment Metadata
+name: experiment_name
+description: "Experiment description"
+model_type: dfm  # dfm, ddfm, arma, tft, xgboost
+
+# Preprocess Configuration
+preprocess:
+  metadata:
+    data_path: data/sample_data.csv
+    target_column: KOGDP...D
+    exclude_columns: [date]
+    sample_start: null
+    sample_end: null
+  
+  data_quality:
+    remove_sparse_rows: true
+    sparse_threshold: 0.8
+    clip_outliers: true
+    outlier_std: 3.0
+  
+  feature_engineering:
+    window_summarizer:
+      enabled: true
+      lags: [1, 2, 3, 5, 10]
+      windows: [5, 10, 20, 40]
+      functions: [mean, std, min, max, median, skew, kurt]
+  
+  global_preprocessing:
+    imputation:
+      method: ffill_bfill  # For non-DFM/DDFM models
+    scaling:
+      enabled: true
+      type: robust
+    transformations:
+      enabled: []
+
+# Series Configuration
+series:
+  - series_id: KO3YEARC
+    frequency: m
+    transformation: chg
+    blocks: [Block_Global, Block_Investment]
+    release: 1
+    scaler: null  # null = use global scaler
+    impute: null  # null = use global or Kalman filter (DFM/DDFM)
+```
+
+### Key Features
+
+1. **Unified Structure**: All experiment settings in one file
+2. **Conditional Imputation**: 
+   - DFM/DDFM: Uses Kalman filter (built-in, no explicit imputation needed)
+   - Other models: Uses explicit imputation (ffill_bfill, mean, median, forecaster)
+3. **Per-Series Overrides**: Each series can override global scaler/impute settings
+4. **Window Summarizer**: Automatic feature engineering with lags and rolling windows
+
+## Model Configs (`config/model/`)
+
+Model-specific hyperparameters:
+- `dfm.yaml` - DFM model parameters
+- `ddfm.yaml` - DDFM model parameters
+- `arma.yaml` - ARMA model parameters
+- `tft.yaml` - Temporal Fusion Transformer parameters
+- `xgboost.yaml` - XGBoost parameters
+
+## Preprocess Configs (`config/preprocess/`)
+
+**Note**: These are now templates/reference files. In the new structure, preprocessing is embedded in experiment configs. These files are kept for:
+- Reference and documentation
+- Backward compatibility
+- Template generation
+
+## Series Configs (`config/series/`)
+
+**Note**: These are legacy files. In the new structure, series configurations are embedded in experiment configs. These files are kept for:
+- Reference
+- Migration purposes
+- Template generation
+
+## Configuration Hierarchy
+
+1. **Experiment Config** (primary) - Contains all settings
+2. **Model Config** (referenced) - Model-specific hyperparameters
+3. **Preprocess Config** (legacy) - Reference templates only
+4. **Series Config** (legacy) - Reference templates only
+
+## Migration Guide
+
+### Old Structure
+```yaml
+# experiment/{id}.yaml
+data:
+  path: data/sample_data.csv
+preprocessing:
+  config_path: config/preprocess/{id}.yaml
+series: [KO3YEARC, ...]
+
+# preprocess/{id}.yaml (separate file)
+imputation:
+  method: ffill_bfill
+scaling:
+  type: robust
+
+# series/{series_id}.yaml (separate files)
+frequency: m
+transformation: chg
+blocks: [Block_Global]
+```
+
+### New Structure
+```yaml
+# experiment/{id}.yaml (unified)
+preprocess:
+  global_preprocessing:
+    imputation:
+      method: ffill_bfill
+    scaling:
+      type: robust
+series:
+  - series_id: KO3YEARC
+    frequency: m
+    transformation: chg
+    blocks: [Block_Global]
+```
+
+## Best Practices
+
+1. **Use Experiment Configs**: All new experiments should use the unified structure
+2. **Model Type Awareness**: 
+   - DFM/DDFM: Imputation handled by Kalman filter
+   - Other models: Specify explicit imputation method
+3. **Per-Series Overrides**: Use sparingly, prefer global settings
+4. **Window Summarizer**: Enable for feature-rich models (XGBoost, TFT)
+5. **Scaling**: Use RobustScaler for economic data with outliers
+
+## UI Integration
+
+The UI left sidebar displays the config tree:
+```
+Experiment
+├── Name, Model Type, Description
+├── Preprocess
+│   ├── Metadata (data path, target, exclude columns)
+│   ├── Data Quality
+│   ├── Feature Engineering (Window Summarizer)
+│   └── Global Preprocessing (imputation, scaling)
+└── Series
+    ├── KO3YEARC (frequency, transformation, blocks, release, scaler, impute)
+    └── [Other series...]
+```
+
+Right side: Training progress and monitoring.
+

dfm_python-0.4.5/config/experiment/default.yaml

@@ -0,0 +1,82 @@
+# Default Experiment Configuration
+# Unified experiment config with embedded preprocessing and series configurations
+
+# Hydra defaults - specify which model config to use
+defaults:
+  - model: dfm  # Options: dfm, ddfm, arma, tft, xgboost
+  - _self_
+
+# ============================================================================
+# Experiment Metadata
+# ============================================================================
+name: default_experiment
+description: "Default experiment configuration template"
+model_type: dfm  # dfm, ddfm, arma, tft, xgboost
+
+# ============================================================================
+# Preprocess Configuration
+# ============================================================================
+preprocess:
+  # Metadata
+  metadata:
+    data_path: data/sample_data.csv  # Path to input data file
+    target_column: null  # Target series for nowcasting/forecasting (e.g., KOGDP...D)
+    # Window Summarizer features (add_feature)
+    add_feature:  # Features to add via Window Summarizer
+      - lag      # Lag features
+      - mean     # Rolling mean
+      - std      # Rolling standard deviation
+      - min      # Rolling minimum
+      - max      # Rolling maximum
+      - median   # Rolling median
+      - skew     # Rolling skewness
+      - kurt     # Rolling kurtosis
+    # Window Summarizer parameters
+    lags: [1, 2, 3, 5, 10]  # Lag values: t-1, t-2, t-3, t-5, t-10
+    windows: [5, 10, 20, 40]  # Rolling window sizes
+  
+  # Series Configuration
+  # All series with their individual configurations
+  series:
+    # If empty, uses all columns from data (except excluded columns)
+    # Example series configurations:
+    # - series_id: KO3YEARC
+    #   frequency: m  # Options: d, w, m, q, sa, a
+    #   transformation: chg  # Options: lin, pch, pca, chg, ch1, cha
+    #   block:  # Block names this series loads on (must include Block_Global)
+    #     - Block_Global
+    #     - Block_Investment
+    #   release: 1  # Day of month when data is released (1-31, or negative for days before month end)
+    #   scaler: null  # Per-series scaler (StandardScaler, RobustScaler, None/null)
+    #                # null = use default (ffill and bfill for non-DFM/DDFM, Kalman filter for DFM/DDFM)
+    #   impute: null  # Per-series imputation (bfill, ffill, forecaster, dfm, ddfm)
+    #                # bfill/ffill: Backward/forward fill
+    #                # forecaster: Forecaster-based imputation
+    #                # dfm/ddfm: Use model's built-in Kalman filter imputation (on the fly)
+    #                # null: Use default (ffill and bfill for non-DFM/DDFM, Kalman filter for DFM/DDFM)
+
+# ============================================================================
+# Model Training Parameters
+# ============================================================================
+# These override model config defaults if specified
+max_iter: 5000      # Maximum EM iterations (for DFM)
+threshold: 1.0e-05  # Convergence threshold (for DFM)
+
+# For DDFM models:
+# encoder_layers: [64, 32]
+# num_factors: 2
+# epochs: 100
+# learning_rate: 0.001
+# batch_size: 32
+
+# ============================================================================
+# Output Configuration
+# ============================================================================
+output:
+  path: outputs/${experiment.name}  # Output directory (uses experiment name)
+  save_model: true      # Save trained model
+  save_plots: true      # Save visualization plots
+  save_results: true    # Save training results
+
+# Forecast configuration (optional)
+forecast_horizon: null  # Number of periods ahead to forecast