dfm-python 0.4.8__py3-none-any.whl → 0.4.51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dfm_python/__init__.py +18 -4
- dfm_python/config/adapter.py +2 -1
- dfm_python/config/schema.py +13 -8
- dfm_python/data/dataloader.py +3 -3
- dfm_python/data/utils.py +1 -1
- dfm_python/encoder/vae.py +0 -15
- dfm_python/lightning/__init__.py +5 -2
- dfm_python/lightning/data_module.py +24 -67
- dfm_python/lightning/scaling.py +26 -32
- dfm_python/logger/inference_logger.py +1 -1
- dfm_python/logger/logger.py +1 -1
- dfm_python/logger/train_logger.py +1 -1
- dfm_python/models/base.py +2 -0
- dfm_python/models/ddfm.py +300 -424
- dfm_python/models/dfm.py +6 -103
- dfm_python/nowcast/__init__.py +37 -0
- dfm_python/nowcast/dataview.py +143 -0
- dfm_python/nowcast/helpers.py +356 -0
- dfm_python/nowcast/nowcast.py +1263 -0
- dfm_python/nowcast/splitters.py +441 -0
- dfm_python/nowcast/transformers.py +330 -0
- dfm_python/nowcast/utils.py +285 -0
- dfm_python/ssm/em.py +53 -565
- dfm_python/ssm/kalman.py +27 -350
- dfm_python/ssm/utils.py +15 -63
- dfm_python/trainer/__init__.py +3 -10
- dfm_python/trainer/ddfm.py +4 -5
- dfm_python/trainer/dfm.py +3 -3
- dfm_python/utils/__init__.py +14 -28
- dfm_python/utils/data.py +3 -63
- dfm_python/utils/time.py +3 -3
- {dfm_python-0.4.8.dist-info → dfm_python-0.4.51.dist-info}/METADATA +110 -9
- dfm_python-0.4.51.dist-info/RECORD +49 -0
- dfm_python-0.4.8.dist-info/RECORD +0 -42
- {dfm_python-0.4.8.dist-info → dfm_python-0.4.51.dist-info}/WHEEL +0 -0
- {dfm_python-0.4.8.dist-info → dfm_python-0.4.51.dist-info}/licenses/LICENSE +0 -0
dfm_python/__init__.py
CHANGED
|
@@ -6,6 +6,7 @@ This package implements a comprehensive Dynamic Factor Model framework with supp
|
|
|
6
6
|
- Tent kernel aggregation for low-to-high frequency mapping
|
|
7
7
|
- Expectation-Maximization (EM) algorithm for parameter estimation
|
|
8
8
|
- Kalman filtering and smoothing for factor extraction
|
|
9
|
+
- News decomposition for nowcasting
|
|
9
10
|
- Deep Dynamic Factor Models (DDFM) with nonlinear encoders (requires PyTorch)
|
|
10
11
|
|
|
11
12
|
The package implements a clock-based approach to mixed-frequency DFMs, where all latent
|
|
@@ -22,6 +23,7 @@ Key Features:
|
|
|
22
23
|
- Flexible block structure for factor modeling
|
|
23
24
|
- Robust handling of missing data (internal spline interpolation)
|
|
24
25
|
- Automatic standardization and data clipping
|
|
26
|
+
- News decomposition for forecast updates
|
|
25
27
|
|
|
26
28
|
Example (Standard Lightning Pattern):
|
|
27
29
|
>>> from dfm_python import DFM, DFMDataModule, DFMTrainer
|
|
@@ -65,7 +67,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
|
|
|
65
67
|
For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
|
|
66
68
|
"""
|
|
67
69
|
|
|
68
|
-
__version__ = "0.4.
|
|
70
|
+
__version__ = "0.4.51"
|
|
69
71
|
|
|
70
72
|
# ============================================================================
|
|
71
73
|
# PUBLIC API DEFINITION
|
|
@@ -79,7 +81,7 @@ __version__ = "0.4.8"
|
|
|
79
81
|
# 2. High-level API: DFM, DDFM, module-level convenience functions
|
|
80
82
|
# 3. Core utilities: TimeIndex, diagnostics
|
|
81
83
|
# 4. Models: BaseFactorModel, DDFM (low-level)
|
|
82
|
-
# 5.
|
|
84
|
+
# 5. Nowcasting: Nowcast, result classes, para_const
|
|
83
85
|
# 6. Data & Results: DFMResult
|
|
84
86
|
# ============================================================================
|
|
85
87
|
|
|
@@ -108,7 +110,16 @@ from .lightning import (
|
|
|
108
110
|
EMAlgorithm, # Module class
|
|
109
111
|
)
|
|
110
112
|
|
|
111
|
-
# Nowcasting
|
|
113
|
+
# Nowcasting (from nowcast/ subpackage)
|
|
114
|
+
from .nowcast import (
|
|
115
|
+
Nowcast,
|
|
116
|
+
NowcastResult,
|
|
117
|
+
)
|
|
118
|
+
from .nowcast.helpers import (
|
|
119
|
+
para_const,
|
|
120
|
+
NewsDecompResult,
|
|
121
|
+
BacktestResult,
|
|
122
|
+
)
|
|
112
123
|
|
|
113
124
|
# Model implementations
|
|
114
125
|
from .models.base import BaseFactorModel
|
|
@@ -119,9 +130,11 @@ from .models.ddfm import DDFM, DDFMModel
|
|
|
119
130
|
|
|
120
131
|
__all__ = [
|
|
121
132
|
# Core classes
|
|
122
|
-
'DFM',
|
|
133
|
+
'DFM', 'Nowcast',
|
|
123
134
|
# Model base and implementations
|
|
124
135
|
'BaseFactorModel',
|
|
136
|
+
# Nowcast result classes
|
|
137
|
+
'NowcastResult', 'NewsDecompResult', 'BacktestResult',
|
|
125
138
|
# Constants
|
|
126
139
|
'DEFAULT_BLOCK_NAME',
|
|
127
140
|
# Config sources
|
|
@@ -129,6 +142,7 @@ __all__ = [
|
|
|
129
142
|
'MergedConfigSource', 'make_config_source',
|
|
130
143
|
# Low-level API (functional interface - advanced usage)
|
|
131
144
|
'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
|
|
145
|
+
'para_const', # Internal utility for nowcasting
|
|
132
146
|
]
|
|
133
147
|
|
|
134
148
|
# DDFM high-level API (PyTorch is mandatory)
|
dfm_python/config/adapter.py
CHANGED
|
@@ -214,7 +214,8 @@ class YamlSource:
|
|
|
214
214
|
frequency=series_data.get('frequency', 'm'),
|
|
215
215
|
transformation=series_data.get('transformation', 'lin'),
|
|
216
216
|
blocks=series_data.get('blocks', []),
|
|
217
|
-
release_date=release_date
|
|
217
|
+
release_date=release_date,
|
|
218
|
+
scaler=series_data.get('scaler', None) # Optional, for per-series scaling
|
|
218
219
|
))
|
|
219
220
|
|
|
220
221
|
# If no series loaded from separate files, try to get from main config
|
dfm_python/config/schema.py
CHANGED
|
@@ -200,6 +200,11 @@ class SeriesConfig:
|
|
|
200
200
|
- Positive value (1-31): Day of month when data is released
|
|
201
201
|
- Negative value: Days before end of previous month when data is released
|
|
202
202
|
Example: 25 = released on 25th of each month, -5 = released 5 days before end of previous month
|
|
203
|
+
scaler : str, optional
|
|
204
|
+
Scaler type for this series: 'standard', 'robust', 'minmax', 'maxabs',
|
|
205
|
+
'quantile', or None/null (no scaling). Used by create_scaling_transformer_from_config()
|
|
206
|
+
to automatically create per-series scaling transformers. If None, uses default
|
|
207
|
+
scaling strategy (typically 'standard').
|
|
203
208
|
"""
|
|
204
209
|
# Required fields (no defaults)
|
|
205
210
|
frequency: str
|
|
@@ -210,6 +215,7 @@ class SeriesConfig:
|
|
|
210
215
|
series_name: Optional[str] = None # Optional metadata for display
|
|
211
216
|
units: Optional[str] = None # Optional metadata for display only (used in news.py output)
|
|
212
217
|
release_date: Optional[int] = None # Release date for pseudo real-time nowcasting
|
|
218
|
+
scaler: Optional[str] = None # Scaler type: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or None
|
|
213
219
|
|
|
214
220
|
def __post_init__(self):
|
|
215
221
|
"""Validate fields after initialization."""
|
|
@@ -302,7 +308,6 @@ class BaseModelConfig:
|
|
|
302
308
|
nan_method: int = 2 # Missing data handling method (1-5). Preprocessing step before Kalman Filter-based handling
|
|
303
309
|
nan_k: int = 3 # Spline parameter for NaN interpolation (cubic spline)
|
|
304
310
|
clock: str = 'm' # Base frequency for nowcasting (global clock): 'd', 'w', 'm', 'q', 'sa', 'a' (defaults to 'm' for monthly)
|
|
305
|
-
scaler: Optional[str] = 'standard' # Unified scaler type for all series: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or None (no scaling). Default: 'standard' for unified scaling.
|
|
306
311
|
|
|
307
312
|
# ========================================================================
|
|
308
313
|
# Internal cache (not user-configurable)
|
|
@@ -655,7 +660,6 @@ class DFMConfig(BaseModelConfig):
|
|
|
655
660
|
'nan_method': data.get('nan_method', 2),
|
|
656
661
|
'nan_k': data.get('nan_k', 3),
|
|
657
662
|
'clock': data.get('clock', 'm'),
|
|
658
|
-
'scaler': data.get('scaler', 'standard'), # Unified scaler for all series (default: 'standard')
|
|
659
663
|
}
|
|
660
664
|
|
|
661
665
|
@classmethod
|
|
@@ -712,11 +716,11 @@ class DDFMConfig(BaseModelConfig):
|
|
|
712
716
|
# ========================================================================
|
|
713
717
|
encoder_layers: Optional[List[int]] = None # Hidden layer dimensions for encoder (default: [64, 32])
|
|
714
718
|
num_factors: Optional[int] = None # Number of factors (inferred from config if None)
|
|
715
|
-
activation: str = '
|
|
719
|
+
activation: str = 'tanh' # Activation function ('tanh', 'relu', 'sigmoid', default: 'tanh')
|
|
716
720
|
use_batch_norm: bool = True # Use batch normalization in encoder (default: True)
|
|
717
721
|
learning_rate: float = 0.001 # Learning rate for Adam optimizer (default: 0.001)
|
|
718
722
|
epochs: int = 100 # Number of training epochs (default: 100)
|
|
719
|
-
batch_size: int =
|
|
723
|
+
batch_size: int = 32 # Batch size for training (default: 32)
|
|
720
724
|
factor_order: int = 1 # VAR lag order for factor dynamics (1 or 2, default: 1)
|
|
721
725
|
use_idiosyncratic: bool = True # Model idio components with AR(1) dynamics (default: True)
|
|
722
726
|
min_obs_idio: int = 5 # Minimum observations for idio AR(1) estimation (default: 5)
|
|
@@ -746,15 +750,15 @@ class DDFMConfig(BaseModelConfig):
|
|
|
746
750
|
def _extract_ddfm(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
747
751
|
"""Extract DDFM-specific parameters from config dict."""
|
|
748
752
|
base_params = cls._extract_base(data)
|
|
749
|
-
# Handle both direct keys and ddfm_ prefix format
|
|
753
|
+
# Handle both direct keys and ddfm_ prefix format (for backward compatibility)
|
|
750
754
|
base_params.update({
|
|
751
755
|
'encoder_layers': data.get('encoder_layers') or data.get('ddfm_encoder_layers', None),
|
|
752
756
|
'num_factors': data.get('num_factors') or data.get('ddfm_num_factors', None),
|
|
753
|
-
'activation': data.get('activation') or data.get('ddfm_activation', '
|
|
757
|
+
'activation': data.get('activation') or data.get('ddfm_activation', 'tanh'),
|
|
754
758
|
'use_batch_norm': data.get('use_batch_norm', data.get('ddfm_use_batch_norm', True)),
|
|
755
759
|
'learning_rate': data.get('learning_rate', data.get('ddfm_learning_rate', 0.001)),
|
|
756
760
|
'epochs': data.get('epochs', data.get('ddfm_epochs', 100)),
|
|
757
|
-
'batch_size': data.get('batch_size', data.get('ddfm_batch_size',
|
|
761
|
+
'batch_size': data.get('batch_size', data.get('ddfm_batch_size', 32)),
|
|
758
762
|
'factor_order': data.get('factor_order', data.get('ddfm_factor_order', 1)),
|
|
759
763
|
'use_idiosyncratic': data.get('use_idiosyncratic', data.get('ddfm_use_idiosyncratic', True)),
|
|
760
764
|
'min_obs_idio': data.get('min_obs_idio', data.get('ddfm_min_obs_idio', 5)),
|
|
@@ -828,7 +832,8 @@ class DDFMConfig(BaseModelConfig):
|
|
|
828
832
|
transformation=series_cfg.get('transformation', 'lin'),
|
|
829
833
|
blocks=series_blocks,
|
|
830
834
|
units=series_cfg.get('units', None), # Optional, for display only
|
|
831
|
-
release_date=series_cfg.get('release_date', None) # Optional, for nowcasting
|
|
835
|
+
release_date=series_cfg.get('release_date', None), # Optional, for nowcasting
|
|
836
|
+
scaler=series_cfg.get('scaler', None) # Optional, for per-series scaling
|
|
832
837
|
))
|
|
833
838
|
|
|
834
839
|
# Convert blocks_dict to dict of block properties
|
dfm_python/data/dataloader.py
CHANGED
|
@@ -101,7 +101,7 @@ def create_dfm_dataloader(
|
|
|
101
101
|
|
|
102
102
|
def create_ddfm_dataloader(
|
|
103
103
|
dataset: DDFMDataset,
|
|
104
|
-
batch_size: int =
|
|
104
|
+
batch_size: int = 32,
|
|
105
105
|
shuffle: bool = True,
|
|
106
106
|
num_workers: int = 0,
|
|
107
107
|
pin_memory: bool = True
|
|
@@ -115,8 +115,8 @@ def create_ddfm_dataloader(
|
|
|
115
115
|
----------
|
|
116
116
|
dataset : DDFMDataset
|
|
117
117
|
DDFM dataset instance with windowed sequences
|
|
118
|
-
batch_size : int, default
|
|
119
|
-
Batch size for training
|
|
118
|
+
batch_size : int, default 32
|
|
119
|
+
Batch size for training
|
|
120
120
|
shuffle : bool, default True
|
|
121
121
|
Whether to shuffle samples (typically True for training)
|
|
122
122
|
num_workers : int, default 0
|
dfm_python/data/utils.py
CHANGED
|
@@ -209,7 +209,7 @@ def load_data(
|
|
|
209
209
|
Time : TimeIndex
|
|
210
210
|
Time index for the data (aligned to clock frequency)
|
|
211
211
|
Z : np.ndarray
|
|
212
|
-
Original untransformed data (T x N), same as X
|
|
212
|
+
Original untransformed data (T x N), same as X (for backward compatibility)
|
|
213
213
|
|
|
214
214
|
Raises
|
|
215
215
|
------
|
dfm_python/encoder/vae.py
CHANGED
|
@@ -454,21 +454,6 @@ def extract_decoder_params(decoder) -> Tuple[np.ndarray, np.ndarray]:
|
|
|
454
454
|
# Decoder weight is already (N x m), so no transpose needed
|
|
455
455
|
C = weight
|
|
456
456
|
|
|
457
|
-
# Check for NaN in extracted C matrix (indicates numerical instability during training)
|
|
458
|
-
if np.any(np.isnan(C)):
|
|
459
|
-
nan_count = np.sum(np.isnan(C))
|
|
460
|
-
nan_ratio = nan_count / C.size
|
|
461
|
-
from ..logger import get_logger
|
|
462
|
-
_logger = get_logger(__name__)
|
|
463
|
-
_logger.warning(
|
|
464
|
-
f"extract_decoder_params: C matrix contains {nan_count}/{C.size} NaN values ({nan_ratio:.1%}). "
|
|
465
|
-
f"This indicates the decoder weights contain NaN, likely due to numerical instability during training. "
|
|
466
|
-
f"Possible causes: (1) learning rate too high, (2) gradient explosion, (3) unstable training."
|
|
467
|
-
)
|
|
468
|
-
# Replace NaN with zeros to prevent further issues
|
|
469
|
-
C = np.nan_to_num(C, nan=0.0)
|
|
470
|
-
_logger.warning("Replaced NaN values in C matrix with zeros.")
|
|
471
|
-
|
|
472
457
|
return C, bias
|
|
473
458
|
|
|
474
459
|
|
dfm_python/lightning/__init__.py
CHANGED
|
@@ -28,14 +28,15 @@ from .scaling import (
|
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
# DFMTrainingState is defined in models.dfm and exported here for convenience
|
|
31
|
+
# Note: dfm_module.py has been removed - DFM class implements EM directly
|
|
31
32
|
from ..models.dfm import DFMTrainingState
|
|
32
33
|
|
|
33
34
|
__all__ = [
|
|
34
35
|
# Kalman filter
|
|
35
|
-
'KalmanFilter',
|
|
36
|
+
'KalmanFilter', # New module class
|
|
36
37
|
'KalmanFilterState',
|
|
37
38
|
# EM algorithm
|
|
38
|
-
'EMAlgorithm',
|
|
39
|
+
'EMAlgorithm', # New module class
|
|
39
40
|
'EMStepParams',
|
|
40
41
|
# Data handling
|
|
41
42
|
'DFMDataModule',
|
|
@@ -49,5 +50,7 @@ __all__ = [
|
|
|
49
50
|
'NoScalingStrategy',
|
|
50
51
|
# Training state (defined in models.dfm)
|
|
51
52
|
'DFMTrainingState',
|
|
53
|
+
# DDFMLightningModule removed: deprecated and unused (DDFM uses DDFMModel directly)
|
|
54
|
+
# DDFMTrainingState removed: duplicate (also defined in models.ddfm)
|
|
52
55
|
]
|
|
53
56
|
|
|
@@ -43,11 +43,13 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
|
|
|
43
43
|
|
|
44
44
|
This function depends on sktime's pipeline structure to traverse
|
|
45
45
|
TransformerPipeline and sklearn transformers (StandardScaler, etc.).
|
|
46
|
+
Note: TabularToSeriesAdaptor support is for backward compatibility only.
|
|
46
47
|
|
|
47
48
|
Parameters
|
|
48
49
|
----------
|
|
49
50
|
transformer : Any
|
|
50
|
-
Transformer to search (
|
|
51
|
+
Transformer to search (can be any scaler, StandardScaler directly, or TabularToSeriesAdaptor for backward compatibility).
|
|
52
|
+
TransformerPipeline, sklearn Pipeline, etc.)
|
|
51
53
|
|
|
52
54
|
Returns
|
|
53
55
|
-------
|
|
@@ -58,7 +60,9 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
|
|
|
58
60
|
if transformer is None:
|
|
59
61
|
return None
|
|
60
62
|
|
|
61
|
-
# Check if transformer is
|
|
63
|
+
# Check if transformer is TabularToSeriesAdaptor (backward compatibility only)
|
|
64
|
+
# Note: New code should use StandardScaler() directly per sktime docs
|
|
65
|
+
# This check is kept for backward compatibility with old code that may still use TabularToSeriesAdaptor
|
|
62
66
|
if hasattr(transformer, 'transformer'):
|
|
63
67
|
# Recursively search the wrapped transformer
|
|
64
68
|
wrapped = transformer.transformer
|
|
@@ -74,7 +78,8 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
|
|
|
74
78
|
if scaler is not None:
|
|
75
79
|
return scaler
|
|
76
80
|
|
|
77
|
-
# Check if transformer is
|
|
81
|
+
# Check if transformer is ColumnTransformer (backward compatibility only - not recommended for new code)
|
|
82
|
+
# Unified scaling with StandardScaler is preferred per sktime docs
|
|
78
83
|
if hasattr(transformer, 'transformers'):
|
|
79
84
|
for name, trans, cols in transformer.transformers:
|
|
80
85
|
scaler = _get_scaler(trans)
|
|
@@ -256,17 +261,15 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
256
261
|
|
|
257
262
|
This DataModule handles data loading for DFM/DDFM models.
|
|
258
263
|
|
|
259
|
-
**Important**:
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
264
|
+
**Important**: This package expects preprocessed data. All preprocessing (imputation,
|
|
265
|
+
scaling, feature engineering, etc.) must be done by the user before passing data to
|
|
266
|
+
this DataModule. Users are responsible for creating their own preprocessing pipelines
|
|
267
|
+
using sktime or other tools.
|
|
263
268
|
|
|
264
269
|
**Usage Pattern**:
|
|
265
|
-
-
|
|
270
|
+
- Provide already-preprocessed data (standardized, no missing values, feature-engineered)
|
|
266
271
|
- If `pipeline=None`, a passthrough transformer is used by default (no-op)
|
|
267
272
|
- Users can optionally provide their preprocessing pipeline to extract statistics (Mx/Wx)
|
|
268
|
-
- For better performance, users can preprocess data (imputation, scaling) before passing,
|
|
269
|
-
but it's not required - models will handle missing data automatically
|
|
270
273
|
|
|
271
274
|
For linear DFM, this uses DFMDataset which returns full sequences.
|
|
272
275
|
For DDFM, use DDFMDataModule which uses DDFMDataset with windowing.
|
|
@@ -308,10 +311,9 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
308
311
|
data_path : str or Path, optional
|
|
309
312
|
Path to data file (CSV). If None, data must be provided.
|
|
310
313
|
data : np.ndarray or pd.DataFrame, optional
|
|
311
|
-
|
|
312
|
-
-
|
|
313
|
-
-
|
|
314
|
-
- Standardized/scaled data (mean=0, std=1) is recommended for better performance
|
|
314
|
+
Preprocessed data array or DataFrame. Data must be:
|
|
315
|
+
- Standardized/scaled (mean=0, std=1 recommended for DFM/DDFM)
|
|
316
|
+
- No missing values (NaN-free)
|
|
315
317
|
- Feature-engineered if needed
|
|
316
318
|
If None, data_path must be provided.
|
|
317
319
|
time_index : TimeIndex, optional
|
|
@@ -415,22 +417,20 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
415
417
|
This method is called by Lightning to set up the data module.
|
|
416
418
|
It loads preprocessed data and extracts statistics from the pipeline for forecasting/nowcasting.
|
|
417
419
|
|
|
418
|
-
**Important**:
|
|
419
|
-
|
|
420
|
-
- **DDFM**: Uses state-space model and MCMC procedure
|
|
420
|
+
**Important**: Data must be fully preprocessed by the user before passing to this DataModule.
|
|
421
|
+
All preprocessing (imputation, scaling, feature engineering, etc.) is the user's responsibility.
|
|
421
422
|
|
|
422
423
|
**Pipeline Purpose**:
|
|
423
424
|
- The pipeline is used to extract statistics (e.g., Mx/Wx from StandardScaler) needed for
|
|
424
425
|
forecasting and nowcasting operations, not to preprocess the data
|
|
426
|
+
- Data is already preprocessed - the pipeline is fitted to extract standardization parameters
|
|
425
427
|
- The pipeline is fitted on the **entire dataset** to extract statistics
|
|
426
428
|
- If `val_split` is provided, it's used for validation during training (e.g., DDFM neural network)
|
|
427
429
|
|
|
428
430
|
**Usage**:
|
|
429
|
-
-
|
|
431
|
+
- Provide preprocessed data (standardized, no missing values)
|
|
430
432
|
- Optionally provide pipeline (e.g., TransformerPipeline with StandardScaler) to extract Mx/Wx for predictions
|
|
431
433
|
- If pipeline=None, uses passthrough (no statistics extracted, Mx/Wx computed from data)
|
|
432
|
-
- For better performance, users can preprocess data (imputation, scaling) before passing,
|
|
433
|
-
but it's not required
|
|
434
434
|
"""
|
|
435
435
|
# Load data if not already provided
|
|
436
436
|
if self.data is None:
|
|
@@ -451,9 +451,6 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
451
451
|
self.time_index = Time
|
|
452
452
|
|
|
453
453
|
# Convert to pandas DataFrame if needed
|
|
454
|
-
# Note: DFM/DDFM can handle NaN values implicitly via Kalman filter
|
|
455
|
-
# (DFM uses handle_missing_data() in Kalman filter, DDFM uses state-space model)
|
|
456
|
-
# So we don't validate NaN here - let the models handle it
|
|
457
454
|
if isinstance(self.data, np.ndarray):
|
|
458
455
|
series_ids = self.config.get_series_ids()
|
|
459
456
|
X_df = pd.DataFrame(self.data, columns=series_ids)
|
|
@@ -508,10 +505,7 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
508
505
|
f"automatically extracted from the pipeline."
|
|
509
506
|
) from e
|
|
510
507
|
|
|
511
|
-
#
|
|
512
|
-
# So we don't validate NaN here - let the models handle it
|
|
513
|
-
|
|
514
|
-
# Ensure output is pandas DataFrame with compatible index
|
|
508
|
+
# Ensure output is pandas DataFrame
|
|
515
509
|
if not isinstance(X_transformed, pd.DataFrame):
|
|
516
510
|
# If transformer returned numpy array, convert to pandas
|
|
517
511
|
if isinstance(X_transformed, np.ndarray):
|
|
@@ -538,45 +532,6 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
538
532
|
# Create generic column names
|
|
539
533
|
X_transformed = pd.DataFrame(X_transformed,
|
|
540
534
|
columns=[f'feature_{i}' for i in range(n_cols)])
|
|
541
|
-
|
|
542
|
-
# Ensure index is compatible (DatetimeIndex, PeriodIndex, or RangeIndex)
|
|
543
|
-
# Use input DataFrame index if length matches
|
|
544
|
-
if len(X_transformed) == len(X_df):
|
|
545
|
-
if isinstance(X_df.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
|
|
546
|
-
X_transformed.index = X_df.index
|
|
547
|
-
else:
|
|
548
|
-
# Try to convert to DatetimeIndex
|
|
549
|
-
try:
|
|
550
|
-
X_transformed.index = pd.to_datetime(X_df.index)
|
|
551
|
-
except (ValueError, TypeError):
|
|
552
|
-
# Fallback to RangeIndex
|
|
553
|
-
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
554
|
-
else:
|
|
555
|
-
# Length doesn't match - use RangeIndex
|
|
556
|
-
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
557
|
-
|
|
558
|
-
# Ensure DataFrame index is compatible (DatetimeIndex, PeriodIndex, or RangeIndex)
|
|
559
|
-
# This handles cases where pipeline output has incompatible index
|
|
560
|
-
if isinstance(X_transformed, pd.DataFrame):
|
|
561
|
-
if not isinstance(X_transformed.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
|
|
562
|
-
# Try to use input DataFrame index if length matches
|
|
563
|
-
if len(X_transformed) == len(X_df):
|
|
564
|
-
if isinstance(X_df.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
|
|
565
|
-
X_transformed.index = X_df.index
|
|
566
|
-
else:
|
|
567
|
-
# Try to convert to DatetimeIndex
|
|
568
|
-
try:
|
|
569
|
-
X_transformed.index = pd.to_datetime(X_df.index)
|
|
570
|
-
except (ValueError, TypeError):
|
|
571
|
-
# Fallback to RangeIndex
|
|
572
|
-
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
573
|
-
else:
|
|
574
|
-
# Try to convert existing index
|
|
575
|
-
try:
|
|
576
|
-
X_transformed.index = pd.to_datetime(X_transformed.index)
|
|
577
|
-
except (ValueError, TypeError):
|
|
578
|
-
# Fallback to RangeIndex
|
|
579
|
-
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
580
535
|
elif hasattr(X_transformed, 'to_numpy'):
|
|
581
536
|
# Has to_numpy method (pandas DataFrame) - convert to numpy then back to DataFrame
|
|
582
537
|
# Try to preserve column names if available
|
|
@@ -617,7 +572,9 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
617
572
|
# This is optional - some pipelines may not have standardization
|
|
618
573
|
# Mx and Wx are already initialized in __init__
|
|
619
574
|
try:
|
|
620
|
-
# Extract scaler from pipeline (
|
|
575
|
+
# Extract scaler from pipeline (works with TransformerPipeline, StandardScaler directly, or TabularToSeriesAdaptor for backward compatibility)
|
|
576
|
+
# Supports any scaler type: StandardScaler, MinMaxScaler, RobustScaler, etc.
|
|
577
|
+
# Note: New code should use StandardScaler() directly per sktime docs (unified scaling)
|
|
621
578
|
scaler = _get_scaler(pipeline_to_use)
|
|
622
579
|
|
|
623
580
|
if scaler is not None:
|
dfm_python/lightning/scaling.py
CHANGED
|
@@ -63,7 +63,7 @@ class ScalingStrategy(Protocol):
|
|
|
63
63
|
class MyScalingStrategy:
|
|
64
64
|
def get_scaler(self, series: SeriesConfig, series_index: int,
|
|
65
65
|
column_name: str) -> Optional[str]:
|
|
66
|
-
# Custom logic (unified scaling
|
|
66
|
+
# Custom logic (not recommended - use unified scaling instead)
|
|
67
67
|
if 'volatility' in column_name.lower():
|
|
68
68
|
return 'robust'
|
|
69
69
|
elif 'return' in column_name.lower():
|
|
@@ -191,25 +191,23 @@ def create_scaling_transformer_from_config(
|
|
|
191
191
|
strategy: Optional[ScalingStrategy] = None,
|
|
192
192
|
column_names: Optional[List[str]] = None
|
|
193
193
|
) -> Any:
|
|
194
|
-
"""Create a unified scaling transformer
|
|
194
|
+
"""Create a unified scaling transformer (StandardScaler for all series).
|
|
195
195
|
|
|
196
|
-
This function returns a raw sklearn
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
TransformerPipeline.
|
|
196
|
+
This function returns a raw sklearn StandardScaler that can be used directly
|
|
197
|
+
in TransformerPipeline. Per sktime docs, sklearn transformers are automatically
|
|
198
|
+
applied per series instance when used in TransformerPipeline.
|
|
200
199
|
|
|
201
|
-
**
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
as it ensures all series contribute proportionally to factor extraction.
|
|
200
|
+
**Default behavior**: Unified scaling (StandardScaler for all series) is the
|
|
201
|
+
recommended approach for factor models as it ensures all series contribute
|
|
202
|
+
proportionally to factor extraction.
|
|
205
203
|
|
|
206
204
|
Parameters
|
|
207
205
|
----------
|
|
208
206
|
config : DFMConfig
|
|
209
|
-
DFM configuration containing
|
|
210
|
-
The scaler type is read from config.scaler (default: 'standard').
|
|
207
|
+
DFM configuration containing series definitions (used for validation)
|
|
211
208
|
strategy : Optional[ScalingStrategy], default None
|
|
212
|
-
Custom scaling strategy. If None, uses
|
|
209
|
+
Custom scaling strategy. If None, uses DefaultScalingStrategy which
|
|
210
|
+
returns 'standard' for unified scaling.
|
|
213
211
|
column_names : Optional[List[str]], default None
|
|
214
212
|
Column names in data (used for validation). If None, uses
|
|
215
213
|
config.get_series_ids().
|
|
@@ -217,9 +215,9 @@ def create_scaling_transformer_from_config(
|
|
|
217
215
|
Returns
|
|
218
216
|
-------
|
|
219
217
|
Any
|
|
220
|
-
Raw sklearn
|
|
221
|
-
|
|
222
|
-
|
|
218
|
+
Raw sklearn StandardScaler (can be used directly in TransformerPipeline).
|
|
219
|
+
Per sktime docs, sklearn transformers are applied per series instance
|
|
220
|
+
automatically when used in TransformerPipeline.
|
|
223
221
|
|
|
224
222
|
Examples
|
|
225
223
|
--------
|
|
@@ -267,25 +265,21 @@ def create_scaling_transformer_from_config(
|
|
|
267
265
|
"""
|
|
268
266
|
_check_sklearn()
|
|
269
267
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
scaler_type = strategy.get_scaler(first_series, 0, first_column_name)
|
|
280
|
-
else:
|
|
281
|
-
scaler_type = 'standard'
|
|
268
|
+
if strategy is None:
|
|
269
|
+
strategy = DefaultScalingStrategy()
|
|
270
|
+
|
|
271
|
+
# Get scaler type from strategy (default: 'standard' for unified scaling)
|
|
272
|
+
# For unified scaling, all series use the same scaler
|
|
273
|
+
if len(config.series) > 0:
|
|
274
|
+
first_series = config.series[0]
|
|
275
|
+
first_column_name = config.get_series_ids()[0] if config.series else "series_0"
|
|
276
|
+
scaler_type = strategy.get_scaler(first_series, 0, first_column_name)
|
|
282
277
|
else:
|
|
283
|
-
|
|
284
|
-
scaler_type = 'standard'
|
|
278
|
+
scaler_type = 'standard' # Default if no series
|
|
285
279
|
|
|
286
280
|
# Return raw sklearn scaler (works directly in TransformerPipeline per sktime docs)
|
|
287
281
|
# Per sktime: sklearn transformers are applied per series instance automatically
|
|
288
|
-
if scaler_type is None
|
|
282
|
+
if scaler_type is None:
|
|
289
283
|
# No scaling - return passthrough
|
|
290
284
|
from sktime.transformations.series.func_transform import FunctionTransformer
|
|
291
285
|
return FunctionTransformer(func=lambda x: x, inverse_func=lambda x: x)
|
|
@@ -295,7 +289,7 @@ def create_scaling_transformer_from_config(
|
|
|
295
289
|
|
|
296
290
|
|
|
297
291
|
# ============================================================================
|
|
298
|
-
# Convenience Function: Simple Uniform Scaling (No
|
|
292
|
+
# Convenience Function: Simple Uniform Scaling (No ColumnTransformer)
|
|
299
293
|
# ============================================================================
|
|
300
294
|
|
|
301
295
|
def create_uniform_scaling_transformer(
|
dfm_python/logger/logger.py
CHANGED
dfm_python/models/base.py
CHANGED
|
@@ -91,6 +91,7 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
91
91
|
self._result: Optional[BaseResult] = None
|
|
92
92
|
self.training_state: Optional[Any] = None
|
|
93
93
|
self._data_module: Optional[Any] = None
|
|
94
|
+
self._nowcast: Optional[Any] = None
|
|
94
95
|
|
|
95
96
|
@property
|
|
96
97
|
def config(self) -> DFMConfig:
|
|
@@ -493,6 +494,7 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
493
494
|
self._config = None
|
|
494
495
|
self._data_module = None
|
|
495
496
|
self._result = None
|
|
497
|
+
self._nowcast = None
|
|
496
498
|
if hasattr(self, 'training_state'):
|
|
497
499
|
self.training_state = None
|
|
498
500
|
return self
|