dfm-python 0.4.5__py3-none-any.whl → 0.4.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dfm_python/__init__.py +4 -18
- dfm_python/config/adapter.py +1 -2
- dfm_python/config/schema.py +8 -13
- dfm_python/data/dataloader.py +3 -3
- dfm_python/data/utils.py +1 -1
- dfm_python/encoder/vae.py +15 -0
- dfm_python/lightning/__init__.py +2 -5
- dfm_python/lightning/data_module.py +67 -24
- dfm_python/lightning/scaling.py +32 -26
- dfm_python/logger/inference_logger.py +1 -1
- dfm_python/logger/logger.py +30 -16
- dfm_python/logger/train_logger.py +1 -1
- dfm_python/models/base.py +0 -2
- dfm_python/models/ddfm.py +442 -301
- dfm_python/models/dfm.py +103 -6
- dfm_python/ssm/em.py +509 -50
- dfm_python/ssm/kalman.py +301 -27
- dfm_python/ssm/utils.py +18 -15
- dfm_python/trainer/__init__.py +10 -3
- dfm_python/trainer/ddfm.py +5 -4
- dfm_python/trainer/dfm.py +3 -3
- dfm_python/utils/__init__.py +28 -14
- dfm_python/utils/data.py +63 -3
- dfm_python/utils/time.py +3 -3
- {dfm_python-0.4.5.dist-info → dfm_python-0.4.7.dist-info}/METADATA +9 -110
- dfm_python-0.4.7.dist-info/RECORD +42 -0
- dfm_python/nowcast/__init__.py +0 -37
- dfm_python/nowcast/dataview.py +0 -143
- dfm_python/nowcast/helpers.py +0 -356
- dfm_python/nowcast/nowcast.py +0 -1263
- dfm_python/nowcast/splitters.py +0 -431
- dfm_python/nowcast/transformers.py +0 -330
- dfm_python/nowcast/utils.py +0 -285
- dfm_python-0.4.5.dist-info/RECORD +0 -49
- {dfm_python-0.4.5.dist-info → dfm_python-0.4.7.dist-info}/WHEEL +0 -0
- {dfm_python-0.4.5.dist-info → dfm_python-0.4.7.dist-info}/licenses/LICENSE +0 -0
dfm_python/__init__.py
CHANGED
|
@@ -6,7 +6,6 @@ This package implements a comprehensive Dynamic Factor Model framework with supp
|
|
|
6
6
|
- Tent kernel aggregation for low-to-high frequency mapping
|
|
7
7
|
- Expectation-Maximization (EM) algorithm for parameter estimation
|
|
8
8
|
- Kalman filtering and smoothing for factor extraction
|
|
9
|
-
- News decomposition for nowcasting
|
|
10
9
|
- Deep Dynamic Factor Models (DDFM) with nonlinear encoders (requires PyTorch)
|
|
11
10
|
|
|
12
11
|
The package implements a clock-based approach to mixed-frequency DFMs, where all latent
|
|
@@ -23,7 +22,6 @@ Key Features:
|
|
|
23
22
|
- Flexible block structure for factor modeling
|
|
24
23
|
- Robust handling of missing data (internal spline interpolation)
|
|
25
24
|
- Automatic standardization and data clipping
|
|
26
|
-
- News decomposition for forecast updates
|
|
27
25
|
|
|
28
26
|
Example (Standard Lightning Pattern):
|
|
29
27
|
>>> from dfm_python import DFM, DFMDataModule, DFMTrainer
|
|
@@ -67,7 +65,7 @@ Note: DFMConfig and SeriesConfig are internal implementation details.
|
|
|
67
65
|
For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
|
|
68
66
|
"""
|
|
69
67
|
|
|
70
|
-
__version__ = "0.4.
|
|
68
|
+
__version__ = "0.4.7"
|
|
71
69
|
|
|
72
70
|
# ============================================================================
|
|
73
71
|
# PUBLIC API DEFINITION
|
|
@@ -81,7 +79,7 @@ __version__ = "0.4.5"
|
|
|
81
79
|
# 2. High-level API: DFM, DDFM, module-level convenience functions
|
|
82
80
|
# 3. Core utilities: TimeIndex, diagnostics
|
|
83
81
|
# 4. Models: BaseFactorModel, DDFM (low-level)
|
|
84
|
-
# 5.
|
|
82
|
+
# 5. Data & Results: DFMResult
|
|
85
83
|
# 6. Data & Results: DFMResult
|
|
86
84
|
# ============================================================================
|
|
87
85
|
|
|
@@ -110,16 +108,7 @@ from .lightning import (
|
|
|
110
108
|
EMAlgorithm, # Module class
|
|
111
109
|
)
|
|
112
110
|
|
|
113
|
-
# Nowcasting
|
|
114
|
-
from .nowcast import (
|
|
115
|
-
Nowcast,
|
|
116
|
-
NowcastResult,
|
|
117
|
-
)
|
|
118
|
-
from .nowcast.helpers import (
|
|
119
|
-
para_const,
|
|
120
|
-
NewsDecompResult,
|
|
121
|
-
BacktestResult,
|
|
122
|
-
)
|
|
111
|
+
# Nowcasting removed - use src.nowcast instead
|
|
123
112
|
|
|
124
113
|
# Model implementations
|
|
125
114
|
from .models.base import BaseFactorModel
|
|
@@ -130,11 +119,9 @@ from .models.ddfm import DDFM, DDFMModel
|
|
|
130
119
|
|
|
131
120
|
__all__ = [
|
|
132
121
|
# Core classes
|
|
133
|
-
'DFM',
|
|
122
|
+
'DFM',
|
|
134
123
|
# Model base and implementations
|
|
135
124
|
'BaseFactorModel',
|
|
136
|
-
# Nowcast result classes
|
|
137
|
-
'NowcastResult', 'NewsDecompResult', 'BacktestResult',
|
|
138
125
|
# Constants
|
|
139
126
|
'DEFAULT_BLOCK_NAME',
|
|
140
127
|
# Config sources
|
|
@@ -142,7 +129,6 @@ __all__ = [
|
|
|
142
129
|
'MergedConfigSource', 'make_config_source',
|
|
143
130
|
# Low-level API (functional interface - advanced usage)
|
|
144
131
|
'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse', 'diagnose_series', 'print_series_diagnosis',
|
|
145
|
-
'para_const', # Internal utility for nowcasting
|
|
146
132
|
]
|
|
147
133
|
|
|
148
134
|
# DDFM high-level API (PyTorch is mandatory)
|
dfm_python/config/adapter.py
CHANGED
|
@@ -214,8 +214,7 @@ class YamlSource:
|
|
|
214
214
|
frequency=series_data.get('frequency', 'm'),
|
|
215
215
|
transformation=series_data.get('transformation', 'lin'),
|
|
216
216
|
blocks=series_data.get('blocks', []),
|
|
217
|
-
release_date=release_date
|
|
218
|
-
scaler=series_data.get('scaler', None) # Optional, for per-series scaling
|
|
217
|
+
release_date=release_date
|
|
219
218
|
))
|
|
220
219
|
|
|
221
220
|
# If no series loaded from separate files, try to get from main config
|
dfm_python/config/schema.py
CHANGED
|
@@ -200,11 +200,6 @@ class SeriesConfig:
|
|
|
200
200
|
- Positive value (1-31): Day of month when data is released
|
|
201
201
|
- Negative value: Days before end of previous month when data is released
|
|
202
202
|
Example: 25 = released on 25th of each month, -5 = released 5 days before end of previous month
|
|
203
|
-
scaler : str, optional
|
|
204
|
-
Scaler type for this series: 'standard', 'robust', 'minmax', 'maxabs',
|
|
205
|
-
'quantile', or None/null (no scaling). Used by create_scaling_transformer_from_config()
|
|
206
|
-
to automatically create per-series scaling transformers. If None, uses default
|
|
207
|
-
scaling strategy (typically 'standard').
|
|
208
203
|
"""
|
|
209
204
|
# Required fields (no defaults)
|
|
210
205
|
frequency: str
|
|
@@ -215,7 +210,6 @@ class SeriesConfig:
|
|
|
215
210
|
series_name: Optional[str] = None # Optional metadata for display
|
|
216
211
|
units: Optional[str] = None # Optional metadata for display only (used in news.py output)
|
|
217
212
|
release_date: Optional[int] = None # Release date for pseudo real-time nowcasting
|
|
218
|
-
scaler: Optional[str] = None # Scaler type: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or None
|
|
219
213
|
|
|
220
214
|
def __post_init__(self):
|
|
221
215
|
"""Validate fields after initialization."""
|
|
@@ -308,6 +302,7 @@ class BaseModelConfig:
|
|
|
308
302
|
nan_method: int = 2 # Missing data handling method (1-5). Preprocessing step before Kalman Filter-based handling
|
|
309
303
|
nan_k: int = 3 # Spline parameter for NaN interpolation (cubic spline)
|
|
310
304
|
clock: str = 'm' # Base frequency for nowcasting (global clock): 'd', 'w', 'm', 'q', 'sa', 'a' (defaults to 'm' for monthly)
|
|
305
|
+
scaler: Optional[str] = 'standard' # Unified scaler type for all series: 'standard', 'robust', 'minmax', 'maxabs', 'quantile', or None (no scaling). Default: 'standard' for unified scaling.
|
|
311
306
|
|
|
312
307
|
# ========================================================================
|
|
313
308
|
# Internal cache (not user-configurable)
|
|
@@ -660,6 +655,7 @@ class DFMConfig(BaseModelConfig):
|
|
|
660
655
|
'nan_method': data.get('nan_method', 2),
|
|
661
656
|
'nan_k': data.get('nan_k', 3),
|
|
662
657
|
'clock': data.get('clock', 'm'),
|
|
658
|
+
'scaler': data.get('scaler', 'standard'), # Unified scaler for all series (default: 'standard')
|
|
663
659
|
}
|
|
664
660
|
|
|
665
661
|
@classmethod
|
|
@@ -716,11 +712,11 @@ class DDFMConfig(BaseModelConfig):
|
|
|
716
712
|
# ========================================================================
|
|
717
713
|
encoder_layers: Optional[List[int]] = None # Hidden layer dimensions for encoder (default: [64, 32])
|
|
718
714
|
num_factors: Optional[int] = None # Number of factors (inferred from config if None)
|
|
719
|
-
activation: str = '
|
|
715
|
+
activation: str = 'relu' # Activation function ('tanh', 'relu', 'sigmoid', default: 'relu' to match original DDFM)
|
|
720
716
|
use_batch_norm: bool = True # Use batch normalization in encoder (default: True)
|
|
721
717
|
learning_rate: float = 0.001 # Learning rate for Adam optimizer (default: 0.001)
|
|
722
718
|
epochs: int = 100 # Number of training epochs (default: 100)
|
|
723
|
-
batch_size: int =
|
|
719
|
+
batch_size: int = 100 # Batch size for training (default: 100 to match original DDFM)
|
|
724
720
|
factor_order: int = 1 # VAR lag order for factor dynamics (1 or 2, default: 1)
|
|
725
721
|
use_idiosyncratic: bool = True # Model idio components with AR(1) dynamics (default: True)
|
|
726
722
|
min_obs_idio: int = 5 # Minimum observations for idio AR(1) estimation (default: 5)
|
|
@@ -750,15 +746,15 @@ class DDFMConfig(BaseModelConfig):
|
|
|
750
746
|
def _extract_ddfm(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
751
747
|
"""Extract DDFM-specific parameters from config dict."""
|
|
752
748
|
base_params = cls._extract_base(data)
|
|
753
|
-
# Handle both direct keys and ddfm_ prefix format
|
|
749
|
+
# Handle both direct keys and ddfm_ prefix format
|
|
754
750
|
base_params.update({
|
|
755
751
|
'encoder_layers': data.get('encoder_layers') or data.get('ddfm_encoder_layers', None),
|
|
756
752
|
'num_factors': data.get('num_factors') or data.get('ddfm_num_factors', None),
|
|
757
|
-
'activation': data.get('activation') or data.get('ddfm_activation', '
|
|
753
|
+
'activation': data.get('activation') or data.get('ddfm_activation', 'relu'),
|
|
758
754
|
'use_batch_norm': data.get('use_batch_norm', data.get('ddfm_use_batch_norm', True)),
|
|
759
755
|
'learning_rate': data.get('learning_rate', data.get('ddfm_learning_rate', 0.001)),
|
|
760
756
|
'epochs': data.get('epochs', data.get('ddfm_epochs', 100)),
|
|
761
|
-
'batch_size': data.get('batch_size', data.get('ddfm_batch_size',
|
|
757
|
+
'batch_size': data.get('batch_size', data.get('ddfm_batch_size', 100)),
|
|
762
758
|
'factor_order': data.get('factor_order', data.get('ddfm_factor_order', 1)),
|
|
763
759
|
'use_idiosyncratic': data.get('use_idiosyncratic', data.get('ddfm_use_idiosyncratic', True)),
|
|
764
760
|
'min_obs_idio': data.get('min_obs_idio', data.get('ddfm_min_obs_idio', 5)),
|
|
@@ -832,8 +828,7 @@ class DDFMConfig(BaseModelConfig):
|
|
|
832
828
|
transformation=series_cfg.get('transformation', 'lin'),
|
|
833
829
|
blocks=series_blocks,
|
|
834
830
|
units=series_cfg.get('units', None), # Optional, for display only
|
|
835
|
-
release_date=series_cfg.get('release_date', None)
|
|
836
|
-
scaler=series_cfg.get('scaler', None) # Optional, for per-series scaling
|
|
831
|
+
release_date=series_cfg.get('release_date', None) # Optional, for nowcasting
|
|
837
832
|
))
|
|
838
833
|
|
|
839
834
|
# Convert blocks_dict to dict of block properties
|
dfm_python/data/dataloader.py
CHANGED
|
@@ -101,7 +101,7 @@ def create_dfm_dataloader(
|
|
|
101
101
|
|
|
102
102
|
def create_ddfm_dataloader(
|
|
103
103
|
dataset: DDFMDataset,
|
|
104
|
-
batch_size: int =
|
|
104
|
+
batch_size: int = 100,
|
|
105
105
|
shuffle: bool = True,
|
|
106
106
|
num_workers: int = 0,
|
|
107
107
|
pin_memory: bool = True
|
|
@@ -115,8 +115,8 @@ def create_ddfm_dataloader(
|
|
|
115
115
|
----------
|
|
116
116
|
dataset : DDFMDataset
|
|
117
117
|
DDFM dataset instance with windowed sequences
|
|
118
|
-
batch_size : int, default
|
|
119
|
-
Batch size for training
|
|
118
|
+
batch_size : int, default 100
|
|
119
|
+
Batch size for training (matches original DDFM)
|
|
120
120
|
shuffle : bool, default True
|
|
121
121
|
Whether to shuffle samples (typically True for training)
|
|
122
122
|
num_workers : int, default 0
|
dfm_python/data/utils.py
CHANGED
|
@@ -209,7 +209,7 @@ def load_data(
|
|
|
209
209
|
Time : TimeIndex
|
|
210
210
|
Time index for the data (aligned to clock frequency)
|
|
211
211
|
Z : np.ndarray
|
|
212
|
-
Original untransformed data (T x N), same as X
|
|
212
|
+
Original untransformed data (T x N), same as X
|
|
213
213
|
|
|
214
214
|
Raises
|
|
215
215
|
------
|
dfm_python/encoder/vae.py
CHANGED
|
@@ -454,6 +454,21 @@ def extract_decoder_params(decoder) -> Tuple[np.ndarray, np.ndarray]:
|
|
|
454
454
|
# Decoder weight is already (N x m), so no transpose needed
|
|
455
455
|
C = weight
|
|
456
456
|
|
|
457
|
+
# Check for NaN in extracted C matrix (indicates numerical instability during training)
|
|
458
|
+
if np.any(np.isnan(C)):
|
|
459
|
+
nan_count = np.sum(np.isnan(C))
|
|
460
|
+
nan_ratio = nan_count / C.size
|
|
461
|
+
from ..logger import get_logger
|
|
462
|
+
_logger = get_logger(__name__)
|
|
463
|
+
_logger.warning(
|
|
464
|
+
f"extract_decoder_params: C matrix contains {nan_count}/{C.size} NaN values ({nan_ratio:.1%}). "
|
|
465
|
+
f"This indicates the decoder weights contain NaN, likely due to numerical instability during training. "
|
|
466
|
+
f"Possible causes: (1) learning rate too high, (2) gradient explosion, (3) unstable training."
|
|
467
|
+
)
|
|
468
|
+
# Replace NaN with zeros to prevent further issues
|
|
469
|
+
C = np.nan_to_num(C, nan=0.0)
|
|
470
|
+
_logger.warning("Replaced NaN values in C matrix with zeros.")
|
|
471
|
+
|
|
457
472
|
return C, bias
|
|
458
473
|
|
|
459
474
|
|
dfm_python/lightning/__init__.py
CHANGED
|
@@ -28,15 +28,14 @@ from .scaling import (
|
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
# DFMTrainingState is defined in models.dfm and exported here for convenience
|
|
31
|
-
# Note: dfm_module.py has been removed - DFM class implements EM directly
|
|
32
31
|
from ..models.dfm import DFMTrainingState
|
|
33
32
|
|
|
34
33
|
__all__ = [
|
|
35
34
|
# Kalman filter
|
|
36
|
-
'KalmanFilter',
|
|
35
|
+
'KalmanFilter',
|
|
37
36
|
'KalmanFilterState',
|
|
38
37
|
# EM algorithm
|
|
39
|
-
'EMAlgorithm',
|
|
38
|
+
'EMAlgorithm',
|
|
40
39
|
'EMStepParams',
|
|
41
40
|
# Data handling
|
|
42
41
|
'DFMDataModule',
|
|
@@ -50,7 +49,5 @@ __all__ = [
|
|
|
50
49
|
'NoScalingStrategy',
|
|
51
50
|
# Training state (defined in models.dfm)
|
|
52
51
|
'DFMTrainingState',
|
|
53
|
-
# DDFMLightningModule removed: deprecated and unused (DDFM uses DDFMModel directly)
|
|
54
|
-
# DDFMTrainingState removed: duplicate (also defined in models.ddfm)
|
|
55
52
|
]
|
|
56
53
|
|
|
@@ -43,13 +43,11 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
|
|
|
43
43
|
|
|
44
44
|
This function depends on sktime's pipeline structure to traverse
|
|
45
45
|
TransformerPipeline and sklearn transformers (StandardScaler, etc.).
|
|
46
|
-
Note: TabularToSeriesAdaptor support is for backward compatibility only.
|
|
47
46
|
|
|
48
47
|
Parameters
|
|
49
48
|
----------
|
|
50
49
|
transformer : Any
|
|
51
|
-
Transformer to search (
|
|
52
|
-
TransformerPipeline, sklearn Pipeline, etc.)
|
|
50
|
+
Transformer to search (StandardScaler, TransformerPipeline, sklearn Pipeline, etc.)
|
|
53
51
|
|
|
54
52
|
Returns
|
|
55
53
|
-------
|
|
@@ -60,9 +58,7 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
|
|
|
60
58
|
if transformer is None:
|
|
61
59
|
return None
|
|
62
60
|
|
|
63
|
-
# Check if transformer is
|
|
64
|
-
# Note: New code should use StandardScaler() directly per sktime docs
|
|
65
|
-
# This check is kept for backward compatibility with old code that may still use TabularToSeriesAdaptor
|
|
61
|
+
# Check if transformer is wrapped (TabularToSeriesAdaptor or similar)
|
|
66
62
|
if hasattr(transformer, 'transformer'):
|
|
67
63
|
# Recursively search the wrapped transformer
|
|
68
64
|
wrapped = transformer.transformer
|
|
@@ -78,8 +74,7 @@ def _get_scaler(transformer: Any) -> Optional[Any]:
|
|
|
78
74
|
if scaler is not None:
|
|
79
75
|
return scaler
|
|
80
76
|
|
|
81
|
-
# Check if transformer is
|
|
82
|
-
# Unified scaling with StandardScaler is preferred per sktime docs
|
|
77
|
+
# Check if transformer is ColumnEnsembleTransformer (or ColumnTransformer)
|
|
83
78
|
if hasattr(transformer, 'transformers'):
|
|
84
79
|
for name, trans, cols in transformer.transformers:
|
|
85
80
|
scaler = _get_scaler(trans)
|
|
@@ -261,15 +256,17 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
261
256
|
|
|
262
257
|
This DataModule handles data loading for DFM/DDFM models.
|
|
263
258
|
|
|
264
|
-
**Important**:
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
259
|
+
**Important**: DFM and DDFM can handle missing data (NaN values) implicitly:
|
|
260
|
+
- **DFM**: Uses Kalman filter's `handle_missing_data()` method to skip NaN observations
|
|
261
|
+
- **DDFM**: Uses state-space model and MCMC procedure to handle missing data through
|
|
262
|
+
idiosyncratic component estimation
|
|
268
263
|
|
|
269
264
|
**Usage Pattern**:
|
|
270
|
-
-
|
|
265
|
+
- Data can contain NaN values - models will handle them implicitly
|
|
271
266
|
- If `pipeline=None`, a passthrough transformer is used by default (no-op)
|
|
272
267
|
- Users can optionally provide their preprocessing pipeline to extract statistics (Mx/Wx)
|
|
268
|
+
- For better performance, users can preprocess data (imputation, scaling) before passing,
|
|
269
|
+
but it's not required - models will handle missing data automatically
|
|
273
270
|
|
|
274
271
|
For linear DFM, this uses DFMDataset which returns full sequences.
|
|
275
272
|
For DDFM, use DDFMDataModule which uses DDFMDataset with windowing.
|
|
@@ -311,9 +308,10 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
311
308
|
data_path : str or Path, optional
|
|
312
309
|
Path to data file (CSV). If None, data must be provided.
|
|
313
310
|
data : np.ndarray or pd.DataFrame, optional
|
|
314
|
-
|
|
315
|
-
-
|
|
316
|
-
-
|
|
311
|
+
Data array or DataFrame. Can contain NaN values - DFM/DDFM will handle them:
|
|
312
|
+
- DFM: Uses Kalman filter to implicitly handle missing data
|
|
313
|
+
- DDFM: Uses state-space model and MCMC to handle missing data
|
|
314
|
+
- Standardized/scaled data (mean=0, std=1) is recommended for better performance
|
|
317
315
|
- Feature-engineered if needed
|
|
318
316
|
If None, data_path must be provided.
|
|
319
317
|
time_index : TimeIndex, optional
|
|
@@ -417,20 +415,22 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
417
415
|
This method is called by Lightning to set up the data module.
|
|
418
416
|
It loads preprocessed data and extracts statistics from the pipeline for forecasting/nowcasting.
|
|
419
417
|
|
|
420
|
-
**Important**:
|
|
421
|
-
|
|
418
|
+
**Important**: DFM and DDFM can handle missing data (NaN values) implicitly:
|
|
419
|
+
- **DFM**: Uses Kalman filter's `handle_missing_data()` method
|
|
420
|
+
- **DDFM**: Uses state-space model and MCMC procedure
|
|
422
421
|
|
|
423
422
|
**Pipeline Purpose**:
|
|
424
423
|
- The pipeline is used to extract statistics (e.g., Mx/Wx from StandardScaler) needed for
|
|
425
424
|
forecasting and nowcasting operations, not to preprocess the data
|
|
426
|
-
- Data is already preprocessed - the pipeline is fitted to extract standardization parameters
|
|
427
425
|
- The pipeline is fitted on the **entire dataset** to extract statistics
|
|
428
426
|
- If `val_split` is provided, it's used for validation during training (e.g., DDFM neural network)
|
|
429
427
|
|
|
430
428
|
**Usage**:
|
|
431
|
-
-
|
|
429
|
+
- Data can contain NaN values - models will handle them automatically
|
|
432
430
|
- Optionally provide pipeline (e.g., TransformerPipeline with StandardScaler) to extract Mx/Wx for predictions
|
|
433
431
|
- If pipeline=None, uses passthrough (no statistics extracted, Mx/Wx computed from data)
|
|
432
|
+
- For better performance, users can preprocess data (imputation, scaling) before passing,
|
|
433
|
+
but it's not required
|
|
434
434
|
"""
|
|
435
435
|
# Load data if not already provided
|
|
436
436
|
if self.data is None:
|
|
@@ -451,6 +451,9 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
451
451
|
self.time_index = Time
|
|
452
452
|
|
|
453
453
|
# Convert to pandas DataFrame if needed
|
|
454
|
+
# Note: DFM/DDFM can handle NaN values implicitly via Kalman filter
|
|
455
|
+
# (DFM uses handle_missing_data() in Kalman filter, DDFM uses state-space model)
|
|
456
|
+
# So we don't validate NaN here - let the models handle it
|
|
454
457
|
if isinstance(self.data, np.ndarray):
|
|
455
458
|
series_ids = self.config.get_series_ids()
|
|
456
459
|
X_df = pd.DataFrame(self.data, columns=series_ids)
|
|
@@ -505,7 +508,10 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
505
508
|
f"automatically extracted from the pipeline."
|
|
506
509
|
) from e
|
|
507
510
|
|
|
508
|
-
#
|
|
511
|
+
# Note: DFM/DDFM can handle NaN values implicitly via Kalman filter
|
|
512
|
+
# So we don't validate NaN here - let the models handle it
|
|
513
|
+
|
|
514
|
+
# Ensure output is pandas DataFrame with compatible index
|
|
509
515
|
if not isinstance(X_transformed, pd.DataFrame):
|
|
510
516
|
# If transformer returned numpy array, convert to pandas
|
|
511
517
|
if isinstance(X_transformed, np.ndarray):
|
|
@@ -532,6 +538,45 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
532
538
|
# Create generic column names
|
|
533
539
|
X_transformed = pd.DataFrame(X_transformed,
|
|
534
540
|
columns=[f'feature_{i}' for i in range(n_cols)])
|
|
541
|
+
|
|
542
|
+
# Ensure index is compatible (DatetimeIndex, PeriodIndex, or RangeIndex)
|
|
543
|
+
# Use input DataFrame index if length matches
|
|
544
|
+
if len(X_transformed) == len(X_df):
|
|
545
|
+
if isinstance(X_df.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
|
|
546
|
+
X_transformed.index = X_df.index
|
|
547
|
+
else:
|
|
548
|
+
# Try to convert to DatetimeIndex
|
|
549
|
+
try:
|
|
550
|
+
X_transformed.index = pd.to_datetime(X_df.index)
|
|
551
|
+
except (ValueError, TypeError):
|
|
552
|
+
# Fallback to RangeIndex
|
|
553
|
+
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
554
|
+
else:
|
|
555
|
+
# Length doesn't match - use RangeIndex
|
|
556
|
+
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
557
|
+
|
|
558
|
+
# Ensure DataFrame index is compatible (DatetimeIndex, PeriodIndex, or RangeIndex)
|
|
559
|
+
# This handles cases where pipeline output has incompatible index
|
|
560
|
+
if isinstance(X_transformed, pd.DataFrame):
|
|
561
|
+
if not isinstance(X_transformed.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
|
|
562
|
+
# Try to use input DataFrame index if length matches
|
|
563
|
+
if len(X_transformed) == len(X_df):
|
|
564
|
+
if isinstance(X_df.index, (pd.DatetimeIndex, pd.PeriodIndex, pd.RangeIndex)):
|
|
565
|
+
X_transformed.index = X_df.index
|
|
566
|
+
else:
|
|
567
|
+
# Try to convert to DatetimeIndex
|
|
568
|
+
try:
|
|
569
|
+
X_transformed.index = pd.to_datetime(X_df.index)
|
|
570
|
+
except (ValueError, TypeError):
|
|
571
|
+
# Fallback to RangeIndex
|
|
572
|
+
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
573
|
+
else:
|
|
574
|
+
# Try to convert existing index
|
|
575
|
+
try:
|
|
576
|
+
X_transformed.index = pd.to_datetime(X_transformed.index)
|
|
577
|
+
except (ValueError, TypeError):
|
|
578
|
+
# Fallback to RangeIndex
|
|
579
|
+
X_transformed.index = pd.RangeIndex(start=0, stop=len(X_transformed))
|
|
535
580
|
elif hasattr(X_transformed, 'to_numpy'):
|
|
536
581
|
# Has to_numpy method (pandas DataFrame) - convert to numpy then back to DataFrame
|
|
537
582
|
# Try to preserve column names if available
|
|
@@ -572,9 +617,7 @@ class DFMDataModule(lightning_pl.LightningDataModule):
|
|
|
572
617
|
# This is optional - some pipelines may not have standardization
|
|
573
618
|
# Mx and Wx are already initialized in __init__
|
|
574
619
|
try:
|
|
575
|
-
# Extract scaler from pipeline (
|
|
576
|
-
# Supports any scaler type: StandardScaler, MinMaxScaler, RobustScaler, etc.
|
|
577
|
-
# Note: New code should use StandardScaler() directly per sktime docs (unified scaling)
|
|
620
|
+
# Extract scaler from pipeline (supports TransformerPipeline, StandardScaler, etc.)
|
|
578
621
|
scaler = _get_scaler(pipeline_to_use)
|
|
579
622
|
|
|
580
623
|
if scaler is not None:
|
dfm_python/lightning/scaling.py
CHANGED
|
@@ -63,7 +63,7 @@ class ScalingStrategy(Protocol):
|
|
|
63
63
|
class MyScalingStrategy:
|
|
64
64
|
def get_scaler(self, series: SeriesConfig, series_index: int,
|
|
65
65
|
column_name: str) -> Optional[str]:
|
|
66
|
-
# Custom logic (
|
|
66
|
+
# Custom logic (unified scaling is typically preferred)
|
|
67
67
|
if 'volatility' in column_name.lower():
|
|
68
68
|
return 'robust'
|
|
69
69
|
elif 'return' in column_name.lower():
|
|
@@ -191,23 +191,25 @@ def create_scaling_transformer_from_config(
|
|
|
191
191
|
strategy: Optional[ScalingStrategy] = None,
|
|
192
192
|
column_names: Optional[List[str]] = None
|
|
193
193
|
) -> Any:
|
|
194
|
-
"""Create a unified scaling transformer
|
|
194
|
+
"""Create a unified scaling transformer from model-level scaler configuration.
|
|
195
195
|
|
|
196
|
-
This function returns a raw sklearn StandardScaler
|
|
197
|
-
in TransformerPipeline. Per sktime docs, sklearn
|
|
198
|
-
applied per series instance when used in
|
|
196
|
+
This function returns a raw sklearn scaler (StandardScaler, RobustScaler, etc.)
|
|
197
|
+
that can be used directly in TransformerPipeline. Per sktime docs, sklearn
|
|
198
|
+
transformers are automatically applied per series instance when used in
|
|
199
|
+
TransformerPipeline.
|
|
199
200
|
|
|
200
|
-
**
|
|
201
|
-
|
|
202
|
-
|
|
201
|
+
**Unified scaling**: The scaler type is now specified at the model level
|
|
202
|
+
(in model config YAML) rather than per-series. This ensures all series use
|
|
203
|
+
the same scaling method, which is the recommended approach for factor models
|
|
204
|
+
as it ensures all series contribute proportionally to factor extraction.
|
|
203
205
|
|
|
204
206
|
Parameters
|
|
205
207
|
----------
|
|
206
208
|
config : DFMConfig
|
|
207
|
-
DFM configuration containing
|
|
209
|
+
DFM configuration containing model-level scaler setting (config.scaler).
|
|
210
|
+
The scaler type is read from config.scaler (default: 'standard').
|
|
208
211
|
strategy : Optional[ScalingStrategy], default None
|
|
209
|
-
Custom scaling strategy. If None, uses
|
|
210
|
-
returns 'standard' for unified scaling.
|
|
212
|
+
Custom scaling strategy. If None, uses the scaler type from config.scaler.
|
|
211
213
|
column_names : Optional[List[str]], default None
|
|
212
214
|
Column names in data (used for validation). If None, uses
|
|
213
215
|
config.get_series_ids().
|
|
@@ -215,9 +217,9 @@ def create_scaling_transformer_from_config(
|
|
|
215
217
|
Returns
|
|
216
218
|
-------
|
|
217
219
|
Any
|
|
218
|
-
Raw sklearn StandardScaler
|
|
219
|
-
Per sktime docs, sklearn transformers
|
|
220
|
-
automatically when used in TransformerPipeline.
|
|
220
|
+
Raw sklearn scaler (StandardScaler, RobustScaler, etc.) that can be used
|
|
221
|
+
directly in TransformerPipeline. Per sktime docs, sklearn transformers
|
|
222
|
+
are applied per series instance automatically when used in TransformerPipeline.
|
|
221
223
|
|
|
222
224
|
Examples
|
|
223
225
|
--------
|
|
@@ -265,21 +267,25 @@ def create_scaling_transformer_from_config(
|
|
|
265
267
|
"""
|
|
266
268
|
_check_sklearn()
|
|
267
269
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
270
|
+
# Get scaler type from model-level config (config.scaler)
|
|
271
|
+
# Unified scaling: all series use the same scaler specified at model level
|
|
272
|
+
if hasattr(config, 'scaler') and config.scaler is not None:
|
|
273
|
+
scaler_type = config.scaler
|
|
274
|
+
elif strategy is not None:
|
|
275
|
+
# Fallback to strategy if config.scaler not available
|
|
276
|
+
if len(config.series) > 0:
|
|
277
|
+
first_series = config.series[0]
|
|
278
|
+
first_column_name = config.get_series_ids()[0] if config.series else "series_0"
|
|
279
|
+
scaler_type = strategy.get_scaler(first_series, 0, first_column_name)
|
|
280
|
+
else:
|
|
281
|
+
scaler_type = 'standard'
|
|
277
282
|
else:
|
|
278
|
-
|
|
283
|
+
# Default to 'standard' for unified scaling
|
|
284
|
+
scaler_type = 'standard'
|
|
279
285
|
|
|
280
286
|
# Return raw sklearn scaler (works directly in TransformerPipeline per sktime docs)
|
|
281
287
|
# Per sktime: sklearn transformers are applied per series instance automatically
|
|
282
|
-
if scaler_type is None:
|
|
288
|
+
if scaler_type is None or scaler_type == 'null':
|
|
283
289
|
# No scaling - return passthrough
|
|
284
290
|
from sktime.transformations.series.func_transform import FunctionTransformer
|
|
285
291
|
return FunctionTransformer(func=lambda x: x, inverse_func=lambda x: x)
|
|
@@ -289,7 +295,7 @@ def create_scaling_transformer_from_config(
|
|
|
289
295
|
|
|
290
296
|
|
|
291
297
|
# ============================================================================
|
|
292
|
-
# Convenience Function: Simple Uniform Scaling (No ColumnTransformer)
|
|
298
|
+
# Convenience Function: Simple Uniform Scaling (No ColumnEnsembleTransformer/ColumnTransformer)
|
|
293
299
|
# ============================================================================
|
|
294
300
|
|
|
295
301
|
def create_uniform_scaling_transformer(
|
dfm_python/logger/logger.py
CHANGED
|
@@ -14,6 +14,11 @@ def get_logger(name: str) -> logging.Logger:
|
|
|
14
14
|
This is the standard way to get a logger in the DFM package.
|
|
15
15
|
All modules should use: _logger = get_logger(__name__)
|
|
16
16
|
|
|
17
|
+
The logger uses hierarchical configuration:
|
|
18
|
+
- Child loggers (e.g., 'dfm_python.models.dfm') inherit from parent logger ('dfm_python')
|
|
19
|
+
- Parent logger ('dfm_python') is configured once with handlers
|
|
20
|
+
- Child loggers propagate to parent (default behavior)
|
|
21
|
+
|
|
17
22
|
Parameters
|
|
18
23
|
----------
|
|
19
24
|
name : str
|
|
@@ -26,21 +31,27 @@ def get_logger(name: str) -> logging.Logger:
|
|
|
26
31
|
"""
|
|
27
32
|
logger = logging.getLogger(name)
|
|
28
33
|
|
|
29
|
-
#
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
34
|
+
# Ensure package-level logger is configured (only once)
|
|
35
|
+
package_logger = logging.getLogger('dfm_python')
|
|
36
|
+
|
|
37
|
+
# Configure package logger if not already configured
|
|
38
|
+
# Use a flag to avoid re-configuring if already done
|
|
39
|
+
if not hasattr(package_logger, '_dfm_configured'):
|
|
40
|
+
# Configure root logger for dfm_python package
|
|
41
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
42
|
+
handler.setFormatter(
|
|
43
|
+
logging.Formatter(
|
|
44
|
+
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
45
|
+
datefmt='%Y-%m-%d %H:%M:%S'
|
|
41
46
|
)
|
|
42
|
-
|
|
43
|
-
|
|
47
|
+
)
|
|
48
|
+
package_logger.addHandler(handler)
|
|
49
|
+
package_logger.setLevel(logging.INFO)
|
|
50
|
+
# Mark as configured to avoid duplicate handlers
|
|
51
|
+
package_logger._dfm_configured = True
|
|
52
|
+
|
|
53
|
+
# Ensure child logger propagates to parent (default, but explicit for clarity)
|
|
54
|
+
logger.propagate = True
|
|
44
55
|
|
|
45
56
|
return logger
|
|
46
57
|
|
|
@@ -51,7 +62,7 @@ def setup_logging(
|
|
|
51
62
|
) -> None:
|
|
52
63
|
"""Setup package-wide logging configuration.
|
|
53
64
|
|
|
54
|
-
|
|
65
|
+
Alias for configure_logging().
|
|
55
66
|
|
|
56
67
|
Parameters
|
|
57
68
|
----------
|
|
@@ -93,8 +104,11 @@ def configure_logging(
|
|
|
93
104
|
logger = logging.getLogger('dfm_python')
|
|
94
105
|
logger.setLevel(level)
|
|
95
106
|
|
|
96
|
-
# Remove existing handlers
|
|
107
|
+
# Remove existing handlers to avoid duplicates
|
|
97
108
|
logger.handlers.clear()
|
|
109
|
+
# Clear the configuration flag so it can be reconfigured
|
|
110
|
+
if hasattr(logger, '_dfm_configured'):
|
|
111
|
+
delattr(logger, '_dfm_configured')
|
|
98
112
|
|
|
99
113
|
# Add console handler
|
|
100
114
|
console_handler = logging.StreamHandler(sys.stdout)
|
dfm_python/models/base.py
CHANGED
|
@@ -91,7 +91,6 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
91
91
|
self._result: Optional[BaseResult] = None
|
|
92
92
|
self.training_state: Optional[Any] = None
|
|
93
93
|
self._data_module: Optional[Any] = None
|
|
94
|
-
self._nowcast: Optional[Any] = None
|
|
95
94
|
|
|
96
95
|
@property
|
|
97
96
|
def config(self) -> DFMConfig:
|
|
@@ -494,7 +493,6 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
494
493
|
self._config = None
|
|
495
494
|
self._data_module = None
|
|
496
495
|
self._result = None
|
|
497
|
-
self._nowcast = None
|
|
498
496
|
if hasattr(self, 'training_state'):
|
|
499
497
|
self.training_state = None
|
|
500
498
|
return self
|