dfm-python 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dfm_python/config/results.py +6 -4
- dfm_python/config/schema.py +7 -2
- dfm_python/models/base.py +114 -171
- dfm_python/models/ddfm.py +30 -180
- dfm_python/models/dfm.py +3 -4
- dfm_python/models/utils.py +62 -59
- dfm_python/utils/data.py +5 -0
- dfm_python/utils/statespace.py +6 -2
- {dfm_python-0.5.0.dist-info → dfm_python-0.5.1.dist-info}/METADATA +36 -13
- {dfm_python-0.5.0.dist-info → dfm_python-0.5.1.dist-info}/RECORD +12 -12
- {dfm_python-0.5.0.dist-info → dfm_python-0.5.1.dist-info}/WHEEL +0 -0
- {dfm_python-0.5.0.dist-info → dfm_python-0.5.1.dist-info}/licenses/LICENSE +0 -0
dfm_python/config/results.py
CHANGED
|
@@ -313,12 +313,14 @@ class NowcastResult:
|
|
|
313
313
|
Examples
|
|
314
314
|
--------
|
|
315
315
|
>>> from dfm_python import DFM
|
|
316
|
+
>>> import numpy as np
|
|
316
317
|
>>> model = DFM()
|
|
317
318
|
>>> trainer.fit(model, data_module)
|
|
318
|
-
>>> #
|
|
319
|
-
>>>
|
|
320
|
-
>>>
|
|
321
|
-
>>>
|
|
319
|
+
>>> # Update state with new data, then predict
|
|
320
|
+
>>> X_std = np.random.randn(10, 5) # Standardized data
|
|
321
|
+
>>> model.update(X_std)
|
|
322
|
+
>>> forecast = model.predict(horizon=1)
|
|
323
|
+
>>> print(f"Forecast: {forecast[0, 0]}")
|
|
322
324
|
"""
|
|
323
325
|
target_series: str
|
|
324
326
|
target_period: datetime
|
dfm_python/config/schema.py
CHANGED
|
@@ -398,6 +398,11 @@ class BaseModelConfig:
|
|
|
398
398
|
f"Block '{block_name}' validation failed: AR lag must be at least 1, got {ar_lag}. "
|
|
399
399
|
f"Please set ar_lag >= 1 for block '{block_name}'."
|
|
400
400
|
)
|
|
401
|
+
if ar_lag > 2:
|
|
402
|
+
raise ValueError(
|
|
403
|
+
f"Block '{block_name}' validation failed: AR lag must be at most 2 (maximum supported VAR order is VAR(2)), got {ar_lag}. "
|
|
404
|
+
f"Please set ar_lag to 1 (VAR(1)) or 2 (VAR(2)) for block '{block_name}'."
|
|
405
|
+
)
|
|
401
406
|
|
|
402
407
|
# Auto-generate series_id if not provided and convert blocks to indices
|
|
403
408
|
n_blocks = len(self.block_names)
|
|
@@ -608,7 +613,7 @@ class DFMConfig(BaseModelConfig):
|
|
|
608
613
|
# ========================================================================
|
|
609
614
|
# EM Algorithm Parameters (HOW - controls the algorithm)
|
|
610
615
|
# ========================================================================
|
|
611
|
-
ar_lag: int = 1 # Number of lags in AR transition equation (lookback window)
|
|
616
|
+
ar_lag: int = 1 # Number of lags in AR transition equation (lookback window). Must be 1 or 2 (maximum supported order is VAR(2))
|
|
612
617
|
threshold: float = 1e-5 # EM convergence threshold
|
|
613
618
|
max_iter: int = 5000 # Maximum EM iterations
|
|
614
619
|
|
|
@@ -717,7 +722,7 @@ class DDFMConfig(BaseModelConfig):
|
|
|
717
722
|
learning_rate: float = 0.001 # Learning rate for Adam optimizer (default: 0.001)
|
|
718
723
|
epochs: int = 100 # Number of training epochs (default: 100)
|
|
719
724
|
batch_size: int = 100 # Batch size for training (default: 100 to match original DDFM)
|
|
720
|
-
factor_order: int = 1 # VAR lag order for factor dynamics
|
|
725
|
+
factor_order: int = 1 # VAR lag order for factor dynamics. Must be 1 or 2 (maximum supported order is VAR(2), default: 1)
|
|
721
726
|
use_idiosyncratic: bool = True # Model idio components with AR(1) dynamics (default: True)
|
|
722
727
|
min_obs_idio: int = 5 # Minimum observations for idio AR(1) estimation (default: 5)
|
|
723
728
|
|
dfm_python/models/base.py
CHANGED
|
@@ -16,16 +16,10 @@ import torch
|
|
|
16
16
|
import pytorch_lightning as pl
|
|
17
17
|
import pandas as pd
|
|
18
18
|
|
|
19
|
-
if TYPE_CHECKING:
|
|
20
|
-
from ..config.results import NowcastResult
|
|
21
|
-
|
|
22
19
|
from ..config import DFMConfig, make_config_source, ConfigSource, MergedConfigSource
|
|
23
20
|
from ..config.results import BaseResult
|
|
24
21
|
from ..config.schema import SeriesConfig, DEFAULT_BLOCK_NAME
|
|
25
22
|
from ..logger import get_logger
|
|
26
|
-
from .utils import create_nowcast_result
|
|
27
|
-
from ..utils.time import parse_timestamp, get_latest_time
|
|
28
|
-
from ..utils.data import create_data_view
|
|
29
23
|
|
|
30
24
|
_logger = get_logger(__name__)
|
|
31
25
|
|
|
@@ -194,28 +188,6 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
194
188
|
|
|
195
189
|
return config
|
|
196
190
|
|
|
197
|
-
def _get_data_from_datamodule(self) -> Tuple[Any, Optional[np.ndarray], Optional[np.ndarray]]:
|
|
198
|
-
"""Get processed data and standardization parameters from DataModule.
|
|
199
|
-
|
|
200
|
-
Returns
|
|
201
|
-
-------
|
|
202
|
-
X_torch : torch.Tensor
|
|
203
|
-
Processed data tensor (T x N)
|
|
204
|
-
Mx : np.ndarray or None
|
|
205
|
-
Mean values for unstandardization (N,)
|
|
206
|
-
Wx : np.ndarray or None
|
|
207
|
-
Standard deviation values for unstandardization (N,)
|
|
208
|
-
|
|
209
|
-
Raises
|
|
210
|
-
------
|
|
211
|
-
ValueError
|
|
212
|
-
If DataModule is not available
|
|
213
|
-
"""
|
|
214
|
-
data_module = self._get_datamodule()
|
|
215
|
-
X_torch = data_module.get_processed_data()
|
|
216
|
-
Mx, Wx = data_module.get_std_params()
|
|
217
|
-
return X_torch, Mx, Wx
|
|
218
|
-
|
|
219
191
|
def _get_datamodule(self):
|
|
220
192
|
"""Get DataModule from model or trainer.
|
|
221
193
|
|
|
@@ -250,7 +222,7 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
250
222
|
) -> np.ndarray:
|
|
251
223
|
"""Forecast factors using VAR dynamics.
|
|
252
224
|
|
|
253
|
-
Supports VAR(1) and VAR(2) factor dynamics.
|
|
225
|
+
Supports VAR(1) and VAR(2) factor dynamics (maximum supported order is VAR(2)).
|
|
254
226
|
|
|
255
227
|
Parameters
|
|
256
228
|
----------
|
|
@@ -259,7 +231,7 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
259
231
|
A : np.ndarray
|
|
260
232
|
Transition matrix. For VAR(1): (m x m), for VAR(2): (m x 2m)
|
|
261
233
|
p : int
|
|
262
|
-
VAR order
|
|
234
|
+
VAR order. Must be 1 or 2 (maximum supported order is VAR(2))
|
|
263
235
|
horizon : int
|
|
264
236
|
Number of periods to forecast
|
|
265
237
|
Z_prev : np.ndarray, optional
|
|
@@ -297,7 +269,7 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
297
269
|
else:
|
|
298
270
|
raise ValueError(
|
|
299
271
|
f"{self.__class__.__name__} prediction failed: unsupported VAR order {p}. "
|
|
300
|
-
f"
|
|
272
|
+
f"Maximum supported VAR order is VAR(2). Please use factor_order=1 (VAR(1)) or factor_order=2 (VAR(2))"
|
|
301
273
|
)
|
|
302
274
|
return Z_forecast
|
|
303
275
|
|
|
@@ -428,13 +400,22 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
428
400
|
Dict[str, torch.Tensor]
|
|
429
401
|
Dictionary of torch tensors: A, C, Q, R, Z_0, V_0
|
|
430
402
|
"""
|
|
403
|
+
# Ensure V_0 is always 2D (defensive check for edge cases)
|
|
404
|
+
V_0 = result.V_0
|
|
405
|
+
if isinstance(V_0, np.ndarray):
|
|
406
|
+
if V_0.ndim == 0:
|
|
407
|
+
V_0 = np.atleast_2d(V_0)
|
|
408
|
+
elif V_0.ndim == 1:
|
|
409
|
+
# If 1D, reshape to (m x m) - assume it's a single value for m=1 case
|
|
410
|
+
V_0 = np.atleast_2d(V_0).T if V_0.shape[0] == 1 else np.atleast_2d(V_0)
|
|
411
|
+
|
|
431
412
|
return {
|
|
432
413
|
'A': torch.tensor(result.A, dtype=torch.float32),
|
|
433
414
|
'C': torch.tensor(C if C is not None else result.C, dtype=torch.float32),
|
|
434
415
|
'Q': torch.tensor(result.Q, dtype=torch.float32),
|
|
435
416
|
'R': torch.tensor(R if R is not None else result.R, dtype=torch.float32),
|
|
436
417
|
'Z_0': torch.tensor(result.Z_0, dtype=torch.float32),
|
|
437
|
-
'V_0': torch.tensor(
|
|
418
|
+
'V_0': torch.tensor(V_0, dtype=torch.float32)
|
|
438
419
|
}
|
|
439
420
|
|
|
440
421
|
def _update_factor_state_dfm(
|
|
@@ -494,33 +475,66 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
494
475
|
"""
|
|
495
476
|
if not hasattr(self, 'encoder') or self.encoder is None:
|
|
496
477
|
_logger.warning(
|
|
497
|
-
f"{self.__class__.__name__}
|
|
498
|
-
"
|
|
478
|
+
f"{self.__class__.__name__} update(): Encoder not available, "
|
|
479
|
+
"cannot update factor state"
|
|
499
480
|
)
|
|
500
481
|
return None
|
|
501
482
|
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
483
|
+
try:
|
|
484
|
+
# Extract factors using encoder
|
|
485
|
+
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
|
486
|
+
self.encoder.eval()
|
|
487
|
+
X_tensor = torch.tensor(X_recent_std, device=device, dtype=torch.float32)
|
|
488
|
+
with torch.no_grad():
|
|
489
|
+
factors_raw = self.encoder(X_tensor).cpu().numpy() # (T x m)
|
|
490
|
+
|
|
491
|
+
# Validate factors shape
|
|
492
|
+
if factors_raw.ndim != 2 or factors_raw.shape[0] == 0:
|
|
493
|
+
_logger.warning(
|
|
494
|
+
f"{self.__class__.__name__} update(): Invalid factors shape {factors_raw.shape}, "
|
|
495
|
+
"cannot update factor state"
|
|
496
|
+
)
|
|
497
|
+
return None
|
|
498
|
+
|
|
499
|
+
# For DDFM: C = I (identity), R = small noise
|
|
500
|
+
m = factors_raw.shape[1]
|
|
501
|
+
C = np.eye(m)
|
|
502
|
+
R = np.eye(m) * 1e-8
|
|
503
|
+
|
|
504
|
+
# Convert to torch: (m x T) format for Kalman filter
|
|
505
|
+
Y = torch.tensor(factors_raw.T, dtype=torch.float32)
|
|
506
|
+
params = self._result_to_torch_params(result, C=C, R=R)
|
|
507
|
+
|
|
508
|
+
# Run Kalman smoother
|
|
509
|
+
kf = self._get_kalman_filter(kalman_filter)
|
|
510
|
+
kf_result = kf(Y, params['A'], params['C'], params['Q'],
|
|
511
|
+
params['R'], params['Z_0'], params['V_0'])
|
|
512
|
+
|
|
513
|
+
# Handle return value - Kalman filter returns (zsmooth, Vsmooth, VVsmooth, loglik)
|
|
514
|
+
if isinstance(kf_result, tuple) and len(kf_result) >= 1:
|
|
515
|
+
zsmooth = kf_result[0]
|
|
516
|
+
else:
|
|
517
|
+
zsmooth = kf_result
|
|
518
|
+
|
|
519
|
+
# zsmooth is (m x (T+1)), transpose to ((T+1) x m), then get last row
|
|
520
|
+
if not isinstance(zsmooth, torch.Tensor):
|
|
521
|
+
zsmooth = torch.tensor(zsmooth, dtype=torch.float32)
|
|
522
|
+
|
|
523
|
+
zsmooth_T = zsmooth.T # ((T+1) x m)
|
|
524
|
+
if zsmooth_T.ndim == 2 and zsmooth_T.shape[0] > 0:
|
|
525
|
+
return zsmooth_T[-1, :].cpu().numpy()
|
|
526
|
+
else:
|
|
527
|
+
_logger.warning(
|
|
528
|
+
f"{self.__class__.__name__} update(): Invalid zsmooth shape {zsmooth_T.shape}, "
|
|
529
|
+
"cannot extract last state"
|
|
530
|
+
)
|
|
531
|
+
return None
|
|
532
|
+
|
|
533
|
+
except Exception as e:
|
|
534
|
+
_logger.warning(
|
|
535
|
+
f"{self.__class__.__name__} update(): Failed to update factor state: {e}"
|
|
536
|
+
)
|
|
537
|
+
return None
|
|
524
538
|
|
|
525
539
|
@property
|
|
526
540
|
def result(self) -> BaseResult:
|
|
@@ -576,149 +590,78 @@ class BaseFactorModel(pl.LightningModule):
|
|
|
576
590
|
"""
|
|
577
591
|
pass
|
|
578
592
|
|
|
579
|
-
def
|
|
593
|
+
def update(
|
|
580
594
|
self,
|
|
581
|
-
|
|
582
|
-
view_date: Optional[Union[datetime, str]] = None,
|
|
583
|
-
target_period: Optional[Union[datetime, str]] = None,
|
|
595
|
+
X_std: np.ndarray,
|
|
584
596
|
*,
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
"""Calculate nowcast for target series.
|
|
589
|
-
|
|
590
|
-
This method performs nowcasting by:
|
|
591
|
-
1. Creating a data view with masking based on release dates
|
|
592
|
-
2. Re-running Kalman filter with masked data to update factor state
|
|
593
|
-
3. Calling predict() with the updated state
|
|
597
|
+
kalman_filter: Optional[Any] = None
|
|
598
|
+
) -> 'BaseFactorModel':
|
|
599
|
+
"""Update factor state with standardized data.
|
|
594
600
|
|
|
595
|
-
This
|
|
601
|
+
This method permanently updates the last factor state (result.Z[-1, :])
|
|
602
|
+
using the provided standardized data. Users should handle all preprocessing
|
|
603
|
+
(masking, imputation, standardization) before calling this method.
|
|
596
604
|
|
|
597
605
|
Parameters
|
|
598
606
|
----------
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
return_result : bool, default False
|
|
606
|
-
If True, returns NowcastResult with additional information.
|
|
607
|
-
If False, returns only the nowcast value (float).
|
|
608
|
-
horizon : int, default 1
|
|
609
|
-
Forecast horizon. For nowcasting, typically 1 (current period).
|
|
607
|
+
X_std : np.ndarray
|
|
608
|
+
Standardized data array (T x N), where T is number of time periods
|
|
609
|
+
and N is number of series. Data should already be standardized using
|
|
610
|
+
result.Mx and result.Wx.
|
|
611
|
+
kalman_filter : Any, optional
|
|
612
|
+
Kalman filter instance. If None, uses default or model's kalman filter.
|
|
610
613
|
|
|
611
614
|
Returns
|
|
612
615
|
-------
|
|
613
|
-
|
|
614
|
-
|
|
616
|
+
BaseFactorModel
|
|
617
|
+
Self for method chaining
|
|
615
618
|
|
|
616
619
|
Examples
|
|
617
620
|
--------
|
|
618
|
-
>>>
|
|
619
|
-
>>>
|
|
620
|
-
>>> #
|
|
621
|
-
>>>
|
|
622
|
-
>>>
|
|
623
|
-
>>> result = model.nowcast('gdp', view_date='2024-01-15', return_result=True)
|
|
621
|
+
>>> # Update state with new data, then predict
|
|
622
|
+
>>> model.update(X_std).predict(horizon=1)
|
|
623
|
+
>>> # Or update and predict separately
|
|
624
|
+
>>> model.update(X_std)
|
|
625
|
+
>>> forecast = model.predict(horizon=6)
|
|
624
626
|
"""
|
|
625
627
|
self._check_trained()
|
|
626
628
|
|
|
627
|
-
#
|
|
628
|
-
data_module = self._get_datamodule()
|
|
629
|
-
raw_data = data_module.data
|
|
630
|
-
time_index = data_module.time_index
|
|
629
|
+
result = self.result # Use property which ensures non-None after _check_trained()
|
|
631
630
|
|
|
632
|
-
#
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
else:
|
|
641
|
-
view_date_dt = view_date
|
|
642
|
-
|
|
643
|
-
# Create data view with masking
|
|
644
|
-
X_view, Time_view, _ = create_data_view(
|
|
645
|
-
X=X,
|
|
646
|
-
Time=time_index,
|
|
647
|
-
Z=None,
|
|
648
|
-
config=self.config,
|
|
649
|
-
view_date=view_date_dt,
|
|
650
|
-
X_frame=raw_data if isinstance(raw_data, pd.DataFrame) else None
|
|
651
|
-
)
|
|
631
|
+
# Validate input shape
|
|
632
|
+
if not isinstance(X_std, np.ndarray):
|
|
633
|
+
X_std = np.asarray(X_std)
|
|
634
|
+
if X_std.ndim != 2:
|
|
635
|
+
raise ValueError(
|
|
636
|
+
f"{self.__class__.__name__} update(): X_std must be 2D array (T x N), "
|
|
637
|
+
f"got shape {X_std.shape}"
|
|
638
|
+
)
|
|
652
639
|
|
|
653
|
-
#
|
|
654
|
-
|
|
655
|
-
X_view_std = self._standardize_data(X_view, result.Mx, result.Wx)
|
|
656
|
-
X_view_std = np.where(np.isfinite(X_view_std), X_view_std, np.nan)
|
|
640
|
+
# Handle NaN/Inf values
|
|
641
|
+
X_std = np.where(np.isfinite(X_std), X_std, np.nan)
|
|
657
642
|
|
|
658
|
-
# Update factor state
|
|
643
|
+
# Update factor state (model-specific)
|
|
659
644
|
if hasattr(self, 'encoder') and self.encoder is not None:
|
|
660
645
|
# DDFM: Extract factors via encoder, then use Kalman filter
|
|
661
|
-
Z_last_updated = self._update_factor_state_ddfm(
|
|
646
|
+
Z_last_updated = self._update_factor_state_ddfm(
|
|
647
|
+
X_std, result, kalman_filter
|
|
648
|
+
)
|
|
662
649
|
else:
|
|
663
650
|
# DFM: Use Kalman filter directly on standardized data
|
|
664
651
|
Z_last_updated = self._update_factor_state_dfm(
|
|
665
|
-
|
|
652
|
+
X_std, result, kalman_filter or getattr(self, 'kalman', None)
|
|
666
653
|
)
|
|
667
654
|
|
|
668
|
-
|
|
669
|
-
|
|
655
|
+
# Update result.Z[-1, :] permanently
|
|
656
|
+
if Z_last_updated is not None:
|
|
657
|
+
result.Z[-1, :] = Z_last_updated
|
|
658
|
+
else:
|
|
670
659
|
_logger.warning(
|
|
671
|
-
f"{self.__class__.__name__}
|
|
672
|
-
f"
|
|
660
|
+
f"{self.__class__.__name__} update(): Failed to update factor state, "
|
|
661
|
+
f"keeping current state"
|
|
673
662
|
)
|
|
674
663
|
|
|
675
|
-
|
|
676
|
-
original_Z_last = result.Z[-1, :].copy()
|
|
677
|
-
result.Z[-1, :] = Z_last_updated
|
|
678
|
-
|
|
679
|
-
try:
|
|
680
|
-
# Call predict() with updated state
|
|
681
|
-
X_forecast = self.predict(horizon=horizon, return_series=True, return_factors=False)
|
|
682
|
-
|
|
683
|
-
# Extract nowcast value for target series
|
|
684
|
-
series_ids = [s.series_id for s in self.config.series if s.series_id]
|
|
685
|
-
try:
|
|
686
|
-
target_idx = series_ids.index(target_series)
|
|
687
|
-
except ValueError:
|
|
688
|
-
available = ', '.join(str(sid) for sid in series_ids[:5])
|
|
689
|
-
raise ValueError(
|
|
690
|
-
f"{self.__class__.__name__} nowcast failed: target_series '{target_series}' not found. "
|
|
691
|
-
f"Available: {available}..."
|
|
692
|
-
)
|
|
693
|
-
|
|
694
|
-
# Get nowcast value (first period of forecast)
|
|
695
|
-
if isinstance(X_forecast, np.ndarray):
|
|
696
|
-
nowcast_value = float(X_forecast[0, target_idx])
|
|
697
|
-
else:
|
|
698
|
-
nowcast_value = float(X_forecast[0][target_idx])
|
|
699
|
-
|
|
700
|
-
if return_result:
|
|
701
|
-
# Calculate data availability
|
|
702
|
-
data_availability = {
|
|
703
|
-
'n_available': int(np.sum(np.isfinite(X_view))),
|
|
704
|
-
'n_missing': int(np.sum(np.isnan(X_view)))
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
# Create NowcastResult using utility function
|
|
708
|
-
return create_nowcast_result(
|
|
709
|
-
target_series=target_series,
|
|
710
|
-
target_period=target_period,
|
|
711
|
-
view_date=view_date_dt,
|
|
712
|
-
nowcast_value=nowcast_value,
|
|
713
|
-
factors_at_view=Z_last_updated,
|
|
714
|
-
data_availability=data_availability,
|
|
715
|
-
dfm_result=result
|
|
716
|
-
)
|
|
717
|
-
else:
|
|
718
|
-
return nowcast_value
|
|
719
|
-
finally:
|
|
720
|
-
# Restore original state
|
|
721
|
-
result.Z[-1, :] = original_Z_last
|
|
664
|
+
return self
|
|
722
665
|
|
|
723
666
|
def get_result(self) -> BaseResult:
|
|
724
667
|
"""Extract result from trained model.
|
dfm_python/models/ddfm.py
CHANGED
|
@@ -32,7 +32,6 @@ from ..encoder.vae import Decoder, Encoder, extract_decoder_params
|
|
|
32
32
|
from ..logger import get_logger
|
|
33
33
|
from ..utils.data import rem_nans_spline
|
|
34
34
|
from ..utils.helpers import (
|
|
35
|
-
find_series_index,
|
|
36
35
|
get_clock_frequency,
|
|
37
36
|
)
|
|
38
37
|
from ..utils.statespace import (
|
|
@@ -40,8 +39,6 @@ from ..utils.statespace import (
|
|
|
40
39
|
estimate_var2,
|
|
41
40
|
)
|
|
42
41
|
from ..utils.time import (
|
|
43
|
-
convert_to_timestamp,
|
|
44
|
-
find_time_index,
|
|
45
42
|
TimeIndex,
|
|
46
43
|
)
|
|
47
44
|
from .base import BaseFactorModel
|
|
@@ -49,6 +46,7 @@ from .utils import (
|
|
|
49
46
|
estimate_var_ddfm,
|
|
50
47
|
validate_factors_ddfm,
|
|
51
48
|
validate_training_data_ddfm,
|
|
49
|
+
forecast_var_factors,
|
|
52
50
|
)
|
|
53
51
|
|
|
54
52
|
if TYPE_CHECKING:
|
|
@@ -73,13 +71,14 @@ class DDFMModel:
|
|
|
73
71
|
This class implements a DDFM with:
|
|
74
72
|
- Nonlinear encoder (MLP) to extract factors from observations
|
|
75
73
|
- Linear decoder for interpretability
|
|
76
|
-
- Linear factor dynamics (VAR)
|
|
74
|
+
- Linear factor dynamics (VAR) - supports VAR(1) and VAR(2), maximum order is VAR(2)
|
|
77
75
|
- Kalman filtering for final smoothing
|
|
78
76
|
|
|
79
77
|
The model is trained using gradient descent (Adam optimizer) to minimize
|
|
80
78
|
reconstruction error, then factor dynamics are estimated via OLS, and
|
|
81
79
|
final smoothing is performed using Kalman filter.
|
|
82
80
|
|
|
81
|
+
Note: Maximum supported VAR order for factor dynamics is VAR(2).
|
|
83
82
|
"""
|
|
84
83
|
|
|
85
84
|
def __init__(
|
|
@@ -120,7 +119,7 @@ class DDFMModel:
|
|
|
120
119
|
batch_size : int
|
|
121
120
|
Batch size for training. Default: 100 (matches original DDFM)
|
|
122
121
|
factor_order : int
|
|
123
|
-
VAR lag order for factor dynamics
|
|
122
|
+
VAR lag order for factor dynamics. Must be 1 or 2 (maximum supported order is VAR(2)). Default: 1
|
|
124
123
|
use_idiosyncratic : bool
|
|
125
124
|
Whether to model idiosyncratic components with AR(1) dynamics. Default: True
|
|
126
125
|
min_obs_idio : int
|
|
@@ -141,7 +140,7 @@ class DDFMModel:
|
|
|
141
140
|
if factor_order not in [1, 2]:
|
|
142
141
|
raise ValueError(
|
|
143
142
|
f"DDFM initialization failed: factor_order must be 1 or 2, got {factor_order}. "
|
|
144
|
-
f"
|
|
143
|
+
f"Maximum supported VAR order is VAR(2). Please provide factor_order=1 (VAR(1)) or factor_order=2 (VAR(2))"
|
|
145
144
|
)
|
|
146
145
|
|
|
147
146
|
self.encoder_layers = encoder_layers or [64, 32]
|
|
@@ -212,45 +211,16 @@ class DDFMModel:
|
|
|
212
211
|
)
|
|
213
212
|
|
|
214
213
|
# Extract parameters
|
|
215
|
-
A = self._result.A
|
|
214
|
+
A = self._result.A
|
|
216
215
|
C = self._result.C
|
|
217
216
|
Wx = self._result.Wx
|
|
218
217
|
Mx = self._result.Mx
|
|
219
|
-
Z_last = self._result.Z[-1, :]
|
|
220
|
-
p = self._result.p
|
|
218
|
+
Z_last = self._result.Z[-1, :]
|
|
219
|
+
p = self._result.p
|
|
221
220
|
|
|
222
|
-
#
|
|
223
|
-
if p ==
|
|
224
|
-
|
|
225
|
-
Z_forecast = np.zeros((horizon, Z_last.shape[0]))
|
|
226
|
-
Z_forecast[0, :] = A @ Z_last
|
|
227
|
-
for h in range(1, horizon):
|
|
228
|
-
Z_forecast[h, :] = A @ Z_forecast[h - 1, :]
|
|
229
|
-
elif p == 2:
|
|
230
|
-
# VAR(2): f_t = A1 @ f_{t-1} + A2 @ f_{t-2}
|
|
231
|
-
# Need last two factor values
|
|
232
|
-
if self._result.Z.shape[0] < 2:
|
|
233
|
-
# Fallback to VAR(1) if not enough history
|
|
234
|
-
Z_forecast = np.zeros((horizon, Z_last.shape[0]))
|
|
235
|
-
A1 = A[:, :Z_last.shape[0]]
|
|
236
|
-
Z_forecast[0, :] = A1 @ Z_last
|
|
237
|
-
for h in range(1, horizon):
|
|
238
|
-
Z_forecast[h, :] = A1 @ Z_forecast[h - 1, :]
|
|
239
|
-
else:
|
|
240
|
-
Z_prev = self._result.Z[-2, :] # f_{t-2}
|
|
241
|
-
A1 = A[:, :Z_last.shape[0]]
|
|
242
|
-
A2 = A[:, Z_last.shape[0]:]
|
|
243
|
-
Z_forecast = np.zeros((horizon, Z_last.shape[0]))
|
|
244
|
-
Z_forecast[0, :] = A1 @ Z_last + A2 @ Z_prev
|
|
245
|
-
if horizon > 1:
|
|
246
|
-
Z_forecast[1, :] = A1 @ Z_forecast[0, :] + A2 @ Z_last
|
|
247
|
-
for h in range(2, horizon):
|
|
248
|
-
Z_forecast[h, :] = A1 @ Z_forecast[h - 1, :] + A2 @ Z_forecast[h - 2, :]
|
|
249
|
-
else:
|
|
250
|
-
raise ValueError(
|
|
251
|
-
f"DDFM prediction failed: unsupported VAR order {p}. "
|
|
252
|
-
f"Only VAR(1) and VAR(2) are supported. Please use factor_order=1 or factor_order=2"
|
|
253
|
-
)
|
|
221
|
+
# Forecast factors using VAR dynamics
|
|
222
|
+
Z_prev = self._result.Z[-2, :] if self._result.Z.shape[0] >= 2 and p == 2 else None
|
|
223
|
+
Z_forecast = forecast_var_factors(Z_last, A, p, horizon, Z_prev)
|
|
254
224
|
|
|
255
225
|
# Transform to observations
|
|
256
226
|
X_forecast_std = Z_forecast @ C.T
|
|
@@ -261,76 +231,6 @@ class DDFMModel:
|
|
|
261
231
|
if return_series:
|
|
262
232
|
return X_forecast
|
|
263
233
|
return Z_forecast
|
|
264
|
-
|
|
265
|
-
def get_state(
|
|
266
|
-
self,
|
|
267
|
-
t: Union[int, datetime],
|
|
268
|
-
target_series: str,
|
|
269
|
-
lookback: Optional[int] = None
|
|
270
|
-
) -> Dict[str, Any]:
|
|
271
|
-
"""Get DFM state at time t.
|
|
272
|
-
|
|
273
|
-
Note: Requires data to be stored during fit(). Data is automatically
|
|
274
|
-
stored from DataModule during fit().
|
|
275
|
-
"""
|
|
276
|
-
if not hasattr(self, '_data') or self._data is None:
|
|
277
|
-
raise ValueError(
|
|
278
|
-
f"DDFM get_state failed: model has not been fitted with DataModule yet. "
|
|
279
|
-
f"Please call trainer.fit(model, data_module) first to store data"
|
|
280
|
-
)
|
|
281
|
-
|
|
282
|
-
if lookback is None:
|
|
283
|
-
clock = get_clock_frequency(self._config, 'm')
|
|
284
|
-
lookback = get_periods_per_year(clock)
|
|
285
|
-
|
|
286
|
-
t = convert_to_timestamp(t, self._time, None)
|
|
287
|
-
i_series = find_series_index(self._config, target_series)
|
|
288
|
-
|
|
289
|
-
# Convert data to numpy if needed
|
|
290
|
-
if hasattr(self._data, 'to_numpy'):
|
|
291
|
-
X_data = self._data.to_numpy()
|
|
292
|
-
else:
|
|
293
|
-
X_data = np.asarray(self._data)
|
|
294
|
-
|
|
295
|
-
# For get_state, we use the raw data directly
|
|
296
|
-
X_view = X_data
|
|
297
|
-
|
|
298
|
-
# Placeholder for nowcast (use src.nowcast for actual nowcasting functionality)
|
|
299
|
-
baseline_nowcast = np.nan
|
|
300
|
-
|
|
301
|
-
baseline_forecast, actual_history, residuals, factors_history = [], [], [], []
|
|
302
|
-
t_idx = find_time_index(self._time, t)
|
|
303
|
-
if t_idx is None:
|
|
304
|
-
raise ValueError(
|
|
305
|
-
f"DDFM get_state failed: time {t} not found in model_instance._time. "
|
|
306
|
-
f"Please provide a valid time value that exists in the model's time index"
|
|
307
|
-
)
|
|
308
|
-
|
|
309
|
-
for i in range(max(0, t_idx - lookback + 1), t_idx + 1):
|
|
310
|
-
if i < X_data.shape[0]:
|
|
311
|
-
# Use smoothed value from result if available, otherwise NaN
|
|
312
|
-
if self._result is not None and hasattr(self._result, 'X_sm') and i < self._result.X_sm.shape[0]:
|
|
313
|
-
forecast_val = self._result.X_sm[i, i_series] if i_series < self._result.X_sm.shape[1] else np.nan
|
|
314
|
-
else:
|
|
315
|
-
forecast_val = np.nan
|
|
316
|
-
baseline_forecast.append(forecast_val)
|
|
317
|
-
actual_val = X_data[i, i_series] if i_series < X_data.shape[1] else np.nan
|
|
318
|
-
actual_history.append(actual_val)
|
|
319
|
-
residuals.append(actual_val - forecast_val)
|
|
320
|
-
if self._result is not None and hasattr(self._result, 'Z') and i < self._result.Z.shape[0]:
|
|
321
|
-
factors_history.append(self._result.Z[i, :])
|
|
322
|
-
else:
|
|
323
|
-
factors_history.append(np.array([]))
|
|
324
|
-
|
|
325
|
-
return {
|
|
326
|
-
'time': t,
|
|
327
|
-
'target_series': target_series,
|
|
328
|
-
'baseline_nowcast': baseline_nowcast,
|
|
329
|
-
'baseline_forecast': np.array(baseline_forecast),
|
|
330
|
-
'actual_history': np.array(actual_history),
|
|
331
|
-
'residuals': np.array(residuals),
|
|
332
|
-
'factors_history': factors_history
|
|
333
|
-
}
|
|
334
234
|
|
|
335
235
|
# ============================================================================
|
|
336
236
|
# High-level API Classes
|
|
@@ -346,6 +246,8 @@ class DDFM(BaseFactorModel):
|
|
|
346
246
|
Lightning training patterns. It inherits from BaseFactorModel and implements
|
|
347
247
|
DDFM training using autoencoder and MCMC procedure.
|
|
348
248
|
|
|
249
|
+
Note: Maximum supported VAR order for factor dynamics is VAR(2) (set via factor_order parameter).
|
|
250
|
+
|
|
349
251
|
Example (Standard Lightning Pattern):
|
|
350
252
|
>>> from dfm_python import DDFM, DFMDataModule, DDFMTrainer
|
|
351
253
|
>>> import pandas as pd
|
|
@@ -429,7 +331,7 @@ class DDFM(BaseFactorModel):
|
|
|
429
331
|
batch_size : int, default 100
|
|
430
332
|
Batch size for training (matches original DDFM)
|
|
431
333
|
factor_order : int, default 1
|
|
432
|
-
VAR lag order for factor dynamics
|
|
334
|
+
VAR lag order for factor dynamics. Must be 1 or 2 (maximum supported order is VAR(2))
|
|
433
335
|
use_idiosyncratic : bool, default True
|
|
434
336
|
Whether to model idiosyncratic components
|
|
435
337
|
min_obs_idio : int, default 5
|
|
@@ -459,6 +361,13 @@ class DDFM(BaseFactorModel):
|
|
|
459
361
|
# We create a minimal default block that will be ignored by DDFM
|
|
460
362
|
config = self._initialize_config(config)
|
|
461
363
|
|
|
364
|
+
# Validate factor_order
|
|
365
|
+
if factor_order not in [1, 2]:
|
|
366
|
+
raise ValueError(
|
|
367
|
+
f"DDFM initialization failed: factor_order must be 1 or 2, got {factor_order}. "
|
|
368
|
+
f"Maximum supported VAR order is VAR(2). Please provide factor_order=1 (VAR(1)) or factor_order=2 (VAR(2))"
|
|
369
|
+
)
|
|
370
|
+
|
|
462
371
|
self.encoder_layers = encoder_layers or [64, 32]
|
|
463
372
|
self.activation = activation
|
|
464
373
|
self.use_batch_norm = use_batch_norm
|
|
@@ -1158,6 +1067,12 @@ class DDFM(BaseFactorModel):
|
|
|
1158
1067
|
Q = Q_f
|
|
1159
1068
|
Z_0 = factors[0, :]
|
|
1160
1069
|
V_0 = np.cov(factors.T)
|
|
1070
|
+
# Ensure V_0 is always 2D (np.cov returns scalar when m=1)
|
|
1071
|
+
if V_0.ndim == 0:
|
|
1072
|
+
V_0 = np.atleast_2d(V_0)
|
|
1073
|
+
elif V_0.ndim == 1:
|
|
1074
|
+
# If 1D, reshape to (m x m)
|
|
1075
|
+
V_0 = np.atleast_2d(V_0).T if V_0.shape[0] == 1 else np.atleast_2d(V_0)
|
|
1161
1076
|
|
|
1162
1077
|
# Estimate R from residuals
|
|
1163
1078
|
R_diag = np.var(residuals, axis=0)
|
|
@@ -1217,7 +1132,9 @@ class DDFM(BaseFactorModel):
|
|
|
1217
1132
|
def on_train_start(self) -> None:
|
|
1218
1133
|
"""Called when training starts. Run MCMC training."""
|
|
1219
1134
|
# Get processed data and standardization params from DataModule
|
|
1220
|
-
|
|
1135
|
+
data_module = self._get_datamodule()
|
|
1136
|
+
X_torch = data_module.get_processed_data()
|
|
1137
|
+
Mx, Wx = data_module.get_std_params()
|
|
1221
1138
|
|
|
1222
1139
|
# Early validation: Check data dimensions and model configuration before training
|
|
1223
1140
|
# This catches configuration issues early with clear error messages
|
|
@@ -1496,71 +1413,4 @@ class DDFM(BaseFactorModel):
|
|
|
1496
1413
|
"""Reset model state."""
|
|
1497
1414
|
super().reset()
|
|
1498
1415
|
return self
|
|
1499
|
-
|
|
1500
|
-
# nowcast() is inherited from BaseFactorModel
|
|
1501
|
-
|
|
1502
|
-
def get_state(
|
|
1503
|
-
self,
|
|
1504
|
-
t: Union[int, datetime],
|
|
1505
|
-
target_series: str,
|
|
1506
|
-
lookback: Optional[int] = None
|
|
1507
|
-
) -> Dict[str, Any]:
|
|
1508
|
-
"""Get DFM state at time t.
|
|
1509
|
-
|
|
1510
|
-
Returns state information including baseline forecast, actual history,
|
|
1511
|
-
residuals, and factor history for the specified time period.
|
|
1512
|
-
"""
|
|
1513
|
-
if not hasattr(self, '_data') or self._data is None:
|
|
1514
|
-
raise ValueError(
|
|
1515
|
-
f"DDFM get_state failed: model has not been fitted with DataModule yet. "
|
|
1516
|
-
f"Please call trainer.fit(model, data_module) first to store data"
|
|
1517
|
-
)
|
|
1518
|
-
|
|
1519
|
-
if lookback is None:
|
|
1520
|
-
lookback = 12 # Default lookback
|
|
1521
|
-
|
|
1522
|
-
i_series = find_series_index(self._config, target_series)
|
|
1523
|
-
|
|
1524
|
-
# Convert data to numpy if needed
|
|
1525
|
-
if hasattr(self._data, 'to_numpy'):
|
|
1526
|
-
X_data = self._data.to_numpy()
|
|
1527
|
-
else:
|
|
1528
|
-
X_data = np.asarray(self._data)
|
|
1529
|
-
|
|
1530
|
-
# Placeholder for nowcast (use src.nowcast for actual nowcasting functionality)
|
|
1531
|
-
baseline_nowcast = np.nan
|
|
1532
|
-
|
|
1533
|
-
baseline_forecast, actual_history, residuals, factors_history = [], [], [], []
|
|
1534
|
-
t_idx = find_time_index(self._time, t)
|
|
1535
|
-
if t_idx is None:
|
|
1536
|
-
raise ValueError(
|
|
1537
|
-
f"DDFM get_state failed: time {t} not found in model_instance._time. "
|
|
1538
|
-
f"Please provide a valid time value that exists in the model's time index"
|
|
1539
|
-
)
|
|
1540
|
-
|
|
1541
|
-
for i in range(max(0, t_idx - lookback + 1), t_idx + 1):
|
|
1542
|
-
if i < X_data.shape[0]:
|
|
1543
|
-
# Use smoothed value from result if available, otherwise NaN
|
|
1544
|
-
if self._result is not None and hasattr(self._result, 'X_sm') and i < self._result.X_sm.shape[0]:
|
|
1545
|
-
forecast_val = self._result.X_sm[i, i_series] if i_series < self._result.X_sm.shape[1] else np.nan
|
|
1546
|
-
else:
|
|
1547
|
-
forecast_val = np.nan
|
|
1548
|
-
baseline_forecast.append(forecast_val)
|
|
1549
|
-
actual_val = X_data[i, i_series] if i_series < X_data.shape[1] else np.nan
|
|
1550
|
-
actual_history.append(actual_val)
|
|
1551
|
-
residuals.append(actual_val - forecast_val)
|
|
1552
|
-
if self._result is not None and hasattr(self._result, 'Z') and i < self._result.Z.shape[0]:
|
|
1553
|
-
factors_history.append(self._result.Z[i, :])
|
|
1554
|
-
else:
|
|
1555
|
-
factors_history.append(np.array([]))
|
|
1556
|
-
|
|
1557
|
-
return {
|
|
1558
|
-
'time': t,
|
|
1559
|
-
'target_series': target_series,
|
|
1560
|
-
'baseline_nowcast': baseline_nowcast,
|
|
1561
|
-
'baseline_forecast': np.array(baseline_forecast),
|
|
1562
|
-
'actual_history': np.array(actual_history),
|
|
1563
|
-
'residuals': np.array(residuals),
|
|
1564
|
-
'factors_history': factors_history
|
|
1565
|
-
}
|
|
1566
1416
|
|
dfm_python/models/dfm.py
CHANGED
|
@@ -664,7 +664,9 @@ class DFM(BaseFactorModel):
|
|
|
664
664
|
def on_train_start(self) -> None:
|
|
665
665
|
"""Called when training starts. Run EM algorithm."""
|
|
666
666
|
# Get processed data and standardization params from DataModule
|
|
667
|
-
|
|
667
|
+
data_module = self._get_datamodule()
|
|
668
|
+
X_torch = data_module.get_processed_data()
|
|
669
|
+
Mx, Wx = data_module.get_std_params()
|
|
668
670
|
|
|
669
671
|
# Run EM algorithm
|
|
670
672
|
self.fit_em(X_torch, Mx=Mx, Wx=Wx)
|
|
@@ -863,8 +865,5 @@ class DFM(BaseFactorModel):
|
|
|
863
865
|
"""Reset model state."""
|
|
864
866
|
super().reset()
|
|
865
867
|
return self
|
|
866
|
-
|
|
867
|
-
# nowcast() is inherited from BaseFactorModel
|
|
868
|
-
|
|
869
868
|
|
|
870
869
|
|
dfm_python/models/utils.py
CHANGED
|
@@ -5,16 +5,12 @@ organization and maintainability.
|
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
7
|
import logging
|
|
8
|
-
from typing import Tuple, Optional
|
|
9
|
-
from datetime import datetime
|
|
8
|
+
from typing import Tuple, Optional
|
|
10
9
|
import numpy as np
|
|
11
10
|
import torch
|
|
12
|
-
import pandas as pd
|
|
13
11
|
|
|
14
12
|
from ..logger import get_logger
|
|
15
13
|
from ..utils.statespace import estimate_var1, estimate_var2
|
|
16
|
-
from ..utils.time import parse_timestamp
|
|
17
|
-
from ..config.results import NowcastResult
|
|
18
14
|
|
|
19
15
|
_logger = get_logger(__name__)
|
|
20
16
|
|
|
@@ -32,7 +28,7 @@ def estimate_var_ddfm(
|
|
|
32
28
|
Factors array (T x m), where T is number of time periods and m is number of factors.
|
|
33
29
|
Must be 2D array with at least 2 observations.
|
|
34
30
|
factor_order : int
|
|
35
|
-
VAR order
|
|
31
|
+
VAR order. Must be 1 or 2 (maximum supported order is VAR(2)).
|
|
36
32
|
num_factors : int, optional
|
|
37
33
|
Number of factors. Used for fallback when factors shape is invalid.
|
|
38
34
|
If None, inferred from factors.shape[1].
|
|
@@ -49,7 +45,7 @@ def estimate_var_ddfm(
|
|
|
49
45
|
Raises
|
|
50
46
|
------
|
|
51
47
|
ValueError
|
|
52
|
-
If factor_order is not 1 or 2.
|
|
48
|
+
If factor_order is not 1 or 2 (maximum supported order is VAR(2)).
|
|
53
49
|
"""
|
|
54
50
|
# Validate factors shape - check for 0-dimensional array first
|
|
55
51
|
factors = np.asarray(factors)
|
|
@@ -113,7 +109,10 @@ def estimate_var_ddfm(
|
|
|
113
109
|
elif factor_order == 2:
|
|
114
110
|
A_f, Q_f = estimate_var2(factors)
|
|
115
111
|
else:
|
|
116
|
-
raise ValueError(
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"DDFM VAR estimation failed: factor_order must be 1 or 2, got {factor_order}. "
|
|
114
|
+
f"Maximum supported VAR order is VAR(2). Please set factor_order to 1 (VAR(1)) or 2 (VAR(2))"
|
|
115
|
+
)
|
|
117
116
|
|
|
118
117
|
# Validate Q_f shape
|
|
119
118
|
if Q_f.ndim == 0:
|
|
@@ -307,7 +306,7 @@ def validate_training_data_ddfm(
|
|
|
307
306
|
num_factors : int
|
|
308
307
|
Number of factors to extract
|
|
309
308
|
factor_order : int
|
|
310
|
-
VAR order for factor dynamics
|
|
309
|
+
VAR order for factor dynamics. Must be 1 or 2 (maximum supported order is VAR(2))
|
|
311
310
|
encoder_layers : list
|
|
312
311
|
Encoder hidden layer dimensions
|
|
313
312
|
encoder : torch.nn.Module, optional
|
|
@@ -407,61 +406,65 @@ def validate_training_data_ddfm(
|
|
|
407
406
|
)
|
|
408
407
|
|
|
409
408
|
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
dfm_result: Optional[Any] = None,
|
|
422
|
-
confidence_interval: Optional[Tuple[float, float]] = None
|
|
423
|
-
) -> NowcastResult:
|
|
424
|
-
"""Create NowcastResult object.
|
|
409
|
+
def forecast_var_factors(
|
|
410
|
+
Z_last: np.ndarray,
|
|
411
|
+
A: np.ndarray,
|
|
412
|
+
p: int,
|
|
413
|
+
horizon: int,
|
|
414
|
+
Z_prev: Optional[np.ndarray] = None
|
|
415
|
+
) -> np.ndarray:
|
|
416
|
+
"""Forecast factors using VAR dynamics (standalone function).
|
|
417
|
+
|
|
418
|
+
Supports VAR(1) and VAR(2) factor dynamics (maximum supported order is VAR(2)).
|
|
419
|
+
This is a module-level function that can be used by classes that don't inherit from BaseFactorModel.
|
|
425
420
|
|
|
426
421
|
Parameters
|
|
427
422
|
----------
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
data_availability : Dict[str, int], optional
|
|
439
|
-
Dictionary with 'n_available' and 'n_missing' keys
|
|
440
|
-
dfm_result : BaseResult, optional
|
|
441
|
-
Full DFM/DDFM result for this view
|
|
442
|
-
confidence_interval : Tuple[float, float], optional
|
|
443
|
-
Confidence interval (lower, upper) for the nowcast
|
|
423
|
+
Z_last : np.ndarray
|
|
424
|
+
Last factor state (m,)
|
|
425
|
+
A : np.ndarray
|
|
426
|
+
Transition matrix. For VAR(1): (m x m), for VAR(2): (m x 2m)
|
|
427
|
+
p : int
|
|
428
|
+
VAR order. Must be 1 or 2 (maximum supported order is VAR(2))
|
|
429
|
+
horizon : int
|
|
430
|
+
Number of periods to forecast
|
|
431
|
+
Z_prev : np.ndarray, optional
|
|
432
|
+
Previous factor state for VAR(2) (m,). Required if p == 2.
|
|
444
433
|
|
|
445
434
|
Returns
|
|
446
435
|
-------
|
|
447
|
-
|
|
448
|
-
|
|
436
|
+
np.ndarray
|
|
437
|
+
Forecasted factors (horizon x m)
|
|
449
438
|
"""
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
439
|
+
if p == 1:
|
|
440
|
+
# VAR(1): f_t = A @ f_{t-1}
|
|
441
|
+
Z_forecast = np.zeros((horizon, Z_last.shape[0]))
|
|
442
|
+
Z_forecast[0, :] = A @ Z_last
|
|
443
|
+
for h in range(1, horizon):
|
|
444
|
+
Z_forecast[h, :] = A @ Z_forecast[h - 1, :]
|
|
445
|
+
elif p == 2:
|
|
446
|
+
# VAR(2): f_t = A1 @ f_{t-1} + A2 @ f_{t-2}
|
|
447
|
+
if Z_prev is None:
|
|
448
|
+
# Fallback to VAR(1) if not enough history
|
|
449
|
+
A1 = A[:, :Z_last.shape[0]]
|
|
450
|
+
Z_forecast = np.zeros((horizon, Z_last.shape[0]))
|
|
451
|
+
Z_forecast[0, :] = A1 @ Z_last
|
|
452
|
+
for h in range(1, horizon):
|
|
453
|
+
Z_forecast[h, :] = A1 @ Z_forecast[h - 1, :]
|
|
454
|
+
else:
|
|
455
|
+
A1 = A[:, :Z_last.shape[0]]
|
|
456
|
+
A2 = A[:, Z_last.shape[0]:]
|
|
457
|
+
Z_forecast = np.zeros((horizon, Z_last.shape[0]))
|
|
458
|
+
Z_forecast[0, :] = A1 @ Z_last + A2 @ Z_prev
|
|
459
|
+
if horizon > 1:
|
|
460
|
+
Z_forecast[1, :] = A1 @ Z_forecast[0, :] + A2 @ Z_last
|
|
461
|
+
for h in range(2, horizon):
|
|
462
|
+
Z_forecast[h, :] = A1 @ Z_forecast[h - 1, :] + A2 @ Z_forecast[h - 2, :]
|
|
455
463
|
else:
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
factors_at_view=factors_at_view,
|
|
464
|
-
data_availability=data_availability,
|
|
465
|
-
dfm_result=dfm_result,
|
|
466
|
-
confidence_interval=confidence_interval
|
|
467
|
-
)
|
|
464
|
+
raise ValueError(
|
|
465
|
+
f"VAR forecasting failed: unsupported VAR order {p}. "
|
|
466
|
+
f"Maximum supported VAR order is VAR(2). Please use p=1 (VAR(1)) or p=2 (VAR(2))"
|
|
467
|
+
)
|
|
468
|
+
return Z_forecast
|
|
469
|
+
|
|
470
|
+
|
dfm_python/utils/data.py
CHANGED
|
@@ -527,6 +527,9 @@ def create_data_view(
|
|
|
527
527
|
from ..utils.time import get_latest_time
|
|
528
528
|
from ..utils.helpers import get_series_ids
|
|
529
529
|
|
|
530
|
+
if Time is None:
|
|
531
|
+
raise ValueError("Time index is required for create_data_view")
|
|
532
|
+
|
|
530
533
|
if isinstance(view_date, str):
|
|
531
534
|
view_date = parse_timestamp(view_date)
|
|
532
535
|
elif view_date is None:
|
|
@@ -541,6 +544,8 @@ def create_data_view(
|
|
|
541
544
|
# Prepare time list
|
|
542
545
|
if isinstance(Time, TimeIndex):
|
|
543
546
|
time_list = [to_python_datetime(t) for t in Time]
|
|
547
|
+
elif Time is None:
|
|
548
|
+
raise ValueError("Time index is required for create_data_view")
|
|
544
549
|
else:
|
|
545
550
|
time_list = []
|
|
546
551
|
for t in Time:
|
dfm_python/utils/statespace.py
CHANGED
|
@@ -534,6 +534,8 @@ def _safe_divide(numerator: np.ndarray, denominator: np.ndarray, default: float
|
|
|
534
534
|
def estimate_var1(factors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
535
535
|
"""Estimate VAR(1) dynamics for factors.
|
|
536
536
|
|
|
537
|
+
Note: Maximum supported VAR order is VAR(2). Use estimate_var2() for VAR(2) estimation.
|
|
538
|
+
|
|
537
539
|
Parameters
|
|
538
540
|
----------
|
|
539
541
|
factors : np.ndarray
|
|
@@ -591,6 +593,8 @@ def estimate_var1(factors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
|
591
593
|
def estimate_var2(factors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
592
594
|
"""Estimate VAR(2) dynamics for factors.
|
|
593
595
|
|
|
596
|
+
Note: VAR(2) is the maximum supported VAR order in this implementation.
|
|
597
|
+
|
|
594
598
|
Parameters
|
|
595
599
|
----------
|
|
596
600
|
factors : np.ndarray
|
|
@@ -956,8 +960,8 @@ def estimate_state_space_params(
|
|
|
956
960
|
A2 = None
|
|
957
961
|
else:
|
|
958
962
|
raise NotImplementedError(
|
|
959
|
-
f"Only VAR(1) or VAR(2) for common factors are supported. "
|
|
960
|
-
f"Got factor_order={factor_order}"
|
|
963
|
+
f"Only VAR(1) or VAR(2) for common factors are supported (maximum supported order is VAR(2)). "
|
|
964
|
+
f"Got factor_order={factor_order}. Please use factor_order=1 (VAR(1)) or factor_order=2 (VAR(2))"
|
|
961
965
|
)
|
|
962
966
|
|
|
963
967
|
# Estimate idiosyncratic AR(1) dynamics
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dfm-python
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
|
|
5
5
|
Project-URL: Homepage, https://github.com/yourusername/dfm-python
|
|
6
6
|
Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme
|
|
@@ -160,11 +160,12 @@ trainer.fit(ddfm_model, dm)
|
|
|
160
160
|
Xf, Zf = ddfm_model.predict(horizon=6)
|
|
161
161
|
```
|
|
162
162
|
|
|
163
|
-
### Example 3: Nowcasting
|
|
163
|
+
### Example 3: Nowcasting with update().predict()
|
|
164
164
|
|
|
165
165
|
```python
|
|
166
166
|
from dfm_python import DFM, DFMDataModule, DFMTrainer
|
|
167
167
|
import pandas as pd
|
|
168
|
+
import numpy as np
|
|
168
169
|
|
|
169
170
|
# Load data and train model (same as above)
|
|
170
171
|
df = pd.read_csv('data/macro.csv')
|
|
@@ -179,14 +180,25 @@ model.load_config('config/model/dfm.yaml')
|
|
|
179
180
|
trainer = DFMTrainer(max_iter=10, threshold=1e-4)
|
|
180
181
|
trainer.fit(model, dm)
|
|
181
182
|
|
|
182
|
-
#
|
|
183
|
-
result = model.
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
183
|
+
# Get standardization parameters from trained model
|
|
184
|
+
result = model.result
|
|
185
|
+
Mx = result.Mx # Mean for standardization
|
|
186
|
+
Wx = result.Wx # Standard deviation for standardization
|
|
187
|
+
|
|
188
|
+
# Prepare new data (in practice, this would be real-time incomplete data)
|
|
189
|
+
# Standardize using the same parameters from training
|
|
190
|
+
X_new_raw = df_processed.iloc[-5:].values # Last 5 periods as example
|
|
191
|
+
X_new_std = (X_new_raw - Mx) / Wx
|
|
192
|
+
|
|
193
|
+
# Update model state with new standardized data, then predict
|
|
194
|
+
# Pattern: model.update(X_std).predict(horizon=1)
|
|
195
|
+
X_nowcast, Z_nowcast = model.update(X_new_std).predict(horizon=1)
|
|
187
196
|
|
|
188
|
-
|
|
189
|
-
|
|
197
|
+
# Extract nowcast for target series
|
|
198
|
+
target_idx = df_processed.columns.get_loc('KOEQUIPTE')
|
|
199
|
+
nowcast_value = X_nowcast[0, target_idx]
|
|
200
|
+
|
|
201
|
+
print(f"Nowcast value: {nowcast_value:.6f}")
|
|
190
202
|
```
|
|
191
203
|
|
|
192
204
|
## Configuration
|
|
@@ -294,8 +306,10 @@ trainer.fit(model, datamodule) # Standard Lightning pattern
|
|
|
294
306
|
# Prediction
|
|
295
307
|
Xf, Zf = model.predict(horizon=6) # Forecast future values
|
|
296
308
|
|
|
297
|
-
# Nowcasting
|
|
298
|
-
result = model.
|
|
309
|
+
# Nowcasting (update state with new data, then predict)
|
|
310
|
+
result = model.result
|
|
311
|
+
X_new_std = (X_new_raw - result.Mx) / result.Wx # Standardize new data
|
|
312
|
+
X_nowcast, Z_nowcast = model.update(X_new_std).predict(horizon=1) # Update and predict
|
|
299
313
|
```
|
|
300
314
|
|
|
301
315
|
### Result Objects
|
|
@@ -376,9 +390,18 @@ pytest src/test/test_trainer.py -v
|
|
|
376
390
|
|
|
377
391
|
## Project Status
|
|
378
392
|
|
|
379
|
-
**Version**: 0.
|
|
393
|
+
**Version**: 0.5.1
|
|
380
394
|
**Status**: Stable and production-ready
|
|
381
|
-
**Python**: 3.10+
|
|
395
|
+
**Python**: 3.10+
|
|
396
|
+
|
|
397
|
+
### What's New in 0.5.1
|
|
398
|
+
|
|
399
|
+
- ✨ **New `update()` method**: Replaces legacy `nowcast()` with flexible `update().predict()` pattern
|
|
400
|
+
- 🔧 **Improved API**: Users now control all preprocessing (masking, imputation, standardization)
|
|
401
|
+
- 🧹 **Code cleanup**: Removed legacy code and overengineering (~200+ lines removed)
|
|
402
|
+
- 🐛 **Bug fixes**: Fixed DDFM `update()` method for single-factor models
|
|
403
|
+
- 📚 **Updated tutorials**: All 4 tutorials rewritten with new pattern and VAR(1) configuration
|
|
404
|
+
- ✅ **VAR(1) only**: Simplified to VAR(1) factor dynamics throughout
|
|
382
405
|
|
|
383
406
|
## License
|
|
384
407
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
dfm_python/__init__.py,sha256=C_3NmIr6G7CyDoWB87Osp5b8XZR65LpHmflxMthue74,5683
|
|
2
2
|
dfm_python/config/__init__.py,sha256=NrTwjNzQRcYlk5Ht0hTF8o-6CCl5e4S_JxQAp1mNyCs,1507
|
|
3
3
|
dfm_python/config/adapter.py,sha256=X-mjWlg6ebWp-VAxjAUssgHnmr8AXOvSxhTTVSHOIIk,27475
|
|
4
|
-
dfm_python/config/results.py,sha256=
|
|
5
|
-
dfm_python/config/schema.py,sha256=
|
|
4
|
+
dfm_python/config/results.py,sha256=hzXL6MwaIxzA4uzouh6Bc_57lbpedphN9ErTMgfkpeE,15967
|
|
5
|
+
dfm_python/config/schema.py,sha256=wed_cYz2Nu6tGzQPRZDqHm6GgkeKl6O0AIawzg5yGng,50967
|
|
6
6
|
dfm_python/config/utils.py,sha256=W9bZeoLvfJmjV5dEWAKXJ9PKzJaNp084vpJZLGjxWbU,19298
|
|
7
7
|
dfm_python/data/__init__.py,sha256=KRhXiRN9DYPM1QfgBeXE3PCaOMYNl1kb7E9mEEGSpWE,925
|
|
8
8
|
dfm_python/data/dataloader.py,sha256=cN2epDGZiwjGl5Df6Qr0B72ehzRe9_ZD6s5DqPkdYQ8,3898
|
|
@@ -20,11 +20,11 @@ dfm_python/logger/inference_logger.py,sha256=TmXiWl703boJ3mIR-WgGpy6hfeF8a68OW9v
|
|
|
20
20
|
dfm_python/logger/logger.py,sha256=1YzsUnvAnAK9CTcfttdSIMtgFvylO13LjQPMEsSs9xc,5187
|
|
21
21
|
dfm_python/logger/train_logger.py,sha256=9VYhzwNYH1Rn7IQipKTY5wmVJcX1n6HDgnUZH0hChPY,17732
|
|
22
22
|
dfm_python/models/__init__.py,sha256=3JsYZOqDGkl1eG8Zrc4-5gf7Sw47PzzDg2XBhsudB04,724
|
|
23
|
-
dfm_python/models/base.py,sha256
|
|
24
|
-
dfm_python/models/ddfm.py,sha256=
|
|
25
|
-
dfm_python/models/dfm.py,sha256=
|
|
23
|
+
dfm_python/models/base.py,sha256=-mKK2onDilvPLVRRrW3W7GYsMZxLMHjZWj6n6gL_xnw,30041
|
|
24
|
+
dfm_python/models/ddfm.py,sha256=yZe7pxc81ZOsCwzoJ78UHXhEu5adRxkdTEvq-L17Sho,59415
|
|
25
|
+
dfm_python/models/dfm.py,sha256=r4MnjhGv9bPB2spiluF6cWKw0cuwR_qq7pfw5oz8dlM,33675
|
|
26
26
|
dfm_python/models/mcmc.py,sha256=QXBVxeBHREkc9K_xCGR1Y4sIFUkmX_RbRcgNktojuEc,23978
|
|
27
|
-
dfm_python/models/utils.py,sha256=
|
|
27
|
+
dfm_python/models/utils.py,sha256=LAtvBqUYxnnl4dIaWl6MC-NHSuoyeNkHZppfWB1lCxQ,20193
|
|
28
28
|
dfm_python/ssm/__init__.py,sha256=dFpt8_3jb_foISDNhma_JUAzch0VrKZcaeWP9zwpwoM,1161
|
|
29
29
|
dfm_python/ssm/em.py,sha256=yRYA930YAqOn4B4oxsPER_-vzO8hDPzMYA_eGnn4Sec,63430
|
|
30
30
|
dfm_python/ssm/kalman.py,sha256=KC6Q71VxM2NSdBVB5107QsNUm-_kE59MiyqrzImWz7E,45931
|
|
@@ -33,12 +33,12 @@ dfm_python/trainer/__init__.py,sha256=gc8jRjM-y2XN911LldwYUYHAAuFEvzUJJvXhG6B3Sv
|
|
|
33
33
|
dfm_python/trainer/ddfm.py,sha256=pqaLzxppJnCDXxPPpFRP7CGMMBVO12veaik6XofgBKU,7646
|
|
34
34
|
dfm_python/trainer/dfm.py,sha256=FQy82Jgkt0u5ecsQ3UtbktDmGc8hCET67OimGh0HmOE,6098
|
|
35
35
|
dfm_python/utils/__init__.py,sha256=QJiyCM8NXq4I96rxgrC4A3BiM2aTBHhFLSHQHpvXz5g,4022
|
|
36
|
-
dfm_python/utils/data.py,sha256=
|
|
36
|
+
dfm_python/utils/data.py,sha256=ofY3_z-HZxMq2bT84IL5bw7XGOsP863Abj513XHvM1E,21940
|
|
37
37
|
dfm_python/utils/diagnostics.py,sha256=UwrJfnn8VACATfQEkTcWw9eEJlTik-VCgTZ1dJABZmw,26049
|
|
38
38
|
dfm_python/utils/helpers.py,sha256=iOVwkP5pti-q-HKOAkp6bUPAp7BPV7BBgh8jL76g9go,13049
|
|
39
|
-
dfm_python/utils/statespace.py,sha256=
|
|
39
|
+
dfm_python/utils/statespace.py,sha256=P-GQiCP9HzI2d9PgYst4nmRuocEXr9gG1yzGYsYWkMg,44159
|
|
40
40
|
dfm_python/utils/time.py,sha256=o6NRPuk4Ynk8kPaUnzR6xq3pPAO6aLUTR9N_3hn_N88,35651
|
|
41
|
-
dfm_python-0.5.
|
|
42
|
-
dfm_python-0.5.
|
|
43
|
-
dfm_python-0.5.
|
|
44
|
-
dfm_python-0.5.
|
|
41
|
+
dfm_python-0.5.1.dist-info/METADATA,sha256=pf5lWV4TFMr4lKSNeUrtpHV_-ym3h_EaslIrAiioH8U,12159
|
|
42
|
+
dfm_python-0.5.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
43
|
+
dfm_python-0.5.1.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
|
|
44
|
+
dfm_python-0.5.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|