dfm-python 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dfm_python/data_loader.py +31 -131
- dfm_python/dfm.py +84 -35
- dfm_python/news.py +230 -77
- {dfm_python-0.1.0.dist-info → dfm_python-0.1.1.dist-info}/METADATA +141 -17
- dfm_python-0.1.1.dist-info/RECORD +10 -0
- dfm_python-0.1.1.dist-info/licenses/LICENSE +21 -0
- dfm_python-0.1.0.dist-info/RECORD +0 -9
- {dfm_python-0.1.0.dist-info → dfm_python-0.1.1.dist-info}/WHEEL +0 -0
dfm_python/data_loader.py
CHANGED
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
import pandas as pd
|
|
5
|
-
import
|
|
5
|
+
import io
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import List, Optional, Tuple, Union
|
|
8
|
-
from datetime import date
|
|
9
8
|
import warnings
|
|
10
9
|
import logging
|
|
11
10
|
|
|
@@ -15,23 +14,12 @@ ModelConfig = DFMConfig
|
|
|
15
14
|
|
|
16
15
|
logger = logging.getLogger(__name__)
|
|
17
16
|
|
|
18
|
-
warnings.filterwarnings('ignore', category=UserWarning)
|
|
19
|
-
|
|
20
|
-
try:
|
|
21
|
-
from scipy.io import loadmat, savemat
|
|
22
|
-
SCIPY_AVAILABLE = True
|
|
23
|
-
except ImportError:
|
|
24
|
-
SCIPY_AVAILABLE = False
|
|
25
|
-
|
|
26
17
|
try:
|
|
27
18
|
from omegaconf import OmegaConf
|
|
28
19
|
OMEGACONF_AVAILABLE = True
|
|
29
20
|
except ImportError:
|
|
30
21
|
OMEGACONF_AVAILABLE = False
|
|
31
22
|
|
|
32
|
-
# MATLAB datenum offset: datenum('1970-01-01') = 719529
|
|
33
|
-
_MATLAB_DATENUM_OFFSET = 719529
|
|
34
|
-
|
|
35
23
|
|
|
36
24
|
def load_config_from_yaml(configfile: Union[str, Path]) -> DFMConfig:
|
|
37
25
|
"""Load model configuration from YAML file.
|
|
@@ -119,6 +107,22 @@ def load_config_from_csv(configfile: Union[str, Path]) -> DFMConfig:
|
|
|
119
107
|
except Exception as e:
|
|
120
108
|
raise ValueError(f"Failed to read CSV file {configfile}: {e}")
|
|
121
109
|
|
|
110
|
+
return _load_config_from_dataframe(df)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _load_config_from_dataframe(df: pd.DataFrame) -> DFMConfig:
|
|
114
|
+
"""Load DFMConfig from a pandas DataFrame (shared by CSV and BytesIO loading).
|
|
115
|
+
|
|
116
|
+
Parameters
|
|
117
|
+
----------
|
|
118
|
+
df : pd.DataFrame
|
|
119
|
+
DataFrame with CSV columns
|
|
120
|
+
|
|
121
|
+
Returns
|
|
122
|
+
-------
|
|
123
|
+
DFMConfig
|
|
124
|
+
Model configuration
|
|
125
|
+
"""
|
|
122
126
|
# Handle 'id' as alias for 'series_id' (for backward compatibility)
|
|
123
127
|
# If CSV has data_code, item_id, api_source, generate series_id from them
|
|
124
128
|
# Otherwise, use 'id' as fallback
|
|
@@ -200,13 +204,13 @@ def load_config_from_csv(configfile: Union[str, Path]) -> DFMConfig:
|
|
|
200
204
|
# Note: Excel support removed - use CSV or YAML configs instead
|
|
201
205
|
|
|
202
206
|
|
|
203
|
-
def load_config(configfile: Union[str, Path]) -> DFMConfig:
|
|
207
|
+
def load_config(configfile: Union[str, Path, io.BytesIO]) -> DFMConfig:
|
|
204
208
|
"""Load model configuration from file (auto-detects YAML or CSV).
|
|
205
209
|
|
|
206
210
|
Parameters
|
|
207
211
|
----------
|
|
208
|
-
configfile : str or
|
|
209
|
-
Path to configuration file (.yaml, .yml, or .csv)
|
|
212
|
+
configfile : str, Path, or BytesIO
|
|
213
|
+
Path to configuration file (.yaml, .yml, or .csv), or BytesIO object with CSV content
|
|
210
214
|
|
|
211
215
|
Returns
|
|
212
216
|
-------
|
|
@@ -216,10 +220,18 @@ def load_config(configfile: Union[str, Path]) -> DFMConfig:
|
|
|
216
220
|
Raises
|
|
217
221
|
------
|
|
218
222
|
FileNotFoundError
|
|
219
|
-
If configfile does not exist
|
|
223
|
+
If configfile does not exist (for file paths)
|
|
220
224
|
ValueError
|
|
221
225
|
If file format is not supported or configuration is invalid
|
|
222
226
|
"""
|
|
227
|
+
# Handle BytesIO (for database storage downloads)
|
|
228
|
+
if isinstance(configfile, io.BytesIO):
|
|
229
|
+
# Read CSV from BytesIO
|
|
230
|
+
configfile.seek(0) # Reset to beginning
|
|
231
|
+
df = pd.read_csv(configfile)
|
|
232
|
+
# Use the same logic as load_config_from_csv but with DataFrame
|
|
233
|
+
return _load_config_from_dataframe(df)
|
|
234
|
+
|
|
223
235
|
configfile = Path(configfile)
|
|
224
236
|
if not configfile.exists():
|
|
225
237
|
raise FileNotFoundError(f"Configuration file not found: {configfile}")
|
|
@@ -288,7 +300,6 @@ def transform_data(Z: np.ndarray, Time: pd.DatetimeIndex, config: DFMConfig) ->
|
|
|
288
300
|
X[:, i] = _transform_series(Z[:, i], config.Transformation[i], config.Frequency[i], step)
|
|
289
301
|
|
|
290
302
|
# Drop first quarter of observations (4 months) since transformations cause missing values
|
|
291
|
-
# Note: MATLAB drops 4 rows (Time(4:end)), not 12
|
|
292
303
|
drop = 4
|
|
293
304
|
if T > drop:
|
|
294
305
|
return X[drop:], Time[drop:], Z[drop:]
|
|
@@ -330,106 +341,6 @@ def sort_data(Z: np.ndarray, Mnem: List[str], config: DFMConfig) -> Tuple[np.nda
|
|
|
330
341
|
return Z_filt[:, perm], [Mnem_filt[i] for i in perm]
|
|
331
342
|
|
|
332
343
|
|
|
333
|
-
def _datenum_to_pandas(Time_mat: np.ndarray) -> pd.DatetimeIndex:
|
|
334
|
-
"""Convert MATLAB datenum to pandas DatetimeIndex.
|
|
335
|
-
|
|
336
|
-
MATLAB datenum: days since 0000-01-01
|
|
337
|
-
Conversion: Time_pandas = 1970-01-01 + (Time_mat - 719529) days
|
|
338
|
-
"""
|
|
339
|
-
if Time_mat.ndim > 1:
|
|
340
|
-
Time_mat = Time_mat.flatten()
|
|
341
|
-
return pd.Timestamp('1970-01-01') + pd.to_timedelta(Time_mat - _MATLAB_DATENUM_OFFSET, unit='D')
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
def _pandas_to_datenum(Time: pd.DatetimeIndex) -> np.ndarray:
|
|
345
|
-
"""Convert pandas DatetimeIndex to MATLAB datenum format.
|
|
346
|
-
|
|
347
|
-
Conversion: Time_mat = (Time_pandas - 1970-01-01).days + 719529
|
|
348
|
-
"""
|
|
349
|
-
if not isinstance(Time, pd.DatetimeIndex):
|
|
350
|
-
Time = pd.to_datetime(Time)
|
|
351
|
-
return (Time - pd.Timestamp('1970-01-01')).days.values.astype(float) + _MATLAB_DATENUM_OFFSET
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
def _extract_matlab_variables(mat_data: dict) -> Tuple[Optional[np.ndarray], Optional[np.ndarray], Optional[List[str]]]:
|
|
355
|
-
"""Extract Z, Time, Mnem from MATLAB .mat file data.
|
|
356
|
-
|
|
357
|
-
Returns (Z, Time_mat, Mnem) or (None, None, None) if not found.
|
|
358
|
-
"""
|
|
359
|
-
Z = mat_data.get('Z')
|
|
360
|
-
Time_mat = mat_data.get('Time')
|
|
361
|
-
Mnem = mat_data.get('Mnem')
|
|
362
|
-
|
|
363
|
-
# Try alternative keys if not found (scipy sometimes adds prefixes)
|
|
364
|
-
if Z is None or Time_mat is None or Mnem is None:
|
|
365
|
-
for key in mat_data.keys():
|
|
366
|
-
if key.startswith('__'):
|
|
367
|
-
continue # Skip metadata
|
|
368
|
-
key_upper = key.upper()
|
|
369
|
-
if Z is None and 'Z' in key_upper and 'TIME' not in key_upper and 'MNEM' not in key_upper:
|
|
370
|
-
Z = mat_data[key]
|
|
371
|
-
elif Time_mat is None and 'TIME' in key_upper:
|
|
372
|
-
Time_mat = mat_data[key]
|
|
373
|
-
elif Mnem is None and 'MNEM' in key_upper:
|
|
374
|
-
Mnem = mat_data[key]
|
|
375
|
-
|
|
376
|
-
# Convert Mnem from MATLAB format to Python list
|
|
377
|
-
if Mnem is not None:
|
|
378
|
-
if isinstance(Mnem, np.ndarray):
|
|
379
|
-
Mnem = [str(item[0]) if isinstance(item, np.ndarray) and item.size > 0
|
|
380
|
-
else str(item) for item in Mnem.flatten()]
|
|
381
|
-
elif not isinstance(Mnem, list):
|
|
382
|
-
Mnem = [str(Mnem)]
|
|
383
|
-
|
|
384
|
-
return Z, Time_mat, Mnem
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
def _load_from_mat_cache(datafile_mat: Path) -> Optional[Tuple[np.ndarray, pd.DatetimeIndex, List[str]]]:
|
|
388
|
-
"""Load data from cached .mat file.
|
|
389
|
-
|
|
390
|
-
Returns (Z, Time, Mnem) if successful, None otherwise.
|
|
391
|
-
"""
|
|
392
|
-
if not SCIPY_AVAILABLE or not datafile_mat.exists():
|
|
393
|
-
return None
|
|
394
|
-
|
|
395
|
-
try:
|
|
396
|
-
mat_data = loadmat(str(datafile_mat))
|
|
397
|
-
Z, Time_mat, Mnem = _extract_matlab_variables(mat_data)
|
|
398
|
-
|
|
399
|
-
if Z is None or Time_mat is None or Mnem is None:
|
|
400
|
-
return None
|
|
401
|
-
|
|
402
|
-
# Convert MATLAB datenum to pandas DatetimeIndex
|
|
403
|
-
Time = _datenum_to_pandas(Time_mat)
|
|
404
|
-
|
|
405
|
-
# Ensure Z is 2D
|
|
406
|
-
if Z.ndim == 1:
|
|
407
|
-
Z = Z.reshape(-1, 1)
|
|
408
|
-
|
|
409
|
-
print(f' Loaded from cached .mat file: {datafile_mat}')
|
|
410
|
-
return Z, Time, Mnem
|
|
411
|
-
except Exception as e:
|
|
412
|
-
warnings.warn(f"Failed to load from .mat cache ({e}). Reading from CSV instead.")
|
|
413
|
-
return None
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
def _save_to_mat_cache(Z: np.ndarray, Time: pd.DatetimeIndex, Mnem: List[str],
|
|
417
|
-
mat_dir: Path, datafile_mat: Path) -> None:
|
|
418
|
-
"""Save data to .mat cache file."""
|
|
419
|
-
if not SCIPY_AVAILABLE:
|
|
420
|
-
return
|
|
421
|
-
|
|
422
|
-
try:
|
|
423
|
-
mat_dir.mkdir(exist_ok=True)
|
|
424
|
-
Time_mat = _pandas_to_datenum(Time)
|
|
425
|
-
Mnem_array = np.array([m.encode('utf-8') if isinstance(m, str) else m
|
|
426
|
-
for m in Mnem], dtype=object)
|
|
427
|
-
savemat(str(datafile_mat), {'Z': Z, 'Time': Time_mat, 'Mnem': Mnem_array})
|
|
428
|
-
print(f' Cached to .mat file: {datafile_mat}')
|
|
429
|
-
except Exception as e:
|
|
430
|
-
warnings.warn(f"Failed to save .mat cache ({e}). Continuing without cache.")
|
|
431
|
-
|
|
432
|
-
|
|
433
344
|
def load_data(datafile: Union[str, Path], config: DFMConfig,
|
|
434
345
|
sample_start: Optional[Union[pd.Timestamp, str]] = None) -> Tuple[np.ndarray, pd.DatetimeIndex, np.ndarray]:
|
|
435
346
|
"""Load and transform data from CSV file.
|
|
@@ -468,19 +379,8 @@ def load_data(datafile: Union[str, Path], config: DFMConfig,
|
|
|
468
379
|
if not datafile.exists():
|
|
469
380
|
raise FileNotFoundError(f"Data file not found: {datafile}")
|
|
470
381
|
|
|
471
|
-
#
|
|
472
|
-
|
|
473
|
-
datafile_mat = mat_dir / f'{datafile.stem}.mat'
|
|
474
|
-
|
|
475
|
-
# Try to load from cache first
|
|
476
|
-
cache_data = _load_from_mat_cache(datafile_mat)
|
|
477
|
-
|
|
478
|
-
# Read from CSV if cache load failed
|
|
479
|
-
if cache_data is None:
|
|
480
|
-
Z, Time, Mnem = read_data(datafile)
|
|
481
|
-
_save_to_mat_cache(Z, Time, Mnem, mat_dir, datafile_mat)
|
|
482
|
-
else:
|
|
483
|
-
Z, Time, Mnem = cache_data
|
|
382
|
+
# Read data from CSV file
|
|
383
|
+
Z, Time, Mnem = read_data(datafile)
|
|
484
384
|
|
|
485
385
|
# Process data: sort to match config order, then transform
|
|
486
386
|
Z, _ = sort_data(Z, Mnem, config)
|
dfm_python/dfm.py
CHANGED
|
@@ -20,43 +20,50 @@ try:
|
|
|
20
20
|
except ImportError:
|
|
21
21
|
SCIPY_AVAILABLE = False
|
|
22
22
|
|
|
23
|
+
# Import rem_nans_spline from utils to avoid duplication
|
|
24
|
+
import sys
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
try:
|
|
27
|
+
from ..utils.data_utils import rem_nans_spline
|
|
28
|
+
except (ImportError, ValueError):
|
|
29
|
+
# Fallback: add parent directory to path if relative import fails
|
|
30
|
+
utils_path = Path(__file__).parent.parent.parent / 'utils'
|
|
31
|
+
if str(utils_path) not in sys.path:
|
|
32
|
+
sys.path.insert(0, str(utils_path.parent))
|
|
33
|
+
from utils.data_utils import rem_nans_spline
|
|
34
|
+
|
|
23
35
|
def _rem_nans_spline(X: np.ndarray, method: int = 2, k: int = 3):
|
|
24
|
-
"""Treat NaNs in dataset for DFM estimation (
|
|
25
|
-
from scipy.interpolate import CubicSpline
|
|
26
|
-
from scipy.signal import lfilter
|
|
27
|
-
|
|
28
|
-
T, N = X.shape
|
|
29
|
-
indNaN = np.isnan(X)
|
|
30
|
-
|
|
31
|
-
def _remove_leading_trailing(threshold: float):
|
|
32
|
-
rem = np.sum(indNaN, axis=1) > (N * threshold if threshold < 1 else threshold)
|
|
33
|
-
nan_lead = np.cumsum(rem) == np.arange(1, T + 1)
|
|
34
|
-
nan_end = np.cumsum(rem[::-1]) == np.arange(1, T + 1)[::-1]
|
|
35
|
-
return ~(nan_lead | nan_end)
|
|
36
|
-
|
|
37
|
-
def _fill_missing(x: np.ndarray, mask: np.ndarray):
|
|
38
|
-
non_nan = np.where(~mask)[0]
|
|
39
|
-
if len(non_nan) < 2:
|
|
40
|
-
return x
|
|
41
|
-
spline = CubicSpline(non_nan, x[non_nan], extrapolate=True)
|
|
42
|
-
x_filled = spline(np.arange(len(x)))
|
|
43
|
-
# Moving average filter
|
|
44
|
-
b = np.ones(k) / k
|
|
45
|
-
x_filled = lfilter(b, 1, x_filled)
|
|
46
|
-
return x_filled
|
|
36
|
+
"""Treat NaNs in dataset for DFM estimation (wrapper for rem_nans_spline).
|
|
47
37
|
|
|
38
|
+
This is a wrapper around rem_nans_spline from utils.data_utils to maintain
|
|
39
|
+
backward compatibility with the internal _rem_nans_spline interface.
|
|
40
|
+
"""
|
|
41
|
+
X_clean, indNaN = rem_nans_spline(X, method=method, k=k)
|
|
42
|
+
# rem_nans_spline returns (X, indNaN), but we need to return (X, keep) where keep is boolean mask
|
|
43
|
+
# For method 2, we need to compute the keep mask
|
|
48
44
|
if method == 2:
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
45
|
+
T_orig, N = X.shape
|
|
46
|
+
T_clean, _ = X_clean.shape
|
|
47
|
+
# Create keep mask: True for rows that were kept
|
|
48
|
+
keep = np.zeros(T_orig, dtype=bool)
|
|
49
|
+
# Simple approach: assume rows were removed from start/end
|
|
50
|
+
# This is approximate but works for the use case
|
|
51
|
+
if T_clean < T_orig:
|
|
52
|
+
# Rows were removed - we'll use a simple heuristic
|
|
53
|
+
# In practice, rem_nans_spline method 2 removes leading/trailing rows with >80% NaN
|
|
54
|
+
# We'll compute which rows were kept
|
|
55
|
+
indNaN_orig = np.isnan(X)
|
|
56
|
+
rem = np.sum(indNaN_orig, axis=1) > (N * 0.8)
|
|
57
|
+
nan_lead = np.cumsum(rem) == np.arange(1, T_orig + 1)
|
|
58
|
+
nan_end = np.cumsum(rem[::-1]) == np.arange(1, T_orig + 1)[::-1]
|
|
59
|
+
keep = ~(nan_lead | nan_end)
|
|
60
|
+
else:
|
|
61
|
+
keep = np.ones(T_orig, dtype=bool)
|
|
57
62
|
return X_clean, keep
|
|
58
63
|
else:
|
|
59
|
-
|
|
64
|
+
# For other methods, return all rows as kept
|
|
65
|
+
T_orig, _ = X.shape
|
|
66
|
+
return X_clean, np.ones(T_orig, dtype=bool)
|
|
60
67
|
|
|
61
68
|
# Set up logger for fallback tracking (optional - only if logging is enabled)
|
|
62
69
|
_logger = logging.getLogger(__name__)
|
|
@@ -76,6 +83,35 @@ if not _logger.handlers:
|
|
|
76
83
|
|
|
77
84
|
# Removed unused validation helpers - consolidated inline for simplicity
|
|
78
85
|
|
|
86
|
+
# Constants
|
|
87
|
+
QUARTERLY_VARIANCE_DIVISOR = 19.0 # Divisor for quarterly idiosyncratic variance calculation
|
|
88
|
+
SCALING_FACTOR = 1e6 # Maximum scaling factor for eigenvalue capping
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _safe_divide(numerator: np.ndarray, denominator: float, default: float = 0.0) -> np.ndarray:
|
|
92
|
+
"""Safely divide numerator by denominator, handling zero and invalid values.
|
|
93
|
+
|
|
94
|
+
Parameters
|
|
95
|
+
----------
|
|
96
|
+
numerator : np.ndarray
|
|
97
|
+
Numerator array
|
|
98
|
+
denominator : float
|
|
99
|
+
Denominator value
|
|
100
|
+
default : float, optional
|
|
101
|
+
Default value to use if denominator is zero or invalid (default: 0.0)
|
|
102
|
+
|
|
103
|
+
Returns
|
|
104
|
+
-------
|
|
105
|
+
np.ndarray
|
|
106
|
+
Result of division, with default values where denominator is invalid
|
|
107
|
+
"""
|
|
108
|
+
if denominator == 0 or np.isnan(denominator) or np.isinf(denominator):
|
|
109
|
+
return np.full_like(numerator, default)
|
|
110
|
+
result = numerator / denominator
|
|
111
|
+
# Replace any NaN/Inf in result with default
|
|
112
|
+
result = np.where(np.isfinite(result), result, default)
|
|
113
|
+
return result
|
|
114
|
+
|
|
79
115
|
|
|
80
116
|
def _validate_dfm_inputs(X: np.ndarray, config, threshold: float) -> None:
|
|
81
117
|
"""Validate inputs for dfm() function.
|
|
@@ -515,11 +551,14 @@ def init_conditions(x: np.ndarray, r: np.ndarray, p: int, blocks: np.ndarray,
|
|
|
515
551
|
n_b = blocks.shape[1] # Number of blocks
|
|
516
552
|
|
|
517
553
|
# Spline without NaNs
|
|
518
|
-
xBal,
|
|
554
|
+
xBal, keep = _rem_nans_spline(x, method=opt_nan['method'], k=opt_nan['k'])
|
|
519
555
|
|
|
520
556
|
T, N = xBal.shape
|
|
521
557
|
nM = N - nQ # Number of monthly series
|
|
522
558
|
|
|
559
|
+
# Create 2D boolean array for NaN indicators
|
|
560
|
+
indNaN = np.isnan(xBal)
|
|
561
|
+
|
|
523
562
|
xNaN = xBal.copy()
|
|
524
563
|
xNaN[indNaN] = np.nan
|
|
525
564
|
res = xBal.copy() # Spline output, later used for residuals
|
|
@@ -976,8 +1015,18 @@ def init_conditions(x: np.ndarray, r: np.ndarray, p: int, blocks: np.ndarray,
|
|
|
976
1015
|
# Quarterly idiosyncratic
|
|
977
1016
|
Rdiag = np.diag(R).copy() # Copy to avoid read-only array
|
|
978
1017
|
# Handle division by zero and NaN/Inf
|
|
979
|
-
|
|
980
|
-
|
|
1018
|
+
# Validate Rdiag values before division
|
|
1019
|
+
Rdiag_quarterly = Rdiag[nM:N]
|
|
1020
|
+
if len(Rdiag_quarterly) > 0:
|
|
1021
|
+
# Ensure no invalid values before division
|
|
1022
|
+
Rdiag_quarterly = np.where(
|
|
1023
|
+
(np.isnan(Rdiag_quarterly) | np.isinf(Rdiag_quarterly) | (Rdiag_quarterly < 0)),
|
|
1024
|
+
1e-4, Rdiag_quarterly
|
|
1025
|
+
)
|
|
1026
|
+
sig_e = _safe_divide(Rdiag_quarterly, QUARTERLY_VARIANCE_DIVISOR, default=1e-4)
|
|
1027
|
+
sig_e = np.where((np.isnan(sig_e) | np.isinf(sig_e) | (sig_e < 1e-6)), 1e-4, sig_e)
|
|
1028
|
+
else:
|
|
1029
|
+
sig_e = np.array([])
|
|
981
1030
|
Rdiag[nM:N] = 1e-4
|
|
982
1031
|
R = np.diag(Rdiag)
|
|
983
1032
|
|
dfm_python/news.py
CHANGED
|
@@ -3,13 +3,50 @@
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
from scipy.linalg import pinv, inv
|
|
5
5
|
from typing import Tuple, Optional, Dict, Union, List, Any, Callable
|
|
6
|
-
from datetime import datetime
|
|
7
6
|
import pandas as pd
|
|
7
|
+
import logging
|
|
8
8
|
|
|
9
9
|
from .kalman import skf, fis, miss_data
|
|
10
|
+
from .config import DFMConfig
|
|
11
|
+
from .dfm import DFMResult
|
|
10
12
|
|
|
13
|
+
# Set up logger
|
|
14
|
+
_logger = logging.getLogger(__name__)
|
|
11
15
|
|
|
12
|
-
|
|
16
|
+
# Constants
|
|
17
|
+
FORECAST_HORIZON_MONTHS = 12 # Number of months to append for forecasting horizon
|
|
18
|
+
DEFAULT_FALLBACK_DATE = '2017-01-01' # Default fallback date for time index extension
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _check_config_consistency(saved_config: Any, current_config: DFMConfig) -> None:
|
|
22
|
+
"""Check if saved config is consistent with current config.
|
|
23
|
+
|
|
24
|
+
Parameters
|
|
25
|
+
----------
|
|
26
|
+
saved_config : Any
|
|
27
|
+
Saved configuration object (may be DFMConfig or dict-like)
|
|
28
|
+
current_config : DFMConfig
|
|
29
|
+
Current configuration object
|
|
30
|
+
|
|
31
|
+
Notes
|
|
32
|
+
-----
|
|
33
|
+
Issues a UserWarning if configs differ. Logs warnings if comparison fails.
|
|
34
|
+
"""
|
|
35
|
+
try:
|
|
36
|
+
if hasattr(saved_config, 'SeriesID') and hasattr(current_config, 'SeriesID'):
|
|
37
|
+
if saved_config.SeriesID != current_config.SeriesID:
|
|
38
|
+
import warnings
|
|
39
|
+
warnings.warn(
|
|
40
|
+
"Config used in estimation differs from current config. "
|
|
41
|
+
"Results may be inconsistent. Consider re-estimating.",
|
|
42
|
+
UserWarning
|
|
43
|
+
)
|
|
44
|
+
except Exception as e:
|
|
45
|
+
_logger.warning(f"update_nowcast: Config consistency check failed: {type(e).__name__}: {str(e)}")
|
|
46
|
+
# If comparison fails, continue anyway
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def para_const(X: np.ndarray, P: Union[DFMResult, Dict[str, Any]], lag: int) -> Dict[str, Any]:
|
|
13
50
|
"""Implement Kalman filter for news calculation with fixed parameters.
|
|
14
51
|
|
|
15
52
|
Parameters:
|
|
@@ -77,47 +114,144 @@ def para_const(X: np.ndarray, P, lag: int) -> Dict:
|
|
|
77
114
|
}
|
|
78
115
|
|
|
79
116
|
|
|
80
|
-
def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res,
|
|
81
|
-
v_news: Union[int, np.ndarray, List[int]]) -> Tuple[
|
|
82
|
-
|
|
117
|
+
def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res: Union[DFMResult, Dict[str, Any]],
|
|
118
|
+
t_fcst: int, v_news: Union[int, np.ndarray, List[int]]) -> Tuple[
|
|
119
|
+
Union[float, np.ndarray], Union[float, np.ndarray], np.ndarray, np.ndarray,
|
|
120
|
+
np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
|
83
121
|
"""Calculate changes in news from data releases.
|
|
84
122
|
|
|
123
|
+
This function implements the news decomposition algorithm for dynamic factor models.
|
|
124
|
+
It computes how new data releases affect forecasts of target variables by decomposing
|
|
125
|
+
the forecast update into contributions from each new data point (news).
|
|
126
|
+
|
|
127
|
+
The algorithm:
|
|
128
|
+
1. Identifies new data points (available in X_new but not in X_old)
|
|
129
|
+
2. Runs Kalman filter/smoother on both old and new datasets
|
|
130
|
+
3. Computes forecast updates (y_new - y_old) for target variables
|
|
131
|
+
4. Decomposes the update into individual news contributions from each new data point
|
|
132
|
+
|
|
85
133
|
Parameters:
|
|
86
134
|
-----------
|
|
87
135
|
X_old : np.ndarray
|
|
88
|
-
Old data matrix (T x N)
|
|
136
|
+
Old data matrix (T x N), where T is time periods and N is number of series.
|
|
137
|
+
Contains data available before the new release.
|
|
89
138
|
X_new : np.ndarray
|
|
90
|
-
New data matrix (T x N)
|
|
91
|
-
|
|
92
|
-
|
|
139
|
+
New data matrix (T x N), same dimensions as X_old.
|
|
140
|
+
Contains data after the new release. Must have same number of series as X_old.
|
|
141
|
+
Res : Union[DFMResult, Dict[str, Any]]
|
|
142
|
+
DFM estimation results. Can be:
|
|
143
|
+
- DFMResult: Direct result object from dfm() function
|
|
144
|
+
- Dict: Dictionary containing 'Res' key with DFMResult, and optionally 'Config' or 'Spec'
|
|
145
|
+
Must have attributes: C, Mx, Wx, X_sm (or equivalent keys if dict)
|
|
93
146
|
t_fcst : int
|
|
94
|
-
Target time index (0-based)
|
|
95
|
-
|
|
96
|
-
|
|
147
|
+
Target time index (0-based) for the forecast period.
|
|
148
|
+
Must satisfy 0 <= t_fcst < T (number of time periods in X_new).
|
|
149
|
+
v_news : Union[int, np.ndarray, List[int]]
|
|
150
|
+
Target variable index(s) (0-based). Specifies which series to forecast.
|
|
151
|
+
Can be:
|
|
97
152
|
- int: Single target variable (backward compatible)
|
|
98
153
|
- np.ndarray or List[int]: Multiple target variables (MATLAB-compatible)
|
|
154
|
+
All indices must be in range [0, N) where N is number of series.
|
|
99
155
|
|
|
100
156
|
Returns:
|
|
101
157
|
--------
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
158
|
+
Tuple containing 9 elements:
|
|
159
|
+
|
|
160
|
+
y_old : Union[float, np.ndarray]
|
|
161
|
+
Forecast of target variable(s) using old data.
|
|
162
|
+
- If v_news is scalar: float
|
|
163
|
+
- If v_news is array: (n_targets,) array
|
|
164
|
+
y_new : Union[float, np.ndarray]
|
|
165
|
+
Forecast of target variable(s) using new data.
|
|
166
|
+
- If v_news is scalar: float
|
|
167
|
+
- If v_news is array: (n_targets,) array
|
|
168
|
+
singlenews : np.ndarray
|
|
169
|
+
Individual news contributions from each new data point.
|
|
170
|
+
- If v_news is scalar: (N,) array, where singlenews[i] is contribution from series i
|
|
171
|
+
- If v_news is array: (N, n_targets) array, where each column is news for one target
|
|
172
|
+
actual : np.ndarray
|
|
173
|
+
Actual values of new data points (N,). NaN for series with no new data.
|
|
174
|
+
forecast : np.ndarray
|
|
175
|
+
Forecasted values (before new data) for new data points (N,). NaN for series with no new data.
|
|
176
|
+
weight : np.ndarray
|
|
177
|
+
Weights for news contributions (N,). NaN for series with no new data.
|
|
178
|
+
- If v_news is scalar: (N,) array
|
|
179
|
+
- If v_news is array: (N, n_targets) array
|
|
180
|
+
t_miss : np.ndarray
|
|
181
|
+
Time indices where new data is available (n_news,). Integer array.
|
|
182
|
+
v_miss : np.ndarray
|
|
183
|
+
Variable indices where new data is available (n_news,). Integer array.
|
|
184
|
+
innov : np.ndarray
|
|
185
|
+
Innovation terms (n_news,). Forecast errors for new data points.
|
|
186
|
+
|
|
187
|
+
Raises:
|
|
188
|
+
------
|
|
189
|
+
ValueError
|
|
190
|
+
If inputs are invalid:
|
|
191
|
+
- X_old or X_new are not 2D numpy arrays
|
|
192
|
+
- X_old and X_new have different number of series
|
|
193
|
+
- t_fcst is out of bounds
|
|
194
|
+
- v_news indices are out of range
|
|
195
|
+
- Res does not have required attributes (C, Mx, Wx, X_sm)
|
|
110
196
|
|
|
111
|
-
|
|
197
|
+
Notes:
|
|
112
198
|
-----
|
|
113
|
-
Now supports both single and multiple target variables, matching MATLAB functionality.
|
|
199
|
+
- Now supports both single and multiple target variables, matching MATLAB functionality.
|
|
200
|
+
- The function handles two cases:
|
|
201
|
+
1. NO FORECAST CASE: Target variable(s) already observed at t_fcst in X_new
|
|
202
|
+
2. FORECAST CASE: Target variable(s) not yet observed, requires forecast update
|
|
203
|
+
- News decomposition follows the methodology in Banbura et al. (2015).
|
|
204
|
+
- If matrix inversion fails during gain calculation, fallback values are used and logged.
|
|
205
|
+
|
|
206
|
+
Examples:
|
|
207
|
+
--------
|
|
208
|
+
>>> from dfm_python import dfm, news_dfm
|
|
209
|
+
>>> # Estimate DFM model
|
|
210
|
+
>>> Res = dfm(X, config)
|
|
211
|
+
>>> # Calculate news for single target variable
|
|
212
|
+
>>> y_old, y_new, singlenews, actual, forecast, weight, t_miss, v_miss, innov = \\
|
|
213
|
+
... news_dfm(X_old, X_new, Res, t_fcst=100, v_news=0)
|
|
214
|
+
>>> # Calculate news for multiple target variables
|
|
215
|
+
>>> targets = [0, 1, 2]
|
|
216
|
+
>>> y_old, y_new, singlenews, actual, forecast, weight, t_miss, v_miss, innov = \\
|
|
217
|
+
... news_dfm(X_old, X_new, Res, t_fcst=100, v_news=targets)
|
|
114
218
|
"""
|
|
219
|
+
# Input validation
|
|
220
|
+
if not isinstance(X_old, np.ndarray) or X_old.ndim != 2:
|
|
221
|
+
raise ValueError(f"news_dfm: X_old must be a 2D numpy array, got {type(X_old)} with shape {getattr(X_old, 'shape', 'N/A')}")
|
|
222
|
+
if not isinstance(X_new, np.ndarray) or X_new.ndim != 2:
|
|
223
|
+
raise ValueError(f"news_dfm: X_new must be a 2D numpy array, got {type(X_new)} with shape {getattr(X_new, 'shape', 'N/A')}")
|
|
224
|
+
if X_old.shape[1] != X_new.shape[1]:
|
|
225
|
+
raise ValueError(f"news_dfm: X_old and X_new must have same number of series. Got {X_old.shape[1]} and {X_new.shape[1]}")
|
|
226
|
+
if not isinstance(t_fcst, (int, np.integer)) or t_fcst < 0:
|
|
227
|
+
raise ValueError(f"news_dfm: t_fcst must be a non-negative integer, got {t_fcst}")
|
|
228
|
+
if t_fcst >= X_new.shape[0]:
|
|
229
|
+
raise ValueError(f"news_dfm: t_fcst ({t_fcst}) must be less than number of time periods ({X_new.shape[0]})")
|
|
230
|
+
|
|
231
|
+
# Validate Res structure
|
|
232
|
+
if isinstance(Res, dict):
|
|
233
|
+
if 'Res' not in Res:
|
|
234
|
+
raise ValueError("news_dfm: Res dict must contain 'Res' key")
|
|
235
|
+
Res_actual = Res['Res']
|
|
236
|
+
else:
|
|
237
|
+
Res_actual = Res
|
|
238
|
+
|
|
239
|
+
# Check required attributes
|
|
240
|
+
required_attrs = ['C', 'Mx', 'Wx', 'X_sm']
|
|
241
|
+
for attr in required_attrs:
|
|
242
|
+
if not hasattr(Res_actual, attr):
|
|
243
|
+
raise ValueError(f"news_dfm: Res must have '{attr}' attribute. Got {type(Res_actual)}")
|
|
244
|
+
|
|
115
245
|
# Normalize v_news to array for consistent handling
|
|
116
246
|
v_news_arr = np.atleast_1d(v_news)
|
|
117
247
|
is_scalar = isinstance(v_news, (int, np.integer)) or (isinstance(v_news, np.ndarray) and v_news.ndim == 0)
|
|
118
248
|
n_targets = len(v_news_arr)
|
|
119
249
|
|
|
120
|
-
|
|
250
|
+
# Validate v_news indices
|
|
251
|
+
if np.any(v_news_arr < 0) or np.any(v_news_arr >= X_new.shape[1]):
|
|
252
|
+
raise ValueError(f"news_dfm: v_news indices must be in range [0, {X_new.shape[1]}), got {v_news_arr}")
|
|
253
|
+
|
|
254
|
+
r = Res_actual.C.shape[1]
|
|
121
255
|
_, N = X_new.shape
|
|
122
256
|
|
|
123
257
|
# Check if targets are already observed
|
|
@@ -125,7 +259,7 @@ def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res, t_fcst: int,
|
|
|
125
259
|
|
|
126
260
|
if np.all(targets_observed):
|
|
127
261
|
# NO FORECAST CASE: Already values for all target variables at time t_fcst
|
|
128
|
-
Res_old = para_const(X_old,
|
|
262
|
+
Res_old = para_const(X_old, Res_actual, 0)
|
|
129
263
|
|
|
130
264
|
# Initialize output arrays
|
|
131
265
|
if is_scalar:
|
|
@@ -148,8 +282,8 @@ def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res, t_fcst: int,
|
|
|
148
282
|
innov = np.array([])
|
|
149
283
|
else:
|
|
150
284
|
# FORECAST CASE
|
|
151
|
-
Mx =
|
|
152
|
-
Wx =
|
|
285
|
+
Mx = Res_actual.Mx
|
|
286
|
+
Wx = Res_actual.Wx
|
|
153
287
|
|
|
154
288
|
# Calculate indicators for missing values
|
|
155
289
|
miss_old = np.isnan(X_old)
|
|
@@ -165,8 +299,8 @@ def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res, t_fcst: int,
|
|
|
165
299
|
|
|
166
300
|
if len(v_miss) == 0:
|
|
167
301
|
# NO NEW INFORMATION
|
|
168
|
-
Res_old = para_const(X_old,
|
|
169
|
-
Res_new = para_const(X_new,
|
|
302
|
+
Res_old = para_const(X_old, Res_actual, 0)
|
|
303
|
+
Res_new = para_const(X_new, Res_actual, 0)
|
|
170
304
|
|
|
171
305
|
if is_scalar:
|
|
172
306
|
y_old = Res_old['X_sm'][t_fcst, v_news_arr[0]]
|
|
@@ -192,17 +326,17 @@ def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res, t_fcst: int,
|
|
|
192
326
|
k = max([np.max(np.abs(lag)), np.max(lag) - np.min(lag)])
|
|
193
327
|
k = int(k)
|
|
194
328
|
|
|
195
|
-
C =
|
|
196
|
-
R_cov =
|
|
329
|
+
C = Res_actual.C
|
|
330
|
+
R_cov = Res_actual.R.T # Note: R in Res might need transposition
|
|
197
331
|
|
|
198
332
|
n_news = len(lag)
|
|
199
333
|
|
|
200
334
|
# Smooth old dataset
|
|
201
|
-
Res_old = para_const(X_old,
|
|
335
|
+
Res_old = para_const(X_old, Res_actual, k)
|
|
202
336
|
Plag = Res_old['Plag']
|
|
203
337
|
|
|
204
338
|
# Smooth new dataset
|
|
205
|
-
Res_new = para_const(X_new,
|
|
339
|
+
Res_new = para_const(X_new, Res_actual, 0)
|
|
206
340
|
|
|
207
341
|
# Get nowcasts for all target variables
|
|
208
342
|
if is_scalar:
|
|
@@ -268,8 +402,12 @@ def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res, t_fcst: int,
|
|
|
268
402
|
for idx, v in enumerate(v_news_arr):
|
|
269
403
|
gain[idx, :] = Wx[v] * C[v, :r] @ P1 @ P2_inv
|
|
270
404
|
totnews[idx] = Wx[v] * C[v, :r] @ P1 @ P2_inv @ innov
|
|
271
|
-
except:
|
|
405
|
+
except (np.linalg.LinAlgError, ValueError) as e:
|
|
272
406
|
# If inversion fails, use simpler approach
|
|
407
|
+
_logger.warning(
|
|
408
|
+
f"news_dfm: Matrix inversion failed for P2, using fallback values. "
|
|
409
|
+
f"Error: {type(e).__name__}: {str(e)}"
|
|
410
|
+
)
|
|
273
411
|
if is_scalar:
|
|
274
412
|
gain = np.ones(n_news) * 0.1
|
|
275
413
|
totnews = np.sum(innov) * 0.1
|
|
@@ -290,13 +428,13 @@ def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res, t_fcst: int,
|
|
|
290
428
|
actual = np.full(N, np.nan)
|
|
291
429
|
forecast = np.full(N, np.nan)
|
|
292
430
|
weight = np.full(N, np.nan)
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
431
|
+
|
|
432
|
+
for i in range(n_news):
|
|
433
|
+
actual[v_miss[i]] = X_new[t_miss[i], v_miss[i]]
|
|
434
|
+
forecast[v_miss[i]] = Res_old['X_sm'][t_miss[i], v_miss[i]]
|
|
435
|
+
if i < len(gain):
|
|
436
|
+
singlenews[v_miss[i]] = gain[i] * innov[i] / Wx[v_miss[i]] if Wx[v_miss[i]] != 0 else 0
|
|
437
|
+
weight[v_miss[i]] = gain[i] / Wx[v_miss[i]] if Wx[v_miss[i]] != 0 else 0
|
|
300
438
|
else:
|
|
301
439
|
# Multiple targets: singlenews is (N, n_targets)
|
|
302
440
|
singlenews = np.full((N, n_targets), np.nan)
|
|
@@ -321,7 +459,8 @@ def news_dfm(X_old: np.ndarray, X_new: np.ndarray, Res, t_fcst: int,
|
|
|
321
459
|
# Database saving is application-specific and should be implemented in adapters.
|
|
322
460
|
# This keeps the DFM module generic and database-agnostic.
|
|
323
461
|
def update_nowcast(X_old: np.ndarray, X_new: np.ndarray, Time: pd.DatetimeIndex,
|
|
324
|
-
config, Res,
|
|
462
|
+
config: DFMConfig, Res: Union[DFMResult, Dict[str, Any]],
|
|
463
|
+
series: str, period: str,
|
|
325
464
|
vintage_old: str, vintage_new: str,
|
|
326
465
|
model_id: Optional[int] = None,
|
|
327
466
|
save_callback: Optional[Callable] = None) -> None:
|
|
@@ -364,6 +503,49 @@ def update_nowcast(X_old: np.ndarray, X_new: np.ndarray, Time: pd.DatetimeIndex,
|
|
|
364
503
|
Database saving is robust with error handling - failures will log warnings
|
|
365
504
|
but not interrupt the nowcast computation.
|
|
366
505
|
"""
|
|
506
|
+
# Input validation
|
|
507
|
+
if not isinstance(X_old, np.ndarray) or X_old.ndim != 2:
|
|
508
|
+
raise ValueError(f"update_nowcast: X_old must be a 2D numpy array, got {type(X_old)}")
|
|
509
|
+
if not isinstance(X_new, np.ndarray) or X_new.ndim != 2:
|
|
510
|
+
raise ValueError(f"update_nowcast: X_new must be a 2D numpy array, got {type(X_new)}")
|
|
511
|
+
if X_old.shape[1] != X_new.shape[1]:
|
|
512
|
+
raise ValueError(f"update_nowcast: X_old and X_new must have same number of series. Got {X_old.shape[1]} and {X_new.shape[1]}")
|
|
513
|
+
if not isinstance(series, str):
|
|
514
|
+
raise ValueError(f"update_nowcast: series must be a string, got {type(series)}")
|
|
515
|
+
if not isinstance(period, str):
|
|
516
|
+
raise ValueError(f"update_nowcast: period must be a string, got {type(period)}")
|
|
517
|
+
if not hasattr(config, 'SeriesID'):
|
|
518
|
+
raise ValueError(f"update_nowcast: config must have 'SeriesID' attribute")
|
|
519
|
+
if series not in config.SeriesID:
|
|
520
|
+
raise ValueError(f"update_nowcast: series '{series}' not found in config.SeriesID")
|
|
521
|
+
|
|
522
|
+
# Validate period format
|
|
523
|
+
try:
|
|
524
|
+
i_series = config.SeriesID.index(series)
|
|
525
|
+
freq = config.Frequency[i_series]
|
|
526
|
+
if freq == 'm':
|
|
527
|
+
# Format: YYYYmMM (e.g., "2016m12")
|
|
528
|
+
parts = period.split('m')
|
|
529
|
+
if len(parts) != 2:
|
|
530
|
+
raise ValueError(f"update_nowcast: period '{period}' must be in format 'YYYYmMM' for monthly frequency")
|
|
531
|
+
year, month = int(parts[0]), int(parts[1])
|
|
532
|
+
if not (1 <= month <= 12):
|
|
533
|
+
raise ValueError(f"update_nowcast: month must be between 1 and 12, got {month}")
|
|
534
|
+
elif freq == 'q':
|
|
535
|
+
# Format: YYYYqQ (e.g., "2016q4")
|
|
536
|
+
parts = period.split('q')
|
|
537
|
+
if len(parts) != 2:
|
|
538
|
+
raise ValueError(f"update_nowcast: period '{period}' must be in format 'YYYYqQ' for quarterly frequency")
|
|
539
|
+
year, q = int(parts[0]), int(parts[1])
|
|
540
|
+
if not (1 <= q <= 4):
|
|
541
|
+
raise ValueError(f"update_nowcast: quarter must be between 1 and 4, got {q}")
|
|
542
|
+
else:
|
|
543
|
+
raise ValueError(f"update_nowcast: Unsupported frequency '{freq}' for period validation")
|
|
544
|
+
except (ValueError, IndexError) as e:
|
|
545
|
+
if isinstance(e, ValueError) and "not found" not in str(e):
|
|
546
|
+
raise ValueError(f"update_nowcast: Invalid period format '{period}': {str(e)}")
|
|
547
|
+
raise
|
|
548
|
+
|
|
367
549
|
# Handle dict format (from saved results) - extract Res and check config consistency
|
|
368
550
|
if isinstance(Res, dict):
|
|
369
551
|
if 'Spec' in Res and 'Res' in Res:
|
|
@@ -371,47 +553,18 @@ def update_nowcast(X_old: np.ndarray, X_new: np.ndarray, Time: pd.DatetimeIndex,
|
|
|
371
553
|
saved_config = Res['Spec']
|
|
372
554
|
Res = Res['Res']
|
|
373
555
|
# Simple consistency check: compare SeriesID
|
|
374
|
-
|
|
375
|
-
if hasattr(saved_config, 'SeriesID') and hasattr(config, 'SeriesID'):
|
|
376
|
-
if saved_config.SeriesID != config.SeriesID:
|
|
377
|
-
import warnings
|
|
378
|
-
warnings.warn(
|
|
379
|
-
"Config used in estimation differs from current config. "
|
|
380
|
-
"Results may be inconsistent. Consider re-estimating.",
|
|
381
|
-
UserWarning
|
|
382
|
-
)
|
|
383
|
-
except Exception:
|
|
384
|
-
pass # If comparison fails, continue anyway
|
|
556
|
+
_check_config_consistency(saved_config, config)
|
|
385
557
|
elif 'Config' in Res and 'Res' in Res:
|
|
386
558
|
# New format with Config key
|
|
387
559
|
saved_config = Res['Config']
|
|
388
560
|
Res = Res['Res']
|
|
389
|
-
|
|
390
|
-
if hasattr(saved_config, 'SeriesID') and hasattr(config, 'SeriesID'):
|
|
391
|
-
if saved_config.SeriesID != config.SeriesID:
|
|
392
|
-
import warnings
|
|
393
|
-
warnings.warn(
|
|
394
|
-
"Config used in estimation differs from current config. "
|
|
395
|
-
"Results may be inconsistent. Consider re-estimating.",
|
|
396
|
-
UserWarning
|
|
397
|
-
)
|
|
398
|
-
except Exception:
|
|
399
|
-
pass
|
|
561
|
+
_check_config_consistency(saved_config, config)
|
|
400
562
|
elif 'Res' in Res:
|
|
401
563
|
Res = Res['Res']
|
|
402
564
|
|
|
403
565
|
# Also check if DFMResult has a Config attribute (for future compatibility)
|
|
404
566
|
if hasattr(Res, 'Config') and hasattr(config, 'SeriesID'):
|
|
405
|
-
|
|
406
|
-
if hasattr(Res.Config, 'SeriesID') and Res.Config.SeriesID != config.SeriesID:
|
|
407
|
-
import warnings
|
|
408
|
-
warnings.warn(
|
|
409
|
-
"Config in Res differs from current config. "
|
|
410
|
-
"Results may be inconsistent.",
|
|
411
|
-
UserWarning
|
|
412
|
-
)
|
|
413
|
-
except Exception:
|
|
414
|
-
pass
|
|
567
|
+
_check_config_consistency(Res.Config, config)
|
|
415
568
|
|
|
416
569
|
# Make sure datasets are same size
|
|
417
570
|
N = X_new.shape[1]
|
|
@@ -422,20 +575,20 @@ def update_nowcast(X_old: np.ndarray, X_new: np.ndarray, Time: pd.DatetimeIndex,
|
|
|
422
575
|
X_old = np.vstack([X_old, np.full((T_new - T_old, N), np.nan)])
|
|
423
576
|
|
|
424
577
|
# Append 1 year (12 months) for forecasting horizon
|
|
425
|
-
X_old = np.vstack([X_old, np.full((
|
|
426
|
-
X_new = np.vstack([X_new, np.full((
|
|
578
|
+
X_old = np.vstack([X_old, np.full((FORECAST_HORIZON_MONTHS, N), np.nan)])
|
|
579
|
+
X_new = np.vstack([X_new, np.full((FORECAST_HORIZON_MONTHS, N), np.nan)])
|
|
427
580
|
|
|
428
581
|
# Extend time index
|
|
429
582
|
last_date = Time.iloc[-1] if hasattr(Time, 'iloc') else Time[-1]
|
|
430
583
|
if hasattr(last_date, 'month'):
|
|
431
584
|
new_dates = pd.date_range(
|
|
432
585
|
start=last_date + pd.DateOffset(months=1),
|
|
433
|
-
periods=
|
|
586
|
+
periods=FORECAST_HORIZON_MONTHS,
|
|
434
587
|
freq='MS'
|
|
435
588
|
)
|
|
436
589
|
else:
|
|
437
590
|
# Fallback if not pandas datetime
|
|
438
|
-
new_dates = pd.date_range(start=
|
|
591
|
+
new_dates = pd.date_range(start=DEFAULT_FALLBACK_DATE, periods=FORECAST_HORIZON_MONTHS, freq='MS')
|
|
439
592
|
|
|
440
593
|
# Handle both Index and Series types
|
|
441
594
|
if isinstance(Time, pd.Series):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dfm-python
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Dynamic Factor Model (DFM) estimation and nowcasting in Python
|
|
5
5
|
Project-URL: Homepage, https://github.com/yourusername/dfm-python
|
|
6
6
|
Project-URL: Documentation, https://github.com/yourusername/dfm-python#readme
|
|
@@ -8,6 +8,7 @@ Project-URL: Repository, https://github.com/yourusername/dfm-python
|
|
|
8
8
|
Project-URL: Issues, https://github.com/yourusername/dfm-python/issues
|
|
9
9
|
Author: DFM Python Contributors
|
|
10
10
|
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
11
12
|
Keywords: dfm,dynamic-factor-model,econometrics,forecasting,nowcasting,time-series
|
|
12
13
|
Classifier: Development Status :: 4 - Beta
|
|
13
14
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -97,6 +98,67 @@ factors = Res.Z # Extracted factors
|
|
|
97
98
|
loadings = Res.C # Factor loadings
|
|
98
99
|
```
|
|
99
100
|
|
|
101
|
+
### Complete Example with Sample Data
|
|
102
|
+
|
|
103
|
+
The package includes sample data and configuration files for testing. Here's a complete working example:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from dfm_python import load_config, load_data, dfm
|
|
107
|
+
import pandas as pd
|
|
108
|
+
import numpy as np
|
|
109
|
+
|
|
110
|
+
# Load configuration (YAML or CSV format)
|
|
111
|
+
config = load_config('config/example_config.yaml')
|
|
112
|
+
# Alternative: config = load_config('spec/example_spec.csv')
|
|
113
|
+
|
|
114
|
+
# Load sample data
|
|
115
|
+
X, Time, Z = load_data(
|
|
116
|
+
'data/sample_data.csv',
|
|
117
|
+
config,
|
|
118
|
+
sample_start=pd.Timestamp('2000-01-01')
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
print(f"Data shape: {X.shape}")
|
|
122
|
+
print(f"Time range: {Time[0]} to {Time[-1]}")
|
|
123
|
+
print(f"Number of series: {len(config.series)}")
|
|
124
|
+
|
|
125
|
+
# Estimate DFM model
|
|
126
|
+
print("\nEstimating DFM model...")
|
|
127
|
+
Res = dfm(X, config, threshold=1e-4, max_iter=1000)
|
|
128
|
+
|
|
129
|
+
# Access results
|
|
130
|
+
print(f"\nExtracted {Res.Z.shape[1]} factors")
|
|
131
|
+
print(f"Factor loadings shape: {Res.C.shape}")
|
|
132
|
+
print(f"Transition matrix shape: {Res.A.shape}")
|
|
133
|
+
|
|
134
|
+
# Display factor estimates
|
|
135
|
+
factors = Res.Z # Factor estimates (T x r)
|
|
136
|
+
loadings = Res.C # Factor loadings (N x r)
|
|
137
|
+
transition = Res.A # Transition matrix
|
|
138
|
+
covariance = Res.Q # Factor covariance
|
|
139
|
+
|
|
140
|
+
print(f"\nFirst few factor values:")
|
|
141
|
+
print(factors[:5, :])
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
**Expected Output:**
|
|
145
|
+
```
|
|
146
|
+
Loading data...
|
|
147
|
+
Data shape: (128, 10)
|
|
148
|
+
Time range: 2000-05-01 00:00:00 to 2010-12-01 00:00:00
|
|
149
|
+
Number of series: 10
|
|
150
|
+
|
|
151
|
+
Estimating DFM model...
|
|
152
|
+
Extracted 4 factors
|
|
153
|
+
Factor loadings shape: (10, 4)
|
|
154
|
+
Transition matrix shape: (4, 4)
|
|
155
|
+
|
|
156
|
+
First few factor values:
|
|
157
|
+
[[ 0.123 -0.045 0.012 0.008]
|
|
158
|
+
[ 0.145 -0.032 0.015 0.011]
|
|
159
|
+
...]
|
|
160
|
+
```
|
|
161
|
+
|
|
100
162
|
### Configuration
|
|
101
163
|
|
|
102
164
|
The DFM module uses a configuration object (`DFMConfig`) that defines:
|
|
@@ -107,21 +169,32 @@ The DFM module uses a configuration object (`DFMConfig`) that defines:
|
|
|
107
169
|
|
|
108
170
|
#### YAML Configuration Example
|
|
109
171
|
|
|
172
|
+
The package includes a sample configuration file (`config/example_config.yaml`) that demonstrates the full configuration structure:
|
|
173
|
+
|
|
110
174
|
```yaml
|
|
111
|
-
# config.yaml
|
|
175
|
+
# config/example_config.yaml
|
|
112
176
|
model:
|
|
113
177
|
series:
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
transformation: "pch"
|
|
117
|
-
blocks: [Global]
|
|
118
|
-
series_2:
|
|
178
|
+
gdp_real:
|
|
179
|
+
series_name: "Real GDP (Quarterly)"
|
|
119
180
|
frequency: "q"
|
|
120
181
|
transformation: "pca"
|
|
121
|
-
|
|
182
|
+
category: "GDP"
|
|
183
|
+
units: "Index (2000=100)"
|
|
184
|
+
blocks: [Global, Consumption, Investment]
|
|
185
|
+
|
|
186
|
+
consumption:
|
|
187
|
+
series_name: "Consumption (Monthly)"
|
|
188
|
+
frequency: "m"
|
|
189
|
+
transformation: "pch"
|
|
190
|
+
category: "Consumption"
|
|
191
|
+
units: "Index (2000=100)"
|
|
192
|
+
blocks: [Global, Consumption]
|
|
122
193
|
|
|
123
194
|
blocks:
|
|
124
195
|
Global:
|
|
196
|
+
factors: 2
|
|
197
|
+
Consumption:
|
|
125
198
|
factors: 1
|
|
126
199
|
Investment:
|
|
127
200
|
factors: 1
|
|
@@ -134,27 +207,39 @@ dfm:
|
|
|
134
207
|
|
|
135
208
|
#### CSV Configuration Example
|
|
136
209
|
|
|
210
|
+
The package also includes a CSV configuration file (`spec/example_spec.csv`) for simpler setups:
|
|
211
|
+
|
|
137
212
|
```csv
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
213
|
+
series_id,series_name,frequency,transformation,category,units,Block_Global,Block_Consumption,Block_Investment
|
|
214
|
+
gdp_real,Real GDP (Quarterly),q,pca,GDP,Index (2000=100),1,1,1
|
|
215
|
+
consumption,Consumption (Monthly),m,pch,Consumption,Index (2000=100),1,1,0
|
|
216
|
+
investment,Investment (Monthly),m,pch,Investment,Index (2000=100),1,0,1
|
|
141
217
|
```
|
|
142
218
|
|
|
219
|
+
**Note:** The first block (Global) must always have value `1` for all series, as it represents the common factor that all series load on.
|
|
220
|
+
|
|
143
221
|
### Data Format
|
|
144
222
|
|
|
145
223
|
CSV data files should have:
|
|
146
224
|
- First column: Date (YYYY-MM-DD format)
|
|
147
|
-
- Subsequent columns: One per series (matching
|
|
148
|
-
- Column names:
|
|
225
|
+
- Subsequent columns: One per series (matching `series_id` from config)
|
|
226
|
+
- Column names: Must match `series_id` from configuration exactly
|
|
227
|
+
|
|
228
|
+
The package includes sample data (`data/sample_data.csv`) with 10 time series spanning 2000-2010:
|
|
149
229
|
|
|
150
|
-
Example:
|
|
151
230
|
```csv
|
|
152
|
-
Date,
|
|
153
|
-
2000-01-01,100.0,
|
|
154
|
-
2000-02-01,
|
|
231
|
+
Date,gdp_real,gdp_nominal,consumption,investment,exports,imports,unemployment,industrial_production,retail_sales,consumer_confidence
|
|
232
|
+
2000-01-01,100.0,105.0,98.5,95.0,102.0,100.5,5.2,98.0,97.5,100.0
|
|
233
|
+
2000-02-01,100.2,105.3,98.7,95.2,102.2,100.7,5.1,98.3,97.8,100.2
|
|
155
234
|
...
|
|
156
235
|
```
|
|
157
236
|
|
|
237
|
+
**Important Notes:**
|
|
238
|
+
- Date column must be named `Date` (case-sensitive)
|
|
239
|
+
- Series column names must exactly match `series_id` values in the configuration
|
|
240
|
+
- Missing values should be represented as empty cells or `NaN`
|
|
241
|
+
- The data supports mixed frequencies (monthly and quarterly series are handled automatically)
|
|
242
|
+
|
|
158
243
|
## Core Components
|
|
159
244
|
|
|
160
245
|
### DFMConfig
|
|
@@ -239,6 +324,45 @@ The DFM module is designed to be **generic and application-agnostic**:
|
|
|
239
324
|
|
|
240
325
|
This separation allows the core module to be used in any context while keeping application-specific code separate.
|
|
241
326
|
|
|
327
|
+
## Testing with Sample Data
|
|
328
|
+
|
|
329
|
+
The package includes sample data and configuration files for testing and demonstration:
|
|
330
|
+
|
|
331
|
+
- **Data**: `data/sample_data.csv` - 10 time series from 2000-2010
|
|
332
|
+
- **Config (YAML)**: `config/example_config.yaml` - Full configuration example
|
|
333
|
+
- **Config (CSV)**: `spec/example_spec.csv` - Simplified CSV configuration
|
|
334
|
+
|
|
335
|
+
To test the package with sample data:
|
|
336
|
+
|
|
337
|
+
```python
|
|
338
|
+
from dfm_python import load_config, load_data, dfm
|
|
339
|
+
import pandas as pd
|
|
340
|
+
|
|
341
|
+
# Load sample configuration
|
|
342
|
+
config = load_config('config/example_config.yaml')
|
|
343
|
+
|
|
344
|
+
# Load sample data
|
|
345
|
+
X, Time, Z = load_data('data/sample_data.csv', config)
|
|
346
|
+
|
|
347
|
+
# Estimate model
|
|
348
|
+
result = dfm(X, config, threshold=1e-4)
|
|
349
|
+
|
|
350
|
+
# Check results
|
|
351
|
+
print(f"Factors extracted: {result.Z.shape[1]}")
|
|
352
|
+
print(f"Factor loadings shape: {result.C.shape}")
|
|
353
|
+
print(f"Number of series: {result.C.shape[0]}")
|
|
354
|
+
print(f"Number of time periods: {result.Z.shape[0]}")
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
You can also run the test suite:
|
|
358
|
+
|
|
359
|
+
```bash
|
|
360
|
+
# From project root
|
|
361
|
+
python -m pytest src/test/
|
|
362
|
+
# Or run specific test
|
|
363
|
+
python src/test/test_synthetic.py
|
|
364
|
+
```
|
|
365
|
+
|
|
242
366
|
## Requirements
|
|
243
367
|
|
|
244
368
|
- Python >= 3.12
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
dfm_python/__init__.py,sha256=_74eT10hQse0xb22IsiTNKLe7daY_0YbKHjMkdhO3U4,961
|
|
2
|
+
dfm_python/config.py,sha256=dBhBO-ckKlzm0DPImOhdP9eDo1m0Nfh2vnanPUGMVfs,19005
|
|
3
|
+
dfm_python/data_loader.py,sha256=v81w3V_IkV9i6V5WYZOlky8mRTyPTaTRfhDKQY2bBLQ,14507
|
|
4
|
+
dfm_python/dfm.py,sha256=MMIOrkWKrN5u8eXl3526JLHb8wVDMyc8hH3Uak7wbVg,108301
|
|
5
|
+
dfm_python/kalman.py,sha256=RS9obW1eT7Q6L5NeXk9vB4ZYZbg96d0yYm4gk6NaO1M,14712
|
|
6
|
+
dfm_python/news.py,sha256=wLT3ppauByLZpDRhN3VMhCk6rIgxFJcFjJJG5GwFrdI,30150
|
|
7
|
+
dfm_python-0.1.1.dist-info/METADATA,sha256=IeXFro6fYWKjajX6xYb-y6SjVBMEuCYDXm-_3tTjNf0,11035
|
|
8
|
+
dfm_python-0.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
9
|
+
dfm_python-0.1.1.dist-info/licenses/LICENSE,sha256=6e87br6LCDHjy2hdIR1-sdBQy351qdmSEbOg6jQ7ZXY,1063
|
|
10
|
+
dfm_python-0.1.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 ywcpar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
dfm_python/__init__.py,sha256=_74eT10hQse0xb22IsiTNKLe7daY_0YbKHjMkdhO3U4,961
|
|
2
|
-
dfm_python/config.py,sha256=dBhBO-ckKlzm0DPImOhdP9eDo1m0Nfh2vnanPUGMVfs,19005
|
|
3
|
-
dfm_python/data_loader.py,sha256=M0NytBWjtwWYIreD6Gsztt99XIn9hYQe2hVI8L5YZT8,18290
|
|
4
|
-
dfm_python/dfm.py,sha256=1zJCz5DberMnsdVq43rhoSYK5nI4ERADNEJEOlpjnUY,106078
|
|
5
|
-
dfm_python/kalman.py,sha256=RS9obW1eT7Q6L5NeXk9vB4ZYZbg96d0yYm4gk6NaO1M,14712
|
|
6
|
-
dfm_python/news.py,sha256=x55URP3VrrOxcmNq0eHl9X63ReqMMPOQ3Bj0oiN3-ew,21893
|
|
7
|
-
dfm_python-0.1.0.dist-info/METADATA,sha256=_ACla-WRfk8_DnPZA8YrxIH7XI_Xmyvr_gDH2KwH3Lw,6939
|
|
8
|
-
dfm_python-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
9
|
-
dfm_python-0.1.0.dist-info/RECORD,,
|
|
File without changes
|