dfm-python 0.5.59__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dfm_python/__init__.py +8 -3
- dfm_python/config/__init__.py +2 -8
- dfm_python/config/adapter.py +3 -10
- dfm_python/config/constants.py +35 -0
- dfm_python/config/schema/model.py +46 -85
- dfm_python/config/schema/params.py +59 -89
- dfm_python/dataset/ddfm_dataset.py +85 -47
- dfm_python/dataset/dfm_dataset.py +209 -477
- dfm_python/encoder/pca.py +3 -0
- dfm_python/functional/em.py +1104 -758
- dfm_python/logger/dfm_logger.py +39 -0
- dfm_python/models/__init__.py +8 -4
- dfm_python/models/base.py +32 -251
- dfm_python/models/{ddfm.py → ddfm/ddfm.py} +281 -54
- dfm_python/models/dfm/__init__.py +39 -0
- dfm_python/models/dfm/dfm.py +1087 -0
- dfm_python/models/dfm/initialization.py +1154 -0
- dfm_python/models/dfm/mixed_freq.py +387 -0
- dfm_python/{numeric → models/dfm}/tent.py +99 -22
- dfm_python/numeric/builder.py +150 -137
- dfm_python/numeric/estimator.py +253 -157
- dfm_python/numeric/stability.py +204 -515
- dfm_python/numeric/validator.py +193 -561
- dfm_python/ssm/kalman.py +549 -448
- dfm_python/utils/__init__.py +32 -56
- dfm_python/utils/misc.py +36 -13
- dfm_python/utils/validation.py +1 -0
- dfm_python-0.6.0.dist-info/METADATA +56 -0
- dfm_python-0.6.0.dist-info/RECORD +55 -0
- dfm_python/functional/dfm_block.py +0 -485
- dfm_python/models/dfm.py +0 -1897
- dfm_python-0.5.59.dist-info/METADATA +0 -380
- dfm_python-0.5.59.dist-info/RECORD +0 -52
- {dfm_python-0.5.59.dist-info → dfm_python-0.6.0.dist-info}/WHEEL +0 -0
dfm_python/__init__.py
CHANGED
|
@@ -32,7 +32,9 @@ Example:
|
|
|
32
32
|
>>> df_processed = df[[col for col in df.columns if col != 'date']]
|
|
33
33
|
>>>
|
|
34
34
|
>>> # Step 2: Create Dataset
|
|
35
|
-
>>>
|
|
35
|
+
>>> from dfm_python.config import DFMConfig
|
|
36
|
+
>>> config = DFMConfig.from_yaml('config/default.yaml')
|
|
37
|
+
>>> dataset = DFMDataset(config=config, data=df_processed)
|
|
36
38
|
>>>
|
|
37
39
|
>>> # Step 3: Create model and load config
|
|
38
40
|
>>> model = DFM()
|
|
@@ -61,7 +63,7 @@ Note: DFMConfig uses frequency dict to specify series (column names -> frequenci
|
|
|
61
63
|
For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
|
|
62
64
|
"""
|
|
63
65
|
|
|
64
|
-
__version__ = "0.5.
|
|
66
|
+
__version__ = "0.5.60"
|
|
65
67
|
|
|
66
68
|
# ============================================================================
|
|
67
69
|
# PUBLIC API DEFINITION
|
|
@@ -102,7 +104,10 @@ from .models.base import BaseFactorModel
|
|
|
102
104
|
from .models.dfm import DFM
|
|
103
105
|
|
|
104
106
|
# DDFM high-level API (PyTorch is mandatory)
|
|
105
|
-
|
|
107
|
+
try:
|
|
108
|
+
from .models.ddfm.ddfm import DDFM
|
|
109
|
+
except ImportError:
|
|
110
|
+
DDFM = None
|
|
106
111
|
|
|
107
112
|
__all__ = [
|
|
108
113
|
# Core classes
|
dfm_python/config/__init__.py
CHANGED
|
@@ -81,12 +81,7 @@ from .types import (
|
|
|
81
81
|
)
|
|
82
82
|
# Import lazily to avoid circular dependencies
|
|
83
83
|
try:
|
|
84
|
-
from ..
|
|
85
|
-
except ImportError:
|
|
86
|
-
compute_idio_lengths = None
|
|
87
|
-
|
|
88
|
-
try:
|
|
89
|
-
from ..numeric.tent import get_agg_structure, group_by_freq
|
|
84
|
+
from ..models.dfm.tent import get_agg_structure, group_by_freq
|
|
90
85
|
except ImportError:
|
|
91
86
|
get_agg_structure = None
|
|
92
87
|
group_by_freq = None
|
|
@@ -100,7 +95,7 @@ def get_periods_per_year(frequency: str) -> int:
|
|
|
100
95
|
__all__ = [
|
|
101
96
|
# Base classes
|
|
102
97
|
'BaseModelConfig', 'BaseResult',
|
|
103
|
-
# 'DEFAULT_BLOCK_NAME', # Removed to avoid circular import
|
|
98
|
+
# 'DEFAULT_BLOCK_NAME', # Removed to avoid circular import
|
|
104
99
|
# Model-specific configs (from schema.py)
|
|
105
100
|
'DFMConfig', 'DDFMConfig',
|
|
106
101
|
# State-space parameters
|
|
@@ -117,7 +112,6 @@ __all__ = [
|
|
|
117
112
|
'FREQUENCY_HIERARCHY',
|
|
118
113
|
'PERIODS_PER_YEAR',
|
|
119
114
|
'get_periods_per_year',
|
|
120
|
-
'compute_idio_lengths',
|
|
121
115
|
'get_agg_structure',
|
|
122
116
|
'group_by_freq',
|
|
123
117
|
# Type definitions (from types.py)
|
dfm_python/config/adapter.py
CHANGED
|
@@ -303,7 +303,6 @@ class YamlSource:
|
|
|
303
303
|
_EXCLUDED_KEYS = {
|
|
304
304
|
'defaults', '_target_', '_recursive_', '_convert_',
|
|
305
305
|
'series', 'blocks', 'data', 'output', 'description', 'name', 'target', 'model',
|
|
306
|
-
'nan_method', 'nan_k' # Internal constants, not user-configurable
|
|
307
306
|
}
|
|
308
307
|
main_settings = {k: v for k, v in cfg_dict.items() if k not in _EXCLUDED_KEYS}
|
|
309
308
|
|
|
@@ -509,18 +508,16 @@ def make_config_source(
|
|
|
509
508
|
# ============================================================================
|
|
510
509
|
|
|
511
510
|
# Model type constants (exported for use in schema/model.py)
|
|
512
|
-
|
|
511
|
+
# NOTE: KDFM is legacy and has been removed from this codebase.
|
|
513
512
|
MODEL_TYPE_DDFM = 'ddfm'
|
|
514
513
|
MODEL_TYPE_DFM = 'dfm'
|
|
515
514
|
|
|
516
515
|
# Model type detection patterns
|
|
517
|
-
_KDFM_TYPE_ALIASES = {'kdfm', 'kernelized'}
|
|
518
516
|
_DDFM_TYPE_ALIASES = {'ddfm', 'deep'}
|
|
519
|
-
_KDFM_PARAMS = {'ar_order', 'ma_order', 'structural_method', 'structural_reg_weight'}
|
|
520
517
|
_DDFM_PARAMS = {'encoder_layers', 'epochs', 'learning_rate', 'batch_size'}
|
|
521
518
|
|
|
522
519
|
def detect_config_type(data: Dict[str, Any]) -> str:
|
|
523
|
-
"""Detect config type (DFM
|
|
520
|
+
"""Detect config type (DFM or DDFM) from data dictionary.
|
|
524
521
|
|
|
525
522
|
Parameters
|
|
526
523
|
----------
|
|
@@ -530,7 +527,7 @@ def detect_config_type(data: Dict[str, Any]) -> str:
|
|
|
530
527
|
Returns
|
|
531
528
|
-------
|
|
532
529
|
str
|
|
533
|
-
'
|
|
530
|
+
'ddfm' or 'dfm'
|
|
534
531
|
|
|
535
532
|
Raises
|
|
536
533
|
------
|
|
@@ -545,8 +542,6 @@ def detect_config_type(data: Dict[str, Any]) -> str:
|
|
|
545
542
|
model_type = data.get('model_type', '').lower()
|
|
546
543
|
|
|
547
544
|
# Check explicit model type
|
|
548
|
-
if model_type in _KDFM_TYPE_ALIASES:
|
|
549
|
-
return MODEL_TYPE_KDFM
|
|
550
545
|
if model_type in _DDFM_TYPE_ALIASES:
|
|
551
546
|
return MODEL_TYPE_DDFM
|
|
552
547
|
if model_type == MODEL_TYPE_DFM:
|
|
@@ -559,8 +554,6 @@ def detect_config_type(data: Dict[str, Any]) -> str:
|
|
|
559
554
|
|
|
560
555
|
# Check for model-specific parameters
|
|
561
556
|
keys = set(data.keys())
|
|
562
|
-
if keys & _KDFM_PARAMS:
|
|
563
|
-
return MODEL_TYPE_KDFM
|
|
564
557
|
if any(k.startswith('ddfm_') for k in keys) or (keys & _DDFM_PARAMS):
|
|
565
558
|
return MODEL_TYPE_DDFM
|
|
566
559
|
|
dfm_python/config/constants.py
CHANGED
|
@@ -35,6 +35,29 @@ DEFAULT_VARIANCE_FALLBACK = 1.0 # Default variance fallback value for numerical
|
|
|
35
35
|
# Maximum eigenvalues
|
|
36
36
|
MAX_EIGENVALUE = 1e3 # Maximum eigenvalue cap (reduced to 1e3 to improve condition number: 1e3/1e-6 = 1e9)
|
|
37
37
|
|
|
38
|
+
# ============================================================================
|
|
39
|
+
# Performance and Numerical Health Guardrails
|
|
40
|
+
# ============================================================================
|
|
41
|
+
|
|
42
|
+
# Time budgets (hard limits to prevent pathological runs)
|
|
43
|
+
# Single constraint: 10 minutes for all datasets
|
|
44
|
+
MAX_ESTEP_SECONDS = 600.0 # Maximum E-step time (Kalman filter + smoother) - 10 minutes
|
|
45
|
+
MAX_MSTEP_SECONDS = 600.0 # Maximum M-step time (parameter updates) - 10 minutes
|
|
46
|
+
MAX_EM_ITERATION_SECONDS = 1200.0 # Maximum total EM iteration time - 20 minutes
|
|
47
|
+
MAX_FIRST_ITERATION_SECONDS = 600.0 # Maximum first iteration E-step time (diagnostic) - 10 minutes
|
|
48
|
+
|
|
49
|
+
# Numerical health thresholds (fail fast on pathological regimes)
|
|
50
|
+
# Note: Condition number check is more lenient during initialization (first iteration)
|
|
51
|
+
MAX_CONDITION_NUMBER_SMOOTHER = 1e8 # Maximum condition number before refusing to smooth (increased from 1e7)
|
|
52
|
+
MAX_CONDITION_NUMBER_INIT = 1e10 # Maximum condition number during initialization (more lenient)
|
|
53
|
+
MAX_STABILIZATION_AMOUNT = 1e-3 # Maximum stabilization amount before error (increased from 1e-4 to 0.1% diagonal loading)
|
|
54
|
+
MAX_STABILIZATION_AMOUNT_INIT = 5e-3 # Maximum stabilization during initialization (more lenient)
|
|
55
|
+
MAX_STD_UNSCALED = 100.0 # Maximum standard deviation for unscaled data (indicates missing preprocessing)
|
|
56
|
+
MIN_STD_SCALED = 1e-4 # Minimum standard deviation for scaled data (indicates degenerate series)
|
|
57
|
+
|
|
58
|
+
# Fallback tracking
|
|
59
|
+
MAX_FALLBACKS_PER_ITERATION = 0 # Maximum allowed fallbacks per iteration (0 = fail on any fallback)
|
|
60
|
+
|
|
38
61
|
# Eigenvalue stability thresholds
|
|
39
62
|
DEFAULT_EIGENVALUE_MAX_MAGNITUDE = 1.0 # Default maximum eigenvalue magnitude for stability checks
|
|
40
63
|
DEFAULT_EIGENVALUE_WARN_THRESHOLD = 0.99 # Default warning threshold for near-unstable eigenvalues
|
|
@@ -566,5 +589,17 @@ __all__ = [
|
|
|
566
589
|
# Frequency hierarchy and tent kernels
|
|
567
590
|
'FREQUENCY_HIERARCHY',
|
|
568
591
|
'MAX_TENT_SIZE',
|
|
592
|
+
# Performance and numerical health guardrails
|
|
593
|
+
'MAX_ESTEP_SECONDS',
|
|
594
|
+
'MAX_MSTEP_SECONDS',
|
|
595
|
+
'MAX_EM_ITERATION_SECONDS',
|
|
596
|
+
'MAX_FIRST_ITERATION_SECONDS',
|
|
597
|
+
'MAX_CONDITION_NUMBER_SMOOTHER',
|
|
598
|
+
'MAX_CONDITION_NUMBER_INIT',
|
|
599
|
+
'MAX_STABILIZATION_AMOUNT',
|
|
600
|
+
'MAX_STABILIZATION_AMOUNT_INIT',
|
|
601
|
+
'MAX_STD_UNSCALED',
|
|
602
|
+
'MIN_STD_SCALED',
|
|
603
|
+
'MAX_FALLBACKS_PER_ITERATION',
|
|
569
604
|
]
|
|
570
605
|
|
|
@@ -65,8 +65,6 @@ from ..constants import (
|
|
|
65
65
|
MIN_EIGENVALUE,
|
|
66
66
|
MAX_EIGENVALUE,
|
|
67
67
|
MIN_DIAGONAL_VARIANCE,
|
|
68
|
-
DEFAULT_NAN_METHOD,
|
|
69
|
-
DEFAULT_NAN_K,
|
|
70
68
|
DEFAULT_CLOCK_FREQUENCY,
|
|
71
69
|
DEFAULT_KDFM_AR_ORDER,
|
|
72
70
|
DEFAULT_KDFM_MA_ORDER,
|
|
@@ -138,7 +136,6 @@ class BaseModelConfig:
|
|
|
138
136
|
# ========================================================================
|
|
139
137
|
clock: str = 'm' # Required: Base frequency for latent factors (global clock): 'd', 'w', 'm', 'q', 'sa', 'a' (defaults to 'm' for monthly)
|
|
140
138
|
target_scaler: Optional[ScalerType] = None # Fitted sklearn scaler instance (StandardScaler, RobustScaler, etc.) for target series only. Must be a fitted scaler object (call .fit() on target data first). Pass scaler object directly, not string. Feature series are assumed to be manually preprocessed. If None, target series are assumed to be already in the desired scale.
|
|
141
|
-
# Note: nan_method and nan_k are internal constants (DEFAULT_NAN_METHOD, DEFAULT_NAN_K) used during initialization only
|
|
142
139
|
|
|
143
140
|
def __post_init__(self):
|
|
144
141
|
"""Validate basic model structure.
|
|
@@ -294,9 +291,6 @@ class DFMConfig(BaseModelConfig):
|
|
|
294
291
|
# - If mixed frequencies detected: augment_idio=True, augment_idio_slow=True (required for tent kernel)
|
|
295
292
|
# These are properties, not config fields - automatically determined from frequency configuration
|
|
296
293
|
|
|
297
|
-
# Kalman Filter Type
|
|
298
|
-
use_cholesky_filter: bool = False # Use CholeskyKalmanFilter for better numerical stability (default: False)
|
|
299
|
-
|
|
300
294
|
# Tent Kernel Weights (for mixed-frequency aggregation)
|
|
301
295
|
tent_weights: Optional[Union[Dict[str, List[float]], Dict[str, np.ndarray]]] = None # Required for mixed-frequency data: tent weights for frequency pairs. Format: {'freq_pair': [weights]} or {'freq': [weights]}. Example: {'m:w': [1, 2, 1]} or {'m': [1, 2, 1]}. Must be specified in config for all slower-frequency pairs.
|
|
302
296
|
|
|
@@ -379,7 +373,11 @@ class DFMConfig(BaseModelConfig):
|
|
|
379
373
|
object.__setattr__(self, '_augment_idio_slow', is_mixed_freq)
|
|
380
374
|
|
|
381
375
|
def to_em_config(self) -> 'EMConfig':
|
|
382
|
-
"""Create EMConfig from DFMConfig consolidated parameters.
|
|
376
|
+
"""Create EMConfig from DFMConfig consolidated parameters.
|
|
377
|
+
|
|
378
|
+
Automatically passes all numerical stability parameters (regularization, ar_clip,
|
|
379
|
+
damping_factor, data_clip) from DFMConfig to EMConfig for use in EM algorithm.
|
|
380
|
+
"""
|
|
383
381
|
from ...functional.em import EMConfig
|
|
384
382
|
from ..constants import DEFAULT_REGULARIZATION, VAR_STABILITY_THRESHOLD
|
|
385
383
|
|
|
@@ -397,6 +395,10 @@ class DFMConfig(BaseModelConfig):
|
|
|
397
395
|
regularization=reg_scale,
|
|
398
396
|
min_norm=min_eigenval,
|
|
399
397
|
max_eigenval=max_eigenval,
|
|
398
|
+
# Pass numerical stability parameters from DFMConfig
|
|
399
|
+
ar_clip=self.ar_clip,
|
|
400
|
+
damping_factor=self.damping_factor,
|
|
401
|
+
data_clip=self.data_clip,
|
|
400
402
|
# Other parameters use defaults from EMConfig
|
|
401
403
|
)
|
|
402
404
|
|
|
@@ -416,16 +418,32 @@ class DFMConfig(BaseModelConfig):
|
|
|
416
418
|
series_ids = list(self.frequency.keys()) if columns is None else columns
|
|
417
419
|
|
|
418
420
|
# Build blocks array from block series lists (N x B matrix)
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
}
|
|
421
|
+
# Default global block behavior:
|
|
422
|
+
# If config omitted blocks.series (empty list) and we have runtime columns,
|
|
423
|
+
# interpret a single empty Block_Global as "all series in this block".
|
|
424
|
+
block_series_sets: Dict[str, set] = {}
|
|
425
|
+
default_global_applied = False
|
|
426
|
+
for name in self.block_names:
|
|
427
|
+
series_list = self.blocks[name].get('series', [])
|
|
428
|
+
if (
|
|
429
|
+
columns is not None
|
|
430
|
+
and (series_list is None or len(series_list) == 0)
|
|
431
|
+
and len(self.block_names) == 1
|
|
432
|
+
and name == "Block_Global"
|
|
433
|
+
):
|
|
434
|
+
block_series_sets[name] = set(series_ids)
|
|
435
|
+
default_global_applied = True
|
|
436
|
+
else:
|
|
437
|
+
block_series_sets[name] = set(series_list or [])
|
|
438
|
+
|
|
423
439
|
blocks_list = [
|
|
424
440
|
[1 if series_id in block_series_sets[name] else 0 for name in self.block_names]
|
|
425
441
|
for series_id in series_ids
|
|
426
442
|
]
|
|
427
443
|
|
|
428
444
|
self._cached_blocks = np.array(blocks_list, dtype=int)
|
|
445
|
+
|
|
446
|
+
# Quick sanity on membership counts for the first block (if any)
|
|
429
447
|
return self._cached_blocks
|
|
430
448
|
|
|
431
449
|
@classmethod
|
|
@@ -453,7 +471,6 @@ class DFMConfig(BaseModelConfig):
|
|
|
453
471
|
'data_clip': data.get('data_clip', None),
|
|
454
472
|
'regularization': data.get('regularization', None),
|
|
455
473
|
'damping_factor': data.get('damping_factor', None),
|
|
456
|
-
'use_cholesky_filter': data.get('use_cholesky_filter', False),
|
|
457
474
|
'tent_weights': data.get('tent_weights', None),
|
|
458
475
|
}
|
|
459
476
|
|
|
@@ -462,14 +479,14 @@ class DFMConfig(BaseModelConfig):
|
|
|
462
479
|
return result
|
|
463
480
|
|
|
464
481
|
@classmethod
|
|
465
|
-
def from_dict(cls, data: Dict[str, Any]) -> Union['DFMConfig', 'DDFMConfig'
|
|
466
|
-
"""Create DFMConfig
|
|
482
|
+
def from_dict(cls, data: Dict[str, Any]) -> Union['DFMConfig', 'DDFMConfig']:
|
|
483
|
+
"""Create DFMConfig or DDFMConfig from dictionary.
|
|
467
484
|
|
|
468
485
|
Expected format: {'frequency': {'column_name': 'frequency'}, 'blocks': {...}, ...}
|
|
469
486
|
|
|
470
487
|
Also accepts estimation parameters: threshold, max_iter, etc.
|
|
471
488
|
"""
|
|
472
|
-
from ...config.adapter import detect_config_type, MODEL_TYPE_DDFM,
|
|
489
|
+
from ...config.adapter import detect_config_type, MODEL_TYPE_DDFM, _normalize_blocks_dict
|
|
473
490
|
from ...utils.errors import ConfigurationError
|
|
474
491
|
|
|
475
492
|
# Extract base params (handles frequency conversion from series if needed)
|
|
@@ -481,14 +498,24 @@ class DFMConfig(BaseModelConfig):
|
|
|
481
498
|
if config_type == MODEL_TYPE_DDFM:
|
|
482
499
|
return DDFMConfig(**base_params, **DDFMConfig._extract_ddfm(data))
|
|
483
500
|
|
|
484
|
-
if config_type == MODEL_TYPE_KDFM:
|
|
485
|
-
return KDFMConfig(**base_params, **KDFMConfig._extract_kdfm(data))
|
|
486
|
-
|
|
487
501
|
# Handle blocks for DFM
|
|
488
502
|
from ...config.adapter import _raise_config_error, _is_dict_like
|
|
489
503
|
blocks_dict = data.get('blocks', {})
|
|
490
504
|
if not blocks_dict:
|
|
491
|
-
|
|
505
|
+
# Default behavior: single global block.
|
|
506
|
+
# This enables minimal configs like:
|
|
507
|
+
# clock: d
|
|
508
|
+
# num_factors: 2
|
|
509
|
+
# and lets the model bind series names from the dataset at runtime.
|
|
510
|
+
default_num_factors = int(data.get('num_factors', 1) or 1)
|
|
511
|
+
if default_num_factors < 1:
|
|
512
|
+
_raise_config_error(f"num_factors must be >= 1, got {default_num_factors}")
|
|
513
|
+
blocks_dict = {
|
|
514
|
+
"Block_Global": {
|
|
515
|
+
"num_factors": default_num_factors,
|
|
516
|
+
"series": [], # to be bound from dataset columns
|
|
517
|
+
}
|
|
518
|
+
}
|
|
492
519
|
if not _is_dict_like(blocks_dict):
|
|
493
520
|
_raise_config_error(f"blocks must be a dict, got {type(blocks_dict)}")
|
|
494
521
|
|
|
@@ -599,72 +626,6 @@ class DDFMConfig(BaseModelConfig):
|
|
|
599
626
|
)
|
|
600
627
|
|
|
601
628
|
|
|
602
|
-
@dataclass
|
|
603
|
-
class KDFMConfig(BaseModelConfig):
|
|
604
|
-
"""KDFM configuration dataclass.
|
|
605
|
-
|
|
606
|
-
This dataclass contains all configuration parameters for the KDFM model.
|
|
607
|
-
It inherits from BaseModelConfig and adds KDFM-specific parameters.
|
|
608
|
-
|
|
609
|
-
Note: KDFM does not use blocks structure (unlike DFM). Only frequency dict is needed.
|
|
610
|
-
"""
|
|
611
|
-
# VARMA parameters
|
|
612
|
-
ar_order: int = DEFAULT_KDFM_AR_ORDER # VAR order p
|
|
613
|
-
ma_order: int = DEFAULT_KDFM_MA_ORDER # MA order q (0 = pure VAR)
|
|
614
|
-
|
|
615
|
-
# Structural identification
|
|
616
|
-
structural_method: str = 'cholesky' # 'cholesky', 'full', 'low_rank'
|
|
617
|
-
structural_rank: Optional[int] = None # For low-rank parameterization
|
|
618
|
-
|
|
619
|
-
# Training parameters (use constants for defaults)
|
|
620
|
-
learning_rate: float = DEFAULT_LEARNING_RATE
|
|
621
|
-
max_epochs: int = DEFAULT_MAX_EPOCHS
|
|
622
|
-
batch_size: int = DEFAULT_BATCH_SIZE
|
|
623
|
-
weight_decay: float = DEFAULT_REGULARIZATION_SCALE
|
|
624
|
-
grad_clip_val: float = DEFAULT_GRAD_CLIP_VAL
|
|
625
|
-
|
|
626
|
-
# Regularization
|
|
627
|
-
structural_reg_weight: float = DEFAULT_STRUCTURAL_REG_WEIGHT # Weight for structural loss
|
|
628
|
-
use_regularization: bool = True
|
|
629
|
-
regularization_scale: float = DEFAULT_REGULARIZATION_SCALE
|
|
630
|
-
|
|
631
|
-
# ========================================================================
|
|
632
|
-
# Factory Methods
|
|
633
|
-
# ========================================================================
|
|
634
|
-
|
|
635
|
-
@classmethod
|
|
636
|
-
def _extract_kdfm(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
637
|
-
"""Extract KDFM-specific parameters from config dict."""
|
|
638
|
-
# Don't extract base params here - they're already in base_params from from_dict
|
|
639
|
-
kdfm_params = cls._extract_params(data, {
|
|
640
|
-
'ar_order': DEFAULT_KDFM_AR_ORDER,
|
|
641
|
-
'ma_order': DEFAULT_KDFM_MA_ORDER,
|
|
642
|
-
'structural_method': 'cholesky',
|
|
643
|
-
'structural_rank': None,
|
|
644
|
-
'learning_rate': DEFAULT_LEARNING_RATE,
|
|
645
|
-
'max_epochs': DEFAULT_MAX_EPOCHS,
|
|
646
|
-
'batch_size': DEFAULT_BATCH_SIZE,
|
|
647
|
-
'weight_decay': DEFAULT_REGULARIZATION_SCALE,
|
|
648
|
-
'grad_clip_val': DEFAULT_GRAD_CLIP_VAL,
|
|
649
|
-
'structural_reg_weight': DEFAULT_STRUCTURAL_REG_WEIGHT,
|
|
650
|
-
'use_regularization': True,
|
|
651
|
-
'regularization_scale': DEFAULT_REGULARIZATION_SCALE,
|
|
652
|
-
})
|
|
653
|
-
return kdfm_params
|
|
654
|
-
|
|
655
|
-
@classmethod
|
|
656
|
-
def from_dict(cls, data: Dict[str, Any]) -> 'KDFMConfig':
|
|
657
|
-
"""Create KDFMConfig from dictionary (delegates to DFMConfig.from_dict for type detection)."""
|
|
658
|
-
result = DFMConfig.from_dict(data)
|
|
659
|
-
if isinstance(result, KDFMConfig):
|
|
660
|
-
return result
|
|
661
|
-
from ..utils.errors import ConfigurationError
|
|
662
|
-
raise ConfigurationError(
|
|
663
|
-
f"Expected KDFMConfig but got {type(result).__name__}",
|
|
664
|
-
details=f"Result type: {type(result).__name__}, expected: KDFMConfig"
|
|
665
|
-
)
|
|
666
|
-
|
|
667
|
-
|
|
668
629
|
# ============================================================================
|
|
669
630
|
# Validation Functions
|
|
670
631
|
# ============================================================================
|
|
@@ -8,97 +8,12 @@ import numpy as np
|
|
|
8
8
|
from typing import Optional, Any
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
@dataclass
|
|
12
|
-
class DFMStateSpaceParams:
|
|
13
|
-
"""State-space model parameters for DFM (fitted during model training).
|
|
14
|
-
|
|
15
|
-
These parameters are computed during DFM.fit() and represent the fitted
|
|
16
|
-
state-space model structure. Contains only the state-space parameters,
|
|
17
|
-
not training metadata (loglik, num_iter, converged) which are stored
|
|
18
|
-
separately in checkpoint dicts.
|
|
19
|
-
|
|
20
|
-
Parameters
|
|
21
|
-
----------
|
|
22
|
-
A : np.ndarray
|
|
23
|
-
Transition matrix (m x m) - VAR dynamics for factors
|
|
24
|
-
C : np.ndarray
|
|
25
|
-
Observation matrix (N x m) - factor loadings
|
|
26
|
-
Q : np.ndarray
|
|
27
|
-
Process noise covariance (m x m) - innovation covariance
|
|
28
|
-
R : np.ndarray
|
|
29
|
-
Observation noise covariance (N x N) - typically diagonal
|
|
30
|
-
Z_0 : np.ndarray
|
|
31
|
-
Initial state mean (m,) - initial factor values
|
|
32
|
-
V_0 : np.ndarray
|
|
33
|
-
Initial state covariance (m x m) - initial uncertainty
|
|
34
|
-
"""
|
|
35
|
-
A: np.ndarray # Transition matrix (m x m)
|
|
36
|
-
C: np.ndarray # Observation matrix (N x m)
|
|
37
|
-
Q: np.ndarray # Process noise covariance (m x m)
|
|
38
|
-
R: np.ndarray # Observation noise covariance (N x N)
|
|
39
|
-
Z_0: np.ndarray # Initial state mean (m,)
|
|
40
|
-
V_0: np.ndarray # Initial state covariance (m x m)
|
|
41
|
-
|
|
42
|
-
@classmethod
|
|
43
|
-
def from_model(cls, model: Any) -> 'DFMStateSpaceParams':
|
|
44
|
-
"""Create DFMStateSpaceParams from DFM model instance.
|
|
45
|
-
|
|
46
|
-
Parameters
|
|
47
|
-
----------
|
|
48
|
-
model : DFM
|
|
49
|
-
DFM model instance with fitted parameters
|
|
50
|
-
|
|
51
|
-
Returns
|
|
52
|
-
-------
|
|
53
|
-
DFMStateSpaceParams
|
|
54
|
-
State-space parameters dataclass
|
|
55
|
-
"""
|
|
56
|
-
# Get parameters from model (stored after fit)
|
|
57
|
-
# First check if training_state is already DFMStateSpaceParams (new format)
|
|
58
|
-
if hasattr(model, 'training_state') and model.training_state is not None:
|
|
59
|
-
if isinstance(model.training_state, cls):
|
|
60
|
-
# Already a DFMStateSpaceParams instance
|
|
61
|
-
return model.training_state
|
|
62
|
-
# Backward compatibility: If old format training_state (has A, C, Q, R, Z_0, V_0 attributes)
|
|
63
|
-
if hasattr(model.training_state, 'A'):
|
|
64
|
-
return cls(
|
|
65
|
-
A=model.training_state.A,
|
|
66
|
-
C=model.training_state.C,
|
|
67
|
-
Q=model.training_state.Q,
|
|
68
|
-
R=model.training_state.R,
|
|
69
|
-
Z_0=model.training_state.Z_0,
|
|
70
|
-
V_0=model.training_state.V_0
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
# Get from model attributes (after _update_parameters is called)
|
|
74
|
-
return cls(
|
|
75
|
-
A=getattr(model, 'A', None),
|
|
76
|
-
C=getattr(model, 'C', None),
|
|
77
|
-
Q=getattr(model, 'Q', None),
|
|
78
|
-
R=getattr(model, 'R', None),
|
|
79
|
-
Z_0=getattr(model, 'Z_0', None),
|
|
80
|
-
V_0=getattr(model, 'V_0', None)
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
def apply_to_model(self, model: Any) -> None:
|
|
84
|
-
"""Apply state-space parameters to DFM model instance.
|
|
85
|
-
|
|
86
|
-
Parameters
|
|
87
|
-
----------
|
|
88
|
-
model : DFM
|
|
89
|
-
DFM model instance to update
|
|
90
|
-
"""
|
|
91
|
-
model._update_parameters(
|
|
92
|
-
self.A, self.C, self.Q, self.R, self.Z_0, self.V_0
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
|
|
96
11
|
@dataclass
|
|
97
12
|
class DFMModelState:
|
|
98
|
-
"""DFM model structure
|
|
13
|
+
"""DFM model state including structure, mixed-frequency parameters, and state-space parameters.
|
|
99
14
|
|
|
100
|
-
Consolidates
|
|
101
|
-
|
|
15
|
+
Consolidates all DFM model state for checkpointing: model structure, mixed-frequency
|
|
16
|
+
configuration, and fitted state-space parameters.
|
|
102
17
|
|
|
103
18
|
Attributes
|
|
104
19
|
----------
|
|
@@ -128,6 +43,18 @@ class DFMModelState:
|
|
|
128
43
|
Indicator for idiosyncratic components
|
|
129
44
|
max_lag_size : int, optional
|
|
130
45
|
Maximum lag size for state dimension
|
|
46
|
+
A : np.ndarray, optional
|
|
47
|
+
Transition matrix (m x m) - VAR dynamics for factors
|
|
48
|
+
C : np.ndarray, optional
|
|
49
|
+
Observation matrix (N x m) - factor loadings
|
|
50
|
+
Q : np.ndarray, optional
|
|
51
|
+
Process noise covariance (m x m) - innovation covariance
|
|
52
|
+
R : np.ndarray, optional
|
|
53
|
+
Observation noise covariance (N x N) - typically diagonal
|
|
54
|
+
Z_0 : np.ndarray, optional
|
|
55
|
+
Initial state mean (m,) - initial factor values
|
|
56
|
+
V_0 : np.ndarray, optional
|
|
57
|
+
Initial state covariance (m x m) - initial uncertainty
|
|
131
58
|
"""
|
|
132
59
|
num_factors: int
|
|
133
60
|
r: np.ndarray
|
|
@@ -142,6 +69,13 @@ class DFMModelState:
|
|
|
142
69
|
frequencies: Optional[np.ndarray] = None
|
|
143
70
|
idio_indicator: Optional[np.ndarray] = None
|
|
144
71
|
max_lag_size: Optional[int] = None
|
|
72
|
+
# State-space parameters (fitted during training)
|
|
73
|
+
A: Optional[np.ndarray] = None
|
|
74
|
+
C: Optional[np.ndarray] = None
|
|
75
|
+
Q: Optional[np.ndarray] = None
|
|
76
|
+
R: Optional[np.ndarray] = None
|
|
77
|
+
Z_0: Optional[np.ndarray] = None
|
|
78
|
+
V_0: Optional[np.ndarray] = None
|
|
145
79
|
|
|
146
80
|
@classmethod
|
|
147
81
|
def from_model(cls, model: Any) -> 'DFMModelState':
|
|
@@ -157,6 +91,27 @@ class DFMModelState:
|
|
|
157
91
|
DFMModelState
|
|
158
92
|
Model state dataclass
|
|
159
93
|
"""
|
|
94
|
+
# Get state-space parameters from training_state if available
|
|
95
|
+
A = C = Q = R = Z_0 = V_0 = None
|
|
96
|
+
if hasattr(model, 'training_state') and model.training_state is not None:
|
|
97
|
+
# Handle both old DFMStateSpaceParams format and new DFMModelState format
|
|
98
|
+
ts = model.training_state
|
|
99
|
+
if hasattr(ts, 'A'):
|
|
100
|
+
A = ts.A
|
|
101
|
+
C = ts.C
|
|
102
|
+
Q = ts.Q
|
|
103
|
+
R = ts.R
|
|
104
|
+
Z_0 = ts.Z_0
|
|
105
|
+
V_0 = ts.V_0
|
|
106
|
+
else:
|
|
107
|
+
# Fallback: get from model attributes directly
|
|
108
|
+
A = getattr(model, 'A', None)
|
|
109
|
+
C = getattr(model, 'C', None)
|
|
110
|
+
Q = getattr(model, 'Q', None)
|
|
111
|
+
R = getattr(model, 'R', None)
|
|
112
|
+
Z_0 = getattr(model, 'Z_0', None)
|
|
113
|
+
V_0 = getattr(model, 'V_0', None)
|
|
114
|
+
|
|
160
115
|
return cls(
|
|
161
116
|
num_factors=model.num_factors,
|
|
162
117
|
r=model.r,
|
|
@@ -170,7 +125,13 @@ class DFMModelState:
|
|
|
170
125
|
tent_weights_dict=getattr(model, '_tent_weights_dict', None),
|
|
171
126
|
frequencies=getattr(model, '_frequencies', None),
|
|
172
127
|
idio_indicator=getattr(model, '_idio_indicator', None),
|
|
173
|
-
max_lag_size=getattr(model, '_max_lag_size', None)
|
|
128
|
+
max_lag_size=getattr(model, '_max_lag_size', None),
|
|
129
|
+
A=A,
|
|
130
|
+
C=C,
|
|
131
|
+
Q=Q,
|
|
132
|
+
R=R,
|
|
133
|
+
Z_0=Z_0,
|
|
134
|
+
V_0=V_0
|
|
174
135
|
)
|
|
175
136
|
|
|
176
137
|
def apply_to_model(self, model: Any) -> None:
|
|
@@ -194,6 +155,12 @@ class DFMModelState:
|
|
|
194
155
|
model._frequencies = self.frequencies
|
|
195
156
|
model._idio_indicator = self.idio_indicator
|
|
196
157
|
model._max_lag_size = self.max_lag_size
|
|
158
|
+
|
|
159
|
+
# Apply state-space parameters if available
|
|
160
|
+
if self.A is not None and self.C is not None and self.Q is not None:
|
|
161
|
+
model._update_parameters(
|
|
162
|
+
self.A, self.C, self.Q, self.R, self.Z_0, self.V_0
|
|
163
|
+
)
|
|
197
164
|
|
|
198
165
|
@dataclass
|
|
199
166
|
class DDFMTrainingState:
|
|
@@ -277,3 +244,6 @@ class DDFMStateSpaceParams:
|
|
|
277
244
|
H: np.ndarray # Observation matrix (N x m) - theta in paper
|
|
278
245
|
R: np.ndarray # Observation noise covariance (N x N)
|
|
279
246
|
|
|
247
|
+
# Backward compatibility alias
|
|
248
|
+
DFMStateSpaceParams = DFMModelState
|
|
249
|
+
|