dfm-python 0.5.55__py3-none-any.whl → 0.5.57__py3-none-any.whl

dfm_python/__init__.py

@@ -23,40 +23,36 @@ Key Features:
     - Robust handling of missing data (internal spline interpolation)
     - Automatic standardization and data clipping
 
-Example (Standard Lightning Pattern):
-    >>> from dfm_python import DFM, DFMDataModule, DFMTrainer
+Example:
+    >>> from dfm_python import DFM, DFMDataset
     >>> import pandas as pd
     >>> 
     >>> # Step 1: Load and preprocess data
     >>> df = pd.read_csv('data/your_data.csv')
     >>> df_processed = df[[col for col in df.columns if col != 'date']]
     >>> 
-    >>> # Step 2: Create DataModule
-    >>> dm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
-    >>> dm.setup()
+    >>> # Step 2: Create Dataset
+    >>> dataset = DFMDataset(config_path='config/default.yaml', data=df_processed)
     >>> 
     >>> # Step 3: Create model and load config
     >>> model = DFM()
     >>> model.load_config('config/default.yaml')
     >>> 
-    >>> # Step 4: Create trainer and fit (standard Lightning pattern)
-    >>> trainer = DFMTrainer(max_epochs=100)  # DEFAULT_MAX_EPOCHS
-    >>> trainer.fit(model, dm)
+    >>> # Step 4: Fit model
+    >>> model.fit(X=dataset.get_processed_data(), dataset=dataset)
     >>> 
     >>> # Step 5: Predict
     >>> Xf, Zf = model.predict(horizon=6)
     >>> 
-    >>> # Or use DDFM (same pattern)
-    >>> from dfm_python import DDFM, DDFMTrainer
+    >>> # Or use DDFM
+    >>> from dfm_python import DDFM, DDFMDataset
     >>> 
-    >>> dm_ddfm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
-    >>> dm_ddfm.setup()
+    >>> dataset_ddfm = DDFMDataset(config_path='config/default.yaml', data=df_processed)
     >>> 
     >>> ddfm_model = DDFM(encoder_layers=[64, 32], num_factors=2)
     >>> ddfm_model.load_config('config/default.yaml')
     >>> 
-    >>> trainer_ddfm = DDFMTrainer(max_epochs=100)  # DEFAULT_MAX_EPOCHS
-    >>> trainer_ddfm.fit(ddfm_model, dm_ddfm)
+    >>> ddfm_model.train(dataset=dataset_ddfm)
     >>> Xf, Zf = ddfm_model.predict(horizon=6)
     
 Note: DFMConfig uses frequency dict to specify series (column names -> frequencies).
@@ -65,7 +61,7 @@ Note: DFMConfig uses frequency dict to specify series (column names -> frequenci
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.5.55"
+__version__ = "0.5.56"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -92,20 +88,15 @@ from .config import (
 from .config import DFMConfig
 
 # Results
-from .config import DFMResult, DDFMResult, KDFMResult, BaseResult
+from .config import DFMResult, DDFMResult, BaseResult
 
 # Utilities (from utils/ subpackage)
 from .utils.metric import calculate_rmse
 
-# DataModule classes (includes both Lightning and custom implementations)
+# Dataset classes
 # Users can import these directly from dfm_python
-from .datamodule import (
-    DFMDataModule,
-    DDFMDataModule,  # DDFM-specific DataModule
-    KDFMDataModule,  # KDFM-specific DataModule
-    DDFMDataset,  # DDFM Dataset class (usually not needed directly)
-    KDFMDataset,  # KDFM Dataset class (usually not needed directly)
-)
+from .dataset.dfm_dataset import DFMDataset
+from .dataset.ddfm_dataset import DDFMDataset
 
 # Model implementations
 from .models.base import BaseFactorModel
@@ -114,9 +105,6 @@ from .models.dfm import DFM
 # DDFM high-level API (PyTorch is mandatory)
 from .models.ddfm import DDFM
 
-# KDFM high-level API
-from .models.kdfm import KDFM
-
 __all__ = [
     # Core classes
     'DFM',
@@ -126,7 +114,7 @@ __all__ = [
     'ConfigSource', 'YamlSource',
     'make_config_source',
     # Low-level API (functional interface - advanced usage)
-    'BaseResult', 'DFMResult', 'DDFMResult', 'KDFMResult', 'calculate_rmse',
+    'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse',
 ]
 
 # DDFM high-level API (PyTorch is mandatory)
@@ -134,27 +122,9 @@ __all__.extend([
     'DDFM',  # High-level API class
 ])
 
-# KDFM high-level API
-__all__.extend([
-    'KDFM',  # High-level API class
-])
-
-# Lightning modules (mandatory dependency)
+# Dataset classes
 __all__.extend([
-    'DFMDataModule',
-    'DDFMDataModule',  # DDFM-specific DataModule
-    'KDFMDataModule',  # KDFM-specific DataModule
-    'DDFMDataset',  # DDFM Dataset class (usually not needed directly)
-    'KDFMDataset',  # KDFM Dataset class (usually not needed directly)
+    'DFMDataset',  # DFM Dataset class
+    'DDFMDataset',  # DDFM Dataset class
 ])
 
-# Trainer classes (mandatory dependency)
-from .trainer import DFMTrainer, DDFMTrainer, KDFMTrainer
-
-__all__.extend([
-    'DFMTrainer',
-    'DDFMTrainer',
-    'KDFMTrainer',
-])
-
-

dfm_python/config/__init__.py

@@ -8,8 +8,9 @@ Note: Series are specified via frequency dict mapping column names to frequencie
 """
 
 from .schema import (
-    BaseModelConfig, DFMConfig, DDFMConfig, KDFMConfig,
-    BaseResult, DFMResult, DDFMResult, KDFMResult, FitParams,
+    BaseModelConfig, DFMConfig, DDFMConfig,
+    BaseResult, DFMResult, DDFMResult,
+    DFMStateSpaceParams, DDFMStateSpaceParams,
 )
 # DEFAULT_BLOCK_NAME is imported lazily where needed to avoid circular imports
 from .constants import (
@@ -45,8 +46,6 @@ from .types import (
     ForecastResult,
     CoefficientMatrix,
     CovarianceMatrix,
-    IRFArray,
-    CompanionMatrix,
     SeriesID,
     Frequency,
     DatasetName,
@@ -57,7 +56,6 @@ from .types import (
     ResultDict,
     ForecastDict,
     MetricsDict,
-    IRFResultDict,
     CheckpointDict,
     ConfigDict,
     Device,
@@ -105,11 +103,11 @@ __all__ = [
     'BaseModelConfig', 'BaseResult',
     # 'DEFAULT_BLOCK_NAME',  # Removed to avoid circular import - import directly from functional.dfm_block
     # Model-specific configs (from schema.py)
-    'DFMConfig', 'DDFMConfig', 'KDFMConfig',
-    # Parameter overrides
-    'FitParams',
+    'DFMConfig', 'DDFMConfig',
+    # State-space parameters
+    'DFMStateSpaceParams', 'DDFMStateSpaceParams',
     # Model-specific results (from results.py)
-    'DFMResult', 'DDFMResult', 'KDFMResult',
+    'DFMResult', 'DDFMResult',
     # Utilities
     'validate_frequency',
     # IO
@@ -137,8 +135,6 @@ __all__ = [
     'ForecastResult',
     'CoefficientMatrix',
     'CovarianceMatrix',
-    'IRFArray',
-    'CompanionMatrix',
     'SeriesID',
     'Frequency',
     'DatasetName',
@@ -149,7 +145,6 @@ __all__ = [
     'ResultDict',
     'ForecastDict',
     'MetricsDict',
-    'IRFResultDict',
     'CheckpointDict',
     'ConfigDict',
     'Device',

dfm_python/config/adapter.py

@@ -118,7 +118,12 @@ def _extract_frequency_dict(
     data: Dict[str, Any],
     clock: str = DEFAULT_CLOCK_FREQUENCY
 ) -> Optional[Dict[str, str]]:
-    """Extract frequency dict from data, handling both new API and legacy formats.
+    """Extract frequency dict from data, handling multiple formats.
+    
+    Supports:
+    1. Grouped format: {'w': [series1, series2, ...], 'm': [series3, ...]}
+    2. Individual format: {'series1': 'w', 'series2': 'm', ...}
+    3. Legacy format: {'series': [{'series_id': ..., 'frequency': ...}, ...]}
     
     Parameters
     ----------
@@ -132,12 +137,63 @@ def _extract_frequency_dict(
     Optional[Dict[str, str]]
         Frequency dict mapping column names to frequencies, or None if not found
     """
-    if 'frequency' in data:
-        # New API: frequency dict directly
-        return data['frequency']
-    elif 'series' in data:
-        # Legacy: convert series list/dict to frequency dict
-        return _convert_series_to_frequency_dict(data['series'], clock)
+    if 'frequency' not in data:
+        # Legacy: check for 'series' key
+        if 'series' in data:
+            return _convert_series_to_frequency_dict(data['series'], clock)
+        return None
+    
+    freq_data = data['frequency']
+    
+    # Check if it's in grouped format: {'w': [...], 'm': [...]}
+    if _is_dict_like(freq_data):
+        from .constants import VALID_FREQUENCIES
+        
+        # Check if all keys are valid frequency codes (grouped format)
+        keys_are_frequencies = all(
+            isinstance(k, str) and k in VALID_FREQUENCIES 
+            for k in freq_data.keys()
+        )
+        
+        # Check if values are lists (grouped format)
+        values_are_lists = all(
+            isinstance(v, (list, tuple)) for v in freq_data.values()
+        )
+        
+        if keys_are_frequencies and values_are_lists:
+            # Grouped format: convert to individual format
+            result = {}
+            for freq, series_list in freq_data.items():
+                if not isinstance(series_list, (list, tuple)):
+                    _raise_config_error(
+                        f"Frequency '{freq}' must map to a list of series names, got {type(series_list)}"
+                    )
+                for series_name in series_list:
+                    if not isinstance(series_name, str):
+                        _raise_config_error(
+                            f"Series names must be strings, got {type(series_name)} in frequency '{freq}'"
+                        )
+                    if series_name in result:
+                        _raise_config_error(
+                            f"Series '{series_name}' appears in multiple frequency groups"
+                        )
+                    result[series_name] = freq
+            return result
+        else:
+            # Individual format: already in correct format, validate structure
+            result = {}
+            for series_name, freq in freq_data.items():
+                if not isinstance(series_name, str):
+                    _raise_config_error(
+                        f"Frequency dict keys must be strings (series names), got {type(series_name)}"
+                    )
+                if not isinstance(freq, str):
+                    _raise_config_error(
+                        f"Frequency dict values must be strings (frequency codes), got {type(freq)} for series '{series_name}'"
+                    )
+                result[series_name] = freq
+            return result
+    
     return None
 
 

dfm_python/config/constants.py

@@ -23,15 +23,17 @@ DEFAULT_MIN_DELTA = 1e-6  # Minimum change for improvement
 # ============================================================================
 
 # Minimum eigenvalues and variances
-MIN_EIGENVALUE = 1e-8  # Minimum eigenvalue for positive definite matrices
+MIN_EIGENVALUE = 1e-6  # Minimum eigenvalue for positive definite matrices (increased from 1e-8 for better conditioning)
 MIN_DIAGONAL_VARIANCE = 1e-8  # Minimum variance for diagonal elements
 MIN_OBSERVATION_NOISE = 1e-4  # Minimum observation noise for measurement error (used in EM updates)
+DEFAULT_DDFM_OBSERVATION_NOISE = 1e-15  # Default observation noise for DDFM state-space (matches original TensorFlow)
 MIN_FACTOR_VARIANCE = 1e-10  # Minimum variance for factors
 MIN_STD = 1e-8  # Minimum standard deviation
+MIN_STD_FOR_SCALE_CHECK = 1e-10  # Minimum standard deviation for scale ratio computation (more lenient than MIN_STD)
 DEFAULT_VARIANCE_FALLBACK = 1.0  # Default variance fallback value for numerical stability
 
 # Maximum eigenvalues
-MAX_EIGENVALUE = 1e6  # Maximum eigenvalue cap
+MAX_EIGENVALUE = 1e3  # Maximum eigenvalue cap (reduced to 1e3 to improve condition number: 1e3/1e-6 = 1e9)
 
 # Eigenvalue stability thresholds
 DEFAULT_EIGENVALUE_MAX_MAGNITUDE = 1.0  # Default maximum eigenvalue magnitude for stability checks
@@ -44,8 +46,11 @@ DEFAULT_CLEAN_INF = MAX_EIGENVALUE  # Default value for Inf replacement in clean
 # Identity matrix defaults
 DEFAULT_IDENTITY_SCALE = 1.0  # Default scale for create_scaled_identity(n, 1.0)
 DEFAULT_ZERO_VALUE = 0.0  # Default zero value for explicit zero assignments
-DEFAULT_XAVIER_GAIN = 1.0  # Default gain for Xavier initialization
-DEFAULT_OUTPUT_LAYER_GAIN = 0.1  # Default gain for output layer Xavier initialization (smaller for stability)
+DEFAULT_INF_VALUE = float('inf')  # Default infinity value for loss/scale comparisons
+DEFAULT_XAVIER_GAIN = 1.0  # Default gain for Xavier initialization (matches TensorFlow GlorotNormal)
+DEFAULT_OUTPUT_LAYER_GAIN = 1.0  # Default gain for output layer Xavier initialization (matches original TensorFlow DDFM GlorotNormal for all layers)
+# NOTE: Original TensorFlow DDFM uses GlorotNormal (gain=1.0) for all layers including output layer.
+# Updated to match original TensorFlow behavior to address training divergence (encoder weights too small, near-zero factors).
 
 # Companion matrix initialization defaults
 DEFAULT_INIT_SCALE = 0.01  # Default initialization scale for companion matrix B and C matrices
@@ -59,6 +64,41 @@ DEFAULT_REGULARIZATION = 1e-6  # Default regularization value
 DEFAULT_CLIP_THRESHOLD = 10.0  # Default clipping threshold (in standard deviations)
 DEFAULT_DATA_CLIP_THRESHOLD = 100.0  # Default data clipping threshold
 
+# Scale validation thresholds (for DDFM scale alignment checks)
+DEFAULT_SCALE_RATIO_MAX = 10.0  # Maximum acceptable scale ratio (prediction std / data std)
+DEFAULT_SCALE_RATIO_MIN = 0.1  # Minimum acceptable scale ratio (prediction std / data std)
+DEFAULT_STANDARDIZATION_MEAN_THRESHOLD = 0.1  # Maximum acceptable absolute mean for standardized data (mean should be ≈0)
+DEFAULT_STANDARDIZATION_STD_MIN = 0.1  # Minimum acceptable std for standardized data (std should be ≈1)
+DEFAULT_STANDARDIZATION_STD_MAX = 10.0  # Maximum acceptable std for standardized data (std should be ≈1)
+DEFAULT_STANDARDIZED_TARGET_STD = 1.0  # Target std for standardized data (std should be ≈1.0 for StandardScaler)
+
+# Variance collapse diagnostics thresholds (for DDFM variance collapse detection)
+DEFAULT_VARIANCE_COLLAPSE_THRESHOLD = 0.1  # Prediction std threshold for variance collapse detection (std < 0.1 indicates collapse)
+DEFAULT_FACTOR_COLLAPSE_THRESHOLD = 0.01  # Factor magnitude threshold for factor collapse detection (|mean| < 0.01 indicates collapse)
+DEFAULT_BATCHNORM_SUPPRESSION_THRESHOLD = 0.01  # BatchNorm running_var threshold for signal suppression detection (var < 0.01 indicates suppression)
+DEFAULT_TIMESTEP_COLLAPSE_THRESHOLD = 0.1  # Per-time-step std threshold for localized collapse detection (std < 0.1 indicates collapse)
+DEFAULT_TIMESTEP_COLLAPSE_RATIO_THRESHOLD = 0.5  # Ratio threshold for localized collapse (ratio > 0.5 indicates widespread collapse)
+DEFAULT_EXPECTED_FACTOR_MAGNITUDE_MIN = 0.1  # Expected minimum factor magnitude (for warning messages)
+DEFAULT_EXPECTED_FACTOR_MAGNITUDE_MAX = 1.0  # Expected maximum factor magnitude (for warning messages)
+
+# DDFM reproduction target values (for debugging and diagnostics)
+DEFAULT_TARGET_PREDICTION_STD = 1.0  # Target prediction std for standardized data (should be ~1.0 for StandardScaler)
+DEFAULT_VARIANCE_COLLAPSE_STD = 0.03  # Observed variance collapse std (std ~0.03 vs target ~1.0)
+DEFAULT_TARGET_CONVERGENCE_ITERATIONS = 10  # Target convergence iterations for exchange_rate dataset (TensorFlow reference)
+DEFAULT_TARGET_DDFM_LOSS = 0.56  # Target DDFM training loss for exchange_rate dataset (TensorFlow reference)
+DEFAULT_DDFM_LOSS_MULTIPLIER = 1.8  # Current loss multiplier vs target (1.00 vs 0.56 = 1.8x)
+
+# Logging precision constants
+DEFAULT_SCALE_LOG_PRECISION = 6  # Default precision for scale logging format strings (e.g., .6f)
+DEFAULT_TIME_LOG_PRECISION = 2  # Default precision for time logging format strings (e.g., .2f)
+DEFAULT_LOSS_LOG_PRECISION = 6  # Default precision for loss/delta logging format strings (e.g., .6f)
+
+# Time conversion constants
+SECONDS_PER_MINUTE = 60  # Seconds per minute (for time conversion)
+
+# Logging level constants
+LOGGING_DEBUG_LEVEL = 10  # logging.DEBUG level for conditional checks
+
 # ============================================================================
 # Training Defaults
 # ============================================================================
@@ -68,15 +108,41 @@ DEFAULT_MAX_ITER = 100  # Default maximum EM iterations (general)
 DEFAULT_EM_MAX_ITER = 5000  # Default maximum EM iterations (DFM-specific)
 DEFAULT_MAX_EPOCHS = 100  # Default maximum training epochs
 DEFAULT_MAX_MCMC_ITER = 200  # Default maximum MCMC iterations
+DEFAULT_N_MC_SAMPLES = 10  # Default number of MC samples per MCMC iteration for DDFM denoising training (matches original TensorFlow epochs=10 default, per experiment/config/model/ddfm.yaml)
+DEFAULT_FACTOR_ORDER = 1  # Default factor order for DDFM (AR(1) dynamics)
+DEFAULT_AR_ORDER_2 = 2  # Default AR order 2 for DDFM forecast (AR(2) dynamics)
 
 # Batch size defaults
 DEFAULT_BATCH_SIZE = 32  # Default batch size for neural networks
-DEFAULT_DDFM_BATCH_SIZE = 100  # Default batch size for DDFM
+DEFAULT_DDFM_WINDOW_SIZE = 100  # Default window size (time-step batch size) for DDFM
+DEFAULT_DDFM_BATCH_SIZE = 100  # Default batch size for DDFM (matches original TensorFlow batch_size=100)
+
+# DDFM target interpolation defaults
+DEFAULT_MIN_TARGET_INTERPOLATE_RATIO = 0.3  # Default minimum target interpolate ratio threshold for DDFM (ratio of missing target values above which interpolation is skipped)
+
+# MCMC training defaults (for DDFM sequential MC processing)
+DEFAULT_MCMC_EPOCHS = 1  # Default epochs per MC sample (MCMC training pattern: one epoch per MC sample)
+DEFAULT_MCMC_VERBOSE = 0  # Default verbosity for MCMC training (silent training)
+
+# Placeholder defaults (for DDFM placeholder MC dataset)
+DEFAULT_PLACEHOLDER_MC_SAMPLES = 1  # Minimal MC samples for placeholder dataset (replaced after on_train_start)
+DEFAULT_PLACEHOLDER_DATA_SHAPE_T = 100  # Default time steps for placeholder data shape
+DEFAULT_PLACEHOLDER_DATA_SHAPE_N = 10  # Default number of series for placeholder data shape
+DEFAULT_PLACEHOLDER_SEED = 42  # Seed for placeholder random state (common test value, placeholder gets replaced)
+DEFAULT_NUM_WORKERS = 0  # Default number of workers for DataLoader (0 = single-threaded)
+DEFAULT_CUDA_DEVICE_INDEX = 0  # Default CUDA device index for GPU operations (0 = first GPU)
+
+# Initialization sample size defaults
+DEFAULT_KDFM_INIT_SAMPLE_SIZE = 100  # Default sample size for KDFM initialization (first N rows used for initialize_from_data)
 
 # Learning rate defaults
 DEFAULT_LEARNING_RATE = 0.001  # Default learning rate
 DEFAULT_DDFM_LEARNING_RATE = 0.005  # Default learning rate for DDFM
 
+# Autoencoder training defaults (matches original TensorFlow pattern)
+DEFAULT_AUTOENCODER_FIT_EPOCHS = 1  # Number of epochs per MC sample (matches original TensorFlow: epochs=1 per autoencoder.fit() call)
+DEFAULT_AUTOENCODER_FIT_VERBOSE = 0  # Verbosity level for autoencoder.fit() (0 = silent, matches original TensorFlow verbose=0)
+
 # Gradient clipping
 DEFAULT_GRAD_CLIP_VAL = 1.0  # Default gradient clipping value
 
@@ -86,6 +152,22 @@ DEFAULT_WEIGHT_DECAY = 0.0  # Default weight decay (L2 regularization)
 # Learning rate decay
 DEFAULT_LR_DECAY_RATE = 0.96  # Default exponential decay rate for learning rate
 
+# Optimizer defaults
+VALID_OPTIMIZERS = {'Adam', 'AdamW', 'SGD'}  # Valid optimizer types for DDFM
+
+# Adam optimizer defaults
+DEFAULT_ADAM_BETA1 = 0.9  # Default beta1 (momentum decay) for Adam optimizer
+DEFAULT_ADAM_BETA2 = 0.999  # Default beta2 (squared gradient decay) for Adam optimizer
+DEFAULT_ADAM_EPS = 1e-7  # Default epsilon for Adam optimizer (matches TensorFlow default for DDFM compatibility)
+DEFAULT_PYTORCH_ADAM_EPS = 1e-8  # PyTorch default epsilon for Adam optimizer (for reference)
+
+# Batch normalization defaults (matching TensorFlow/Keras defaults)
+DEFAULT_BATCH_NORM_MOMENTUM = 0.99  # TensorFlow/Keras BatchNormalization default momentum
+DEFAULT_BATCH_NORM_EPS = 1e-3  # TensorFlow/Keras BatchNormalization default epsilon
+
+# Random seed defaults
+DEFAULT_RANDOM_SEED_MAX = 2**31  # Maximum value for random seed generation (2147483647, 32-bit signed integer max)
+
 # Loss function defaults
 DEFAULT_HUBER_DELTA = 1.0  # Default delta parameter for Huber loss
 HUBER_QUADRATIC_COEFF = 0.5  # Quadratic coefficient for Huber loss (0.5 * a^2 term)
@@ -115,7 +197,11 @@ CHOLESKY_LOG_DET_FACTOR = 2.0  # Factor for log determinant computation from Cho
 # ============================================================================
 
 # Encoder layer defaults
-DEFAULT_ENCODER_LAYERS = [64, 32]  # Default encoder layer sizes
+DEFAULT_ENCODER_LAYERS = [16, 4]  # Default encoder layer sizes (matches experiment config and ddfm.py default parameter)
+DEFAULT_NUM_FACTORS = 3  # Default number of factors for DDFM
+DEFAULT_ACTIVATION = 'relu'  # Default activation function for DDFM
+DEFAULT_DECODER = 'linear'  # Default decoder type for DDFM
+DEFAULT_USE_BATCH_NORM = True  # Default batch normalization setting for DDFM
 
 # ============================================================================
 # Data Processing Defaults
@@ -131,8 +217,9 @@ DEFAULT_START_DATE = datetime(2000, 1, 1)
 # Default window size for DDFM
 DEFAULT_WINDOW_SIZE = 100
 DEFAULT_TENT_KERNEL_SIZE = 5  # Default tent kernel size for slower-frequency series aggregation
-# Note: DEFAULT_BATCH_SIZE is defined above (line 52) as 32 for general neural networks
-# Use DEFAULT_DDFM_BATCH_SIZE (100) for DDFM-specific batch size
+DEFAULT_TENT_KERNEL_REGULARIZATION_MULTIPLIER = 100.0  # Regularization multiplier for tent kernel series (handles ill-conditioning)
+# Note: DEFAULT_BATCH_SIZE is defined above (line 88) as 32 for general neural networks
+# Use DEFAULT_DDFM_WINDOW_SIZE (100) for DDFM-specific window size (time-step batch size)
 
 # Warning/display limits
 MAX_WARNING_ITEMS = 5  # Maximum number of items to show in warning messages
@@ -142,6 +229,9 @@ MAX_ERROR_ITEMS = 20  # Maximum number of items to show in error message details
 DEFAULT_MIN_OBS = 5  # Default minimum observations for estimation
 DEFAULT_MIN_OBS_IDIO = 5  # Default minimum observations for idio estimation
 DEFAULT_MIN_OBS_VAR = 7  # Minimum observations for VAR estimation (order + 5)
+DEFAULT_MIN_OBS_PRETRAIN = 50  # Minimum observations for DDFM pre-training without interpolation
+DEFAULT_MULT_EPOCH_PRETRAIN = 1  # Multiplier for DDFM pre-training epochs
+DEFAULT_PRETRAIN_EPOCHS = 200  # Default number of epochs for DDFM pre-training (matching original paper)
 
 # Dimension validation bounds
 MIN_TIME_STEPS = 1  # Minimum number of time steps (T) required for data
@@ -149,7 +239,9 @@ MIN_VARIABLES = 1  # Minimum number of variables (N) required for data
 MIN_DDFM_TIME_STEPS = 2  # Minimum number of time steps (T) required for DDFM training (DDFM-specific, different from general MIN_TIME_STEPS=1)
 MIN_DDFM_DATASET_SIZE_WARNING = 10  # Minimum dataset size (T) below which DDFM denoising training warns about potential instability
 MIN_ITER_FOR_DELTA_COMPUTATION = 1  # Minimum iteration count required before computing delta (MSE change) in DDFM denoising training
+DEFAULT_MIN_ITER_FOR_CONVERGENCE_CHECK = 2  # Minimum iteration count required before checking convergence in DDFM (check from iteration 3 onwards)
 MIN_EPS_SHAPE_FOR_IDIO = 1  # Minimum eps shape dimension required for idiosyncratic component processing in DDFM
+MIN_SHAPE_FOR_AR2 = 2  # Minimum shape dimension for AR(2) forecast (need at least 2 previous time steps)
 
 # Idiosyncratic component defaults
 DEFAULT_IDIO_STD = 0.1  # Default idiosyncratic standard deviation (when estimation fails)
@@ -193,7 +285,7 @@ DEFAULT_PROGRESS_LOG_INTERVAL = 5  # Default progress logging interval for EM al
 # Precision Defaults
 # ============================================================================
 
-DEFAULT_DTYPE = np.float32  # Default numpy dtype for arrays
+DEFAULT_DTYPE = np.float64  # Default numpy dtype for arrays (float64 for better numerical stability)
 
 # PyTorch dtype (matches DEFAULT_DTYPE)
 try:
@@ -328,6 +420,7 @@ __all__ = [
     'MIN_OBSERVATION_NOISE',
     'MIN_FACTOR_VARIANCE',
     'MIN_STD',
+    'MIN_STD_FOR_SCALE_CHECK',
     'MAX_EIGENVALUE',
     'DEFAULT_EIGENVALUE_MAX_MAGNITUDE',
     'DEFAULT_EIGENVALUE_WARN_THRESHOLD',
@@ -343,19 +436,53 @@ __all__ = [
     'DEFAULT_REGULARIZATION',
     'DEFAULT_CLIP_THRESHOLD',
     'DEFAULT_DATA_CLIP_THRESHOLD',
+    'DEFAULT_SCALE_RATIO_MAX',
+    'DEFAULT_SCALE_RATIO_MIN',
+    'DEFAULT_STANDARDIZATION_MEAN_THRESHOLD',
+    'DEFAULT_STANDARDIZATION_STD_MIN',
+    'DEFAULT_STANDARDIZATION_STD_MAX',
+    'DEFAULT_STANDARDIZED_TARGET_STD',
+    'DEFAULT_TARGET_PREDICTION_STD',
+    'DEFAULT_VARIANCE_COLLAPSE_STD',
+    'DEFAULT_TARGET_CONVERGENCE_ITERATIONS',
+    'DEFAULT_TARGET_DDFM_LOSS',
+    'DEFAULT_DDFM_LOSS_MULTIPLIER',
+    'DEFAULT_SCALE_LOG_PRECISION',
+    'DEFAULT_TIME_LOG_PRECISION',
+    'DEFAULT_LOSS_LOG_PRECISION',
+    'SECONDS_PER_MINUTE',
     'MIN_CONDITION_NUMBER',
     'MAX_CONDITION_NUMBER',
     # Training
     'DEFAULT_MAX_ITER',
     'DEFAULT_MAX_EPOCHS',
     'DEFAULT_MAX_MCMC_ITER',
+    'DEFAULT_N_MC_SAMPLES',
     'DEFAULT_BATCH_SIZE',  # General neural network default (32)
-    'DEFAULT_DDFM_BATCH_SIZE',  # DDFM-specific default (100)
+    'DEFAULT_DDFM_WINDOW_SIZE',  # DDFM-specific window size default (100)
+    'DEFAULT_DDFM_BATCH_SIZE',  # DDFM batch size default (100)
+    'DEFAULT_PLACEHOLDER_MC_SAMPLES',  # Placeholder MC samples default (1)
+    'DEFAULT_PLACEHOLDER_DATA_SHAPE_T',  # Placeholder data shape T default (100)
+    'DEFAULT_PLACEHOLDER_DATA_SHAPE_N',  # Placeholder data shape N default (10)
+    'DEFAULT_PLACEHOLDER_SEED',  # Placeholder random seed default (42)
+    'DEFAULT_NUM_WORKERS',  # DataLoader num_workers default (0)
+    'DEFAULT_CUDA_DEVICE_INDEX',  # CUDA device index default (0 = first GPU)
+    'DEFAULT_KDFM_INIT_SAMPLE_SIZE',  # KDFM initialization sample size default (100)
     'DEFAULT_LEARNING_RATE',
     'DEFAULT_DDFM_LEARNING_RATE',
+    'DEFAULT_AUTOENCODER_FIT_EPOCHS',
+    'DEFAULT_AUTOENCODER_FIT_VERBOSE',
     'DEFAULT_GRAD_CLIP_VAL',
     'DEFAULT_WEIGHT_DECAY',
     'DEFAULT_LR_DECAY_RATE',
+    'VALID_OPTIMIZERS',
+    'DEFAULT_ADAM_BETA1',
+    'DEFAULT_ADAM_BETA2',
+    'DEFAULT_ADAM_EPS',
+    'DEFAULT_TENSORFLOW_ADAM_EPS',
+    'DEFAULT_BATCH_NORM_MOMENTUM',
+    'DEFAULT_BATCH_NORM_EPS',
+    'DEFAULT_RANDOM_SEED_MAX',
     'DEFAULT_HUBER_DELTA',
     'HUBER_QUADRATIC_COEFF',
     'DEFAULT_DDFM_CLIP_RANGE_DEEP',
@@ -368,7 +495,13 @@ __all__ = [
     'DEFAULT_STRUCTURAL_DIAG_SCALE',
     'DEFAULT_CHOLESKY_EPS',
     # Architecture
+    'DEFAULT_FACTOR_ORDER',
+    'DEFAULT_AR_ORDER_2',
     'DEFAULT_ENCODER_LAYERS',
+    'DEFAULT_NUM_FACTORS',
+    'DEFAULT_ACTIVATION',
+    'DEFAULT_DECODER',
+    'DEFAULT_USE_BATCH_NORM',
     # Data processing
     'DEFAULT_NAN_METHOD',
     'DEFAULT_NAN_K',
@@ -378,12 +511,17 @@ __all__ = [
     'DEFAULT_MIN_OBS',
     'DEFAULT_MIN_OBS_IDIO',
     'DEFAULT_MIN_OBS_VAR',
+    'DEFAULT_MIN_OBS_PRETRAIN',
+    'DEFAULT_MULT_EPOCH_PRETRAIN',
+    'DEFAULT_PRETRAIN_EPOCHS',
     # Dimension validation
     'MIN_TIME_STEPS',
     'MIN_VARIABLES',
+    'MIN_SHAPE_FOR_AR2',
     'MIN_DDFM_TIME_STEPS',
     'MIN_DDFM_DATASET_SIZE_WARNING',
     'MIN_ITER_FOR_DELTA_COMPUTATION',
+    'DEFAULT_MIN_ITER_FOR_CONVERGENCE_CHECK',
     'MIN_EPS_SHAPE_FOR_IDIO',
     'DEFAULT_IDIO_STD',
     'DEFAULT_IDIO_RHO0',
@@ -408,6 +546,7 @@ __all__ = [
     'DEFAULT_LOG_INTERVAL',
     'DEFAULT_PROGRESS_LOG_INTERVAL',
     'DEFAULT_TENT_KERNEL_SIZE',
+    'DEFAULT_TENT_KERNEL_REGULARIZATION_MULTIPLIER',
     'MAX_ERROR_ITEMS',
     # IRF
     'DEFAULT_IRF_HORIZON',

dfm_python/config/schema/__init__.py

@@ -1,17 +1,19 @@
 """Configuration schema package for DFM models.
 
 This package contains:
-- model.py: BaseModelConfig, DFMConfig, DDFMConfig, KDFMConfig - model configurations
-- results.py: BaseResult, DFMResult, DDFMResult, KDFMResult - result structures
+- model.py: BaseModelConfig, DFMConfig, DDFMConfig - model configurations
+- results.py: BaseResult, DFMResult, DDFMResult - result structures
 
 Note: Series are specified via frequency dict mapping column names to frequencies.
 """
 
-from .model import BaseModelConfig, DFMConfig, DDFMConfig, KDFMConfig
-from .results import BaseResult, DFMResult, DDFMResult, KDFMResult, FitParams
+from .model import BaseModelConfig, DFMConfig, DDFMConfig
+from .results import BaseResult, DFMResult, DDFMResult
+from .params import DFMStateSpaceParams, DDFMStateSpaceParams
 
 __all__ = [
-    'BaseModelConfig', 'DFMConfig', 'DDFMConfig', 'KDFMConfig',
-    'BaseResult', 'DFMResult', 'DDFMResult', 'KDFMResult', 'FitParams',
+    'BaseModelConfig', 'DFMConfig', 'DDFMConfig',
+    'BaseResult', 'DFMResult', 'DDFMResult',
+    'DFMStateSpaceParams', 'DDFMStateSpaceParams',
 ]