dfm-python 0.5.55__py3-none-any.whl → 0.5.56__py3-none-any.whl

dfm_python/__init__.py

@@ -23,40 +23,36 @@ Key Features:
     - Robust handling of missing data (internal spline interpolation)
     - Automatic standardization and data clipping
 
-Example (Standard Lightning Pattern):
-    >>> from dfm_python import DFM, DFMDataModule, DFMTrainer
+Example:
+    >>> from dfm_python import DFM, DFMDataset
     >>> import pandas as pd
     >>> 
     >>> # Step 1: Load and preprocess data
     >>> df = pd.read_csv('data/your_data.csv')
     >>> df_processed = df[[col for col in df.columns if col != 'date']]
     >>> 
-    >>> # Step 2: Create DataModule
-    >>> dm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
-    >>> dm.setup()
+    >>> # Step 2: Create Dataset
+    >>> dataset = DFMDataset(config_path='config/default.yaml', data=df_processed)
     >>> 
     >>> # Step 3: Create model and load config
     >>> model = DFM()
     >>> model.load_config('config/default.yaml')
     >>> 
-    >>> # Step 4: Create trainer and fit (standard Lightning pattern)
-    >>> trainer = DFMTrainer(max_epochs=100)  # DEFAULT_MAX_EPOCHS
-    >>> trainer.fit(model, dm)
+    >>> # Step 4: Fit model
+    >>> model.fit(X=dataset.get_processed_data(), dataset=dataset)
     >>> 
     >>> # Step 5: Predict
     >>> Xf, Zf = model.predict(horizon=6)
     >>> 
-    >>> # Or use DDFM (same pattern)
-    >>> from dfm_python import DDFM, DDFMTrainer
+    >>> # Or use DDFM
+    >>> from dfm_python import DDFM, DDFMDataset
     >>> 
-    >>> dm_ddfm = DFMDataModule(config_path='config/default.yaml', data=df_processed)
-    >>> dm_ddfm.setup()
+    >>> dataset_ddfm = DDFMDataset(config_path='config/default.yaml', data=df_processed)
     >>> 
     >>> ddfm_model = DDFM(encoder_layers=[64, 32], num_factors=2)
     >>> ddfm_model.load_config('config/default.yaml')
     >>> 
-    >>> trainer_ddfm = DDFMTrainer(max_epochs=100)  # DEFAULT_MAX_EPOCHS
-    >>> trainer_ddfm.fit(ddfm_model, dm_ddfm)
+    >>> ddfm_model.train(dataset=dataset_ddfm)
     >>> Xf, Zf = ddfm_model.predict(horizon=6)
     
 Note: DFMConfig uses frequency dict to specify series (column names -> frequencies).
@@ -65,7 +61,7 @@ Note: DFMConfig uses frequency dict to specify series (column names -> frequenci
 For detailed documentation, see the README.md file and the tutorial notebooks/scripts.
 """
 
-__version__ = "0.5.55"
+__version__ = "0.5.56"
 
 # ============================================================================
 # PUBLIC API DEFINITION
@@ -92,20 +88,15 @@ from .config import (
 from .config import DFMConfig
 
 # Results
-from .config import DFMResult, DDFMResult, KDFMResult, BaseResult
+from .config import DFMResult, DDFMResult, BaseResult
 
 # Utilities (from utils/ subpackage)
 from .utils.metric import calculate_rmse
 
-# DataModule classes (includes both Lightning and custom implementations)
+# Dataset classes
 # Users can import these directly from dfm_python
-from .datamodule import (
-    DFMDataModule,
-    DDFMDataModule,  # DDFM-specific DataModule
-    KDFMDataModule,  # KDFM-specific DataModule
-    DDFMDataset,  # DDFM Dataset class (usually not needed directly)
-    KDFMDataset,  # KDFM Dataset class (usually not needed directly)
-)
+from .dataset.dfm_dataset import DFMDataset
+from .dataset.ddfm_dataset import DDFMDataset
 
 # Model implementations
 from .models.base import BaseFactorModel
@@ -114,9 +105,6 @@ from .models.dfm import DFM
 # DDFM high-level API (PyTorch is mandatory)
 from .models.ddfm import DDFM
 
-# KDFM high-level API
-from .models.kdfm import KDFM
-
 __all__ = [
     # Core classes
     'DFM',
@@ -126,7 +114,7 @@ __all__ = [
     'ConfigSource', 'YamlSource',
     'make_config_source',
     # Low-level API (functional interface - advanced usage)
-    'BaseResult', 'DFMResult', 'DDFMResult', 'KDFMResult', 'calculate_rmse',
+    'BaseResult', 'DFMResult', 'DDFMResult', 'calculate_rmse',
 ]
 
 # DDFM high-level API (PyTorch is mandatory)
@@ -134,27 +122,9 @@ __all__.extend([
     'DDFM',  # High-level API class
 ])
 
-# KDFM high-level API
-__all__.extend([
-    'KDFM',  # High-level API class
-])
-
-# Lightning modules (mandatory dependency)
+# Dataset classes
 __all__.extend([
-    'DFMDataModule',
-    'DDFMDataModule',  # DDFM-specific DataModule
-    'KDFMDataModule',  # KDFM-specific DataModule
-    'DDFMDataset',  # DDFM Dataset class (usually not needed directly)
-    'KDFMDataset',  # KDFM Dataset class (usually not needed directly)
+    'DFMDataset',  # DFM Dataset class
+    'DDFMDataset',  # DDFM Dataset class
 ])
 
-# Trainer classes (mandatory dependency)
-from .trainer import DFMTrainer, DDFMTrainer, KDFMTrainer
-
-__all__.extend([
-    'DFMTrainer',
-    'DDFMTrainer',
-    'KDFMTrainer',
-])
-
-

dfm_python/config/__init__.py

@@ -8,8 +8,9 @@ Note: Series are specified via frequency dict mapping column names to frequencie
 """
 
 from .schema import (
-    BaseModelConfig, DFMConfig, DDFMConfig, KDFMConfig,
-    BaseResult, DFMResult, DDFMResult, KDFMResult, FitParams,
+    BaseModelConfig, DFMConfig, DDFMConfig,
+    BaseResult, DFMResult, DDFMResult,
+    DFMFitParams, DDFMFitParams, FitParams,  # FitParams is backward compatibility alias
 )
 # DEFAULT_BLOCK_NAME is imported lazily where needed to avoid circular imports
 from .constants import (
@@ -105,11 +106,12 @@ __all__ = [
     'BaseModelConfig', 'BaseResult',
     # 'DEFAULT_BLOCK_NAME',  # Removed to avoid circular import - import directly from functional.dfm_block
     # Model-specific configs (from schema.py)
-    'DFMConfig', 'DDFMConfig', 'KDFMConfig',
+    'DFMConfig', 'DDFMConfig',
     # Parameter overrides
-    'FitParams',
+    'DFMFitParams', 'DDFMFitParams',
+    'FitParams',  # Backward compatibility alias
     # Model-specific results (from results.py)
-    'DFMResult', 'DDFMResult', 'KDFMResult',
+    'DFMResult', 'DDFMResult',
     # Utilities
     'validate_frequency',
     # IO

dfm_python/config/constants.py

@@ -26,8 +26,10 @@ DEFAULT_MIN_DELTA = 1e-6  # Minimum change for improvement
 MIN_EIGENVALUE = 1e-8  # Minimum eigenvalue for positive definite matrices
 MIN_DIAGONAL_VARIANCE = 1e-8  # Minimum variance for diagonal elements
 MIN_OBSERVATION_NOISE = 1e-4  # Minimum observation noise for measurement error (used in EM updates)
+DEFAULT_DDFM_OBSERVATION_NOISE = 1e-15  # Default observation noise for DDFM state-space (matches original TensorFlow)
 MIN_FACTOR_VARIANCE = 1e-10  # Minimum variance for factors
 MIN_STD = 1e-8  # Minimum standard deviation
+MIN_STD_FOR_SCALE_CHECK = 1e-10  # Minimum standard deviation for scale ratio computation (more lenient than MIN_STD)
 DEFAULT_VARIANCE_FALLBACK = 1.0  # Default variance fallback value for numerical stability
 
 # Maximum eigenvalues
@@ -44,8 +46,11 @@ DEFAULT_CLEAN_INF = MAX_EIGENVALUE  # Default value for Inf replacement in clean
 # Identity matrix defaults
 DEFAULT_IDENTITY_SCALE = 1.0  # Default scale for create_scaled_identity(n, 1.0)
 DEFAULT_ZERO_VALUE = 0.0  # Default zero value for explicit zero assignments
-DEFAULT_XAVIER_GAIN = 1.0  # Default gain for Xavier initialization
-DEFAULT_OUTPUT_LAYER_GAIN = 0.1  # Default gain for output layer Xavier initialization (smaller for stability)
+DEFAULT_INF_VALUE = float('inf')  # Default infinity value for loss/scale comparisons
+DEFAULT_XAVIER_GAIN = 1.0  # Default gain for Xavier initialization (matches TensorFlow GlorotNormal)
+DEFAULT_OUTPUT_LAYER_GAIN = 1.0  # Default gain for output layer Xavier initialization (matches original TensorFlow DDFM GlorotNormal for all layers)
+# NOTE: Original TensorFlow DDFM uses GlorotNormal (gain=1.0) for all layers including output layer.
+# Updated to match original TensorFlow behavior to address training divergence (encoder weights too small, near-zero factors).
 
 # Companion matrix initialization defaults
 DEFAULT_INIT_SCALE = 0.01  # Default initialization scale for companion matrix B and C matrices
@@ -59,6 +64,41 @@ DEFAULT_REGULARIZATION = 1e-6  # Default regularization value
 DEFAULT_CLIP_THRESHOLD = 10.0  # Default clipping threshold (in standard deviations)
 DEFAULT_DATA_CLIP_THRESHOLD = 100.0  # Default data clipping threshold
 
+# Scale validation thresholds (for DDFM scale alignment checks)
+DEFAULT_SCALE_RATIO_MAX = 10.0  # Maximum acceptable scale ratio (prediction std / data std)
+DEFAULT_SCALE_RATIO_MIN = 0.1  # Minimum acceptable scale ratio (prediction std / data std)
+DEFAULT_STANDARDIZATION_MEAN_THRESHOLD = 0.1  # Maximum acceptable absolute mean for standardized data (mean should be ≈0)
+DEFAULT_STANDARDIZATION_STD_MIN = 0.1  # Minimum acceptable std for standardized data (std should be ≈1)
+DEFAULT_STANDARDIZATION_STD_MAX = 10.0  # Maximum acceptable std for standardized data (std should be ≈1)
+DEFAULT_STANDARDIZED_TARGET_STD = 1.0  # Target std for standardized data (std should be ≈1.0 for StandardScaler)
+
+# Variance collapse diagnostics thresholds (for DDFM variance collapse detection)
+DEFAULT_VARIANCE_COLLAPSE_THRESHOLD = 0.1  # Prediction std threshold for variance collapse detection (std < 0.1 indicates collapse)
+DEFAULT_FACTOR_COLLAPSE_THRESHOLD = 0.01  # Factor magnitude threshold for factor collapse detection (|mean| < 0.01 indicates collapse)
+DEFAULT_BATCHNORM_SUPPRESSION_THRESHOLD = 0.01  # BatchNorm running_var threshold for signal suppression detection (var < 0.01 indicates suppression)
+DEFAULT_TIMESTEP_COLLAPSE_THRESHOLD = 0.1  # Per-time-step std threshold for localized collapse detection (std < 0.1 indicates collapse)
+DEFAULT_TIMESTEP_COLLAPSE_RATIO_THRESHOLD = 0.5  # Ratio threshold for localized collapse (ratio > 0.5 indicates widespread collapse)
+DEFAULT_EXPECTED_FACTOR_MAGNITUDE_MIN = 0.1  # Expected minimum factor magnitude (for warning messages)
+DEFAULT_EXPECTED_FACTOR_MAGNITUDE_MAX = 1.0  # Expected maximum factor magnitude (for warning messages)
+
+# DDFM reproduction target values (for debugging and diagnostics)
+DEFAULT_TARGET_PREDICTION_STD = 1.0  # Target prediction std for standardized data (should be ~1.0 for StandardScaler)
+DEFAULT_VARIANCE_COLLAPSE_STD = 0.03  # Observed variance collapse std (std ~0.03 vs target ~1.0)
+DEFAULT_TARGET_CONVERGENCE_ITERATIONS = 10  # Target convergence iterations for exchange_rate dataset (TensorFlow reference)
+DEFAULT_TARGET_DDFM_LOSS = 0.56  # Target DDFM training loss for exchange_rate dataset (TensorFlow reference)
+DEFAULT_DDFM_LOSS_MULTIPLIER = 1.8  # Current loss multiplier vs target (1.00 vs 0.56 = 1.8x)
+
+# Logging precision constants
+DEFAULT_SCALE_LOG_PRECISION = 6  # Default precision for scale logging format strings (e.g., .6f)
+DEFAULT_TIME_LOG_PRECISION = 2  # Default precision for time logging format strings (e.g., .2f)
+DEFAULT_LOSS_LOG_PRECISION = 6  # Default precision for loss/delta logging format strings (e.g., .6f)
+
+# Time conversion constants
+SECONDS_PER_MINUTE = 60  # Seconds per minute (for time conversion)
+
+# Logging level constants
+LOGGING_DEBUG_LEVEL = 10  # logging.DEBUG level for conditional checks
+
 # ============================================================================
 # Training Defaults
 # ============================================================================
@@ -68,15 +108,41 @@ DEFAULT_MAX_ITER = 100  # Default maximum EM iterations (general)
 DEFAULT_EM_MAX_ITER = 5000  # Default maximum EM iterations (DFM-specific)
 DEFAULT_MAX_EPOCHS = 100  # Default maximum training epochs
 DEFAULT_MAX_MCMC_ITER = 200  # Default maximum MCMC iterations
+DEFAULT_N_MC_SAMPLES = 10  # Default number of MC samples per MCMC iteration for DDFM denoising training (matches original TensorFlow epochs=10 default, per experiment/config/model/ddfm.yaml)
+DEFAULT_FACTOR_ORDER = 1  # Default factor order for DDFM (AR(1) dynamics)
+DEFAULT_AR_ORDER_2 = 2  # Default AR order 2 for DDFM forecast (AR(2) dynamics)
 
 # Batch size defaults
 DEFAULT_BATCH_SIZE = 32  # Default batch size for neural networks
-DEFAULT_DDFM_BATCH_SIZE = 100  # Default batch size for DDFM
+DEFAULT_DDFM_WINDOW_SIZE = 100  # Default window size (time-step batch size) for DDFM
+DEFAULT_DDFM_BATCH_SIZE = 100  # Default batch size for DDFM (matches original TensorFlow batch_size=100)
+
+# DDFM target interpolation defaults
+DEFAULT_MIN_TARGET_INTERPOLATE_RATIO = 0.3  # Default minimum target interpolate ratio threshold for DDFM (ratio of missing target values above which interpolation is skipped)
+
+# MCMC training defaults (for DDFM sequential MC processing)
+DEFAULT_MCMC_EPOCHS = 1  # Default epochs per MC sample (MCMC training pattern: one epoch per MC sample)
+DEFAULT_MCMC_VERBOSE = 0  # Default verbosity for MCMC training (silent training)
+
+# Placeholder defaults (for DDFM placeholder MC dataset)
+DEFAULT_PLACEHOLDER_MC_SAMPLES = 1  # Minimal MC samples for placeholder dataset (replaced after on_train_start)
+DEFAULT_PLACEHOLDER_DATA_SHAPE_T = 100  # Default time steps for placeholder data shape
+DEFAULT_PLACEHOLDER_DATA_SHAPE_N = 10  # Default number of series for placeholder data shape
+DEFAULT_PLACEHOLDER_SEED = 42  # Seed for placeholder random state (common test value, placeholder gets replaced)
+DEFAULT_NUM_WORKERS = 0  # Default number of workers for DataLoader (0 = single-threaded)
+DEFAULT_CUDA_DEVICE_INDEX = 0  # Default CUDA device index for GPU operations (0 = first GPU)
+
+# Initialization sample size defaults
+DEFAULT_KDFM_INIT_SAMPLE_SIZE = 100  # Default sample size for KDFM initialization (first N rows used for initialize_from_data)
 
 # Learning rate defaults
 DEFAULT_LEARNING_RATE = 0.001  # Default learning rate
 DEFAULT_DDFM_LEARNING_RATE = 0.005  # Default learning rate for DDFM
 
+# Autoencoder training defaults (matches original TensorFlow pattern)
+DEFAULT_AUTOENCODER_FIT_EPOCHS = 1  # Number of epochs per MC sample (matches original TensorFlow: epochs=1 per autoencoder.fit() call)
+DEFAULT_AUTOENCODER_FIT_VERBOSE = 0  # Verbosity level for autoencoder.fit() (0 = silent, matches original TensorFlow verbose=0)
+
 # Gradient clipping
 DEFAULT_GRAD_CLIP_VAL = 1.0  # Default gradient clipping value
 
@@ -86,6 +152,22 @@ DEFAULT_WEIGHT_DECAY = 0.0  # Default weight decay (L2 regularization)
 # Learning rate decay
 DEFAULT_LR_DECAY_RATE = 0.96  # Default exponential decay rate for learning rate
 
+# Optimizer defaults
+VALID_OPTIMIZERS = {'Adam', 'AdamW', 'SGD'}  # Valid optimizer types for DDFM
+
+# Adam optimizer defaults
+DEFAULT_ADAM_BETA1 = 0.9  # Default beta1 (momentum decay) for Adam optimizer
+DEFAULT_ADAM_BETA2 = 0.999  # Default beta2 (squared gradient decay) for Adam optimizer
+DEFAULT_ADAM_EPS = 1e-7  # Default epsilon for Adam optimizer (matches TensorFlow default for DDFM compatibility)
+DEFAULT_PYTORCH_ADAM_EPS = 1e-8  # PyTorch default epsilon for Adam optimizer (for reference)
+
+# Batch normalization defaults (matching TensorFlow/Keras defaults)
+DEFAULT_BATCH_NORM_MOMENTUM = 0.99  # TensorFlow/Keras BatchNormalization default momentum
+DEFAULT_BATCH_NORM_EPS = 1e-3  # TensorFlow/Keras BatchNormalization default epsilon
+
+# Random seed defaults
+DEFAULT_RANDOM_SEED_MAX = 2**31  # Maximum value for random seed generation (2147483647, 32-bit signed integer max)
+
 # Loss function defaults
 DEFAULT_HUBER_DELTA = 1.0  # Default delta parameter for Huber loss
 HUBER_QUADRATIC_COEFF = 0.5  # Quadratic coefficient for Huber loss (0.5 * a^2 term)
@@ -115,7 +197,11 @@ CHOLESKY_LOG_DET_FACTOR = 2.0  # Factor for log determinant computation from Cho
 # ============================================================================
 
 # Encoder layer defaults
-DEFAULT_ENCODER_LAYERS = [64, 32]  # Default encoder layer sizes
+DEFAULT_ENCODER_LAYERS = [16, 4]  # Default encoder layer sizes (matches experiment config and ddfm.py default parameter)
+DEFAULT_NUM_FACTORS = 3  # Default number of factors for DDFM
+DEFAULT_ACTIVATION = 'relu'  # Default activation function for DDFM
+DEFAULT_DECODER = 'linear'  # Default decoder type for DDFM
+DEFAULT_USE_BATCH_NORM = True  # Default batch normalization setting for DDFM
 
 # ============================================================================
 # Data Processing Defaults
@@ -131,8 +217,8 @@ DEFAULT_START_DATE = datetime(2000, 1, 1)
 # Default window size for DDFM
 DEFAULT_WINDOW_SIZE = 100
 DEFAULT_TENT_KERNEL_SIZE = 5  # Default tent kernel size for slower-frequency series aggregation
-# Note: DEFAULT_BATCH_SIZE is defined above (line 52) as 32 for general neural networks
-# Use DEFAULT_DDFM_BATCH_SIZE (100) for DDFM-specific batch size
+# Note: DEFAULT_BATCH_SIZE is defined above (line 88) as 32 for general neural networks
+# Use DEFAULT_DDFM_WINDOW_SIZE (100) for DDFM-specific window size (time-step batch size)
 
 # Warning/display limits
 MAX_WARNING_ITEMS = 5  # Maximum number of items to show in warning messages
@@ -142,6 +228,9 @@ MAX_ERROR_ITEMS = 20  # Maximum number of items to show in error message details
 DEFAULT_MIN_OBS = 5  # Default minimum observations for estimation
 DEFAULT_MIN_OBS_IDIO = 5  # Default minimum observations for idio estimation
 DEFAULT_MIN_OBS_VAR = 7  # Minimum observations for VAR estimation (order + 5)
+DEFAULT_MIN_OBS_PRETRAIN = 50  # Minimum observations for DDFM pre-training without interpolation
+DEFAULT_MULT_EPOCH_PRETRAIN = 1  # Multiplier for DDFM pre-training epochs
+DEFAULT_PRETRAIN_EPOCHS = 200  # Default number of epochs for DDFM pre-training (matching original paper)
 
 # Dimension validation bounds
 MIN_TIME_STEPS = 1  # Minimum number of time steps (T) required for data
@@ -149,7 +238,9 @@ MIN_VARIABLES = 1  # Minimum number of variables (N) required for data
 MIN_DDFM_TIME_STEPS = 2  # Minimum number of time steps (T) required for DDFM training (DDFM-specific, different from general MIN_TIME_STEPS=1)
 MIN_DDFM_DATASET_SIZE_WARNING = 10  # Minimum dataset size (T) below which DDFM denoising training warns about potential instability
 MIN_ITER_FOR_DELTA_COMPUTATION = 1  # Minimum iteration count required before computing delta (MSE change) in DDFM denoising training
+DEFAULT_MIN_ITER_FOR_CONVERGENCE_CHECK = 2  # Minimum iteration count required before checking convergence in DDFM (check from iteration 3 onwards)
 MIN_EPS_SHAPE_FOR_IDIO = 1  # Minimum eps shape dimension required for idiosyncratic component processing in DDFM
+MIN_SHAPE_FOR_AR2 = 2  # Minimum shape dimension for AR(2) forecast (need at least 2 previous time steps)
 
 # Idiosyncratic component defaults
 DEFAULT_IDIO_STD = 0.1  # Default idiosyncratic standard deviation (when estimation fails)
@@ -328,6 +419,7 @@ __all__ = [
     'MIN_OBSERVATION_NOISE',
     'MIN_FACTOR_VARIANCE',
     'MIN_STD',
+    'MIN_STD_FOR_SCALE_CHECK',
     'MAX_EIGENVALUE',
     'DEFAULT_EIGENVALUE_MAX_MAGNITUDE',
     'DEFAULT_EIGENVALUE_WARN_THRESHOLD',
@@ -343,19 +435,53 @@ __all__ = [
     'DEFAULT_REGULARIZATION',
     'DEFAULT_CLIP_THRESHOLD',
     'DEFAULT_DATA_CLIP_THRESHOLD',
+    'DEFAULT_SCALE_RATIO_MAX',
+    'DEFAULT_SCALE_RATIO_MIN',
+    'DEFAULT_STANDARDIZATION_MEAN_THRESHOLD',
+    'DEFAULT_STANDARDIZATION_STD_MIN',
+    'DEFAULT_STANDARDIZATION_STD_MAX',
+    'DEFAULT_STANDARDIZED_TARGET_STD',
+    'DEFAULT_TARGET_PREDICTION_STD',
+    'DEFAULT_VARIANCE_COLLAPSE_STD',
+    'DEFAULT_TARGET_CONVERGENCE_ITERATIONS',
+    'DEFAULT_TARGET_DDFM_LOSS',
+    'DEFAULT_DDFM_LOSS_MULTIPLIER',
+    'DEFAULT_SCALE_LOG_PRECISION',
+    'DEFAULT_TIME_LOG_PRECISION',
+    'DEFAULT_LOSS_LOG_PRECISION',
+    'SECONDS_PER_MINUTE',
     'MIN_CONDITION_NUMBER',
     'MAX_CONDITION_NUMBER',
     # Training
     'DEFAULT_MAX_ITER',
     'DEFAULT_MAX_EPOCHS',
     'DEFAULT_MAX_MCMC_ITER',
+    'DEFAULT_N_MC_SAMPLES',
     'DEFAULT_BATCH_SIZE',  # General neural network default (32)
-    'DEFAULT_DDFM_BATCH_SIZE',  # DDFM-specific default (100)
+    'DEFAULT_DDFM_WINDOW_SIZE',  # DDFM-specific window size default (100)
+    'DEFAULT_DDFM_BATCH_SIZE',  # DDFM batch size default (100)
+    'DEFAULT_PLACEHOLDER_MC_SAMPLES',  # Placeholder MC samples default (1)
+    'DEFAULT_PLACEHOLDER_DATA_SHAPE_T',  # Placeholder data shape T default (100)
+    'DEFAULT_PLACEHOLDER_DATA_SHAPE_N',  # Placeholder data shape N default (10)
+    'DEFAULT_PLACEHOLDER_SEED',  # Placeholder random seed default (42)
+    'DEFAULT_NUM_WORKERS',  # DataLoader num_workers default (0)
+    'DEFAULT_CUDA_DEVICE_INDEX',  # CUDA device index default (0 = first GPU)
+    'DEFAULT_KDFM_INIT_SAMPLE_SIZE',  # KDFM initialization sample size default (100)
     'DEFAULT_LEARNING_RATE',
     'DEFAULT_DDFM_LEARNING_RATE',
+    'DEFAULT_AUTOENCODER_FIT_EPOCHS',
+    'DEFAULT_AUTOENCODER_FIT_VERBOSE',
     'DEFAULT_GRAD_CLIP_VAL',
     'DEFAULT_WEIGHT_DECAY',
     'DEFAULT_LR_DECAY_RATE',
+    'VALID_OPTIMIZERS',
+    'DEFAULT_ADAM_BETA1',
+    'DEFAULT_ADAM_BETA2',
+    'DEFAULT_ADAM_EPS',
+    'DEFAULT_TENSORFLOW_ADAM_EPS',
+    'DEFAULT_BATCH_NORM_MOMENTUM',
+    'DEFAULT_BATCH_NORM_EPS',
+    'DEFAULT_RANDOM_SEED_MAX',
     'DEFAULT_HUBER_DELTA',
     'HUBER_QUADRATIC_COEFF',
     'DEFAULT_DDFM_CLIP_RANGE_DEEP',
@@ -368,7 +494,13 @@ __all__ = [
     'DEFAULT_STRUCTURAL_DIAG_SCALE',
     'DEFAULT_CHOLESKY_EPS',
     # Architecture
+    'DEFAULT_FACTOR_ORDER',
+    'DEFAULT_AR_ORDER_2',
     'DEFAULT_ENCODER_LAYERS',
+    'DEFAULT_NUM_FACTORS',
+    'DEFAULT_ACTIVATION',
+    'DEFAULT_DECODER',
+    'DEFAULT_USE_BATCH_NORM',
     # Data processing
     'DEFAULT_NAN_METHOD',
     'DEFAULT_NAN_K',
@@ -378,12 +510,18 @@ __all__ = [
     'DEFAULT_MIN_OBS',
     'DEFAULT_MIN_OBS_IDIO',
     'DEFAULT_MIN_OBS_VAR',
+    'DEFAULT_MIN_OBS_PRETRAIN',
+    'DEFAULT_MULT_EPOCH_PRETRAIN',
+    'DEFAULT_PRETRAIN_EPOCHS',
     # Dimension validation
     'MIN_TIME_STEPS',
     'MIN_VARIABLES',
+    'MIN_SHAPE_FOR_AR2',
     'MIN_DDFM_TIME_STEPS',
+    'MIN_SHAPE_FOR_AR2',
     'MIN_DDFM_DATASET_SIZE_WARNING',
     'MIN_ITER_FOR_DELTA_COMPUTATION',
+    'DEFAULT_MIN_ITER_FOR_CONVERGENCE_CHECK',
     'MIN_EPS_SHAPE_FOR_IDIO',
     'DEFAULT_IDIO_STD',
     'DEFAULT_IDIO_RHO0',

dfm_python/config/schema/__init__.py

@@ -1,17 +1,23 @@
 """Configuration schema package for DFM models.
 
 This package contains:
-- model.py: BaseModelConfig, DFMConfig, DDFMConfig, KDFMConfig - model configurations
-- results.py: BaseResult, DFMResult, DDFMResult, KDFMResult - result structures
+- model.py: BaseModelConfig, DFMConfig, DDFMConfig - model configurations
+- results.py: BaseResult, DFMResult, DDFMResult - result structures
 
 Note: Series are specified via frequency dict mapping column names to frequencies.
 """
 
-from .model import BaseModelConfig, DFMConfig, DDFMConfig, KDFMConfig
-from .results import BaseResult, DFMResult, DDFMResult, KDFMResult, FitParams
+from .model import BaseModelConfig, DFMConfig, DDFMConfig
+from .results import BaseResult, DFMResult, DDFMResult
+from .params import DFMFitParams, DDFMFitParams
+
+# Backward compatibility alias
+FitParams = DFMFitParams
 
 __all__ = [
-    'BaseModelConfig', 'DFMConfig', 'DDFMConfig', 'KDFMConfig',
-    'BaseResult', 'DFMResult', 'DDFMResult', 'KDFMResult', 'FitParams',
+    'BaseModelConfig', 'DFMConfig', 'DDFMConfig',
+    'BaseResult', 'DFMResult', 'DDFMResult',
+    'DFMFitParams', 'DDFMFitParams',
+    'FitParams',  # Backward compatibility alias
 ]
 

dfm_python/config/schema/model.py

@@ -46,7 +46,7 @@ from ..constants import (
     DEFAULT_LEARNING_RATE,
     DEFAULT_MAX_EPOCHS,
     DEFAULT_BATCH_SIZE,
-    DEFAULT_DDFM_BATCH_SIZE,
+    DEFAULT_DDFM_WINDOW_SIZE,
     DEFAULT_GRAD_CLIP_VAL,
     DEFAULT_REGULARIZATION_SCALE,
     DEFAULT_STRUCTURAL_REG_WEIGHT,
@@ -495,24 +495,25 @@ class DDFMConfig(BaseModelConfig):
     - Series definitions via frequency dict (column names -> frequencies)
     """
     # ========================================================================
-    # Neural Network Training Parameters
+    # Neural Network Training Hyper Parameters
     # ========================================================================
     encoder_layers: Optional[List[int]] = None  # Hidden layer dimensions for encoder (default: [64, 32])
     num_factors: Optional[int] = None  # Number of factors (inferred from config if None)
     activation: str = 'relu'  # Activation function ('tanh', 'relu', 'sigmoid', default: 'relu' to match original DDFM)
     use_batch_norm: bool = True  # Use batch normalization in encoder (default: True)
     learning_rate: float = 0.001  # Learning rate for Adam optimizer (default: 0.001)
-    epochs: int = 100  # Number of training epochs (default: 100)
-    batch_size: int = 100  # Batch size for training (default: 100 to match original DDFM)
+    n_mc_samples: int = 10  # Number of MC samples per MCMC iteration (default: 10, matching original TensorFlow epochs=10 default, per experiment/config/model/ddfm.yaml)
+    window_size: int = 100  # Window size (time-step batch size) for training (default: 100 to match original DDFM)
     # Note: factor_order removed - factors always use AR(1) dynamics (simplified)
     use_idiosyncratic: bool = True  # Model idio components with AR(1) dynamics (default: True)
     min_obs_idio: int = 5  # Minimum observations for idio AR(1) estimation (default: 5)
     
     # Additional training parameters
-    max_iter: int = DEFAULT_MAX_MCMC_ITER  # Maximum MCMC iterations for iterative factor extraction
+    max_epoch: int = DEFAULT_MAX_MCMC_ITER  # Maximum number of epochs (MCMC iterations). One epoch = one MCMC iteration (MC sampling → training → convergence check)
     tolerance: float = DEFAULT_TOLERANCE  # Convergence tolerance for MCMC iterations
     disp: int = 10  # Display frequency for training progress
     seed: Optional[int] = None  # Random seed for reproducibility
+    lags_input: int = 0  # Number of lags of inputs on encoder (default 0, matching original TensorFlow DDFM)
     
     
     # ========================================================================
@@ -523,21 +524,40 @@ class DDFMConfig(BaseModelConfig):
     def _extract_ddfm(cls, data: Dict[str, Any]) -> Dict[str, Any]:
         """Extract DDFM-specific parameters from config dict."""
         # Don't extract base params here - they're already in base_params from from_dict
+        from ..constants import DEFAULT_N_MC_SAMPLES
         ddfm_params = cls._extract_params(data, {
             'encoder_layers': None,
             'num_factors': None,
             'activation': 'relu',
             'use_batch_norm': True,
             'learning_rate': DEFAULT_LEARNING_RATE,
-            'epochs': DEFAULT_MAX_EPOCHS,
-            'batch_size': DEFAULT_DDFM_BATCH_SIZE,
+            'epochs': DEFAULT_N_MC_SAMPLES,  # Backward compatibility: map 'epochs' to n_mc_samples
+            'n_mc_samples': DEFAULT_N_MC_SAMPLES,  # Preferred name: number of MC samples per MCMC iteration
+            'window_size': DEFAULT_DDFM_WINDOW_SIZE,  # Window size (time-step batch size) for training
             'use_idiosyncratic': True,
             'min_obs_idio': DEFAULT_MIN_OBS_IDIO,
-            'max_iter': DEFAULT_MAX_MCMC_ITER,
+            'max_epoch': DEFAULT_MAX_MCMC_ITER,  # Maximum epochs (MCMC iterations)
             'tolerance': DEFAULT_TOLERANCE,
             'disp': DEFAULT_DISP,
             'seed': None,
+            'lags_input': 0,  # Number of lags (default 0, matching original TensorFlow)
         })
+        # Map 'epochs' from config to 'n_mc_samples' for clarity (backward compatibility)
+        # Always remove 'epochs' if present (even if n_mc_samples is also present)
+        if 'epochs' in ddfm_params:
+            if 'n_mc_samples' not in ddfm_params:
+                ddfm_params['n_mc_samples'] = ddfm_params['epochs']
+            ddfm_params.pop('epochs')  # Always remove 'epochs' to avoid passing it to constructor
+        # Map 'batch_size' from config to 'window_size' for backward compatibility
+        if 'batch_size' in ddfm_params:
+            if 'window_size' not in ddfm_params:
+                ddfm_params['window_size'] = ddfm_params['batch_size']
+            ddfm_params.pop('batch_size')  # Always remove 'batch_size' to avoid passing it to constructor
+        # Only accept 'max_epoch' parameter (no backward compatibility)
+        # Remove any old parameter names if present
+        ddfm_params.pop('max_iter', None)
+        ddfm_params.pop('max_iterations', None)
+        ddfm_params.pop('max_mc_iter', None)
         return ddfm_params
     
     @classmethod
@@ -546,7 +566,7 @@ class DDFMConfig(BaseModelConfig):
         result = DFMConfig.from_dict(data)
         if isinstance(result, DDFMConfig):
             return result
-        from ..utils.errors import ConfigurationError
+        from ...utils.errors import ConfigurationError
         raise ConfigurationError(
             "Expected DDFMConfig but got DFMConfig",
             details=f"Result type: {type(result).__name__}, expected: DDFMConfig"

dfm_python/config/schema/params.py

@@ -0,0 +1,94 @@
+"""Model parameters for DFM and DDFM.
+
+This module defines dataclasses for storing fit-time parameters and state-space parameters.
+"""
+
+from dataclasses import dataclass
+import numpy as np
+from typing import Optional
+
+
+@dataclass
+class DFMFitParams:
+    """Fit-time parameter overrides for DFM estimation.
+    
+    This dataclass allows overriding configuration parameters at fit time
+    without modifying the configuration object itself.
+    
+    Based on DFM.fit() usage, these parameters are passed to the EM algorithm.
+    
+    Attributes
+    ----------
+    max_iter : int, optional
+        Maximum EM iterations (overrides config.max_iter)
+        Used in: kalman_filter.em(max_iter=...)
+    threshold : float, optional
+        Convergence threshold (overrides config.threshold)
+        Used in: kalman_filter.em(threshold=...)
+    regularization_scale : float, optional
+        Regularization scale for matrix operations (overrides EM config)
+        Note: This affects EMConfig.regularization and EMConfig.matrix_regularization
+    """
+    max_iter: Optional[int] = None
+    threshold: Optional[float] = None
+    regularization_scale: Optional[float] = None
+
+@dataclass
+class DDFMTrainingState:
+    """Training state for DDFM model.
+    
+    Stores the current state of DDFM training, including convergence status,
+    loss, factors, and residuals. Used for checkpointing and resuming training.
+    
+    Attributes
+    ----------
+    num_iter : int
+        Current iteration number in MCMC training loop.
+    loss_now : float, optional
+        Current training loss value.
+    converged : bool
+        Whether training has converged.
+    eps : np.ndarray, optional
+        Idiosyncratic residuals (T x num_target_series).
+    factors : np.ndarray, optional
+        Extracted factors (n_mc_samples x T x num_factors) or (T x num_factors).
+    last_neurons : np.ndarray, optional
+        Last layer neurons for MLP decoder (n_mc_samples x T x num_neurons) or (T x num_neurons).
+        For linear decoder, this equals factors.
+    """
+    num_iter: int = 0
+    loss_now: Optional[float] = None
+    converged: bool = False
+    eps: Optional[np.ndarray] = None
+    factors: Optional[np.ndarray] = None
+    last_neurons: Optional[np.ndarray] = None
+
+@dataclass
+class DDFMFitParams:
+    """State-space model parameters for DDFM (created during fit).
+    
+    These parameters are computed during DDFM.build_state_space() and represent
+    the fitted state-space model structure. Naming follows original paper conventions.
+    
+    Parameters
+    ----------
+    F : np.ndarray
+        Transition matrix (m x m) - VAR(1) dynamics for factors (A in paper)
+    Q : np.ndarray
+        Process noise covariance (m x m) - innovation covariance (W in original code)
+    mu_0 : np.ndarray
+        Initial state mean (m,) - initial factor values
+    Sigma_0 : np.ndarray
+        Initial state covariance (m x m) - initial uncertainty
+    H : np.ndarray
+        Observation matrix (N x m) - decoder weights (measurement equation, theta in paper)
+    R : np.ndarray
+        Observation noise covariance (N x N) - typically diagonal, small values
+    """
+    F: np.ndarray  # Transition matrix (m x m) - A in paper
+    Q: np.ndarray  # Process noise covariance (m x m) - W in original code
+    mu_0: np.ndarray  # Initial state mean (m,)
+    Sigma_0: np.ndarray  # Initial state covariance (m x m)
+    H: np.ndarray  # Observation matrix (N x m) - theta in paper
+    R: np.ndarray  # Observation noise covariance (N x N)
+