spotforecast2 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

spotforecast2/processing/n2n_predict.py

@@ -1,43 +1,308 @@
+"""
+End-to-end baseline forecasting using equivalent date method.
+
+This module provides a complete forecasting pipeline using the ForecasterEquivalentDate
+baseline model. It handles data preparation, outlier detection, imputation, model
+training, and prediction in a single integrated function.
+
+Model persistence follows scikit-learn conventions using joblib for efficient
+serialization and deserialization of trained forecasters.
+
+Examples:
+    Basic usage with default parameters:
+
+    >>> from spotforecast2.processing.n2n_predict import n2n_predict
+    >>> predictions = n2n_predict(forecast_horizon=24, verbose=True)
+
+    Using cached models:
+
+    >>> # Load existing models if available, or train new ones
+    >>> predictions = n2n_predict(
+    ...     forecast_horizon=24,
+    ...     force_train=False,
+    ...     model_dir="./models",
+    ...     verbose=True
+    ... )
+
+    Force retraining and update cache:
+
+    >>> predictions = n2n_predict(
+    ...     forecast_horizon=24,
+    ...     force_train=True,
+    ...     model_dir="./models",
+    ...     verbose=True
+    ... )
+"""
+
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
 import pandas as pd
-from typing import List, Optional
 from spotforecast2.forecaster.recursive import ForecasterEquivalentDate
 from spotforecast2.data.fetch_data import fetch_data
 from spotforecast2.preprocessing.curate_data import basic_ts_checks
 from spotforecast2.preprocessing.curate_data import agg_and_resample_data
 from spotforecast2.preprocessing.outlier import mark_outliers
-
 from spotforecast2.preprocessing.split import split_rel_train_val_test
 from spotforecast2.forecaster.utils import predict_multivariate
 from spotforecast2.preprocessing.curate_data import get_start_end
 
+try:
+    from joblib import dump, load
+except ImportError:
+    raise ImportError("joblib is required. Install with: pip install joblib")
+
 try:
     from tqdm.auto import tqdm
 except ImportError:  # pragma: no cover - fallback when tqdm is not installed
     tqdm = None
 
 
+# ============================================================================
+# Model Persistence Functions
+# ============================================================================
+
+
+def _ensure_model_dir(model_dir: Union[str, Path]) -> Path:
+    """Ensure model directory exists.
+
+    Args:
+        model_dir: Directory path for model storage.
+
+    Returns:
+        Path: Validated Path object.
+
+    Raises:
+        OSError: If directory cannot be created.
+    """
+    model_path = Path(model_dir)
+    model_path.mkdir(parents=True, exist_ok=True)
+    return model_path
+
+
+def _get_model_filepath(model_dir: Path, target: str) -> Path:
+    """Get filepath for a single model.
+
+    Args:
+        model_dir: Directory containing models.
+        target: Target variable name.
+
+    Returns:
+        Path: Full filepath for the model.
+
+    Examples:
+        >>> path = _get_model_filepath(Path("./models"), "power")
+        >>> str(path)
+        './models/forecaster_power.joblib'
+    """
+    return model_dir / f"forecaster_{target}.joblib"
+
+
+def _save_forecasters(
+    forecasters: Dict[str, object],
+    model_dir: Union[str, Path],
+    verbose: bool = False,
+) -> Dict[str, Path]:
+    """Save trained forecasters to disk using joblib.
+
+    Follows scikit-learn persistence conventions using joblib for efficient
+    serialization of sklearn-compatible estimators.
+
+    Args:
+        forecasters: Dictionary mapping target names to trained ForecasterEquivalentDate objects.
+        model_dir: Directory to save models. Created if it doesn't exist.
+        verbose: Print progress messages. Default: False.
+
+    Returns:
+        Dict[str, Path]: Dictionary mapping target names to saved model filepaths.
+
+    Raises:
+        OSError: If models cannot be written to disk.
+        TypeError: If forecasters contain non-serializable objects.
+
+    Examples:
+        >>> forecasters = {"power": forecaster_obj}
+        >>> paths = _save_forecasters(forecasters, "./models", verbose=True)
+        >>> print(paths["power"])
+        models/forecaster_power.joblib
+    """
+    model_path = _ensure_model_dir(model_dir)
+    saved_paths = {}
+
+    for target, forecaster in forecasters.items():
+        filepath = _get_model_filepath(model_path, target)
+        try:
+            dump(forecaster, filepath, compress=3)
+            saved_paths[target] = filepath
+            if verbose:
+                print(f"  ✓ Saved forecaster for {target} to {filepath}")
+        except Exception as e:
+            raise OSError(f"Failed to save model for {target}: {e}")
+
+    return saved_paths
+
+
+def _load_forecasters(
+    target_columns: List[str],
+    model_dir: Union[str, Path],
+    verbose: bool = False,
+) -> Tuple[Dict[str, object], List[str]]:
+    """Load trained forecasters from disk using joblib.
+
+    Attempts to load all forecasters for given targets. Missing models
+    are indicated in the return value for selective retraining.
+
+    Args:
+        target_columns: List of target variable names to load.
+        model_dir: Directory containing saved models.
+        verbose: Print progress messages. Default: False.
+
+    Returns:
+        Tuple[Dict[str, object], List[str]]:
+        - forecasters: Dictionary of successfully loaded ForecasterEquivalentDate objects.
+        - missing_targets: List of target names without saved models.
+
+    Examples:
+        >>> forecasters, missing = _load_forecasters(
+        ...     ["power", "energy"],
+        ...     "./models",
+        ...     verbose=True
+        ... )
+        >>> print(missing)
+        ['energy']
+    """
+    model_path = Path(model_dir)
+    forecasters = {}
+    missing_targets = []
+
+    for target in target_columns:
+        filepath = _get_model_filepath(model_path, target)
+        if filepath.exists():
+            try:
+                forecasters[target] = load(filepath)
+                if verbose:
+                    print(f"  ✓ Loaded forecaster for {target} from {filepath}")
+            except Exception as e:
+                if verbose:
+                    print(f"  ✗ Failed to load {target}: {e}")
+                missing_targets.append(target)
+        else:
+            missing_targets.append(target)
+
+    return forecasters, missing_targets
+
+
+def _model_directory_exists(model_dir: Union[str, Path]) -> bool:
+    """Check if model directory exists.
+
+    Args:
+        model_dir: Directory path to check.
+
+    Returns:
+        bool: True if directory exists, False otherwise.
+    """
+    return Path(model_dir).exists()
+
+
+# ============================================================================
+# Main Function
+# ============================================================================
+
+
 def n2n_predict(
     columns: Optional[List[str]] = None,
     forecast_horizon: int = 24,
     contamination: float = 0.01,
     window_size: int = 72,
+    force_train: bool = False,
+    model_dir: Union[str, Path] = "./models_baseline",
     verbose: bool = True,
     show_progress: bool = True,
-) -> pd.DataFrame:
-    """
-    End-to-end prediction function replicating the workflow from 01_base_predictor combined with fetch_data.
+) -> Tuple[pd.DataFrame, Dict]:
+    """End-to-end baseline forecasting using equivalent date method.
+
+    This function implements a complete forecasting pipeline that:
+    1. Loads and validates target data
+    2. Detects and removes outliers
+    3. Imputes missing values
+    4. Splits into train/validation/test sets
+    5. Trains or loads equivalent date forecasters
+    6. Generates multi-step ahead predictions
+
+    Models are persisted to disk following scikit-learn conventions using joblib.
+    Existing models are reused for prediction unless force_train=True.
 
     Args:
-        columns: List of target columns to forecast. If None, uses a default set (defined internally or from data).
-                 Note: fetch_data supports None to return all columns.
-        forecast_horizon: Number of steps to forecast.
-        contamination: Contamination factor for outlier detection.
-        window_size: Window size for weighting (not fully utilized in main flow but kept for consistency).
-        verbose: Whether to print progress logs.
-        show_progress: Show progress bar during training and prediction.
+        columns: List of target columns to forecast. If None, uses all available columns.
+            Default: None.
+        forecast_horizon: Number of time steps to forecast ahead. Default: 24.
+        contamination: Contamination parameter for outlier detection. Default: 0.01.
+        window_size: Rolling window size for gap detection. Default: 72.
+        force_train: Force retraining of all models, ignoring cached models.
+            Default: False.
+        model_dir: Directory for saving/loading trained models.
+            Default: "./models_baseline".
+        verbose: Print progress messages. Default: True.
+        show_progress: Show progress bar during training and prediction. Default: True.
 
     Returns:
-        pd.DataFrame: The multi-output predictions.
+        Tuple containing:
+        - predictions: DataFrame with forecast values for each target variable.
+        - forecasters: Dictionary of trained ForecasterEquivalentDate objects keyed by target.
+
+    Raises:
+        ValueError: If data validation fails or required data cannot be retrieved.
+        ImportError: If required dependencies are not installed.
+        OSError: If models cannot be saved to disk.
+
+    Examples:
+        Basic usage with automatic model caching:
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     forecast_horizon=24,
+        ...     verbose=True
+        ... )
+        >>> print(predictions.shape)
+        (24, 11)
+
+        Load cached models (if available):
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     forecast_horizon=24,
+        ...     force_train=False,
+        ...     model_dir="./saved_models",
+        ...     verbose=True
+        ... )
+
+        Force retraining and update cache:
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     forecast_horizon=24,
+        ...     force_train=True,
+        ...     model_dir="./saved_models",
+        ...     verbose=True
+        ... )
+
+        With specific target columns:
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     columns=["power", "energy"],
+        ...     forecast_horizon=48,
+        ...     force_train=False,
+        ...     verbose=True
+        ... )
+
+    Notes:
+        - Trained models are saved to disk using joblib for fast reuse.
+        - When force_train=False, existing models are loaded and prediction
+          proceeds without retraining. This significantly speeds up prediction
+          for repeated calls with the same configuration.
+        - The model_dir directory is created automatically if it doesn't exist.
+
+    Performance Notes:
+        - First run: Full training (~2-5 minutes depending on data size)
+        - Subsequent runs (force_train=False): Model loading only (~1-2 seconds)
+        - Force retrain (force_train=True): Full training again (~2-5 minutes)
     """
     if columns is not None:
         TARGET = columns
@@ -98,20 +363,66 @@ def n2n_predict(
     end_validation = pd.concat([data_train, data_val]).index[-1]
 
     baseline_forecasters = {}
+    targets_to_train = list(data.columns)
+
+    # Attempt to load cached models if force_train=False
+    if not force_train and _model_directory_exists(model_dir):
+        if verbose:
+            print("  Attempting to load cached models...")
+        cached_forecasters, missing_targets = _load_forecasters(
+            target_columns=list(data.columns),
+            model_dir=model_dir,
+            verbose=verbose,
+        )
+        baseline_forecasters.update(cached_forecasters)
+        targets_to_train = missing_targets
+
+        if len(cached_forecasters) == len(data.columns):
+            if verbose:
+                print(f"  ✓ All {len(data.columns)} forecasters loaded from cache")
+        elif len(cached_forecasters) > 0:
+            if verbose:
+                print(
+                    f"  ✓ Loaded {len(cached_forecasters)} forecasters, "
+                    f"will train {len(targets_to_train)} new ones"
+                )
+
+    # Train missing or forced models
+    if len(targets_to_train) > 0:
+        if force_train and len(baseline_forecasters) > 0:
+            if verbose:
+                print(f"  Force retraining all {len(data.columns)} forecasters...")
+            targets_to_train = list(data.columns)
+            baseline_forecasters.clear()
+
+        target_iter = targets_to_train
+        if show_progress and tqdm is not None:
+            target_iter = tqdm(
+                targets_to_train,
+                desc="Training forecasters",
+                unit="model",
+            )
 
-    target_iter = data.columns
-    if show_progress and tqdm is not None:
-        target_iter = tqdm(data.columns, desc="Training forecasters", unit="model")
+        for target in target_iter:
+            forecaster = ForecasterEquivalentDate(
+                offset=pd.DateOffset(days=1), n_offsets=1
+            )
 
-    for target in target_iter:
-        forecaster = ForecasterEquivalentDate(offset=pd.DateOffset(days=1), n_offsets=1)
+            forecaster.fit(y=data.loc[:end_validation, target])
 
-        forecaster.fit(y=data.loc[:end_validation, target])
+            baseline_forecasters[target] = forecaster
 
-        baseline_forecasters[target] = forecaster
+        # Save newly trained models to disk
+        if verbose:
+            print(f"  Saving {len(targets_to_train)} trained forecasters to disk...")
+        _save_forecasters(
+            forecasters={t: baseline_forecasters[t] for t in targets_to_train},
+            model_dir=model_dir,
+            verbose=verbose,
+        )
 
     if verbose:
-        print("✓ Multi-output baseline system trained")
+        print(f"  ✓ Total forecasters available: {len(baseline_forecasters)}")
 
     # --- Predict ---
     if verbose:
@@ -123,4 +434,4 @@ def n2n_predict(
         show_progress=show_progress,
     )
 
-    return predictions
+    return predictions, baseline_forecasters

spotforecast2/processing/n2n_predict_with_covariates.py

@@ -744,7 +744,7 @@ def n2n_predict_with_covariates(
     force_train: bool = False,
     model_dir: Union[str, Path] = "./forecaster_models",
     verbose: bool = True,
-    show_progress: bool = True,
+    show_progress: bool = False,
 ) -> Tuple[pd.DataFrame, Dict, Dict]:
     """End-to-end recursive forecasting with exogenous covariates.
 
@@ -781,9 +781,9 @@ def n2n_predict_with_covariates(
         force_train: Force retraining of all models, ignoring cached models.
             Default: False.
         model_dir: Directory for saving/loading trained models.
-            Default: "./forecaster_models".
+            Default: "./models_covariates".
         verbose: Print progress messages. Default: True.
-        show_progress: Show progress bar during training. Default: True.
+        show_progress: Show progress bar during training. Default: False.
 
     Returns:
         Tuple containing:
@@ -913,6 +913,10 @@ def n2n_predict_with_covariates(
         """Return sample weights for given index."""
         return custom_weights(index, weights_series)
 
+    # Note: weight_func is a local function and cannot be pickled.
+    # Model persistence is disabled when using weight_func.
+    use_model_persistence = False
+
     # ========================================================================
     # 4. EXOGENOUS FEATURES ENGINEERING
     # ========================================================================
@@ -1070,11 +1074,11 @@ def n2n_predict_with_covariates(
     window_features = RollingFeatures(stats=["mean"], window_sizes=window_size)
     end_validation = pd.concat([data_train, data_val]).index[-1]
 
-    # Attempt to load cached models if force_train=False
+    # Attempt to load cached models if force_train=False and persistence is enabled
     recursive_forecasters = {}
     targets_to_train = target_columns
 
-    if not force_train and _model_directory_exists(model_dir):
+    if use_model_persistence and not force_train and _model_directory_exists(model_dir):
         if verbose:
             print("  Attempting to load cached models...")
         cached_forecasters, missing_targets = _load_forecasters(
@@ -1132,14 +1136,20 @@ def n2n_predict_with_covariates(
             if verbose:
                 print(f"    ✓ Forecaster trained for {target}")
 
-        # Save newly trained models to disk
-        if verbose:
-            print(f"  Saving {len(targets_to_train)} trained forecasters to disk...")
-        _save_forecasters(
-            forecasters={t: recursive_forecasters[t] for t in targets_to_train},
-            model_dir=model_dir,
-            verbose=verbose,
-        )
+        # Save newly trained models to disk (only if persistence is enabled)
+        if use_model_persistence:
+            if verbose:
+                print(
+                    f"  Saving {len(targets_to_train)} trained forecasters to disk..."
+                )
+            _save_forecasters(
+                forecasters={t: recursive_forecasters[t] for t in targets_to_train},
+                model_dir=model_dir,
+                verbose=verbose,
+            )
+        else:
+            if verbose:
+                print("  ⚠ Model persistence disabled (weight_func cannot be pickled)")
 
     if verbose:
         print(f"  ✓ Total forecasters available: {len(recursive_forecasters)}")

{spotforecast2-0.1.0.dist-info → spotforecast2-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: spotforecast2
-Version: 0.1.0
+Version: 0.1.1
 Summary: Forecasting with spot
 Author: bartzbeielstein
 Author-email: bartzbeielstein <32470350+bartzbeielstein@users.noreply.github.com>

{spotforecast2-0.1.0.dist-info → spotforecast2-0.1.1.dist-info}/RECORD

@@ -31,8 +31,8 @@ spotforecast2/preprocessing/imputation.py,sha256=lmH-HumI_QLLm9aMESe_oZq84Axn60w
 spotforecast2/preprocessing/outlier.py,sha256=jZxAR870QtYner7b4gXk6LLGJw0juLq1VU4CGklYd3c,4208
 spotforecast2/preprocessing/split.py,sha256=mzzt5ltUZdVzfWtBBTQjp8E2MyqVdWUFtz7nN11urbU,5011
 spotforecast2/processing/agg_predict.py,sha256=VKlruB0x-eJKokkHyJxR87rZ4m53si3ODbrd0ibPlow,2378
-spotforecast2/processing/n2n_predict.py,sha256=Jkf-fMw2RSKY8-0UDc8D0yiiZxiF9s5DyfeRpfx90ks,4060
-spotforecast2/processing/n2n_predict_with_covariates.py,sha256=wnGSfwUjC5xKgaRyz0N80p4oW6sdbI1bgA1YyVQTA8I,39942
+spotforecast2/processing/n2n_predict.py,sha256=dAj5yXD2JGXSqtl0VDkq0O_8FO_K9BCYG6osbJbWDFg,14494
+spotforecast2/processing/n2n_predict_with_covariates.py,sha256=5a1lYIQE1d-t4ZvSQDoW87G705eiIZxtrCn4w7U2bVw,40420
 spotforecast2/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spotforecast2/utils/__init__.py,sha256=NrMt_xJLe4rbTFbsbgSQYeREohEOiYG5S-97e6Jj07I,1018
 spotforecast2/utils/convert_to_utc.py,sha256=hz8mJUHK9jDLUiN5LdNX5l3KZuOKlklyycB4zFdB9Ng,1405
@@ -42,6 +42,6 @@ spotforecast2/utils/generate_holiday.py,sha256=SHaPvPMt-abis95cChHf5ObyPwCTrzJ87
 spotforecast2/utils/validation.py,sha256=x9ypQzcneDhWJA_piiY4Q3_ogoGd1LTsZ7__MFeG9Fc,21618
 spotforecast2/weather/__init__.py,sha256=1Jco88pl0deNESgNATin83Nf5i9c58pxN7G-vNiOiu0,120
 spotforecast2/weather/weather_client.py,sha256=Ec_ywug6uoa71MfXM8RNbXEvtBtBzr-SUS5xq_HKtZE,9837
-spotforecast2-0.1.0.dist-info/WHEEL,sha256=5DEXXimM34_d4Gx1AuF9ysMr1_maoEtGKjaILM3s4w4,80
-spotforecast2-0.1.0.dist-info/METADATA,sha256=3odonjnM8q3No1KyboAF_Ta41X7LjUGLthDiavC2rMM,3481
-spotforecast2-0.1.0.dist-info/RECORD,,
+spotforecast2-0.1.1.dist-info/WHEEL,sha256=5DEXXimM34_d4Gx1AuF9ysMr1_maoEtGKjaILM3s4w4,80
+spotforecast2-0.1.1.dist-info/METADATA,sha256=TMwW-WMXSoNRVw7oDLU3Ys_8JXhODgvXxXjSeokWaXs,3481
+spotforecast2-0.1.1.dist-info/RECORD,,