spotforecast2 0.1.0__tar.gz → 0.2.0__tar.gz

{spotforecast2-0.1.0 → spotforecast2-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: spotforecast2
-Version: 0.1.0
+Version: 0.2.0
 Summary: Forecasting with spot
 Author: bartzbeielstein
 Author-email: bartzbeielstein <32470350+bartzbeielstein@users.noreply.github.com>

{spotforecast2-0.1.0 → spotforecast2-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spotforecast2"
-version = "0.1.0"
+version = "0.2.0"
 description = "Forecasting with spot"
 readme = "README.md"
 authors = [

spotforecast2-0.2.0/src/spotforecast2/processing/n2n_predict.py

@@ -0,0 +1,444 @@
+"""
+End-to-end baseline forecasting using equivalent date method.
+
+This module provides a complete forecasting pipeline using the ForecasterEquivalentDate
+baseline model. It handles data preparation, outlier detection, imputation, model
+training, and prediction in a single integrated function.
+
+Model persistence follows scikit-learn conventions using joblib for efficient
+serialization and deserialization of trained forecasters.
+
+Examples:
+    Basic usage with default parameters:
+
+    >>> from spotforecast2.processing.n2n_predict import n2n_predict
+    >>> predictions = n2n_predict(forecast_horizon=24, verbose=True)
+
+    Using cached models:
+
+    >>> # Load existing models if available, or train new ones
+    >>> predictions = n2n_predict(
+    ...     forecast_horizon=24,
+    ...     force_train=False,
+    ...     model_dir="./models",
+    ...     verbose=True
+    ... )
+
+    Force retraining and update cache:
+
+    >>> predictions = n2n_predict(
+    ...     forecast_horizon=24,
+    ...     force_train=True,
+    ...     model_dir="./models",
+    ...     verbose=True
+    ... )
+"""
+
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import pandas as pd
+from spotforecast2.forecaster.recursive import ForecasterEquivalentDate
+from spotforecast2.data.fetch_data import fetch_data
+from spotforecast2.preprocessing.curate_data import basic_ts_checks
+from spotforecast2.preprocessing.curate_data import agg_and_resample_data
+from spotforecast2.preprocessing.outlier import mark_outliers
+from spotforecast2.preprocessing.split import split_rel_train_val_test
+from spotforecast2.forecaster.utils import predict_multivariate
+from spotforecast2.preprocessing.curate_data import get_start_end
+
+try:
+    from joblib import dump, load
+except ImportError:
+    raise ImportError("joblib is required. Install with: pip install joblib")
+
+try:
+    from tqdm.auto import tqdm
+except ImportError:  # pragma: no cover - fallback when tqdm is not installed
+    tqdm = None
+
+
+# ============================================================================
+# Model Persistence Functions
+# ============================================================================
+
+
+def _ensure_model_dir(model_dir: Union[str, Path]) -> Path:
+    """Ensure model directory exists.
+
+    Args:
+        model_dir: Directory path for model storage.
+
+    Returns:
+        Path: Validated Path object.
+
+    Raises:
+        OSError: If directory cannot be created.
+    """
+    model_path = Path(model_dir)
+    model_path.mkdir(parents=True, exist_ok=True)
+    return model_path
+
+
+def _get_model_filepath(model_dir: Path, target: str) -> Path:
+    """Get filepath for a single model.
+
+    Args:
+        model_dir: Directory containing models.
+        target: Target variable name.
+
+    Returns:
+        Path: Full filepath for the model.
+
+    Examples:
+        >>> path = _get_model_filepath(Path("./models"), "power")
+        >>> str(path)
+        './models/forecaster_power.joblib'
+    """
+    return model_dir / f"forecaster_{target}.joblib"
+
+
+def _save_forecasters(
+    forecasters: Dict[str, object],
+    model_dir: Union[str, Path],
+    verbose: bool = False,
+) -> Dict[str, Path]:
+    """Save trained forecasters to disk using joblib.
+
+    Follows scikit-learn persistence conventions using joblib for efficient
+    serialization of sklearn-compatible estimators.
+
+    Args:
+        forecasters: Dictionary mapping target names to trained ForecasterEquivalentDate objects.
+        model_dir: Directory to save models. Created if it doesn't exist.
+        verbose: Print progress messages. Default: False.
+
+    Returns:
+        Dict[str, Path]: Dictionary mapping target names to saved model filepaths.
+
+    Raises:
+        OSError: If models cannot be written to disk.
+        TypeError: If forecasters contain non-serializable objects.
+
+    Examples:
+        >>> forecasters = {"power": forecaster_obj}
+        >>> paths = _save_forecasters(forecasters, "./models", verbose=True)
+        >>> print(paths["power"])
+        models/forecaster_power.joblib
+    """
+    model_path = _ensure_model_dir(model_dir)
+    saved_paths = {}
+
+    for target, forecaster in forecasters.items():
+        filepath = _get_model_filepath(model_path, target)
+        try:
+            dump(forecaster, filepath, compress=3)
+            saved_paths[target] = filepath
+            if verbose:
+                print(f"  ✓ Saved forecaster for {target} to {filepath}")
+        except Exception as e:
+            raise OSError(f"Failed to save model for {target}: {e}")
+
+    return saved_paths
+
+
+def _load_forecasters(
+    target_columns: List[str],
+    model_dir: Union[str, Path],
+    verbose: bool = False,
+) -> Tuple[Dict[str, object], List[str]]:
+    """Load trained forecasters from disk using joblib.
+
+    Attempts to load all forecasters for given targets. Missing models
+    are indicated in the return value for selective retraining.
+
+    Args:
+        target_columns: List of target variable names to load.
+        model_dir: Directory containing saved models.
+        verbose: Print progress messages. Default: False.
+
+    Returns:
+        Tuple[Dict[str, object], List[str]]:
+        - forecasters: Dictionary of successfully loaded ForecasterEquivalentDate objects.
+        - missing_targets: List of target names without saved models.
+
+    Examples:
+        >>> forecasters, missing = _load_forecasters(
+        ...     ["power", "energy"],
+        ...     "./models",
+        ...     verbose=True
+        ... )
+        >>> print(missing)
+        ['energy']
+    """
+    model_path = Path(model_dir)
+    forecasters = {}
+    missing_targets = []
+
+    for target in target_columns:
+        filepath = _get_model_filepath(model_path, target)
+        if filepath.exists():
+            try:
+                forecasters[target] = load(filepath)
+                if verbose:
+                    print(f"  ✓ Loaded forecaster for {target} from {filepath}")
+            except Exception as e:
+                if verbose:
+                    print(f"  ✗ Failed to load {target}: {e}")
+                missing_targets.append(target)
+        else:
+            missing_targets.append(target)
+
+    return forecasters, missing_targets
+
+
+def _model_directory_exists(model_dir: Union[str, Path]) -> bool:
+    """Check if model directory exists.
+
+    Args:
+        model_dir: Directory path to check.
+
+    Returns:
+        bool: True if directory exists, False otherwise.
+    """
+    return Path(model_dir).exists()
+
+
+# ============================================================================
+# Main Function
+# ============================================================================
+
+
+def n2n_predict(
+    data: Optional[pd.DataFrame] = None,
+    columns: Optional[List[str]] = None,
+    forecast_horizon: int = 24,
+    contamination: float = 0.01,
+    window_size: int = 72,
+    force_train: bool = False,
+    model_dir: Union[str, Path] = "./models_baseline",
+    verbose: bool = True,
+    show_progress: bool = True,
+) -> Tuple[pd.DataFrame, Dict]:
+    """End-to-end baseline forecasting using equivalent date method.
+
+    This function implements a complete forecasting pipeline that:
+    1. Loads and validates target data
+    2. Detects and removes outliers
+    3. Imputes missing values
+    4. Splits into train/validation/test sets
+    5. Trains or loads equivalent date forecasters
+    6. Generates multi-step ahead predictions
+
+    Models are persisted to disk following scikit-learn conventions using joblib.
+    Existing models are reused for prediction unless force_train=True.
+
+    Args:
+        data: Optional DataFrame with target time series data. If None, fetches data automatically.
+            Default: None.
+        columns: List of target columns to forecast. If None, uses all available columns.
+            Default: None.
+        forecast_horizon: Number of time steps to forecast ahead. Default: 24.
+        contamination: Contamination parameter for outlier detection. Default: 0.01.
+        window_size: Rolling window size for gap detection. Default: 72.
+        force_train: Force retraining of all models, ignoring cached models.
+            Default: False.
+        model_dir: Directory for saving/loading trained models.
+            Default: "./models_baseline".
+        verbose: Print progress messages. Default: True.
+        show_progress: Show progress bar during training and prediction. Default: True.
+
+    Returns:
+        Tuple containing:
+        - predictions: DataFrame with forecast values for each target variable.
+        - forecasters: Dictionary of trained ForecasterEquivalentDate objects keyed by target.
+
+    Raises:
+        ValueError: If data validation fails or required data cannot be retrieved.
+        ImportError: If required dependencies are not installed.
+        OSError: If models cannot be saved to disk.
+
+    Examples:
+        Basic usage with automatic model caching:
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     forecast_horizon=24,
+        ...     verbose=True
+        ... )
+        >>> print(predictions.shape)
+        (24, 11)
+
+        Load cached models (if available):
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     forecast_horizon=24,
+        ...     force_train=False,
+        ...     model_dir="./saved_models",
+        ...     verbose=True
+        ... )
+
+        Force retraining and update cache:
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     forecast_horizon=24,
+        ...     force_train=True,
+        ...     model_dir="./saved_models",
+        ...     verbose=True
+        ... )
+
+        With specific target columns:
+
+        >>> predictions, forecasters = n2n_predict(
+        ...     columns=["power", "energy"],
+        ...     forecast_horizon=48,
+        ...     force_train=False,
+        ...     verbose=True
+        ... )
+
+    Notes:
+        - Trained models are saved to disk using joblib for fast reuse.
+        - When force_train=False, existing models are loaded and prediction
+          proceeds without retraining. This significantly speeds up prediction
+          for repeated calls with the same configuration.
+        - The model_dir directory is created automatically if it doesn't exist.
+
+    Performance Notes:
+        - First run: Full training (~2-5 minutes depending on data size)
+        - Subsequent runs (force_train=False): Model loading only (~1-2 seconds)
+        - Force retrain (force_train=True): Full training again (~2-5 minutes)
+    """
+    if columns is not None:
+        TARGET = columns
+    else:
+        TARGET = None
+
+    if verbose:
+        print("--- Starting n2n_predict ---")
+        print("Fetching data...")
+
+    # Fetch data
+    if data is not None:
+        if TARGET is not None:
+            data = data[TARGET]
+    else:
+        data = fetch_data(columns=TARGET)
+
+    START, END, COV_START, COV_END = get_start_end(
+        data=data,
+        forecast_horizon=forecast_horizon,
+        verbose=verbose,
+    )
+
+    basic_ts_checks(data, verbose=verbose)
+
+    data = agg_and_resample_data(data, verbose=verbose)
+
+    # --- Outlier Handling ---
+    if verbose:
+        print("Handling outliers...")
+
+    # data_old = data.copy() # kept in notebook, maybe useful for debugging but not used logic-wise here
+    data, outliers = mark_outliers(
+        data, contamination=contamination, random_state=1234, verbose=verbose
+    )
+
+    # --- Missing Data (Imputation) ---
+    if verbose:
+        print("Imputing missing data...")
+
+    missing_indices = data.index[data.isnull().any(axis=1)]
+    if verbose:
+        n_missing = len(missing_indices)
+        pct_missing = (n_missing / len(data)) * 100
+        print(f"Number of rows with missing values: {n_missing}")
+        print(f"Percentage of rows with missing values: {pct_missing:.2f}%")
+
+    data = data.ffill()
+    data = data.bfill()
+
+    # --- Train, Val, Test Split ---
+    if verbose:
+        print("Splitting data...")
+    data_train, data_val, data_test = split_rel_train_val_test(
+        data, perc_train=0.8, perc_val=0.2, verbose=verbose
+    )
+
+    # --- Model Fit ---
+    if verbose:
+        print("Fitting models...")
+
+    end_validation = pd.concat([data_train, data_val]).index[-1]
+
+    baseline_forecasters = {}
+    targets_to_train = list(data.columns)
+
+    # Attempt to load cached models if force_train=False
+    if not force_train and _model_directory_exists(model_dir):
+        if verbose:
+            print("  Attempting to load cached models...")
+        cached_forecasters, missing_targets = _load_forecasters(
+            target_columns=list(data.columns),
+            model_dir=model_dir,
+            verbose=verbose,
+        )
+        baseline_forecasters.update(cached_forecasters)
+        targets_to_train = missing_targets
+
+        if len(cached_forecasters) == len(data.columns):
+            if verbose:
+                print(f"  ✓ All {len(data.columns)} forecasters loaded from cache")
+        elif len(cached_forecasters) > 0:
+            if verbose:
+                print(
+                    f"  ✓ Loaded {len(cached_forecasters)} forecasters, "
+                    f"will train {len(targets_to_train)} new ones"
+                )
+
+    # Train missing or forced models
+    if len(targets_to_train) > 0:
+        if force_train and len(baseline_forecasters) > 0:
+            if verbose:
+                print(f"  Force retraining all {len(data.columns)} forecasters...")
+            targets_to_train = list(data.columns)
+            baseline_forecasters.clear()
+
+        target_iter = targets_to_train
+        if show_progress and tqdm is not None:
+            target_iter = tqdm(
+                targets_to_train,
+                desc="Training forecasters",
+                unit="model",
+            )
+
+        for target in target_iter:
+            forecaster = ForecasterEquivalentDate(
+                offset=pd.DateOffset(days=1), n_offsets=1
+            )
+
+            forecaster.fit(y=data.loc[:end_validation, target])
+
+            baseline_forecasters[target] = forecaster
+
+        # Save newly trained models to disk
+        if verbose:
+            print(f"  Saving {len(targets_to_train)} trained forecasters to disk...")
+        _save_forecasters(
+            forecasters={t: baseline_forecasters[t] for t in targets_to_train},
+            model_dir=model_dir,
+            verbose=verbose,
+        )
+
+    if verbose:
+        print(f"  ✓ Total forecasters available: {len(baseline_forecasters)}")
+
+    # --- Predict ---
+    if verbose:
+        print("Generating predictions...")
+
+    predictions = predict_multivariate(
+        baseline_forecasters,
+        steps_ahead=forecast_horizon,
+        show_progress=show_progress,
+    )
+
+    return predictions, baseline_forecasters

{spotforecast2-0.1.0 → spotforecast2-0.2.0}/src/spotforecast2/processing/n2n_predict_with_covariates.py

@@ -727,6 +727,7 @@ def _model_directory_exists(model_dir: Union[str, Path]) -> bool:
 
 
 def n2n_predict_with_covariates(
+    data: Optional[pd.DataFrame] = None,
     forecast_horizon: int = 24,
     contamination: float = 0.01,
     window_size: int = 72,
@@ -744,7 +745,7 @@ def n2n_predict_with_covariates(
     force_train: bool = False,
     model_dir: Union[str, Path] = "./forecaster_models",
     verbose: bool = True,
-    show_progress: bool = True,
+    show_progress: bool = False,
 ) -> Tuple[pd.DataFrame, Dict, Dict]:
     """End-to-end recursive forecasting with exogenous covariates.
 
@@ -763,6 +764,8 @@ def n2n_predict_with_covariates(
     Existing models are reused for prediction unless force_train=True.
 
     Args:
+        data: Optional DataFrame with target time series data. If None, fetches data automatically.
+            Default: None.
         forecast_horizon: Number of time steps to forecast ahead. Default: 24.
         contamination: Contamination parameter for outlier detection. Default: 0.01.
         window_size: Rolling window size for gap detection. Default: 72.
@@ -781,9 +784,9 @@ def n2n_predict_with_covariates(
         force_train: Force retraining of all models, ignoring cached models.
             Default: False.
         model_dir: Directory for saving/loading trained models.
-            Default: "./forecaster_models".
+            Default: "./models_covariates".
         verbose: Print progress messages. Default: True.
-        show_progress: Show progress bar during training. Default: True.
+        show_progress: Show progress bar during training. Default: False.
 
     Returns:
         Tuple containing:
@@ -865,7 +868,8 @@ def n2n_predict_with_covariates(
     if verbose:
         print("\n[1/9] Loading and preparing target data...")
 
-    data = fetch_data()
+    if data is None:
+        data = fetch_data()
     target_columns = data.columns.tolist()
 
     if verbose:
@@ -913,6 +917,10 @@ def n2n_predict_with_covariates(
         """Return sample weights for given index."""
         return custom_weights(index, weights_series)
 
+    # Note: weight_func is a local function and cannot be pickled.
+    # Model persistence is disabled when using weight_func.
+    use_model_persistence = False
+
     # ========================================================================
     # 4. EXOGENOUS FEATURES ENGINEERING
     # ========================================================================
@@ -1070,11 +1078,11 @@ def n2n_predict_with_covariates(
     window_features = RollingFeatures(stats=["mean"], window_sizes=window_size)
     end_validation = pd.concat([data_train, data_val]).index[-1]
 
-    # Attempt to load cached models if force_train=False
+    # Attempt to load cached models if force_train=False and persistence is enabled
     recursive_forecasters = {}
     targets_to_train = target_columns
 
-    if not force_train and _model_directory_exists(model_dir):
+    if use_model_persistence and not force_train and _model_directory_exists(model_dir):
         if verbose:
             print("  Attempting to load cached models...")
         cached_forecasters, missing_targets = _load_forecasters(
@@ -1132,14 +1140,20 @@ def n2n_predict_with_covariates(
             if verbose:
                 print(f"    ✓ Forecaster trained for {target}")
 
-        # Save newly trained models to disk
-        if verbose:
-            print(f"  Saving {len(targets_to_train)} trained forecasters to disk...")
-        _save_forecasters(
-            forecasters={t: recursive_forecasters[t] for t in targets_to_train},
-            model_dir=model_dir,
-            verbose=verbose,
-        )
+        # Save newly trained models to disk (only if persistence is enabled)
+        if use_model_persistence:
+            if verbose:
+                print(
+                    f"  Saving {len(targets_to_train)} trained forecasters to disk..."
+                )
+            _save_forecasters(
+                forecasters={t: recursive_forecasters[t] for t in targets_to_train},
+                model_dir=model_dir,
+                verbose=verbose,
+            )
+        else:
+            if verbose:
+                print("  ⚠ Model persistence disabled (weight_func cannot be pickled)")
 
     if verbose:
         print(f"  ✓ Total forecasters available: {len(recursive_forecasters)}")

spotforecast2-0.1.0/src/spotforecast2/processing/n2n_predict.py

@@ -1,126 +0,0 @@
-import pandas as pd
-from typing import List, Optional
-from spotforecast2.forecaster.recursive import ForecasterEquivalentDate
-from spotforecast2.data.fetch_data import fetch_data
-from spotforecast2.preprocessing.curate_data import basic_ts_checks
-from spotforecast2.preprocessing.curate_data import agg_and_resample_data
-from spotforecast2.preprocessing.outlier import mark_outliers
-
-from spotforecast2.preprocessing.split import split_rel_train_val_test
-from spotforecast2.forecaster.utils import predict_multivariate
-from spotforecast2.preprocessing.curate_data import get_start_end
-
-try:
-    from tqdm.auto import tqdm
-except ImportError:  # pragma: no cover - fallback when tqdm is not installed
-    tqdm = None
-
-
-def n2n_predict(
-    columns: Optional[List[str]] = None,
-    forecast_horizon: int = 24,
-    contamination: float = 0.01,
-    window_size: int = 72,
-    verbose: bool = True,
-    show_progress: bool = True,
-) -> pd.DataFrame:
-    """
-    End-to-end prediction function replicating the workflow from 01_base_predictor combined with fetch_data.
-
-    Args:
-        columns: List of target columns to forecast. If None, uses a default set (defined internally or from data).
-                 Note: fetch_data supports None to return all columns.
-        forecast_horizon: Number of steps to forecast.
-        contamination: Contamination factor for outlier detection.
-        window_size: Window size for weighting (not fully utilized in main flow but kept for consistency).
-        verbose: Whether to print progress logs.
-        show_progress: Show progress bar during training and prediction.
-
-    Returns:
-        pd.DataFrame: The multi-output predictions.
-    """
-    if columns is not None:
-        TARGET = columns
-    else:
-        TARGET = None
-
-    if verbose:
-        print("--- Starting n2n_predict ---")
-        print("Fetching data...")
-
-    # Fetch data
-    data = fetch_data(columns=TARGET)
-
-    START, END, COV_START, COV_END = get_start_end(
-        data=data,
-        forecast_horizon=forecast_horizon,
-        verbose=verbose,
-    )
-
-    basic_ts_checks(data, verbose=verbose)
-
-    data = agg_and_resample_data(data, verbose=verbose)
-
-    # --- Outlier Handling ---
-    if verbose:
-        print("Handling outliers...")
-
-    # data_old = data.copy() # kept in notebook, maybe useful for debugging but not used logic-wise here
-    data, outliers = mark_outliers(
-        data, contamination=contamination, random_state=1234, verbose=verbose
-    )
-
-    # --- Missing Data (Imputation) ---
-    if verbose:
-        print("Imputing missing data...")
-
-    missing_indices = data.index[data.isnull().any(axis=1)]
-    if verbose:
-        n_missing = len(missing_indices)
-        pct_missing = (n_missing / len(data)) * 100
-        print(f"Number of rows with missing values: {n_missing}")
-        print(f"Percentage of rows with missing values: {pct_missing:.2f}%")
-
-    data = data.ffill()
-    data = data.bfill()
-
-    # --- Train, Val, Test Split ---
-    if verbose:
-        print("Splitting data...")
-    data_train, data_val, data_test = split_rel_train_val_test(
-        data, perc_train=0.8, perc_val=0.2, verbose=verbose
-    )
-
-    # --- Model Fit ---
-    if verbose:
-        print("Fitting models...")
-
-    end_validation = pd.concat([data_train, data_val]).index[-1]
-
-    baseline_forecasters = {}
-
-    target_iter = data.columns
-    if show_progress and tqdm is not None:
-        target_iter = tqdm(data.columns, desc="Training forecasters", unit="model")
-
-    for target in target_iter:
-        forecaster = ForecasterEquivalentDate(offset=pd.DateOffset(days=1), n_offsets=1)
-
-        forecaster.fit(y=data.loc[:end_validation, target])
-
-        baseline_forecasters[target] = forecaster
-
-    if verbose:
-        print("✓ Multi-output baseline system trained")
-
-    # --- Predict ---
-    if verbose:
-        print("Generating predictions...")
-
-    predictions = predict_multivariate(
-        baseline_forecasters,
-        steps_ahead=forecast_horizon,
-        show_progress=show_progress,
-    )
-
-    return predictions