spotforecast2 0.0.1__py3-none-any.whl

spotforecast2/preprocessing/__init__.py

@@ -0,0 +1,30 @@
+from .curate_data import (
+    get_start_end,
+    curate_holidays,
+    curate_weather,
+    basic_ts_checks,
+    agg_and_resample_data,
+)
+from .outlier import mark_outliers, manual_outlier_removal
+from .imputation import custom_weights, get_missing_weights
+from .split import split_abs_train_val_test, split_rel_train_val_test
+from ._differentiator import TimeSeriesDifferentiator
+from ._binner import QuantileBinner
+from ._rolling import RollingFeatures
+
+__all__ = [
+    "get_start_end",
+    "curate_holidays",
+    "curate_weather",
+    "basic_ts_checks",
+    "agg_and_resample_data",
+    "mark_outliers",
+    "manual_outlier_removal",
+    "custom_weights",
+    "get_missing_weights",
+    "split_abs_train_val_test",
+    "split_rel_train_val_test",
+    "TimeSeriesDifferentiator",
+    "QuantileBinner",
+    "RollingFeatures",
+]

spotforecast2/preprocessing/_binner.py

@@ -0,0 +1,378 @@
+"""
+QuantileBinner class for binning data into quantile-based bins.
+
+This module contains the QuantileBinner class which bins data into quantile-based bins
+using numpy.percentile with optimized performance using numpy.searchsorted.
+"""
+
+from __future__ import annotations
+import warnings
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.exceptions import NotFittedError
+
+from spotforecast2.exceptions import IgnoredArgumentWarning
+
+
+class QuantileBinner(BaseEstimator, TransformerMixin):
+    """
+    Bin data into quantile-based bins using numpy.percentile.
+
+    This class is similar to sklearn's KBinsDiscretizer but optimized for
+    performance using numpy.searchsorted for fast bin assignment. Bin intervals
+    are defined following the convention: bins[i-1] <= x < bins[i]. Values
+    outside the range are clipped to the first or last bin.
+
+    Args:
+        n_bins: The number of quantile-based bins to create. Must be >= 2.
+        method: The method used to compute quantiles, passed to numpy.percentile.
+            Default is 'linear'. Valid values: "inverse_cdf",
+            "averaged_inverse_cdf", "closest_observation",
+            "interpolated_inverse_cdf", "hazen", "weibull", "linear",
+            "median_unbiased", "normal_unbiased".
+        subsample: Maximum number of samples for computing quantiles. If dataset
+            has more samples, a random subset is used. Default 200000.
+        dtype: Data type for bin indices. Default is numpy.float64.
+        random_state: Random seed for subset generation. Default 789654.
+
+    Attributes:
+        n_bins (int): Number of bins to create.
+        method (str): Quantile computation method.
+        subsample (int): Maximum samples for quantile computation.
+        dtype (type): Data type for bin indices.
+        random_state (int): Random seed.
+        n_bins_ (int): Actual number of bins after fitting (may differ from n_bins
+            if duplicate edges are found).
+        bin_edges_ (np.ndarray): Edges of the bins learned during fitting.
+        internal_edges_ (np.ndarray): Internal edges for optimized bin assignment.
+        intervals_ (dict): Mapping from bin index to (lower, upper) interval bounds.
+
+    Examples:
+        >>> import numpy as np
+        >>> from spotforecast2.preprocessing import QuantileBinner
+        >>>
+        >>> # Basic usage: create 3 quantile bins
+        >>> X = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+        >>> binner = QuantileBinner(n_bins=3)
+        >>> _ = binner.fit(X)
+        >>> result = binner.transform(np.array([1.5, 5.5, 9.5]))
+        >>> print(result)
+        [0. 1. 2.]
+        >>>
+        >>> # Check bin intervals
+        >>> print(binner.n_bins_)
+        3
+        >>> assert len(binner.intervals_) == 3
+        >>>
+        >>> # Use fit_transform for one-step operation
+        >>> X2 = np.array([10, 20, 30, 40, 50])
+        >>> binner2 = QuantileBinner(n_bins=2)
+        >>> bins = binner2.fit_transform(X2)
+        >>> print(bins)
+        [0. 0. 1. 1. 1.]
+    """
+
+    def __init__(
+        self,
+        n_bins: int,
+        method: str = "linear",
+        subsample: int = 200000,
+        dtype: type = np.float64,
+        random_state: int = 789654,
+    ) -> None:
+
+        self._validate_params(n_bins, method, subsample, dtype, random_state)
+
+        self.n_bins = n_bins
+        self.method = method
+        self.subsample = subsample
+        self.dtype = dtype
+        self.random_state = random_state
+        self.n_bins_ = None
+        self.bin_edges_ = None
+        self.internal_edges_ = None
+        self.intervals_ = None
+
+    def _validate_params(
+        self, n_bins: int, method: str, subsample: int, dtype: type, random_state: int
+    ):
+        """
+        Validate parameters passed to the class initializer.
+
+        Args:
+            n_bins: Number of quantile-based bins. Must be int >= 2.
+            method: Quantile computation method for numpy.percentile.
+            subsample: Number of samples for computing quantiles. Must be int >= 1.
+            dtype: Data type for bin indices. Must be a valid numpy dtype.
+            random_state: Random seed for subset generation. Must be int >= 0.
+
+        Raises:
+            ValueError: If n_bins < 2, method is invalid, subsample < 1,
+                random_state < 0, or dtype is not a valid type.
+
+        Examples:
+            >>> import numpy as np
+            >>> from spotforecast2.preprocessing import QuantileBinner
+            >>>
+            >>> # Valid parameters work fine
+            >>> binner = QuantileBinner(n_bins=5, method='linear')
+            >>> assert binner.n_bins == 5
+            >>>
+            >>> # Invalid n_bins raises ValueError
+            >>> try:
+            ...     binner = QuantileBinner(n_bins=1)
+            ... except ValueError as e:
+            ...     assert 'greater than 1' in str(e)
+            >>>
+            >>> # Invalid method raises ValueError
+            >>> try:
+            ...     binner = QuantileBinner(n_bins=3, method='invalid')
+            ... except ValueError as e:
+            ...     assert 'must be one of' in str(e)
+        """
+
+        if not isinstance(n_bins, int) or n_bins < 2:
+            raise ValueError(f"`n_bins` must be an int greater than 1. Got {n_bins}.")
+
+        valid_methods = [
+            "inverse_cdf",
+            "averaged_inverse_cdf",
+            "closest_observation",
+            "interpolated_inverse_cdf",
+            "hazen",
+            "weibull",
+            "linear",
+            "median_unbiased",
+            "normal_unbiased",
+        ]
+        if method not in valid_methods:
+            raise ValueError(f"`method` must be one of {valid_methods}. Got {method}.")
+        if not isinstance(subsample, int) or subsample < 1:
+            raise ValueError(
+                f"`subsample` must be an integer greater than or equal to 1. "
+                f"Got {subsample}."
+            )
+        if not isinstance(random_state, int) or random_state < 0:
+            raise ValueError(
+                f"`random_state` must be an integer greater than or equal to 0. "
+                f"Got {random_state}."
+            )
+        if not isinstance(dtype, type):
+            raise ValueError(f"`dtype` must be a valid numpy dtype. Got {dtype}.")
+
+    def fit(self, X: np.ndarray, y: object = None) -> object:
+        """
+        Learn bin edges based on quantiles from training data.
+
+        Computes quantile-based bin edges using numpy.percentile. If the dataset
+        contains more samples than `subsample`, a random subset is used. Duplicate
+        edges (which can occur with repeated values) are removed automatically.
+
+        Args:
+            X: Training data (1D numpy array) for computing quantiles.
+            y: Ignored.
+
+        Returns:
+            Self for method chaining.
+
+        Raises:
+            ValueError: If input data X is empty.
+
+        Examples:
+            >>> import numpy as np
+            >>> from spotforecast2.preprocessing import QuantileBinner
+            >>>
+            >>> # Fit with basic data
+            >>> X = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+            >>> binner = QuantileBinner(n_bins=3)
+            >>> _ = binner.fit(X)
+            >>> print(binner.n_bins_)
+            3
+            >>> print(len(binner.bin_edges_))
+            4
+            >>>
+            >>> # Fit with repeated values (may reduce number of bins)
+            >>> X_repeated = np.array([1, 1, 1, 2, 2, 2, 3, 3, 3])
+            >>> binner2 = QuantileBinner(n_bins=5)
+            >>> _ = binner2.fit(X_repeated)
+            >>> # n_bins_ may be less than 5 due to duplicates
+            >>> assert binner2.n_bins_ <= 5
+        """
+        # Note: Original implementation expects X, but sklearn TransformerMixin passes y=None.
+        # Adjusted signature to (self, X: np.ndarray, y: object = None)
+
+        if X.size == 0:
+            raise ValueError("Input data `X` cannot be empty.")
+        if len(X) > self.subsample:
+            rng = np.random.default_rng(self.random_state)
+            X = X[rng.integers(0, len(X), self.subsample)]
+
+        bin_edges = np.percentile(
+            a=X, q=np.linspace(0, 100, self.n_bins + 1), method=self.method
+        )
+
+        # Remove duplicate edges (can happen when data has many repeated values)
+        # to ensure bins are always numbered 0 to n_bins_-1
+        self.bin_edges_ = np.unique(bin_edges)
+
+        # Ensure at least 1 bin when all values are identical
+        if len(self.bin_edges_) == 1:
+            # Create artificial edges around the single value
+            self.bin_edges_ = np.array([self.bin_edges_.item(), self.bin_edges_.item()])
+
+        self.n_bins_ = len(self.bin_edges_) - 1
+
+        if self.n_bins_ != self.n_bins:
+            warnings.warn(
+                f"The number of bins has been reduced from {self.n_bins} to "
+                f"{self.n_bins_} due to duplicated edges caused by repeated predicted "
+                f"values.",
+                IgnoredArgumentWarning,
+            )
+
+        # Internal edges for optimized transform with searchsorted
+        self.internal_edges_ = self.bin_edges_[1:-1]
+        self.intervals_ = {
+            int(i): (float(self.bin_edges_[i]), float(self.bin_edges_[i + 1]))
+            for i in range(self.n_bins_)
+        }
+
+        return self
+
+    def transform(self, X: np.ndarray, y: object = None) -> np.ndarray:
+        """
+        Assign new data to learned bins.
+
+        Uses numpy.searchsorted for efficient bin assignment. Values are assigned
+        to bins following the convention: bins[i-1] <= x < bins[i]. Values outside
+        the fitted range are clipped to the first or last bin.
+
+        Args:
+            X: Data to assign to bins (1D numpy array).
+            y: Ignored.
+
+        Returns:
+            Bin indices as numpy array with dtype specified in __init__.
+
+        Raises:
+            NotFittedError: If fit() has not been called yet.
+
+        Examples:
+            >>> import numpy as np
+            >>> from spotforecast2.preprocessing import QuantileBinner
+            >>>
+            >>> # Fit and transform
+            >>> X_train = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+            >>> binner = QuantileBinner(n_bins=3)
+            >>> _ = binner.fit(X_train)
+            >>>
+            >>> X_test = np.array([1.5, 5.5, 9.5])
+            >>> result = binner.transform(X_test)
+            >>> print(result)
+            [0. 1. 2.]
+            >>>
+            >>> # Values outside range are clipped
+            >>> X_extreme = np.array([0, 100])
+            >>> result_extreme = binner.transform(X_extreme)
+            >>> print(result_extreme)  # Both clipped to valid bin indices
+            [0. 2.]
+        """
+
+        if self.bin_edges_ is None:
+            raise NotFittedError(
+                "The model has not been fitted yet. Call 'fit' with training data first."
+            )
+
+        bin_indices = np.searchsorted(self.internal_edges_, X, side="right").astype(
+            self.dtype
+        )
+
+        return bin_indices
+
+    def fit_transform(self, X, y=None, **fit_params):
+        """
+        Fit to data, then transform it.
+
+        Fits transformer to X and y with optional parameters fit_params
+        and returns a transformed version of X.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input samples.
+
+        y :  array-like of shape (n_samples,) or (n_samples, n_outputs), \
+                default=None
+            Target values (None for unsupervised transformations).
+
+        **fit_params : dict
+            Additional fit parameters.
+
+        Returns
+        -------
+        X_new : ndarray array of shape (n_samples, n_features_new)
+            Transformed array.
+        """
+        # fit_transform is usually provided by TransformerMixin but we can implement it
+        # or rely on inheritance. The original implementation had it explicitly.
+
+        self.fit(X, y)
+        return self.transform(X, y)
+
+    def get_params(self, deep=True):
+        """
+        Get parameters of the quantile binner.
+
+        Returns:
+            Dictionary containing n_bins, method, subsample, dtype, and
+            random_state parameters.
+
+        Examples:
+            >>> import numpy as np
+            >>> from spotforecast2.preprocessing import QuantileBinner
+            >>>
+            >>> binner = QuantileBinner(n_bins=5, method='median_unbiased', subsample=1000)
+            >>> params = binner.get_params()
+            >>> print(params['n_bins'])
+            5
+            >>> print(params['method'])
+            median_unbiased
+            >>> print(params['subsample'])
+            1000
+        """
+
+        return {
+            "n_bins": self.n_bins,
+            "method": self.method,
+            "subsample": self.subsample,
+            "dtype": self.dtype,
+            "random_state": self.random_state,
+        }
+
+    def set_params(self, **params):
+        """
+        Set parameters of the QuantileBinner.
+
+        Args:
+            **params: Parameter names and values to set as keyword arguments.
+
+        Returns:
+            None
+
+        Examples:
+            >>> import numpy as np
+            >>> from spotforecast2.preprocessing import QuantileBinner
+            >>>
+            >>> binner = QuantileBinner(n_bins=3)
+            >>> print(binner.n_bins)
+            3
+            >>> binner.set_params(n_bins=5, method='weibull')
+            >>> print(binner.n_bins)
+            5
+            >>> print(binner.method)
+            weibull
+        """
+
+        for param, value in params.items():
+            setattr(self, param, value)
+        return self

spotforecast2/preprocessing/_common.py

@@ -0,0 +1,123 @@
+"""
+Common preprocessing functions and utilities.
+"""
+
+import functools
+from typing import Callable, Any
+import numpy as np
+from numba import njit
+
+
+def _check_X_numpy_ndarray_1d(ensure_1d: bool = True):
+    """
+    Decorator to check if argument `X` is a 1D numpy ndarray.
+
+    Args:
+        ensure_1d : bool, default True
+            If True, ensure X is a 1D array.
+
+    Returns:
+        wrapper : Callable
+            Decorated function.
+    """
+
+    def decorator(func: Callable):
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            # args[0] is self, args[1] is X (if passed positional)
+            # kwargs might contain X
+            X = kwargs.get("X")
+            if X is None and len(args) > 0:
+                X = args[0]
+
+            if X is not None:
+                if not isinstance(X, np.ndarray):
+                    raise TypeError(f"`X` must be a numpy ndarray. Got {type(X)}.")
+                if ensure_1d and X.ndim != 1:
+                    raise ValueError(f"`X` must be a 1D numpy ndarray. Got {X.ndim}D.")
+
+            return func(self, *args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+@njit(cache=True)
+def _np_mean_jit(x: np.ndarray) -> float:
+    """
+    Numba optimized mean function.
+    """
+    return np.nanmean(x)
+
+
+@njit(cache=True)
+def _np_std_jit(x: np.ndarray) -> float:
+    """
+    Numba optimized std function (ddof=1).
+    """
+    return np.nanstd(x)
+
+
+@njit(cache=True)
+def _np_min_jit(x: np.ndarray) -> float:
+    """
+    Numba optimized min function.
+    """
+    return np.nanmin(x)
+
+
+@njit(cache=True)
+def _np_max_jit(x: np.ndarray) -> float:
+    """
+    Numba optimized max function.
+    """
+    return np.nanmax(x)
+
+
+@njit(cache=True)
+def _np_sum_jit(x: np.ndarray) -> float:
+    """
+    Numba optimized sum function.
+    """
+    return np.nansum(x)
+
+
+@njit(cache=True)
+def _np_median_jit(x: np.ndarray) -> float:
+    """
+    Numba optimized median function.
+    """
+    return np.nanmedian(x)
+
+
+def check_valid_quantile(quantile: float | list[float] | tuple[float]) -> None:
+    """
+    Check if quantile is valid (0 <= quantile <= 1).
+    """
+    if isinstance(quantile, (float, int)):
+        if not (0 <= quantile <= 1):
+            raise ValueError(f"Quantile must be between 0 and 1. Got {quantile}.")
+    elif isinstance(quantile, (list, tuple, np.ndarray)):
+        for q in quantile:
+            if not (0 <= q <= 1):
+                raise ValueError(f"Quantiles must be between 0 and 1. Got {q}.")
+    else:
+        raise TypeError(
+            f"Quantile must be a float, list, tuple or numpy array. Got {type(quantile)}."
+        )
+
+
+def check_is_fitted(estimator: Any, attributes: list[str] | None = None) -> None:
+    """
+    Check if estimator is fitted by verifying if attributes exist.
+    """
+    if attributes is None:
+        attributes = []
+
+    for attr in attributes:
+        if not hasattr(estimator, attr):
+            raise ValueError(
+                f"This {type(estimator).__name__} instance is not fitted yet. "
+                f"Call 'fit' with appropriate arguments before using this estimator."
+            )

spotforecast2/preprocessing/_differentiator.py

@@ -0,0 +1,123 @@
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.utils.validation import check_is_fitted
+from ._common import _check_X_numpy_ndarray_1d
+
+
+class TimeSeriesDifferentiator(BaseEstimator, TransformerMixin):
+    """
+    Transforms a time series into a differenced time series.
+
+    Args:
+        order (int, optional): Order of differentiation. Defaults to 1.
+        initial_values (list, numpy ndarray, optional): Values to be used for the inverse transformation (reverting differentiation).
+            If None, the first `order` values of the training data `X` are stored during `fit`.
+
+    Attributes:
+        initial_values_ (list): Values stored for inverse transformation.
+        last_values_ (list): Last values of the differenced time series.
+    """
+
+    def __init__(self, order: int = 1, initial_values: list | np.ndarray | None = None):
+        self.order = order
+        self.initial_values = initial_values
+
+    @_check_X_numpy_ndarray_1d(ensure_1d=True)
+    def fit(self, X: np.ndarray, y: object = None) -> object:
+        """
+        Store initial values if not provided.
+        """
+        if self.order < 1:
+            raise ValueError("`order` must be a positive integer.")
+
+        if self.initial_values is None:
+            if len(X) < self.order:
+                raise ValueError(
+                    f"The time series must have at least {self.order} values "
+                    f"to compute the differentiation of order {self.order}."
+                )
+            self.initial_values_ = list(X[: self.order])
+        else:
+            if len(self.initial_values) != self.order:
+                raise ValueError(
+                    f"The length of `initial_values` must be equal to the order "
+                    f"of differentiation ({self.order})."
+                )
+            self.initial_values_ = list(self.initial_values)
+
+        self.last_values_ = X[-self.order :]
+
+        return self
+
+    @_check_X_numpy_ndarray_1d(ensure_1d=True)
+    def transform(self, X: np.ndarray, y: object = None) -> np.ndarray:
+        """
+        Compute the differences.
+        """
+        if not hasattr(self, "initial_values_") and self.initial_values is not None:
+            self.fit(X)
+        elif not hasattr(self, "initial_values_"):
+            check_is_fitted(self, ["initial_values_"])
+
+        X_diff = np.diff(X, n=self.order)
+        # Pad with NaNs to keep same length
+        X_diff = np.concatenate([np.full(self.order, np.nan), X_diff])
+
+        # Update last values seen (for next window inverse)
+        self.last_values_ = X[-self.order :]
+
+        return X_diff
+
+    def inverse_transform_next_window(self, X: np.ndarray) -> np.ndarray:
+        """
+        Inverse transform for the next window of predictions.
+        """
+        check_is_fitted(self, ["initial_values_", "last_values_"])
+
+        if self.order == 1:
+            result = np.cumsum(X) + self.last_values_[-1]
+        else:
+            # Recursive or iterative approach for higher orders
+            # Simplified: Assuming order 1 is sufficient for now or throwing error
+            raise NotImplementedError(
+                "inverse_transform_next_window not implemented for order > 1"
+            )
+
+        return result
+
+    @_check_X_numpy_ndarray_1d(ensure_1d=True)
+    def inverse_transform(self, X: np.ndarray, y: object = None) -> np.ndarray:
+        """
+        Revert the differences.
+        """
+        check_is_fitted(self, ["initial_values_"])
+
+        # X contains the differenced series (with NaNs at the beginning potentially)
+        # remove NaNs at the start corresponding to order
+        X_clean = X[self.order :]
+
+        if len(X_clean) == 0:
+            # Just return initial values if only NaNs were passed
+            return np.array(self.initial_values_)
+
+        result = list(self.initial_values_)
+
+        if self.order == 1:
+            current_value = result[-1]
+            restored = []
+            for diff_val in X_clean:
+                current_value += diff_val
+                restored.append(current_value)
+            result.extend(restored)
+        else:
+            # Recursive reconstruction for higher orders logic check
+            # For order > 1, np.diff does repeated diffs.
+            # To invert, we need to do repeated cumsum.
+            # But we need appropriate initial values for each level of integration.
+            # This is a simplified version.
+
+            raise NotImplementedError(
+                "Inverse transform for order > 1 is currently not fully implemented in this port."
+            )
+
+        return np.array(result)