spotforecast2 0.0.1__py3-none-any.whl

spotforecast2/preprocessing/_rolling.py

@@ -0,0 +1,136 @@
+import numpy as np
+import pandas as pd
+from typing import List, Any
+from ._common import (
+    _np_mean_jit,
+    _np_std_jit,
+    _np_min_jit,
+    _np_max_jit,
+    _np_sum_jit,
+    _np_median_jit,
+)
+
+
+class RollingFeatures:
+    """
+    Compute rolling features (stats) over a window of the time series.
+    Compatible with scikit-learn transformers API (fit, transform).
+
+    Attributes:
+        stats_funcs (list): List of rolling statistics functions.
+        window_sizes (list): List of window sizes.
+        features_names (list): List of feature names.
+    """
+
+    def __init__(
+        self,
+        stats: str | List[str] | List[Any],
+        window_sizes: int | List[int],
+        features_names: List[str] | None = None,
+    ):
+        """
+        Initialize the rolling features transformer.
+
+        Args:
+            stats (str | List[str] | List[Any]): Rolling statistics to compute.
+            window_sizes (int | List[int]): Window sizes for rolling statistics.
+            features_names (List[str] | None, optional): Names of the features.
+                Defaults to None.
+        """
+        self.stats = stats
+        self.window_sizes = window_sizes
+        self.features_names = features_names
+
+        # Validation and processing logic...
+        self._validate_params()
+
+    def _validate_params(self):
+        """
+        Validate the parameters of the rolling features transformer.
+        """
+        if isinstance(self.window_sizes, int):
+            self.window_sizes = [self.window_sizes]
+
+        if isinstance(self.stats, str):
+            self.stats = [self.stats]
+
+        # Map strings to functions
+        valid_stats = {
+            "mean": _np_mean_jit,
+            "std": _np_std_jit,
+            "min": _np_min_jit,
+            "max": _np_max_jit,
+            "sum": _np_sum_jit,
+            "median": _np_median_jit,
+        }
+
+        self.stats_funcs = []
+        for s in self.stats:
+            if isinstance(s, str):
+                if s not in valid_stats:
+                    raise ValueError(
+                        f"Stat '{s}' not supported. Supported: {list(valid_stats.keys())}"
+                    )
+                self.stats_funcs.append(valid_stats[s])
+            else:
+                self.stats_funcs.append(s)
+
+        if self.features_names is None:
+            self.features_names = []
+            for ws in self.window_sizes:
+                for s in self.stats:
+                    s_name = s if isinstance(s, str) else s.__name__
+                    self.features_names.append(f"roll_{s_name}_{ws}")
+
+    def fit(self, X, y=None):
+        """
+        Fit the rolling features transformer.
+
+        Args:
+            X (np.ndarray): Time series to transform.
+            y (object, optional): Ignored.
+
+        Returns:
+            self: Fitted rolling features transformer.
+        """
+        return self
+
+    def transform(self, X: np.ndarray) -> np.ndarray:
+        """
+        Compute rolling features.
+
+        Args:
+            X (np.ndarray): Time series to transform.
+
+        Returns:
+            np.ndarray: Array with rolling features.
+        """
+        # Assume X is 1D array
+        n_samples = len(X)
+        output = np.full((n_samples, len(self.features_names)), np.nan)
+
+        idx_feature = 0
+        for ws in self.window_sizes:
+            for func in self.stats_funcs:
+                # Naive rolling window loop - can be optimized or use pandas rolling
+                # Using pandas for simplicity and speed if X is convertible
+                series = pd.Series(X)
+                rolled = series.rolling(window=ws).apply(func, raw=True)
+                output[:, idx_feature] = rolled.values
+                idx_feature += 1
+
+        return output
+
+    def transform_batch(self, X: pd.Series) -> pd.DataFrame:
+        """
+        Transform a pandas Series to rolling features DataFrame.
+
+        Args:
+            X (pd.Series): Time series to transform.
+
+        Returns:
+            pd.DataFrame: DataFrame with rolling features.
+        """
+        values = X.to_numpy()
+        transformed = self.transform(values)
+        return pd.DataFrame(transformed, index=X.index, columns=self.features_names)

spotforecast2/preprocessing/curate_data.py

@@ -0,0 +1,254 @@
+import pandas as pd
+
+
+def get_start_end(
+    data: pd.DataFrame,
+    forecast_horizon: int,
+    verbose: bool = True,
+) -> tuple[str, str, str, str]:
+    """Get start and end date strings for data and covariate ranges.
+    Covariate range is extended by the forecast horizon.
+
+    Args:
+        data (pd.DataFrame):
+            The dataset with a datetime index.
+        forecast_horizon (int):
+            The forecast horizon in hours.
+        verbose (bool):
+            Whether to print the determined date ranges.
+
+    Returns:
+        tuple[str, str, str, str]: (data_start, data_end, covariate_start, covariate_end)
+            Date strings in the format "YYYY-MM-DDTHH:MM" for data and covariate ranges.
+
+    Examples:
+        >>> from spotforecast2.preprocessing.curate_data import get_start_end
+        >>> import pandas as pd
+        >>> date_rng = pd.date_range(start='2023-01-01', end='2023-01-10', freq='h')
+        >>> data = pd.DataFrame(date_rng, columns=['date'])
+        >>> data.set_index('date', inplace=True)
+        >>> start, end, cov_start, cov_end = get_start_end(data, forecast_horizon=24, verbose=False)
+        >>> print(start, end, cov_start, cov_end)
+        2023-01-01T00:00 2023-01-10T00:00 2023-01-01T00:00 2023-01-11T00:00
+    """
+    FORECAST_HORIZON = forecast_horizon
+
+    START = data.index.min().strftime("%Y-%m-%dT%H:%M")
+    END = data.index.max().strftime("%Y-%m-%dT%H:%M")
+    if verbose:
+        print(f"Data range: {START} to {END}")
+    # Define covariate range relative to data range
+    COV_START = START
+    # Extend end date by forecast horizon to include future covariates
+    COV_END = (pd.to_datetime(END) + pd.Timedelta(hours=FORECAST_HORIZON)).strftime(
+        "%Y-%m-%dT%H:%M"
+    )
+    if verbose:
+        print(f"Covariate data range: {COV_START} to {COV_END}")
+    return START, END, COV_START, COV_END
+
+
+def curate_holidays(
+    holiday_df: pd.DataFrame, data: pd.DataFrame, forecast_horizon: int
+):
+    """Checks if the holiday dataframe has the correct shape.
+    Args:
+        holiday_df (pd.DataFrame):
+            DataFrame containing holiday information.
+        data (pd.DataFrame):
+            The main dataset.
+        forecast_horizon (int):
+            The forecast horizon in hours.
+
+    Examples:
+        >>> from spotforecast2.data.fetch_data import fetch_data, fetch_holiday_data
+        >>> from spotforecast2.preprocessing.curate_data import get_start_end, curate_holidays
+        >>> data = fetch_data()
+        >>> START, END, COV_START, COV_END = get_start_end(
+        ...     data=data,
+        ...     forecast_horizon=24,
+        ...     verbose=False
+        ... )
+        >>> holiday_df = fetch_holiday_data(
+        ...     start='2023-01-01T00:00',
+        ...     end='2023-01-10T00:00',
+        ...     tz='UTC',
+        ...     freq='h',
+        ...     country_code='DE',
+        ...     state='NW'
+        ... )
+        >>> FORECAST_HORIZON = 24
+        >>> curate_holidays(holiday_df, data, forecast_horizon=FORECAST_HORIZON)
+
+    Raises:
+        AssertionError:
+            If the holiday dataframe does not have the correct number of rows.
+    """
+    try:
+        assert holiday_df.shape[0] == data.shape[0] + forecast_horizon
+        print("Holiday dataframe has correct shape.")
+    except AssertionError:
+        print("Holiday dataframe has wrong shape.")
+
+
+def curate_weather(weather_df: pd.DataFrame, data: pd.DataFrame, forecast_horizon: int):
+    """Checks if the weather dataframe has the correct shape.
+
+    Args:
+        weather_df (pd.DataFrame):
+            DataFrame containing weather information.
+        data (pd.DataFrame):
+            The main dataset.
+        forecast_horizon (int):
+            The forecast horizon in hours.
+
+    Examples:
+        >>> from spotforecast2.data.fetch_data import fetch_data, fetch_weather_data
+        >>> from spotforecast2.preprocessing.curate_data import get_start_end, curate_weather
+        >>> data = fetch_data()
+        >>> START, END, COV_START, COV_END = get_start_end(
+        ...     data=data,
+        ...     forecast_horizon=24,
+        ...     verbose=False
+        ... )
+        >>> weather_df = fetch_weather_data(
+        ...     cov_start=COV_START,
+        ...     cov_end=COV_END,
+        ...     tz='UTC',
+        ...     freq='h',
+        ...     latitude=51.5136,
+        ...     longitude=7.4653
+        ... )
+        >>> FORECAST_HORIZON = 24
+        >>> curate_weather(weather_df, data, forecast_horizon=FORECAST_HORIZON)
+
+    Raises:
+        AssertionError:
+            If the weather dataframe does not have the correct number of rows.
+    """
+    try:
+        assert weather_df.shape[0] == data.shape[0] + forecast_horizon
+        print("Weather dataframe has correct shape.")
+    except AssertionError:
+        print("Weather dataframe has wrong shape.")
+
+
+def basic_ts_checks(data: pd.DataFrame, verbose: bool = False) -> bool:
+    """Checks if the time series data has a datetime index and is sorted.
+
+    Args:
+        data (pd.DataFrame):
+            The main dataset.
+        verbose (bool):
+            Whether to print additional information.
+
+    Examples:
+        >>> from spotforecast2.data.fetch_data import fetch_data
+        >>> from spotforecast2.preprocessing.curate_data import basic_ts_checks
+        >>> data = fetch_data()
+        >>> basic_ts_checks(data)
+
+    Raises:
+        TypeError:
+            If the index is not a datetime index.
+        ValueError:
+            If the datetime index is not sorted in increasing order or is incomplete.
+
+    Returns:
+        bool: True if the datetime index is valid, sorted, and complete.
+    """
+    # Check if the time series data has a datetime index
+    if not pd.api.types.is_datetime64_any_dtype(data.index):
+        raise TypeError("The index is not a datetime index.")
+
+    # Check if the datetime index is sorted
+    if not data.index.is_monotonic_increasing:
+        raise ValueError("The datetime index is not sorted in increasing order.")
+
+    # Check if the index is complete (no missing timestamps)
+    start_date = data.index.min()
+    end_date = data.index.max()
+    complete_date_range = pd.date_range(
+        start=start_date, end=end_date, freq=data.index.freq
+    )
+    is_index_complete = (data.index == complete_date_range).all()
+
+    if not is_index_complete:
+        raise ValueError(
+            "The datetime index has missing timestamps and is not complete."
+        )
+    if verbose:
+        print(
+            "The time series data has a valid datetime index that is sorted and complete."
+        )
+    return True
+
+
+def agg_and_resample_data(
+    data: pd.DataFrame,
+    rule: str = "h",
+    closed: str = "left",
+    label: str = "left",
+    by="mean",
+    verbose: bool = False,
+) -> pd.DataFrame:
+    """
+    Aggregates and resamples the data to (e.g.,hourly) frequency by computing the specified aggregation (e.g. for each hour).
+
+    Args:
+        data (pd.DataFrame):
+            The dataset with a datetime index.
+        rule (str):
+            The resample rule (e.g., 'h' for hourly, 'D' for daily).
+            Default is 'h' which creates an hourly grid.
+        closed (str):
+            Which side of bin interval is closed. Default is 'left'.
+            Using `closed="left", label="left"` specifies that a time interval
+            (e.g., 10:00 to 11:00) is labeled with the start timestamp (10:00).
+            For consumption data, a different representation is usually more common:
+            `closed="left", label="right"`, so the interval is labeled with the end
+            timestamp (11:00), since consumption is typically reported after one hour.
+        label (str):
+            Which bin edge label to use. Default is 'left'.
+            See 'closed' parameter for details on labeling behavior.
+        by (str or callable):
+            Aggregation method to apply (e.g., 'mean', 'sum', 'median').
+            Default is 'mean'.
+            The aggregation serves robustness: if the data were more finely resolved
+            (e.g., quarter-hourly), asfreq would only pick one value (sampling),
+            while .agg("mean") forms the correct average over the hour.
+            If the data is already hourly, .agg doesn't change anything but ensures
+            that no duplicates exist.
+        verbose (bool):
+            Whether to print additional information.
+
+    Returns:
+        pd.DataFrame: Resampled and aggregated dataframe.
+
+    Notes:
+        - resample(rule="h"): Creates an hourly grid
+        - closed/label: Control how time intervals are labeled
+        - .agg({...: by}): Aggregates values within each time bin
+
+    Examples::
+        >>> from spotforecast2.preprocessing.curate_data import agg_and_resample_data
+        >>> import pandas as pd
+        >>> date_rng = pd.date_range(start='2023-01-01', end='2023-01-02', freq='15T')
+        >>> data = pd.DataFrame(date_rng, columns=['date'])
+        >>> data.set_index('date', inplace=True)
+        >>> data['value'] = range(len(data))
+        >>> resampled_data = agg_and_resample_data(data, rule='h', by='mean')
+        >>> print(resampled_data.head())
+    """
+    if verbose:
+        print(f"Original data shape: {data.shape}")
+    # Create aggregation dictionary for all columns
+    agg_dict = {col: by for col in data.columns}
+
+    data = data.resample(rule=rule, closed=closed, label=label).agg(agg_dict)
+    if verbose:
+        print(
+            f"Data resampled with rule='{rule}', closed='{closed}', label='{label}', aggregation='{by}'."
+        )
+        print(f"Resampled data shape: {data.shape}")
+    return data

spotforecast2/preprocessing/imputation.py

@@ -0,0 +1,92 @@
+import pandas as pd
+
+
+def custom_weights(index, weights_series: pd.Series) -> float:
+    """
+    Return 0 if index is in or near any gap.
+
+    Args:
+        index (pd.Index):
+            The index to check.
+        weights_series (pd.Series):
+            Series containing weights.
+
+    Returns:
+        float: The weight corresponding to the index.
+
+    Examples:
+        >>> from spotforecast2.data.fetch_data import fetch_data
+        >>> from spotforecast2.preprocessing.imputation import custom_weights
+        >>> data = fetch_data()
+        >>> _, missing_weights = get_missing_weights(data, window_size=72, verbose=False)
+        >>> for idx in data.index[:5]:
+        ...     weight = custom_weights(idx, missing_weights)
+        ...     print(f"Index: {idx}, Weight: {weight}")
+    """
+    # do plausibility check
+    if isinstance(index, pd.Index):
+        if not index.isin(weights_series.index).all():
+            raise ValueError("Index not found in weights_series.")
+        return weights_series.loc[index].values
+
+    if index not in weights_series.index:
+        raise ValueError("Index not found in weights_series.")
+    return weights_series.loc[index]
+
+
+def get_missing_weights(
+    data: pd.DataFrame, window_size: int = 72, verbose: bool = False
+) -> tuple[pd.DataFrame, pd.Series]:
+    """
+    Return imputed DataFrame and a series indicating missing weights.
+
+    Args:
+        data (pd.DataFrame):
+            The input dataset.
+        window_size (int):
+            The size of the rolling window to consider for missing values.
+        verbose (bool):
+            Whether to print additional information.
+
+    Returns:
+        Tuple[pd.DataFrame, pd.Series]:
+            A tuple containing the forward and backward filled DataFrame and a boolean series where True indicates missing weights.
+
+    Examples:
+        >>> from spotforecast2.data.fetch_data import fetch_data
+        >>> from spotforecast2.preprocessing.imputation import get_missing_weights
+        >>> data = fetch_data()
+        >>> filled_data, missing_weights = get_missing_weights(data, window_size=72, verbose=True)
+
+    """
+    # first perform some checks if dataframe has enough data and if window_size is appropriate
+    if data.shape[0] == 0:
+        raise ValueError("Input data is empty.")
+    if window_size <= 0:
+        raise ValueError("window_size must be a positive integer.")
+    if window_size >= data.shape[0]:
+        raise ValueError("window_size must be smaller than the number of rows in data.")
+
+    missing_indices = data.index[data.isnull().any(axis=1)]
+    n_missing = len(missing_indices)
+    if verbose:
+        pct_missing = (n_missing / len(data)) * 100
+        print(f"Number of rows with missing values: {n_missing}")
+        print(f"Percentage of rows with missing values: {pct_missing:.2f}%")
+        print(f"missing_indices: {missing_indices}")
+    data = data.ffill()
+    data = data.bfill()
+
+    is_missing = pd.Series(0, index=data.index)
+    is_missing.loc[missing_indices] = 1
+    weights_series = 1 - is_missing.rolling(window=window_size + 1, min_periods=1).max()
+    if verbose:
+        n_missing_after = weights_series.isna().sum()
+        pct_missing_after = (n_missing_after / len(data)) * 100
+        print(
+            f"Number of rows with missing weights after processing: {n_missing_after}"
+        )
+        print(
+            f"Percentage of rows with missing weights after processing: {pct_missing_after:.2f}%"
+        )
+    return data, weights_series.isna()

spotforecast2/preprocessing/outlier.py

@@ -0,0 +1,114 @@
+from sklearn.ensemble import IsolationForest
+import numpy as np
+import pandas as pd
+
+
+def mark_outliers(
+    data: pd.DataFrame,
+    contamination: float = 0.1,
+    random_state: int = 1234,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, np.ndarray]:
+    """Marks outliers as NaN in the dataset using Isolation Forest.
+
+    Args:
+        data (pd.DataFrame):
+            The input dataset.
+        contamination (float):
+            The (estimated) proportion of outliers in the dataset.
+        random_state (int):
+            Random seed for reproducibility. Default is 1234.
+        verbose (bool):
+            Whether to print additional information.
+
+    Returns:
+        tuple[pd.DataFrame, np.ndarray]: A tuple containing the modified dataset with outliers marked as NaN and the outlier labels.
+
+    Examples:
+        >>> from spotforecast2.data.fetch_data import fetch_data
+        >>> from spotforecast2.preprocessing.outlier import mark_outliers
+        >>> data = fetch_data()
+        >>> cleaned_data, outlier_labels = mark_outliers(data, contamination=0.1, random_state=42, verbose=True)
+    """
+    for col in data.columns:
+        iso = IsolationForest(contamination=contamination, random_state=random_state)
+        # Fit and predict (-1 for outliers, 1 for inliers)
+        outliers = iso.fit_predict(data[[col]])
+
+        # Mark outliers as NaN
+        data.loc[outliers == -1, col] = np.nan
+
+        pct_outliers = (outliers == -1).mean() * 100
+        if verbose:
+            print(
+                f"Column '{col}': Marked {pct_outliers:.4f}% of data points as outliers."
+            )
+    return data, outliers
+
+
+def manual_outlier_removal(
+    data: pd.DataFrame,
+    column: str,
+    lower_threshold: float | None = None,
+    upper_threshold: float | None = None,
+    verbose: bool = False,
+) -> tuple[pd.DataFrame, int]:
+    """Manual outlier removal function.
+    Args:
+        data (pd.DataFrame):
+            The input dataset.
+        column (str):
+            The column name in which to perform manual outlier removal.
+        lower_threshold (float | None):
+            The lower threshold below which values are considered outliers.
+            If None, no lower threshold is applied.
+        upper_threshold (float | None):
+            The upper threshold above which values are considered outliers.
+            If None, no upper threshold is applied.
+        verbose (bool):
+            Whether to print additional information.
+
+    Returns:
+        tuple[pd.DataFrame, int]: A tuple containing the modified dataset with outliers marked as NaN and the number of outliers marked.
+
+    Examples:
+        >>> from spotforecast2.data.fetch_data import fetch_data
+        >>> from spotforecast2.preprocessing.outlier import manual_outlier_removal
+        >>> data = fetch_data()
+        >>> data, n_manual_outliers = manual_outlier_removal(
+        ...     data,
+        ...     column='ABC',
+        ...     lower_threshold=50,
+        ...     upper_threshold=700,
+        ...     verbose=True
+    """
+    if lower_threshold is None and upper_threshold is None:
+        if verbose:
+            print(f"No thresholds provided for {column}; no outliers marked.")
+        return data, 0
+
+    if lower_threshold is not None and upper_threshold is not None:
+        mask = (data[column] > upper_threshold) | (data[column] < lower_threshold)
+    elif lower_threshold is not None:
+        mask = data[column] < lower_threshold
+    else:
+        mask = data[column] > upper_threshold
+
+    n_manual_outliers = mask.sum()
+
+    data.loc[mask, column] = np.nan
+
+    if verbose:
+        if lower_threshold is not None and upper_threshold is not None:
+            print(
+                f"Manually marked {n_manual_outliers} values > {upper_threshold} or < {lower_threshold} as outliers in {column}."
+            )
+        elif lower_threshold is not None:
+            print(
+                f"Manually marked {n_manual_outliers} values < {lower_threshold} as outliers in {column}."
+            )
+        else:
+            print(
+                f"Manually marked {n_manual_outliers} values > {upper_threshold} as outliers in {column}."
+            )
+    return data, n_manual_outliers