openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/feature_engineering/feature_adder.py

@@ -1,206 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module provides functionality for defining custom feature adders."""
-import inspect
-import re
-from abc import ABC, abstractmethod
-from collections import Counter, namedtuple
-from importlib import import_module
-from typing import Optional, Sequence
-
-import pandas as pd
-
-ParsedFeature = namedtuple("ParsedFeature", ["name", "params"])
-
-
-class FeatureAdder(ABC):
-    """Abstract class that implement the FeatureAdder interface.
-
-    It is the basic block that handles the logic for computing the specific feature and the syntactic sugar to load
-    properly the feature adder according to the feature name.
-
-    """
-
-    @property
-    @abstractmethod
-    def _regex(self) -> str:
-        pass
-
-    @property
-    @abstractmethod
-    def name(self) -> str:
-        """Name of the FeatureAdder."""
-
-    @abstractmethod
-    def required_features(self, feature_names) -> list[str]:
-        """List of features that are required to calculate this feature."""
-
-    def __hash__(self):
-        """Genearate hash of the name of this feature."""
-        return hash(self.name)
-
-    def parse_feature_name(self, feature_name: str) -> Optional[dict[str, str]]:
-        """Parse a feature name.
-
-        If the feature name is taken in charge by the feature adder, the method returns
-        a dictionnary with the potentially parsed parameters contained the feature name. In the
-        case the feature name does not contain parameters an empty dictionary is returned.
-        Otherwise the method returns None.
-
-        Args:
-            feature_name (str): The feature name, this may contain parameter informations.
-
-        Returns:
-            Optional[dict[str, Any]]: The parsed parameters. If the feature name is recognized but has no parameters
-            an empty dictionnary is returned. If the feature name is not recognized, None is
-            returned.
-
-        """
-        reg = self._regex
-        match = re.match(reg, feature_name)
-        return None if match is None else match.groupdict()
-
-    @abstractmethod
-    def apply_features(
-        self, df: pd.DataFrame, parsed_feature_names: Sequence[ParsedFeature]
-    ) -> pd.DataFrame:
-        """Apply or add the features to the input dataframe."""
-
-    def __repr__(self):
-        """Represent as string."""
-        return "%s(<%s>)" % (self.__class__.__name__, self.name)
-
-
-class FeatureDispatcher:
-    """Orchestrator of the feature adders.
-
-    It scans the feature_names to assign to each feature the proper feature adder and launch the effective computing of
-    the features.
-
-    """
-
-    def __init__(self, feature_adders: Sequence[FeatureAdder]):
-        """Initialize feature dispatcher."""
-        self.feature_adders = list(feature_adders)
-        self._check_feature_adder_names_unicity()
-
-    def _check_feature_adder_names_unicity(self):
-        names = Counter(adder.name for adder in self.feature_adders)
-        duplicated_names = []
-        for name, count in names.items():
-            if count > 1:
-                duplicated_names.append(name)
-
-        if len(duplicated_names) > 0:
-            raise RuntimeError(
-                "There is at least one duplicated feature adder name: %s"
-                % duplicated_names
-            )
-
-    def dispatch_features(
-        self, feature_names: list[str]
-    ) -> dict[FeatureAdder, list[ParsedFeature]]:
-        """Dispatch features.
-
-        Args:
-            feature_names: The names of the features to be dispatched.
-
-        Returns:
-            Dictionary with parsed features.
-
-        """
-        recognized_features = set()
-        dispatched_features = {}
-
-        for feature_name in feature_names:
-            for adder_obj in self.feature_adders:
-                parsed_params = adder_obj.parse_feature_name(feature_name)
-                if parsed_params is not None:
-                    if feature_name in recognized_features:
-                        raise RuntimeError(
-                            "Ambiguous feature adder set detected. The feature name"
-                            " '%s' is recognised by more than 1 feature adder"
-                            % feature_names
-                        )
-                    recognized_features.add(feature_name)
-                    features = dispatched_features.setdefault(adder_obj, [])
-                    features.append(ParsedFeature(feature_name, parsed_params))
-
-        return dispatched_features
-
-    def apply_features(
-        self, df: pd.DataFrame, feature_names: list[str]
-    ) -> pd.DataFrame:
-        """Applies features to the input DataFrame.
-
-        Args:
-            df: DataFrame to which the features have to be added.
-            feature_names: Names of the features.
-
-        Returns:
-            DataFrame with the added features.
-
-        """
-        if feature_names is None:
-            return df
-        dispatched_features = self.dispatch_features(feature_names)
-
-        applied_features = set()
-        applied_features_num = 0
-
-        while True:
-            for adder, parsed_features in dispatched_features.items():
-                parsed_feature_names = [pf.name for pf in parsed_features]
-                required_features = adder.required_features(parsed_feature_names)
-
-                if len(set(required_features) - set(df.columns)) == 0:
-                    df = adder.apply_features(df, parsed_features)
-                    applied_features |= set(parsed_feature_names)
-
-            if (
-                len(applied_features) == applied_features_num
-            ):  # No new feature was treated
-                break
-
-            applied_features_num = len(applied_features)
-
-        return df
-
-
-def adders_from_module(module_name: str) -> list[FeatureAdder]:
-    """Load all FeatureAdders classes on the fly from the module.
-
-    Args:
-        module_name: The name of the module from which to import.
-
-    Returns:
-        A list with all loaded FeatureAdders.
-
-    """
-    module = import_module(module_name)
-    feature_adders = []
-
-    for element_name in dir(module):
-        element = getattr(module, element_name)
-        if (
-            isinstance(element, type)
-            and issubclass(element, FeatureAdder)
-            and not inspect.isabstract(element)
-        ):
-            feature_adders.append(element())
-
-    return feature_adders
-
-
-def adders_from_modules(module_names: list[str]) -> list[FeatureAdder]:
-    """Load all FeatureAdders classes on the fly from multiple modules.
-
-    Args:
-        module_names: A list with names of the modules from which to import.
-
-    Returns:
-        A list with all loaded FeatureAdders.
-
-    """
-    return sum((adders_from_module(module_name) for module_name in module_names), [])

openstef/feature_engineering/feature_applicator.py

@@ -1,202 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module defines several FeatureApplicators.
-
-These applicatiors are used to add features to the input data in the corresponding pipelines.
-
-"""
-from abc import ABC, abstractmethod
-from typing import Optional, Union
-
-import numpy as np
-import pandas as pd
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.feature_engineering.apply_features import apply_features
-from openstef.feature_engineering.feature_adder import (
-    FeatureDispatcher,
-    adders_from_modules,
-)
-from openstef.feature_engineering.general import (
-    add_missing_feature_columns,
-    enforce_feature_order,
-    remove_non_requested_feature_columns,
-)
-
-LATENCY_CONFIG = {
-    "day_ahead_electricity_price": 24
-}  # A specific latency is part of a specific feature.
-
-
-class AbstractFeatureApplicator(ABC):
-    """Defines the Applicator interface."""
-
-    def __init__(
-        self,
-        horizons: Union[list[float], str],
-        feature_names: Optional[list[str]] = None,
-        feature_modules: Optional[list[str]] = [],
-    ) -> None:
-        """Initialize abstract feature applicator.
-
-        Args:
-            horizons: list of horizons in hours
-            feature_names:  List of requested features
-            feature_modules: List of modules from which FeatureAdders should be loaded.
-
-        """
-        if not isinstance(horizons, str) and type(horizons) is not list and not None:
-            raise ValueError("horizons must be added as a list")
-
-        self.feature_names = feature_names
-        self.horizons = horizons
-        self.features_adder = adders_from_modules(feature_modules)
-        self.features_dispatcher = FeatureDispatcher(self.features_adder)
-
-    @abstractmethod
-    def add_features(
-        self, df: pd.DataFrame, pj: PredictionJobDataClass = None
-    ) -> pd.DataFrame:
-        """Adds features to an input DataFrame.
-
-        Args:
-            df: DataFrame with input data to which the features have to be added
-            pj: (Optional) A prediction job that is needed for location dependent features,
-                if not specified a default location is used
-
-        Returns:
-            Dataframe with added features.
-
-        """
-
-
-class TrainFeatureApplicator(AbstractFeatureApplicator):
-    """Feature applicator for use during training."""
-
-    def add_features(
-        self,
-        df: pd.DataFrame,
-        pj: PredictionJobDataClass = None,
-        latency_config: dict = None,
-    ) -> pd.DataFrame:
-        """Adds features to an input DataFrame.
-
-        This method is implemented specifically for a model train pipeline. For larger
-        horzions data is invalidated as when they are not available.
-
-        For example:
-            For horzion 24 hours the feature T-720min is not added as the load
-            720 minutes ago is not available 24 hours in advance. In case of a horizon
-            0.25 hours this feature is added as in this case the feature is available.
-
-        Args:
-            df: Input data to which the features will be added.
-            pj: (Optional) A prediction job that is needed for location dependent features,
-                if not specified a default location is used
-            latency_config: (Optional) Invalidate certain features that are not
-                available for a specific horizon due to data latency. Defaults to
-                ``{"day_ahead_electricity_price": 24}``.
-
-        Returns:
-            Input DataFrame with an extra column for every added feature and sorted on the datetime index.
-
-        """
-        # If pj is none add empty dict
-        if pj is None:
-            pj = {}
-
-        if latency_config is None:
-            latency_config = LATENCY_CONFIG
-
-        # Set default horizons if none are provided
-        if self.horizons is None:
-            self.horizons = [0.25, 24]
-
-        # Pre define output variables
-        result = pd.DataFrame()
-
-        if isinstance(self.horizons, str):
-            # copy the custom horizon into the horizon column
-            res = df.copy(deep=True)
-            res["horizon"] = res[self.horizons]
-            result = pd.concat([result, res])
-        else:
-            # Loop over horizons and add corresponding features
-            for horizon in self.horizons:
-                # Deep copy of df is important, because we want a fresh start every iteration!
-                res = apply_features(
-                    df.copy(deep=True),
-                    horizon=horizon,
-                    pj=pj,
-                    feature_names=self.feature_names,
-                )
-                res["horizon"] = horizon
-                result = pd.concat([result, res])
-
-        # Add custom features with the dispatcher
-        result = self.features_dispatcher.apply_features(
-            result, feature_names=self.feature_names
-        )
-
-        # IMPORTANT: sort index to prevent errors when slicing on the (datetime) index
-        # if we don't sort, the duplicated indexes (one per horizon) have large gaps
-        # and slicing will give an exception.
-        result = result.sort_index(axis=0)
-
-        # Invalidate features that are not available for a specific horizon due to data
-        # latency
-        for feature, time in latency_config.items():
-            result.loc[result["horizon"] > time, feature] = np.nan
-
-        # NOTE this is required since apply_features could add additional features
-        if self.feature_names is not None:
-            features = self.feature_names + ["horizon"]
-            result = remove_non_requested_feature_columns(result, features)
-
-        # Sort all features except for the (first) load and (last) horizon columns
-        return enforce_feature_order(result)
-
-
-class OperationalPredictFeatureApplicator(AbstractFeatureApplicator):
-    """Feature applicator for use in operational forecasts."""
-
-    def add_features(
-        self, df: pd.DataFrame, pj: PredictionJobDataClass = None
-    ) -> pd.DataFrame:
-        """Adds features to an input DataFrame.
-
-        This method is implemented specifically for an operational prediction pipeline and will add every available feature.
-
-        Args:
-            df: DataFrame with input data to which the features have to be added
-            pj: (Optional) A prediction job that is needed for location dependent features,
-                if not specified a default location is used
-        Returns:
-           Input DataFrame with an extra column for every added feature.
-
-        """
-        # If pj is none add empty dict
-        if pj is None:
-            pj = {}
-
-        num_horizons = len(self.horizons)
-        if num_horizons != 1:
-            raise ValueError(f"Expected one horizon, got {num_horizons}")
-
-        # Add core features
-        df = apply_features(
-            df, feature_names=self.feature_names, horizon=self.horizons[0], pj=pj
-        )
-        # Add custom features with the dispatcher
-        df = self.features_dispatcher.apply_features(
-            df, feature_names=self.feature_names
-        )
-
-        df = add_missing_feature_columns(df, self.feature_names)
-
-        # NOTE this is required since apply_features could add additional features
-        if self.feature_names is not None:
-            df = remove_non_requested_feature_columns(df, self.feature_names)
-
-        return enforce_feature_order(df)

openstef/feature_engineering/general.py

@@ -1,141 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This modelu contains various helper functions."""
-
-import logging
-
-import numpy as np
-import pandas as pd
-import structlog
-
-from openstef.settings import Settings
-
-
-def add_missing_feature_columns(
-    input_data: pd.DataFrame, features: list[str]
-) -> pd.DataFrame:
-    """Adds feature column for features in the featurelist.
-
-    Add feature columns for features in the feature list if these columns don't
-    exist in the input data. If a column is added, its value is set to NaN.
-    This is especially usefull to make sure the required columns are in place when
-    making a prediction.
-
-    .. note::
-        This function is intended as a final check to prevent errors during predicion.
-        In an ideal world this function is not nescarry.
-
-    Args:
-        input_data: DataFrame with input data and featurs.
-        features: List of requiered features.
-
-    Returns:
-        Input dataframe with missing columns filled with ``np.N=nan``.
-
-    """
-    structlog.configure(
-        wrapper_class=structlog.make_filtering_bound_logger(
-            logging.getLevelName(Settings.log_level)
-        )
-    )
-    logger = structlog.get_logger(__name__)
-
-    if features is None:
-        features = []
-
-    missing_features = [f for f in features if f not in list(input_data)]
-
-    for feature in missing_features:
-        logger.warning(
-            f"Adding NaN column for missing feature: {feature}", missing_feature=feature
-        )
-        input_data[feature] = np.nan
-
-    return input_data
-
-
-def remove_non_requested_feature_columns(
-    input_data: pd.DataFrame, requested_features: list[str]
-) -> pd.DataFrame:
-    """Removes features that are provided in the input data but not in the feature list.
-
-    This should not be nescesarry but serves as an extra failsave for making predicitons
-
-    Args:
-        input_data: DataFrame with features
-        requested_features: List of reuqested features
-
-    Returns:
-        Model input data with features.
-
-    """
-    structlog.configure(
-        wrapper_class=structlog.make_filtering_bound_logger(
-            logging.getLevelName(Settings.log_level)
-        )
-    )
-    logger = structlog.get_logger(__name__)
-
-    if requested_features is None:
-        requested_features = []
-
-    not_requested_features = [
-        f for f in list(input_data) if f not in requested_features
-    ]
-
-    # Do not see "load" or "horizon" as an extra feature as it is no feature
-    if "load" in not_requested_features:
-        not_requested_features.remove("load")
-
-    num_not_requested_features = len(not_requested_features)
-
-    if num_not_requested_features != 0:
-        logger.warning(
-            f"Removing {num_not_requested_features} unrequested features!",
-            num_not_requested_features=num_not_requested_features,
-        )
-
-    return input_data.drop(not_requested_features, axis=1)
-
-
-def enforce_feature_order(input_data: pd.DataFrame) -> pd.DataFrame:
-    """Enforces correct order of features.
-
-    Alphabetically orders the feature columns. The load column remains the first column
-    and the horizons column remains the last column.
-    Everything in between is alphabetically sorted:
-    The order eventually looks like this:
-    ["load"] -- [alphabetically sorted features] -- ['horizon']
-
-    This function assumes the first column contains the to be predicted variable
-    Furthermore the "horizon" is moved to the last position if it is pressent.
-
-    Args:
-        input_data: Input data with features.
-
-    Returns:
-        Properly sorted input data.
-
-    """
-    # Extract first column name
-    first_column_name = input_data.columns.to_list()[
-        0
-    ]  # Most of the time this is "load"
-
-    # Sort columns
-    columns = list(np.sort(input_data.columns.to_list()))
-
-    # Remove first column and add to the start
-    columns.remove(first_column_name)
-    column_order = [first_column_name] + columns
-
-    # If "Horzion" column is available add to the end
-    if "horizon" in columns:
-        # "horizon" is pressent in the training procces
-        # but not in the forecasting process
-        column_order.remove("horizon")
-        column_order = column_order + ["horizon"]
-
-    # Return dataframe with columns in the correct order
-    return input_data.loc[:, column_order]