openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/model/metamodels/grouped_regressor.py

@@ -1,222 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2021 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module defines the grouped regressor."""
-from typing import Any, Callable, Union
-
-import joblib
-import numpy as np
-import pandas as pd
-from pandas.core.groupby.generic import DataFrameGroupBy
-from sklearn.base import BaseEstimator, MetaEstimatorMixin, RegressorMixin, clone
-from sklearn.utils.validation import check_is_fitted
-
-
-class GroupedRegressor(BaseEstimator, RegressorMixin, MetaEstimatorMixin):
-    """Meta-model that trains an instance of the base estimator for each key of a groupby operation applied on the data.
-
-    The base estimator is a sklearn regressor, the groupby is performed on the columns specified in parameters.
-    Moreover fit and predict methods can be performed in parallel for each group key thanks to joblib.
-
-    Example:
-
-    .. code-block:: md
-
-        data =  | index | group | x0 | x1 | x3 | y |
-                |   0   |   1   | .. | .. | .. | . |
-                |   1   |   2   | .. | .. | .. | . |
-                |   2   |   1   | .. | .. | .. | . |
-                |   3   |   2   | .. | .. | .. | . |
-
-                [              X              ][ Y ]
-
-
-    The GroupedRegressor on the data with the group_columns='group' fits 2 models:
-        - The model 1 with the row 0 and 2, columns x0, x1 and x3 as the features and column y as the target.
-        - The model 2 with the row 1 and 3, columns x0, x1 and x3 as the features and column y as the target.
-
-    Args:
-        base_estimator: Regressor .
-
-        group_columns: Name(s) of the column(s) used as the key for groupby operation.
-
-        n_jobs: default=1
-            The maximum number of concurrently running jobs,
-            such as the number of Python worker processes when backend=”multiprocessing”
-            or the size of the thread-pool when backend=”threading
-
-    Attributes:
-        feature_names_:  All input feature (without group_columns).
-
-        estimators_:
-            Dictionnary that stocks fitted estimators for each group.
-            The keys are the keys of grouping and the values are the regressors fitted on the grouped data.
-
-    """
-
-    def __init__(
-        self,
-        base_estimator: RegressorMixin,
-        group_columns: Union[str, int, list[str], list[int]],
-        n_jobs: int = 1,
-    ):
-        """Initialize meta model."""
-        self.base_estimator = base_estimator
-        if type(group_columns) in [int, str]:
-            self.group_columns = [group_columns]
-        else:
-            self.group_columns = group_columns
-        self.n_jobs = n_jobs
-
-    def _get_tags(self):
-        return self.base_estimator._get_tags()
-
-    def _check_group_columns(self, df):
-        if type(self.group_columns) is not list:
-            raise ValueError(
-                "The group columns parameter should be a list, it gets a {}".format(
-                    type(self.group_columns)
-                )
-            )
-        for c in self.group_columns:
-            if c not in list(df.columns):
-                raise ValueError("The group column {} is missing!".format(c))
-
-    def _partial_fit(
-        self, group: Any, df_group: pd.DataFrame, eval_set=None, **kwargs
-    ) -> tuple[Any, BaseEstimator]:
-        estimator = clone(self.base_estimator)
-        X = df_group.loc[:, self.feature_names_]
-        y = df_group.loc[:, "__target__"]
-
-        if eval_set is None:
-            estimator_fitted = estimator.fit(X, y, **kwargs)
-        else:
-            estimator_fitted = estimator.fit(
-                X,
-                y,
-                eval_set=[
-                    (
-                        df.loc[
-                            (df[self.group_columns] == group).to_numpy().flatten(),
-                            self.feature_names_,
-                        ],
-                        df.loc[
-                            (df[self.group_columns] == group).to_numpy().flatten(),
-                            "__target__",
-                        ],
-                    )
-                    for df in eval_set
-                ],
-                **kwargs
-            )
-
-        return (group, estimator_fitted)
-
-    def _partial_predict(self, group, df_group, **kwargs):
-        return self.estimators_[group].predict(df_group, **kwargs)
-
-    @classmethod
-    def grouped_compute(
-        cls,
-        df: pd.DataFrame,
-        group_columns: Union[list[str], list[int]],
-        func: Callable[[tuple, pd.DataFrame], np.array],
-        n_jobs: int = 1,
-        eval_set=None,
-    ) -> tuple[tuple[np.array, ...], DataFrameGroupBy, pd.DataFrame]:
-        """Computes the specified function on each group defined by the grouping columns.
-
-        It is an utility function used to perform fit and predict on each group.
-        The df_res is the final dataframe that aggregate the results for each
-        group. The group_res is a tuple where each field is corresponding to a results for a group. The gb is the
-        grouping object.
-
-        Args:
-            df: DataFrame containing the input data necessary for the computation .
-            group_columns: List of the columns used for the groupby operation
-            func: Function that take the group key and the conrresponding data of this group
-                and perform the computation on this group.
-            n_jobs: The maximum number of concurrently running jobs,
-
-        Returns:
-            The tuple of the results of each group, the grouping dataframe and the global dataframe of results.
-
-        """
-        index_name = df.index.name or "index"
-        df_reset = df.reset_index()
-
-        df_res = pd.DataFrame(index=df_reset.index)
-
-        gb = df_reset.groupby(group_columns)
-
-        if n_jobs > 1:
-            # Preferred scaling is at cluster level (e.g. k8s/serverless) instead of process level
-            group_res = joblib.Parallel(n_jobs=n_jobs)(
-                joblib.delayed(func)(
-                    group, df_group.set_index(index_name).drop(group_columns, axis=1)
-                )
-                for group, df_group in gb
-            )
-        else:
-            group_res = (
-                func(group, df_group.set_index(index_name).drop(group_columns, axis=1))
-                for group, df_group in gb
-            )
-        return group_res, gb, df_res
-
-    def _grouped_predict(self, df: pd.DataFrame, n_jobs: int = 1, **kwargs) -> np.array:
-        group_res, gb, df_res = self.grouped_compute(
-            df,
-            self.group_columns,
-            lambda group, df_group: self._partial_predict(group, df_group, **kwargs),
-            n_jobs,
-        )
-
-        for (group, group_index), result in zip(gb.groups.items(), group_res):
-            df_res.loc[group_index, "__result__"] = np.array(result)
-
-        return df_res["__result__"].to_numpy()
-
-    def _grouped_fit(
-        self, df: pd.DataFrame, n_jobs: int = 1, eval_set=None, **kwargs
-    ) -> dict[Any, BaseEstimator]:
-        group_res, _, _ = self.grouped_compute(
-            df,
-            self.group_columns,
-            lambda group, df_group: self._partial_fit(
-                group, df_group, eval_set=eval_set, **kwargs
-            ),
-            n_jobs,
-        )
-        return dict(group_res)
-
-    def fit(self, x: np.ndarray, y: np.ndarray, eval_set=None, **kwargs):
-        """Fit the model."""
-        df = pd.DataFrame(x).copy(deep=True)
-        self._check_group_columns(df)
-
-        eval_df = None
-        if eval_set is not None:
-            eval_df = []
-            for x_set, y_set in eval_set:
-                self._check_group_columns(x_set)
-                df_set = pd.DataFrame(x_set).copy(deep=True)
-                df_set["__target__"] = y_set
-                eval_df.append(df_set)
-
-        self.feature_names_ = [
-            c for c in list(df.columns) if c not in self.group_columns
-        ]
-        df.loc[:, "__target__"] = y
-        self.estimators_ = self._grouped_fit(
-            df, self.n_jobs, eval_set=eval_df, **kwargs
-        )
-        return self
-
-    def predict(self, x: np.ndarray, **kwargs) -> np.ndarray:
-        """Make a predicion."""
-        check_is_fitted(self)
-        df = pd.DataFrame(x)
-        self._check_group_columns(df)
-        return self._grouped_predict(df, self.n_jobs, **kwargs)

openstef/model/metamodels/missing_values_handler.py

@@ -1,138 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module defines the missing value handler."""
-from typing import Union
-
-import numpy as np
-import pandas as pd
-from sklearn.base import BaseEstimator, MetaEstimatorMixin, RegressorMixin, clone
-from sklearn.impute import SimpleImputer
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import FunctionTransformer
-from sklearn.utils.validation import check_array, check_is_fitted, check_X_y
-
-
-class MissingValuesHandler(BaseEstimator, RegressorMixin, MetaEstimatorMixin):
-    """Class for a meta-models that handles missing values and removes columns filled exclusively by NaN.
-
-    It's a pipeline of:
-
-        - An Imputation transformer for completing missing values.
-        - A Regressor fitted on the filled data.
-
-    Args:
-        base_estimator: Regressor used in the pipeline.
-        missing_values: The placeholder for the missing values. All occurrences of
-            `missing_values` will be imputed. For pandas' dataframes with
-            nullable integer dtypes with missing values, `missing_values`
-            should be set to `np.nan`, since `pd.NA` will be converted to `np.nan`.
-        imputation_strategy:  The imputation strategy.
-            - If None no imputation is performed.
-            - If "mean", then replace missing values using the mean along
-            each column. Can only be used with numeric data.
-            - If "median", then replace missing values using the median along
-            each column. Can only be used with numeric data.
-            - If "most_frequent", then replace missing using the most frequent
-            value along each column. Can be used with strings or numeric data.
-            If there is more than one such value, only the smallest is returned.
-            - If "constant", then replace missing values with fill_value. Can be
-            used with strings or numeric data.
-        fill_value: When strategy == "constant", fill_value is used to replace all
-            occurrences of missing_values.
-            If left to the default, fill_value will be 0 when imputing numerical
-            data and "missing_value" for strings or object data types.
-
-    Attributes:
-        feature_names:
-            All input feature.
-
-        non_null_columns_:
-            Valid features used by the regressor.
-
-        n_features_in_:
-            Number of input features.
-
-        regressor_: RegressorMixin
-            Regressor fitted on valid columns.
-
-        imputer_: SimpleImputer
-            Imputer for missig value fitted on valid columns.
-
-        pipeline_: Pipeline
-            Pipeline that chains the imputer and the regressor.
-
-        feature_importances_: ndarray (n_features_in_, )
-            The feature importances from the regressor for valid features and zero otherwise.
-
-    """
-
-    def __init__(
-        self,
-        base_estimator: RegressorMixin,
-        missing_values: Union[int, float, str, None] = np.nan,
-        imputation_strategy: str = None,
-        fill_value: Union[str, int, float] = None,
-    ):
-        """Initialize missing values handler."""
-        self.base_estimator = base_estimator
-        self.missing_values = missing_values
-        self.imputation_strategy = imputation_strategy
-        self.fill_value = fill_value
-
-    def _get_tags(self):
-        tags = self.base_estimator._get_tags()
-        tags["requires_y"] = True
-        tags["multioutput"] = False
-        tags["allow_nan"] = self.imputation_strategy is not None
-        return tags
-
-    def fit(self, x, y):
-        """Fit model."""
-        _, y = check_X_y(x, y, force_all_finite="allow-nan", y_numeric=True)
-        if type(x) != pd.DataFrame:
-            x = pd.DataFrame(np.asarray(x))
-        self.feature_in_names_ = list(x.columns)
-        self.n_features_in_ = x.shape[1]
-
-        # Remove always null columns
-        is_column_null = x.isnull().all(axis="index")
-        self.non_null_columns_ = list(x.columns[~is_column_null])
-
-        self.regressor_ = clone(self.base_estimator)
-
-        # Build the proper imputation transformer
-        # - Identity function if strategy is None
-        # - SimpleImputer with the dedicated strategy
-        if self.imputation_strategy is None:
-            self.imputer_ = FunctionTransformer(func=self._identity)
-        else:
-            self.imputer_ = SimpleImputer(
-                missing_values=self.missing_values,
-                strategy=self.imputation_strategy,
-                fill_value=self.fill_value,
-            )
-
-        self.pipeline_ = Pipeline(
-            [("imputer", self.imputer_), ("regressor", self.regressor_)]
-        )
-
-        # Fit only on non_null_columns
-        self.pipeline_.fit(x[self.non_null_columns_], y)
-
-        return self
-
-    @classmethod
-    def _identity(cls, x):
-        return x
-
-    def predict(self, x):
-        """Make a prediction."""
-        check_is_fitted(self)
-        check_array(
-            x,
-            force_all_finite="allow-nan",
-        )
-        if type(x) != pd.DataFrame:
-            x = pd.DataFrame(np.array(x))
-        return self.pipeline_.predict(x[self.non_null_columns_])

openstef/model/model_creator.py

@@ -1,214 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import logging
-from typing import Union
-
-import structlog
-
-from openstef.enums import ModelType
-from openstef.model.regressors.arima import ARIMAOpenstfRegressor
-from openstef.model.regressors.custom_regressor import is_custom_type, load_custom_model
-from openstef.model.regressors.gblinear_quantile import GBLinearQuantileOpenstfRegressor
-from openstef.model.regressors.lgbm import LGBMOpenstfRegressor
-from openstef.model.regressors.linear import LinearOpenstfRegressor
-from openstef.model.regressors.linear_quantile import LinearQuantileOpenstfRegressor
-from openstef.model.regressors.regressor import OpenstfRegressor
-from openstef.model.regressors.flatliner import FlatlinerRegressor
-from openstef.model.regressors.xgb import XGBOpenstfRegressor
-from openstef.model.regressors.xgb_quantile import XGBQuantileOpenstfRegressor
-from openstef.model.regressors.xgb_multioutput_quantile import (
-    XGBMultiOutputQuantileOpenstfRegressor,
-)
-from openstef.settings import Settings
-
-structlog.configure(
-    wrapper_class=structlog.make_filtering_bound_logger(
-        logging.getLevelName(Settings.log_level)
-    )
-)
-logger = structlog.get_logger(__name__)
-
-valid_model_kwargs = {
-    ModelType.XGB: [
-        "n_estimators",
-        "objective",
-        "max_depth",
-        "learning_rate",
-        "verbosity",
-        "booster",
-        "tree_method",
-        "gamma",
-        "min_child_weight",
-        "max_delta_step",
-        "subsample",
-        "colsample_bytree",
-        "colsample_bylevel",
-        "colsample_bynode",
-        "reg_alpha",
-        "reg_lambda",
-        "scale_pos_weight",
-        "base_score",
-        "missing",
-        "num_parallel_tree",
-        "kwargs",
-        "random_state",
-        "n_jobs",
-        "monotone_constraints",
-        "interaction_constraints",
-        "importance_type",
-        "gpu_id",
-        "validate_parameters",
-        "early_stopping_rounds",
-    ],
-    ModelType.LGB: [
-        "boosting_type",
-        "objective",
-        "num_leaves",
-        "max_depth",
-        "learning_rate",
-        "n_estimators",
-        "subsample_for_bin",
-        "min_split_gain",
-        "min_child_weight",
-        "min_child_samples",
-        "subsample",
-        "subsample_freq",
-        "colsample_bytree",
-        "reg_alpha",
-        "reg_lambda",
-        "random_state",
-        "n_jobs",
-        "silent",
-        "importance_type",
-        "early_stopping_rounds",
-    ],
-    ModelType.XGB_QUANTILE: [
-        "quantiles",
-        "gamma",
-        "colsample_bytree",
-        "subsample",
-        "min_child_weight",
-        "max_depth",
-        "early_stopping_rounds",
-    ],
-    ModelType.XGB_MULTIOUTPUT_QUANTILE: [
-        "quantiles",
-        "gamma",
-        "colsample_bytree",
-        "subsample",
-        "min_child_weight",
-        "max_depth",
-        "early_stopping_rounds",
-        "arctan_smoothing",
-    ],
-    ModelType.LINEAR: [
-        "missing_values",
-        "imputation_strategy",
-        "fill_value",
-    ],
-    ModelType.FLATLINER: [
-        "quantiles",
-    ],
-    ModelType.LINEAR_QUANTILE: [
-        "alpha",
-        "quantiles",
-        "solver",
-        "missing_values",
-        "imputation_strategy",
-        "fill_value",
-        "weight_scale_percentile",
-        "weight_exponent",
-        "weight_floor",
-        "no_fill_future_values_features",
-    ],
-    ModelType.GBLINEAR_QUANTILE: [
-        "quantiles",
-        "missing_values",
-        "imputation_strategy",
-        "fill_value",
-        "weight_scale_percentile",
-        "weight_exponent",
-        "weight_floor",
-        "no_fill_future_values_features",
-        "clipped_features",
-        "learning_rate",
-        "num_boost_round",
-        "early_stopping_rounds",
-        "reg_alpha",
-        "reg_lambda",
-        "updater",
-        "feature_selector",
-        "top_k",
-    ],
-    ModelType.ARIMA: [
-        "backtest_max_horizon",
-        "order",
-        "seasonal_order",
-        "trend",
-    ],
-}
-
-
-class ModelCreator:
-    """Factory object for creating machine learning models."""
-
-    # Set object mapping
-    MODEL_CONSTRUCTORS = {
-        ModelType.XGB: XGBOpenstfRegressor,
-        ModelType.LGB: LGBMOpenstfRegressor,
-        ModelType.XGB_QUANTILE: XGBQuantileOpenstfRegressor,
-        ModelType.XGB_MULTIOUTPUT_QUANTILE: XGBMultiOutputQuantileOpenstfRegressor,
-        ModelType.LINEAR: LinearOpenstfRegressor,
-        ModelType.LINEAR_QUANTILE: LinearQuantileOpenstfRegressor,
-        ModelType.GBLINEAR_QUANTILE: GBLinearQuantileOpenstfRegressor,
-        ModelType.ARIMA: ARIMAOpenstfRegressor,
-        ModelType.FLATLINER: FlatlinerRegressor,
-    }
-
-    @staticmethod
-    def create_model(model_type: Union[ModelType, str], **kwargs) -> OpenstfRegressor:
-        """Create a machine learning model based on model type.
-
-        Args:
-            model_type: Model type to construct.
-            kwargs: Optional keyword argument to pass to the model.
-
-        Raises:
-            NotImplementedError: When using an invalid model_type.
-
-        Returns:
-            OpenSTEF model
-
-        """
-        try:
-            # This will raise a ValueError when an invalid model_type str is used
-            # and nothing when a MLModelType enum is used.
-            if is_custom_type(model_type):
-                model_class = load_custom_model(model_type)
-                valid_kwargs = model_class.valid_kwargs()
-            else:
-                model_type = ModelType(model_type)
-                model_class = ModelCreator.MODEL_CONSTRUCTORS[model_type]
-                valid_kwargs = valid_model_kwargs[model_type]
-                # Check if model as imported
-                if model_class is None:
-                    raise ImportError(
-                        f"Constructor not available for '{model_type}'. "
-                        "Perhaps you forgot to install an optional dependency? "
-                        "Please refer to the ReadMe for instructions"
-                    )
-        except ValueError as e:
-            valid_types = [t.value for t in ModelType]
-            raise NotImplementedError(
-                f"No constructor for '{model_type}', "
-                f"valid model_types are: {valid_types} "
-                "or import a custom model"
-            ) from e
-
-        # only pass relevant arguments to model constructor to prevent warnings
-        model_kwargs = {
-            key: value for key, value in kwargs.items() if key in valid_kwargs
-        }
-
-        return model_class(**model_kwargs)