openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/model_selection/model_selection.py

@@ -1,311 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import random
-import secrets
-from datetime import timedelta
-from itertools import accumulate
-from typing import Iterable
-
-import numpy as np
-import pandas as pd
-
-AMOUNT_DAY = 96  # Duration of the periods (in T-15) that are in a day (default = 96)
-PERIOD_TIMEDELTA = 1  # Duration of the periods (in days) that will be sampled as validation data for each split.
-PEAK_FRACTION = 0.15
-
-
-def group_kfold(
-    input_data: pd.DataFrame, n_folds: int, randomize_fold_split: bool = True
-) -> pd.DataFrame:
-    """Function to group data into groups, according to the date and the number of folds.
-
-    Each date gets assigned a number between 0 and n_folds.
-
-    Args:
-        input_data: Input data
-        n_folds: Number of folds
-        randomize_fold_split: Indicates if random split needs to be applied
-
-    Returns:
-        Grouped data
-
-    """
-    unique_dates = input_data["dates"].unique()  # dates defines the day (Y-M-D)
-    # Group separators
-    len_data = len(unique_dates)  # number of indices that can be used for splitting
-    size = len_data // n_folds  # size of each fold set
-    rem = (
-        len_data % n_folds
-    )  # remaining number of indices when divided in fold sets of equal size
-    separators = list(
-        accumulate([0] + [size + 1] * rem + [size] * (n_folds - rem))
-    )  # location of seperators
-
-    items = list(unique_dates)
-
-    if randomize_fold_split:
-        random.shuffle(items)  # if random, shuffle the days
-
-    for i, s in enumerate(zip(separators, separators[1:])):
-        group = items[slice(*s)]
-        input_data.loc[
-            input_data[input_data["dates"].isin(group)].index, "random_fold"
-        ] = i
-    return input_data
-
-
-def sample_indices_train_val(
-    data: pd.DataFrame, peaks: pd.DataFrame
-) -> tuple[np.array, np.array]:
-    """Sample indices of given period length assuming the peaks are evenly spreaded.
-
-    Args:
-        data: Clean data with features
-        peaks: Data frame of selected peaks to sample the dates from
-
-    Returns:
-        - List with the start point of each peak
-        - Sorted list with the indices corresponding to the peak
-
-    """
-    sampled = set()
-    peaks_val = []
-
-    for peak in peaks:
-        sampled |= set(data[data.index.date == peak].index)
-        peaks_val.append(peak)
-    return peaks_val, np.sort(list(sampled))
-
-
-def random_sample(all_peaks: np.array, k: int) -> np.array:
-    """Random sampling of numbers out of a np.array.
-
-    Implemented due to security sonar cloud not accepting the random built-in functions.
-
-    Args:
-        all_peaks: List with numbers to sample from
-        k: Number of wanted samples
-
-    Returns:
-        Sorted array with the random samples (dates from the peaks)
-
-    """
-    random_peaks = []
-    all_peaks_list = all_peaks.tolist()
-    for _ in range(k):
-        element_random = secrets.choice(all_peaks_list)
-        all_peaks_list.remove(element_random)
-        random_peaks.append(element_random)
-    return np.array(random_peaks)
-
-
-def split_data_train_validation_test(
-    data_: pd.DataFrame,
-    test_fraction: float = 0.1,
-    validation_fraction: float = 0.15,
-    back_test: bool = False,
-    stratification_min_max: bool = True,
-) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
-    """Split input data into train, test and validation set.
-
-    Function for splitting data with features in a train, test and
-    validation dataset. In an operational setting the following sequence is
-    returned (when using stratification):
-
-    Train >> Validation (and the test is the Train and Validation combined.)
-
-    For a back test (indicated with argument "back_test") the following sequence
-    is returned:
-
-    Train >> Validation >> Test
-
-    The ratios of the different types can be set with test_fraction and
-    validation fraction.
-
-    Args:
-        data_: Cleaned data with features
-        test_fraction : Number between 0 and 1 that indicates the desired
-            fraction of test data.
-        validation_fraction: Number between 0 and 1 that indicates the
-            desired fraction of validation data.
-        back_test: Indicates if data is intended for a back test.
-        stratification_min_max: Indicates if validation data must be sampled as
-            periods, using stratification on min and max values per day.
-            If True, 'extreme days' are ensured to be included in the validation and train sets,
-            ensuring the validation set to be representative of the train set.
-
-    Returns:
-        - Train data.
-        - Validation data.
-        - Test data.
-
-    Raises:
-        ValueError: When the test and validation fractions are too high.
-
-    """
-    test_fraction = test_fraction if back_test else 0
-    train_fraction = 1 - (test_fraction + validation_fraction)
-    if train_fraction < 0:
-        raise ValueError(
-            "Test ({test_fraction}) and validation fraction ({validation_fraction}) too"
-            " high."
-        )
-
-    # Define constants
-    min_days_for_stratification = 4
-    extreme_values_fraction = (
-        0.15  # 15percent of highest and lowest values are considered extremes
-    )
-
-    # Get start date from the index
-    start_date = data_.index.min().to_pydatetime()
-    end_date = data_.index.max().to_pydatetime()
-
-    # Calculate total of quarter hours (PTU's) in input data
-    number_indices = len(data_.index.unique())  # Total number of unique timepoints
-    delta = (
-        data_.index.unique().sort_values()[1] - data_.index.unique().sort_values()[0]
-    )  # Delta t, assumed to be constant throughout DataFrame
-    delta = timedelta(
-        seconds=delta.seconds
-    )  # Convert from pandas timedelta to original python timedelta
-
-    # Determine which dates are in testset
-    if back_test:
-        start_date_test = end_date - np.round(number_indices * test_fraction) * delta
-        test_data = data_[start_date_test:]
-        train_val_data = data_[:start_date_test]
-        operational_score_data = (
-            pd.DataFrame()
-        )  # Empty because a backtest is no operational setting.
-    else:
-        start_date_val = start_date + np.round(number_indices * test_fraction) * delta
-        test_data = data_[
-            :start_date_val
-        ]  # Empty as all data is used for training in an operational setting.
-        train_val_data = data_[start_date_val:]
-        operational_score_data = data_.copy(deep=True).reset_index(
-            drop=True
-        )  # Used to check wether a new operationally train model is better than the old one.
-
-    if stratification_min_max and (
-        len(set(train_val_data.index.date)) >= min_days_for_stratification
-    ):
-        # First how many dates are considers min or max.
-        # Note that this should be at least to, so one can go to the train set
-        # and another to the testset
-        train_val_dates = list(set(train_val_data.index.date))
-        n_days_per_min_max_subset = int(
-            max(extreme_values_fraction * len(set(data_.index.date)), 2)
-        )
-        # Find max_dates
-        max_dates = (
-            train_val_data[["load"]]
-            .resample("1D")
-            .max()
-            .sort_values(by="load", ascending=False)
-            .dropna()
-            .index[:n_days_per_min_max_subset]
-        ).date
-        # Find min_dates, but do not consider the max_dates
-        min_dates_subset = train_val_data.loc[
-            ~np.isin(train_val_data.index.date, max_dates), ["load"]
-        ]
-        min_dates = (
-            min_dates_subset[["load"]]
-            .resample("1D")
-            .min()
-            .sort_values(by="load", ascending=True)
-            .dropna()
-            .index[:n_days_per_min_max_subset]
-        ).date
-        other_dates = [
-            x for x in train_val_dates if x not in min_dates and x not in max_dates
-        ]
-
-        # Divide min, max and other dates fairly over validation and train set, with at least 1 min and max in train and validation
-        val_dates = []
-        train_dates = []
-        for date_set in [max_dates, min_dates, other_dates]:
-            n_days_val = max(1, int(validation_fraction * len(date_set)))
-            val_dates += list(
-                np.random.choice(list(date_set), n_days_val, replace=False)
-            )
-            train_dates += [x for x in date_set if x not in val_dates]
-
-        validation_data = train_val_data[np.isin(train_val_data.index.date, val_dates)]
-        train_data = train_val_data[np.isin(train_val_data.index.date, train_dates)]
-
-    # Default sampling, take a one single validation set.
-    else:
-        if back_test:
-            start_date_train = (
-                start_date + np.round(number_indices * validation_fraction) * delta
-            )
-            end_date_train = end_date - np.round(number_indices * test_fraction) * delta
-            validation_data = data_[:start_date_train]
-            train_data = data_[start_date_train:end_date_train]
-        else:
-            start_date_val = (
-                start_date + np.round(number_indices * test_fraction) * delta
-            )
-            start_date_train = (
-                start_date_val + np.round(number_indices * validation_fraction) * delta
-            )
-            train_data = data_[start_date_train:None]
-            validation_data = data_[start_date_val:start_date_train]
-
-    train_data = train_data.sort_index()
-    validation_data = validation_data.sort_index()
-    test_data = test_data.sort_index()
-
-    return (train_data, validation_data, test_data, operational_score_data)
-
-
-def backtest_split_default(
-    data: pd.DataFrame,
-    n_folds: int,
-    test_fraction: float = 0.15,
-    stratification_min_max: bool = True,
-    randomize_fold_split: bool = False,
-) -> Iterable[tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]]:
-    """Default cross validation strategy.
-
-    Args:
-        data:
-        n_folds:
-        test_fraction:
-        stratification_min_max:
-        randomize_fold_split:
-
-    Returns:
-        Iterable on train, val, test splits
-
-    Notes:
-        We use a generator in order to have lazy estimation and avoid multiple copy of the data.
-
-    """
-    if n_folds > 1:
-        data.index = pd.to_datetime(data.index)
-        data["dates"] = data.index
-        data = group_kfold(data, n_folds, randomize_fold_split)
-
-        for ifold in range(n_folds):
-            test_data = data[data["random_fold"] == ifold].sort_index()
-
-            (train_data, validation_data, _, _) = split_data_train_validation_test(
-                data[data["random_fold"] != ifold].iloc[:, :-2],
-                test_fraction=0,
-                back_test=True,
-                stratification_min_max=stratification_min_max,
-            )
-
-            yield train_data, validation_data, test_data.iloc[:, :-2], pd.DataFrame()
-    else:
-        yield split_data_train_validation_test(
-            data,
-            back_test=True,
-            test_fraction=test_fraction,
-            stratification_min_max=stratification_min_max,
-        )

openstef/monitoring/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0

openstef/monitoring/performance_meter.py

@@ -1,92 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-from collections import OrderedDict
-from time import perf_counter
-
-
-class PerformanceMeter:
-    def __init__(self, logger):
-        self.logger = logger
-        self.levels = OrderedDict()
-        self.level_timers = []
-        self.checkpoint_timers = []
-
-    def start_level(self, level_label: str, level_name: str, **kwargs):
-        """Enters a new level in the performance meter and logs it.
-
-        This function also creates a checkpoint on the newly created level.
-
-        Args:
-            level_label: The label of the new level. This could i.e. be 'task'
-            level_name: The name of the specified level.
-            **kwargs: Any other kwargs are appended to the logging.
-
-        Returns:
-            self
-
-        """
-        self.levels[level_label] = level_name
-
-        self.logger.info(
-            f"{level_label.capitalize()} started",
-            **self.levels,
-            **kwargs,
-        )
-        time = perf_counter()
-        self.level_timers.append(time)
-        self.checkpoint_timers.append(time)
-
-        return self
-
-    def checkpoint(self, name_checkpoint: str, **kwargs):
-        """Creates a timing checkpoint and logs the runtime from the previous one.
-
-        Args:
-            name_checkpoint: The name of the checkpoint. This will be logged as
-                "checkpoint: name_checkpoint"
-            **kwargs: Any other kwargs are appended to the logging.
-
-        Returns:
-            self
-
-        """
-        runtime = round(perf_counter() - self.checkpoint_timers.pop(), ndigits=3)
-        self.logger.info(
-            f"{name_checkpoint.capitalize()} completed",
-            **self.levels,
-            ktp_checkpoint=name_checkpoint,
-            ktp_runtime=runtime,
-            **kwargs,
-        )
-        self.checkpoint_timers.append(perf_counter())
-
-        return self
-
-    def complete_level(self, successful: bool = True, **kwargs):
-        """Completes the most inner level and logs the total runtime of that level.
-
-        Args:
-            successful: Whether the level was successful. Defaults to True.
-            **kwargs: Any other kwargs are appended to the logging.
-
-        Returns:
-            self
-
-        """
-        runtime = round(perf_counter() - self.level_timers.pop(), ndigits=3)
-        self.checkpoint_timers.pop()
-
-        level_label, level_name = self.levels.popitem()
-
-        self.logger.info(
-            f"{level_label.capitalize()} completed",
-            **self.levels,
-            **{level_label: level_name},
-            ktp_runtime=runtime,
-            ktp_successful=int(successful),
-            **kwargs,
-        )
-
-        return self

openstef/monitoring/teams.py

@@ -1,203 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import logging
-from typing import Union
-
-import pandas as pd
-import pymsteams
-import structlog
-from pymsteams import cardsection
-
-from openstef.settings import Settings
-
-
-def post_teams(
-    msg: Union[str, dict],
-    invalid_coefficients: pd.DataFrame = None,
-    coefficients_df: pd.DataFrame = None,
-    url: str = None,
-    proxies: dict = None,
-) -> None:
-    """Post a message to Teams - KTP.
-
-    Note that currently no authentication occurs.
-    Security is given by keeping the URL secret.
-    One should therefore refrain from using more enhanced features such as
-    action buttons.
-
-    Args:
-        msg: For simple messages a string can be passed. For more
-            complex messages pass a dict. The following keys are supported:
-            text, links, sections. Each section can contain the following keys:
-            text, title, images, facts, markdown. Also see:
-            https://docs.microsoft.com/en-us/outlook/actionable-messages/send-via-connectors
-        invalid_coefficients: df of information of invalid
-            coefficients. Defaults to None.
-        coefficients_df: df of new coefficients. Defaults to None.
-        url: webhook url, monitoring by default
-        proxies: Optinonal proxy settings.
-
-    Note:
-        This function is namespace-specific.
-    """
-    if not Settings.post_teams_messages:
-        return
-
-    structlog.configure(
-        wrapper_class=structlog.make_filtering_bound_logger(
-            logging.getLevelName(Settings.log_level)
-        )
-    )
-    logger = structlog.get_logger(__name__)
-    # If no url is passed, give warning and don't send teams message
-    if url is None:
-        logger.warning("Can't post Teams message, no url given.")
-        return
-
-    # Add invalid coefficients and manual coefficients-query to message
-    if invalid_coefficients is not None and coefficients_df is not None:
-        # add invalid coefficient information to message in dict-format
-        invalid_coefficients_text = "".join(
-            [
-                f"\n* **{row.coef_name}**: {round(row.coef_value_new, 2)}, "
-                f"(previous: {round(row.coef_value_last, 2)})"
-                for index, row in invalid_coefficients.iterrows()
-            ]
-        )
-        query = build_sql_query_string(coefficients_df, "energy_split_coefs")
-        query_text = (
-            "If you would like to update the coefficients manually in the "
-            + "database, use this query:"
-        )
-        msg = {
-            "fallback": msg,
-            "title": "Invalid energy splitting coefficients",
-            "text": msg,
-            "sections": [
-                {
-                    "text": invalid_coefficients_text,
-                    "markdown": True,
-                },
-                {
-                    "title": "Manual query",
-                    "text": query_text,
-                    "markdown": True,
-                },
-                {
-                    "text": query,
-                    "markdown": True,
-                },
-            ],
-        }
-
-    card = pymsteams.connectorcard(url)
-
-    # add proxies
-    # NOTE the connectorcard.proxy is passed to the requests library under the hood
-    card.proxies = proxies
-
-    # if msg is string, convert to dict
-    if type(msg) is str:
-        msg = dict(text=msg)
-    card.text(msg.get("text"))
-    card.summary(msg.get("fallback", "-"))
-
-    # set title, color, ...
-    card.color(msg.get("color", "white"))
-    card.title(msg.get("title"))
-
-    link_dicts = msg.get("links", [])  # link_dicts can be single dict or list of dicts
-    if isinstance(link_dicts, dict):  # if single dict
-        card.addLinkButton(link_dicts["buttontext"], link_dicts["buttonurl"])
-    elif isinstance(link_dicts, list):  # if list of dicts
-        for link_dict in link_dicts:
-            card.addLinkButton(link_dict["buttontext"], link_dict["buttonurl"])
-
-    # Add sections
-    for section_dict in msg.get("sections", []):
-        card_section = get_card_section(section_dict=section_dict)
-        card.addSection(card_section)
-
-    card.send()
-
-
-def get_card_section(section_dict: dict) -> cardsection:
-    """Get card section for teams message from dictionary."""
-    card_section = cardsection()
-    card_section.text(section_dict.get("text"))
-    card_section.title(section_dict.get("title"))
-    for image in section_dict.get("images", []):
-        card_section.addImage(image)
-    for fact in section_dict.get("facts", []):
-        card_section.addFact(*fact)
-    if not section_dict.get("markdown", True):
-        card_section.disableMarkdown()
-    if "link" in section_dict:
-        card_section.linkButton(
-            section_dict.get("link").get("buttontext"),
-            section_dict.get("link").get("buttonurl"),
-        )
-    return card_section
-
-
-def build_sql_query_string(df: pd.DataFrame, table: str) -> str:
-    """Build sql insert query string for Teams message output from df.
-
-    Args:
-        df: Df of table values to insert in sql.
-        table: Table to insert df into.
-
-    Returns:
-        Sql query of insert statement.
-
-    """
-    # round all values to two decimals
-    df = df.round(2)
-    # convert datetime to string format
-    datetime_columns = df.columns[
-        df.columns.isin(["date_start", "date_end", "created"])
-    ]
-    for col in datetime_columns:
-        df[col] = df[col].astype("str")
-
-    sql_texts = [
-        "```  \nINSERT INTO "
-        + table
-        + " ("
-        + str(", ".join(df.columns))
-        + ") VALUES  \n"
-    ]
-    for index, row in df.iterrows():
-        if index != df.index[0]:
-            sql_texts.append(",  \n")  # 2 spaces and \n create a new line
-        sql_texts.append(str(tuple(row.values)))
-    sql_texts.append("  \n```")
-    query = "".join(sql_texts)
-    return query
-
-
-def format_message(title: str, params: dict, fallback=None, color=None) -> dict:
-    if color is None:
-        color = "#046b00"  # green
-    if fallback is None:
-        fallback = title
-
-    # format allparams using limited precision for floats
-    # make keys bold (**key**)
-    values = []
-    for k, v in params.items():
-        if type(v) is float:
-            values.append(f"**{k}**: {v:0.3f}")
-            continue
-        values.append(f"**{k}**: {v}")
-    # join all {params}: {value}  pairs with a new line
-    text = "".join([f"* {v}   \n" for v in values])
-
-    msg = {
-        "fallback": fallback,
-        "title": title,
-        "text": text,
-        "color": color,
-    }
-    return msg

openstef/pipeline/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0