openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/tasks/split_forecast.py

@@ -1,273 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-
-"""This module contains the CRON job that is periodically executed to make prognoses of solar features.
-
-These features are usefull for splitting the load
-in solar and wind contributions.
-This is achieved by carrying out the folowing steps:
-  1. Get the wind and solar reference data for the specific location of the
-     customer
-  2. Get the TDCV (Typical Domestic Consumption Values) data
-  3. Fit a linear combination of above time series to the historic load data to
-     determine the contributions of each energy source.
-  4. Write the resulting coeficients to the SQL database.
-
-Example:
-    This module is meant to be called directly from a CRON job. A description of
-    the CRON job can be found in the /k8s/CronJobs folder.
-    Alternatively this code can be run directly by running::
-
-        $ python split_forecast.py
-
-"""
-import logging
-from datetime import datetime
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import scipy.optimize
-import structlog
-
-import openstef.monitoring.teams as monitoring
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import ModelType
-from openstef.settings import Settings
-from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
-from openstef.tasks.utils.taskcontext import TaskContext
-
-COEF_MAX_FRACTION_DIFF = 0.3
-
-
-def main(config=None, database=None):
-    taskname = Path(__file__).name.replace(".py", "")
-
-    if database is None or config is None:
-        raise RuntimeError(
-            "Please specifiy a config object and/or database connection object. These"
-            " can be found in the openstef-dbc package."
-        )
-
-    with TaskContext(taskname, config, database) as context:
-        model_type = [ml.value for ml in ModelType]
-
-        PredictionJobLoop(
-            context,
-            model_type=model_type,
-        ).map(split_forecast_task, context)
-
-
-def split_forecast_task(
-    pj: PredictionJobDataClass,
-    context: TaskContext,
-) -> pd.DataFrame:
-    """Function that caries out the energy splitting for a specific prediction job with id pid.
-
-    Args:
-        pid: Prediction job id
-
-    Returns:
-        Energy splitting coefficients.
-
-    """
-    structlog.configure(
-        wrapper_class=structlog.make_filtering_bound_logger(
-            logging.getLevelName(Settings.log_level)
-        )
-    )
-    logger = structlog.get_logger(__name__)
-
-    logger.info("Start splitting energy", pid=pj["id"])
-
-    # Get input for splitting
-    input_split_function = context.database.get_input_energy_splitting(pj)
-
-    # Old split method;
-    # find_components() gives two things:
-    # - the split components (load, solar, wind, consumption, Inschatting (=sum of others) )
-    # - coefdict: coefficients of each component; these are not yet an output of Dazls. Lets discuss with JM if we want that
-
-    # Carry out the splitting
-    components, coefdict = find_components(input_split_function)
-
-    # Calculate mean absolute error (MAE)
-    # TODO: use a standard metric function for this
-    error = components[["load", "Inschatting"]].diff(axis=1).iloc[:, 1]
-    mae = error.abs().mean()
-    coefdict.update({"MAE": mae})
-    coefsdf = convert_coefdict_to_coefsdf(pj, input_split_function, coefdict)
-
-    # Get the coefs of previous runs and check if new coefs are valid
-    last_coefsdict = context.database.get_energy_split_coefs(pj)
-    last_coefsdf = convert_coefdict_to_coefsdf(pj, input_split_function, last_coefsdict)
-    invalid_coefs = determine_invalid_coefs(coefsdf, last_coefsdf)
-    if not invalid_coefs.empty:
-        # If coefs not valid, do not update the coefs in the db and send teams
-        # message that something strange is happening
-        monitoring.post_teams(
-            f"New splitting coefficient(s) for pid **{pj['id']}** deviate strongly "
-            "from previously stored coefficients.",
-            url=context.config.teams_monitoring_url,
-            invalid_coefficients=invalid_coefs,
-            coefficients_df=coefsdf,
-        )
-        # Use the last known coefficients for further processing
-        return last_coefsdf
-    else:
-        # Save Results
-        context.database.write_energy_splitting_coefficients(
-            coefsdf, if_exists="append"
-        )
-        logger.info(
-            "Succesfully wrote energy split coefficients to database", pid=pj["id"]
-        )
-        return coefsdf
-
-
-def determine_invalid_coefs(
-    new_coefs: pd.DataFrame, last_coefs: pd.DataFrame
-) -> pd.DataFrame:
-    """Determine which new coefficients are valid and return them.
-
-    Args:
-        new_coefs: df of new coefficients for standard load
-            profiles (i.e. wind, solar, household)
-        last_coefs: df of last coefficients for standard load
-            profiles (i.e. wind, solar, household)
-
-    Returns:
-        Dataframe with invalid coefficients
-
-    """
-    merged_coefs = pd.merge(
-        last_coefs, new_coefs, on="coef_name", how="left", suffixes=["_last", "_new"]
-    )
-    # calculate difference between new and last coefficients, if no new
-    # coefficient, set difference to inf
-    # If coefficient name is not present in new coefficients list, fail. If coefficient
-    # name is not present in last coefficients list, add it.
-    merged_coefs["difference"] = (
-        (merged_coefs.coef_value_last - merged_coefs.coef_value_new)
-        .abs()
-        .fillna(np.inf)
-    )
-    # Check if the absolute difference between last coefficients and new coefficients
-    # is more than COEF_MAX_FRACTION_DIFF x absolute value of last coefficient
-    invalid_coefs = merged_coefs[
-        merged_coefs.difference
-        > (COEF_MAX_FRACTION_DIFF * merged_coefs.coef_value_last).abs()
-    ]
-    return invalid_coefs
-
-
-def convert_coefdict_to_coefsdf(
-    pj: PredictionJobDataClass, input_split_function: pd.DataFrame, coefdict: dict
-) -> pd.DataFrame:
-    """Convert dictionary of coefficients to dataframe with additional data for db storage.
-
-    Args:
-        pj: prediction job
-        input_split_function: df of columns of standard load profiles,
-            i.e. wind, solar, household
-        coefdict: dict of coefficient per standard load profile
-
-    Returns:
-        DataFrame of coefficients to insert in sql
-
-    """
-    #
-    sql_column_labels = ["pid", "date_start", "date_end", "created"]
-    sql_colum_values = [
-        pj["id"],
-        input_split_function.index.min().date(),
-        input_split_function.index.max().date(),
-        datetime.utcnow(),
-    ]
-    coefsdf = pd.DataFrame(
-        {"coef_name": list(coefdict.keys()), "coef_value": list(coefdict.values())}
-    )
-    for i, column in enumerate(sql_column_labels):
-        coefsdf[column] = sql_colum_values[i]
-
-    return coefsdf
-
-
-def find_components(
-    df: pd.DataFrame, zero_bound: bool = True
-) -> tuple[pd.DataFrame, dict]:
-    """Function that does the actual energy splitting.
-
-    Args:
-        df: Input data. The dataframe should contain these columns
-            in exactly this order: [load, wind_ref, pv_ref, mulitple tdcv colums]
-        zero_bound: If zero_bound is True coefficients can't be negative.
-
-    Returns:
-        tuple:
-            - DataFrame containing the wind and solar components
-            - Dict with the coefficients that result from the fitting
-
-    """
-    load = df.iloc[:, 0]
-    wind_ref = df.iloc[:, 1]
-    pv_ref = df.iloc[:, 2]
-
-    # Define scaler
-    nedu_scaler = (load.max() - load.min()) / 10
-
-    # Come up with inital guess for the fitting
-    p_wind_guess = 1.0
-    ppv_guess = 1.0
-    p0 = [p_wind_guess, ppv_guess] + (len(df.columns) - 3) * [nedu_scaler]
-
-    # Define fitting bounds
-    if zero_bound:
-        bounds = (0, "inf")
-    else:
-        bounds = ("-inf", "inf")
-
-    # Define function to fit
-    def weighted_sum(x, *args):
-        if len(x) != len(args):
-            raise ValueError("Length of args should match len of x")
-        weights = np.array([v for v in args])
-        return np.dot(x.T, weights)
-
-    # Carry out fitting
-    # See https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html # noqa
-    coefs, cov = scipy.optimize.curve_fit(
-        weighted_sum,
-        xdata=df.iloc[:, 1:].values.T,
-        ydata=load.values,
-        p0=p0,
-        bounds=bounds,
-        method="trf",
-    )
-
-    # Set 'almost zero' to zero
-    coefs[coefs < 0.1] = 0
-
-    # Reconstuct historical load
-    hist = weighted_sum(df.iloc[:, 1:].values.T, *coefs)
-    histp0 = weighted_sum(df.iloc[:, 1:].values.T, *p0)
-
-    # Make a nice dataframe to return the components
-    components = df.iloc[:, [0]].copy()
-    components["Inschatting"] = hist.T
-    components["p0"] = histp0.T
-    components["Windopwek"] = wind_ref * coefs[0]
-    components["Zonne-opwek"] = pv_ref * coefs[1]
-    components["StandaardVerbruik"] = (df.iloc[:, 3:] * coefs[2:]).sum(axis=1)
-    components["Residu"] = -1 * components.iloc[:, 0:2].diff(axis=1).iloc[:, 1]
-
-    # Make nice dictinary to return coefficents
-    coefdict = {name: value for name, value in zip(df.columns[1:], coefs)}
-
-    # Return result
-    return components, coefdict
-
-
-if __name__ == "__main__":
-    main()

openstef/tasks/train_model.py

@@ -1,224 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module contains the CRON job that is periodically executed to retrain the prognosis models.
-
-For this the folowing steps are caried out:
-  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
-  2. Apply features
-  3. Train and Test the new model
-  4. Check if new model performs better than the old model
-  5. Store the model if it performs better
-  6. Send slack message to inform the users
-
-Example:
-    This module is meant to be called directly from a CRON job. A description of
-    the CRON job can be found in the /k8s/CronJobs folder.
-    Alternatively this code can be run directly by running::
-
-        $ python model_train.py
-
-"""
-from datetime import datetime, timedelta
-from pathlib import Path
-
-import pandas as pd
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import ModelType, PipelineType
-from openstef.exceptions import (
-    InputDataOngoingZeroFlatlinerError,
-    SkipSaveTrainingForecasts,
-)
-from openstef.model.serializer import MLflowSerializer
-from openstef.pipeline.train_model import (
-    MAXIMUM_MODEL_AGE,
-    train_model_pipeline,
-    train_pipeline_step_load_model,
-)
-from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
-from openstef.tasks.utils.taskcontext import TaskContext
-
-TRAINING_PERIOD_DAYS: int = 120
-DEFAULT_CHECK_MODEL_AGE: bool = True
-
-
-def train_model_task(
-    pj: PredictionJobDataClass,
-    context: TaskContext,
-    check_old_model_age: bool = DEFAULT_CHECK_MODEL_AGE,
-    datetime_start: datetime = None,
-    datetime_end: datetime = None,
-) -> None:
-    """Train model task.
-
-    Top level task that trains a new model and makes sure the best available model is
-    stored. On this task level all database and context manager dependencies are resolved.
-
-    Expected prediction job keys:  "id", "model", "lat", "lon", "name"
-
-    Args:
-        pj: Prediction job
-        context: Contect object that holds a config manager and a
-            database connection.
-        check_old_model_age: check if model is too young to be retrained
-        datetime_start: Start
-        datetime_end: End
-
-    Raises:
-        SkipSaveTrainingForecasts: If old model is better or too young, you don't need to save the traing forcast.
-        InputDataOngoingZeroFlatlinerError: If all recent load measurements are zero.
-
-    """
-    # Check pipeline types
-    if PipelineType.TRAIN not in pj.pipelines_to_run:
-        context.logger.info(
-            "Skip this PredictionJob because train pipeline is not specified in the pj."
-        )
-        return
-
-    # TODO: Improve implementation by using a field in the database and leveraging the
-    #       `pipelines_to_run` attribute of the `PredictionJobDataClass` object. This
-    #       would require a change to the MySQL datamodel.
-    if (
-        context.config.externally_posted_forecasts_pids
-        and pj.id in context.config.externally_posted_forecasts_pids
-    ):
-        context.logger.info(
-            "Skip this PredictionJob because its forecasts are posted by an external process."
-        )
-        return
-
-    # Get the paths for storing model and reports from the config manager
-    mlflow_tracking_uri = context.config.paths_mlflow_tracking_uri
-    context.logger.debug(f"MLflow tracking uri: {mlflow_tracking_uri}")
-    artifact_folder = context.config.paths_artifact_folder
-    context.logger.debug(f"Artifact folder: {artifact_folder}")
-
-    context.perf_meter.checkpoint("Added metadata to PredictionJob")
-
-    # Check the model age before retrieving the input data to speed up train job.
-    # (The exact same model age check is also part of the "train_model_pipeline".)
-
-    # Initialize serializer
-    serializer = MLflowSerializer(mlflow_tracking_uri=mlflow_tracking_uri)
-
-    # Get old model and age
-    _, _, old_model_age = train_pipeline_step_load_model(pj, serializer)
-
-    # Check old model age and continue yes/no
-    if (old_model_age < MAXIMUM_MODEL_AGE) and check_old_model_age:
-        context.perf_meter.checkpoint(
-            f"Old model is younger than {MAXIMUM_MODEL_AGE} days, skip training"
-        )
-        if pj.save_train_forecasts:
-            raise SkipSaveTrainingForecasts
-        return
-
-    # Define start and end of the training input data
-    training_period_days_to_fetch = (
-        TRAINING_PERIOD_DAYS
-        if pj.data_balancing_ratio is None
-        else int(pj.data_balancing_ratio * TRAINING_PERIOD_DAYS)
-    )
-
-    if datetime_end is None:
-        datetime_end = datetime.utcnow()
-    if datetime_start is None:
-        datetime_start = datetime_end - timedelta(days=training_period_days_to_fetch)
-
-    # Get training input data from database
-    input_data = context.database.get_model_input(
-        pid=pj["id"],
-        location=[pj["lat"], pj["lon"]],
-        datetime_start=datetime_start,
-        datetime_end=datetime_end,
-    )
-
-    # If data balancing is enabled, fetch data from 1 year ago and combine it with the
-    # current data
-    if pj.data_balancing_ratio is not None:
-        # Because the data is from the past, we can use the data from the "future"
-        balanced_datetime_start = datetime_end - timedelta(days=365)
-        balanced_datetime_end = balanced_datetime_start + timedelta(
-            days=training_period_days_to_fetch
-        )
-
-        balanced_input_data = context.database.get_model_input(
-            pid=pj["id"],
-            location=[pj["lat"], pj["lon"]],
-            datetime_start=balanced_datetime_start,
-            datetime_end=balanced_datetime_end,
-        )
-
-        input_data = pd.concat(
-            [
-                balanced_input_data,
-                input_data,
-            ]
-        )
-
-    context.perf_meter.checkpoint("Retrieved timeseries input")
-
-    # Excecute the model training pipeline
-    try:
-        data_sets = train_model_pipeline(
-            pj,
-            input_data,
-            check_old_model_age=check_old_model_age,
-            mlflow_tracking_uri=mlflow_tracking_uri,
-            artifact_folder=artifact_folder,
-        )
-
-        if data_sets:
-            context.perf_meter.checkpoint("Model trained")
-        else:
-            context.perf_meter.checkpoint("Model not trained")
-
-        if pj.save_train_forecasts:
-            if data_sets is None:
-                raise RuntimeError("Forecasts were not retrieved")
-            if not hasattr(context.database, "write_train_forecasts"):
-                raise RuntimeError(
-                    "Database connector does dot support 'write_train_forecasts' while "
-                    "'save_train_forecasts option was activated.'"
-                )
-            context.database.write_train_forecasts(pj, data_sets)
-            context.logger.debug(f"Saved Forecasts from trained model on datasets")
-    except SkipSaveTrainingForecasts:
-        context.logger.debug(f"Skip saving forecasts")
-    except InputDataOngoingZeroFlatlinerError:
-        if (
-            context.config.known_zero_flatliners
-            and pj.id in context.config.known_zero_flatliners
-        ):
-            context.logger.info(
-                "No model was trained for this known zero flatliner. No model needs to be trained either, since the fallback forecasts are sufficient."
-            )
-            return
-        else:
-            raise InputDataOngoingZeroFlatlinerError(
-                'All recent load measurements are zero. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
-            )
-
-
-def main(model_type=None, config=None, database=None):
-    if database is None or config is None:
-        raise RuntimeError(
-            "Please specifiy a config object and/or database connection object. These"
-            " can be found in the openstef-dbc package."
-        )
-
-    if model_type is None:
-        model_type = [ml.value for ml in ModelType]
-
-    taskname = Path(__file__).name.replace(".py", "")
-    datetime_now = datetime.utcnow()
-    with TaskContext(taskname, config, database) as context:
-        PredictionJobLoop(context, model_type=model_type).map(
-            train_model_task, context, datetime_end=datetime_now
-        )
-
-
-if __name__ == "__main__":
-    main()

openstef/tasks/utils/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0

openstef/tasks/utils/dependencies.py

@@ -1,107 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-import random
-from typing import Iterable, Sequence, Set, Union
-
-import networkx as nx
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-
-NodeIdType = Union[str, int]
-EdgeType = tuple[NodeIdType, NodeIdType]
-
-
-def has_dependencies(pjs: Iterable[PredictionJobDataClass]) -> bool:
-    """Test whether some prediction jobs have dependencies information.
-
-    Args:
-        pjs: The list of prediction jobs
-
-    Returns:
-        True if some dependency information was found.
-
-    """
-    for pj in pjs:
-        if pj.depends_on is not None and len(pj.depends_on) > 0:
-            return True
-    return False
-
-
-def build_graph_structure(
-    pjs: Iterable[PredictionJobDataClass],
-) -> tuple[Set[NodeIdType], Set[EdgeType]]:
-    """Build the graph of dependencies between prediction jobs.
-
-    Args:
-        pjs: The Iterable of prediction jobs
-
-    Returns:
-        - The set of node ids of the graph
-        - The set of edges in the graph
-
-    """
-    nodes = set()
-    edges = set()
-
-    for pj in pjs:
-        nodes.add(pj["id"])
-        if pj.depends_on is not None:
-            for j in pj.depends_on:
-                edges.add((j, pj["id"]))
-
-    return nodes, edges
-
-
-def build_nx_graph(
-    nodes: Iterable[NodeIdType], edges: Iterable[EdgeType]
-) -> nx.DiGraph:
-    """Build a networkx Directed Graph.
-
-    Args:
-        nodes: The sequence of node ids
-        edges: The sequence of edges
-
-    Returns:
-        The dependency graph
-
-    """
-    graph = nx.DiGraph()
-    graph.add_nodes_from(nodes)
-    graph.add_edges_from(edges)
-    return graph
-
-
-def find_groups(
-    pjs: Sequence[PredictionJobDataClass], randomize_groups: bool = False
-) -> tuple[nx.DiGraph, list[list[PredictionJobDataClass]]]:
-    """Find a sequence of prediction job groups respecting dependencies.
-
-    Compute groups of prediction jobs such that the prediction jobs in a group
-    depend of at least one prediction job in the previous group and does not depend
-    on a prediction job in the following groups.
-    This means that all the prediction jobs in a group can be run in parallel and that
-    if groups are treated in the given order, the dependencies of a prediction job have
-    already been treated when the prediction job is run.
-
-    Args:
-        pjs: The sequence of prediction jobs
-        randomize_groups: Wether subgroups should be randomized.
-
-    Returns:
-        - The dependency graph
-        - The list of prediction job groups
-
-    """
-    nodes, edges = build_graph_structure(pjs)
-    graph = build_nx_graph(nodes, edges)
-    groups = list(nx.topological_generations(graph))
-
-    if randomize_groups:
-        for group in groups:
-            random.shuffle(group)
-
-    # Convert groups of pj ids to groups of pjs
-    pj_id_map = {pj["id"]: i for i, pj in enumerate(pjs)}
-    pj_groups = [[pjs[pj_id_map[pj_id]] for pj_id in group] for group in groups]
-    return graph, pj_groups