openstef 3.4.56__py3-none-any.whl → 4.0.0a3__py3-none-any.whl

openstef/tasks/create_basecase_forecast.py

@@ -1,118 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module should be executed once every day.
-
-For all prediction_jobs, it will create a 'basecase' forecast which is less accurate, but (almost) always available.
-For now, it uses the load a week earlier.
-Missing datapoints are interpolated.
-
-Example:
-    This module is meant to be called directly from a CRON job. A description of the
-    CRON job can be found in the /k8s/CronJobs folder.
-
-    Alternatively this code can be run directly by running:
-
-        $ python create_basecase_forecast.py
-
-"""
-from datetime import datetime, timedelta
-from pathlib import Path
-
-import pandas as pd
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import PipelineType
-from openstef.pipeline.create_basecase_forecast import create_basecase_forecast_pipeline
-from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
-from openstef.tasks.utils.taskcontext import TaskContext
-
-T_BEHIND_DAYS: int = 15
-T_AHEAD_DAYS: int = 14
-
-
-def create_basecase_forecast_task(
-    pj: PredictionJobDataClass,
-    context: TaskContext,
-    t_behind_days=T_BEHIND_DAYS,
-    t_ahead_days=T_AHEAD_DAYS,
-) -> None:
-    """Top level task that creates a basecase forecast.
-
-    On this task level all database and context manager dependencies are resolved.
-
-    Args:
-        pj: Prediction job
-        context: Contect object that holds a config manager and a database connection
-        t_behind_days: number of days included as history. This is used to generated lagged features for the to-be-forecasted period
-        t_ahead_days: number of days a basecase forecast is created for
-
-    """
-    # Check pipeline types
-    if PipelineType.FORECAST not in pj.pipelines_to_run:
-        context.logger.info(
-            "Skip this PredictionJob because forecast pipeline is not specified in the pj."
-        )
-        return
-
-    # TODO: Improve implementation by using a field in the database and leveraging the
-    #       `pipelines_to_run` attribute of the `PredictionJobDataClass` object. This
-    #       would require a change to the MySQL datamodel.
-    if (
-        context.config.externally_posted_forecasts_pids
-        and pj.id in context.config.externally_posted_forecasts_pids
-    ):
-        context.logger.info(
-            "Skip this PredictionJob because its forecasts are posted by an external process."
-        )
-        return
-
-    # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
-    datetime_end = datetime.utcnow() + timedelta(days=t_ahead_days)
-
-    # Retrieve input data
-    input_data = context.database.get_model_input(
-        pid=pj["id"],
-        location=[pj["lat"], pj["lon"]],
-        datetime_start=datetime_start,
-        datetime_end=datetime_end,
-    )
-
-    # Make basecase forecast using the corresponding pipeline
-    basecase_forecast = create_basecase_forecast_pipeline(pj, input_data)
-
-    # Do not store basecase forecasts for moments within the prediction job's horizon.
-    # Those should be updated by regular forecast process.
-    basecase_forecast = basecase_forecast.loc[
-        basecase_forecast.index
-        > (
-            pd.to_datetime(datetime.utcnow(), utc=True)
-            + timedelta(minutes=pj.horizon_minutes)
-        ),
-        :,
-    ]
-
-    # Write basecase forecast to the database
-    context.database.write_forecast(basecase_forecast, t_ahead_series=True)
-
-
-def main(config: object = None, database: object = None, **kwargs):
-    taskname = Path(__file__).name.replace(".py", "")
-
-    if database is None or config is None:
-        raise RuntimeError(
-            "Please specifiy a config object and/or database connection object. These"
-            " can be found in the openstef-dbc package."
-        )
-
-    with TaskContext(taskname, config, database) as context:
-        model_type = ["xgb", "xgb_quantile", "lgb"]
-
-        PredictionJobLoop(context, model_type=model_type).map(
-            create_basecase_forecast_task, context, **kwargs
-        )
-
-
-if __name__ == "__main__":
-    main()

openstef/tasks/create_components_forecast.py

@@ -1,162 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module contains the CRON job that is periodically executed to make the components prognoses.
-
-This code assumes trained models are available from the persistent storage.
-If these are not available run model_train.py to train all models.
-To provide the prognoses the following steps are carried out:
-  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
-  2. Apply features
-  3. Load model
-  4. Make component prediction
-  5. Write prediction to the database
-  6. Send Teams message if something goes wrong
-
-Example:
-    This module is meant to be called directly from a CRON job. A description of
-    the CRON job can be found in the /k8s/CronJobs folder.
-    Alternatively this code can be run directly by running::
-
-        $ python create_components_forecast.py
-
-"""
-import logging
-from datetime import datetime, timedelta, timezone
-from pathlib import Path
-
-import pandas as pd
-import structlog
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import ModelType
-from openstef.exceptions import ComponentForecastTooShortHorizonError
-from openstef.pipeline.create_component_forecast import (
-    create_components_forecast_pipeline,
-)
-from openstef.settings import Settings
-from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
-from openstef.tasks.utils.taskcontext import TaskContext
-
-T_BEHIND_DAYS = 0
-T_AHEAD_DAYS = 3
-
-
-def create_components_forecast_task(
-    pj: PredictionJobDataClass,
-    context: TaskContext,
-    t_behind_days: int = T_BEHIND_DAYS,
-    t_ahead_days: int = T_AHEAD_DAYS,
-) -> None:
-    """Top level task that creates a components forecast.
-
-    On this task level all database and context manager dependencies are resolved.
-
-    Args:
-        pj: Prediction job
-        context: Contect object that holds a config manager and a database connection
-        t_behind_days: number of days in the past that the component forecast is created for
-        t_ahead_days: number of days in the future that the component forecast is created for
-
-    Raises:
-        ComponentForecastTooShortHorizonError: If the forecast horizon is too short
-         (less than 30 minutes in advance)
-
-    """
-    structlog.configure(
-        wrapper_class=structlog.make_filtering_bound_logger(
-            logging.getLevelName(Settings.log_level)
-        )
-    )
-    logger = structlog.get_logger(__name__)
-    if pj["train_components"] == 0:
-        context.logger.info(
-            "Skip prediction job", train_components=pj["train_components"]
-        )
-        return
-
-    # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
-    datetime_end = datetime.utcnow() + timedelta(days=t_ahead_days)
-
-    logger.info(
-        "Get predicted load", datetime_start=datetime_start, datetime_end=datetime_end
-    )
-    # Get most recent load forecast as input_data,
-    # we use a regular forecast as input point for creating component forecasts
-    input_data = context.database.get_predicted_load(
-        pj, start_time=datetime_start, end_time=datetime_end
-    )
-    # Check if input_data is not empty
-    if len(input_data) == 0:
-        logger.warning(f"No forecast found. Skipping pid", pid=pj["id"])
-        return
-
-    logger.info("retrieving weather data")
-    # TODO make openstef_dbc function to retrieve inputdata for component forecast in one call,
-    #  this will make this function much shorter
-    # Get required weather data
-    weather_data = context.database.get_weather_data(
-        [pj["lat"], pj["lon"]],
-        [
-            "radiation",
-            "windspeed_100m",
-        ],  # These variables are used when determing the splitting coeficients, and should therefore be reused when making the component forcasts.
-        datetime_start=datetime_start,
-        datetime_end=datetime_end,
-    )
-
-    # Make forecast for the demand, wind and pv components
-    forecasts = create_components_forecast_pipeline(pj, input_data, weather_data)
-
-    ## Perform sanity check on index
-    if not isinstance(forecasts.index, pd.core.indexes.datetimes.DatetimeIndex):
-        raise ValueError(
-            f"Index is not datetime. Received forecasts:{forecasts.head()}"
-        )
-
-    # save forecast to database #######################################################
-    context.database.write_forecast(forecasts)
-    logger.debug("Written forecast to database")
-
-    # Check if forecast was complete enough, otherwise raise exception
-    if forecasts.index.max() < datetime.utcnow().replace(
-        tzinfo=timezone.utc
-    ) + timedelta(hours=30):
-        # Check which input data is missing the most.
-        # Do this by counting the NANs for (load)forecast, radiation and windspeed
-        max_index = forecasts.index.max()
-        n_nas = dict(
-            nans_load_forecast=input_data.loc[max_index:, "forecast"].isna().sum(),
-            nans_radiation=weather_data.loc[max_index:, "radiation"].isna().sum(),
-            nans_windspeed_100m=weather_data.loc[max_index:, "windspeed_100m"]
-            .isna()
-            .sum(),
-        )
-        max_na = max(n_nas, key=n_nas.get)
-
-        raise ComponentForecastTooShortHorizonError(
-            f"Could not make component forecast for two days ahead, probably input data is missing, {max_na}: {n_nas[max_na]}"
-        )
-
-
-def main(config: object = None, database: object = None, **kwargs):
-    taskname = Path(__file__).name.replace(".py", "")
-
-    if database is None or config is None:
-        raise RuntimeError(
-            "Please specifiy a config object and/or database connection object. These"
-            " can be found in the openstef-dbc package."
-        )
-
-    with TaskContext(taskname, config, database) as context:
-        model_type = [ml.value for ml in ModelType]
-
-        PredictionJobLoop(
-            context,
-            model_type=model_type,
-        ).map(create_components_forecast_task, context, **kwargs)
-
-
-if __name__ == "__main__":
-    main()

openstef/tasks/create_forecast.py

@@ -1,145 +0,0 @@
-# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <korte.termijn.prognoses@alliander.com> # noqa E501>
-#
-# SPDX-License-Identifier: MPL-2.0
-"""This module contains the CRON job that is periodically executed to make prognoses and save them in to the database.
-
-This code assumes trained models are available from the persistent storage. If these
-are not available run model_train.py to train all models.
-To provide the prognoses the folowing steps are carried out:
-  1. Get historic training data (TDCV, Load, Weather and day_ahead_electricity_price price data)
-  2. Apply features
-  3. Load model
-  4. Make prediction
-  5. Write prediction to the database
-  6. Send Teams message if something goes wrong
-
-Example:
-    This module is meant to be called directly from a CRON job.
-    Alternatively this code can be run directly by running::
-
-        $ python create_forecast.py
-
-"""
-from datetime import datetime, timedelta
-from pathlib import Path
-
-from openstef.data_classes.prediction_job import PredictionJobDataClass
-from openstef.enums import BiddingZone, ModelType, PipelineType
-from openstef.exceptions import InputDataOngoingZeroFlatlinerError
-from openstef.pipeline.create_forecast import create_forecast_pipeline
-from openstef.tasks.utils.predictionjobloop import PredictionJobLoop
-from openstef.tasks.utils.taskcontext import TaskContext
-from openstef.validation.validation import detect_ongoing_zero_flatliner
-
-T_BEHIND_DAYS: int = 14
-
-
-def create_forecast_task(
-    pj: PredictionJobDataClass, context: TaskContext, t_behind_days: int = T_BEHIND_DAYS
-) -> None:
-    """Top level task that creates a forecast.
-
-    On this task level all database and context manager dependencies are resolved.
-
-    Expected prediction job keys; "id", "lat", "lon", "resolution_minutes",
-        "horizon_minutes", "type", "name", "quantiles"
-
-    Args:
-        pj: Prediction job
-        context: Contect object that holds a config manager and a database connection
-        t_behind_days: number of days included as history. This is used to generated lagged features for the to-be-forecasted period
-
-    """
-    # Check pipeline types
-    if PipelineType.FORECAST not in pj.pipelines_to_run:
-        context.logger.info(
-            "Skip this PredictionJob because forecast pipeline is not specified in the pj."
-        )
-        return
-
-    # TODO: Improve implementation by using a field in the database and leveraging the
-    #       `pipelines_to_run` attribute of the `PredictionJobDataClass` object. This
-    #       would require a change to the MySQL datamodel.
-    if (
-        context.config.externally_posted_forecasts_pids
-        and pj.id in context.config.externally_posted_forecasts_pids
-    ):
-        context.logger.info(
-            "Skip this PredictionJob because its forecasts are posted by an external process."
-        )
-        return
-
-    # Extract mlflow tracking URI and trained models folder
-    mlflow_tracking_uri = context.config.paths_mlflow_tracking_uri
-
-    # Define datetime range for input data
-    datetime_start = datetime.utcnow() - timedelta(days=t_behind_days)
-    datetime_end = datetime.utcnow() + timedelta(seconds=pj.horizon_minutes * 60)
-
-    # Retrieve input data
-    input_data = context.database.get_model_input(
-        pid=pj["id"],
-        location=[pj["lat"], pj["lon"]],
-        datetime_start=datetime_start,
-        datetime_end=datetime_end,
-        market_price=pj.electricity_bidding_zone.value,
-    )
-
-    # Add APX price to the input data for backward compatibility,remove this line when all models are retrained
-    if pj.electricity_bidding_zone == BiddingZone.NL:
-        input_data["APX"] = input_data["day_ahead_electricity_price"]
-
-    try:
-        # Make forecast with the forecast pipeline
-        forecast = create_forecast_pipeline(
-            pj, input_data, mlflow_tracking_uri=mlflow_tracking_uri
-        )
-    except (InputDataOngoingZeroFlatlinerError, LookupError) as e:
-        if (
-            context.config.known_zero_flatliners
-            and pj.id in context.config.known_zero_flatliners
-        ):
-            context.logger.info(
-                "No forecasts were made for this known zero flatliner prediction job. No forecasts need to be made either, since the fallback forecasts are sufficient."
-            )
-            return
-        elif isinstance(e, InputDataOngoingZeroFlatlinerError):
-            raise InputDataOngoingZeroFlatlinerError(
-                'All recent load measurements are zero. Check the load profile of this pid as well as related/neighbouring prediction jobs. Afterwards, consider adding this pid to the "known_zero_flatliners" app_setting and possibly removing other pids from the same app_setting.'
-            ) from e
-        elif isinstance(e, LookupError):
-            zero_flatliner_ongoing = detect_ongoing_zero_flatliner(
-                load=input_data.iloc[:, 0],
-                duration_threshold_minutes=pj.flatliner_threshold_minutes,
-            )
-            if zero_flatliner_ongoing:
-                raise LookupError(
-                    'Model not found. Consider checking for a zero flatliner and adding this pid to the "known_zero_flatliners" app_setting. For zero flatliners, no model can be trained.'
-                ) from e
-            else:
-                raise e
-
-    # Write forecast to the database
-    context.database.write_forecast(forecast, t_ahead_series=True)
-
-
-def main(model_type=None, config=None, database=None, **kwargs):
-    taskname = Path(__file__).name.replace(".py", "")
-
-    if database is None or config is None:
-        raise RuntimeError(
-            "Please specify a config object and/or database connection object. These"
-            " can be found in the openstef-dbc package."
-        )
-
-    with TaskContext(taskname, config, database) as context:
-        if model_type is None:
-            model_type = [ml.value for ml in ModelType]
-
-        PredictionJobLoop(context, model_type=model_type).map(
-            create_forecast_task, context, **kwargs
-        )
-
-
-if __name__ == "__main__":
-    main()