openstef-models 4.0.0__py3-none-any.whl

openstef_models/__init__.py

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+"""Core models for OpenSTEF."""
+
+import logging
+
+# Set up logging configuration
+root_logger = logging.getLogger(name=__name__)
+if not root_logger.handlers:
+    root_logger.addHandler(logging.NullHandler())
+
+__all__ = []

openstef_models/explainability/__init__.py

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Explainability utilities for OpenSTEF.
+
+Tools for feature importance, attribution and model interpretation.
+"""
+
+from .mixins import ContributionsMixin, ExplainableForecaster
+from .plotters import ContributionsPlotter, FeatureImportancePlotter
+
+__all__ = [
+    "ContributionsMixin",
+    "ContributionsPlotter",
+    "ExplainableForecaster",
+    "FeatureImportancePlotter",
+]

openstef_models/explainability/mixins.py

@@ -0,0 +1,126 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Mixins for adding explainability features to forecasting models.
+
+Provides base classes that enable models to expose feature importance scores
+and generate visualization plots.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Literal
+
+import pandas as pd
+import plotly.graph_objects as go
+
+from openstef_core.datasets import ForecastInputDataset, TimeSeriesDataset
+from openstef_core.types import Q, Quantile
+from openstef_models.explainability.plotters.contributions_plotter import ContributionsPlotter
+from openstef_models.explainability.plotters.feature_importance_plotter import FeatureImportancePlotter
+
+
+class ExplainableForecaster(ABC):
+    """Mixin for forecasters that can explain feature importance.
+
+    Provides a standardized interface for accessing and visualizing feature
+    importance scores across different forecasting models.
+    """
+
+    @property
+    @abstractmethod
+    def feature_importances(self) -> pd.DataFrame:
+        """Get feature importance scores for this model.
+
+        Returns DataFrame with feature names as index and quantiles as columns.
+        Each quantile represents the importance distribution across multiple
+        model training runs or folds.
+
+        Returns:
+            DataFrame with feature names as index and quantile columns.
+            Values represent normalized importance scores summing to 1.0.
+
+        Note:
+            The returned DataFrame must have feature names as index and quantile
+            columns in format 'quantile_PXX' (e.g., 'quantile_P50', 'quantile_P95').
+            All quantile values must be between 0 and 1.
+        """
+        raise NotImplementedError
+
+    def plot_feature_importances(self, quantile: Quantile = Q(0.5)) -> go.Figure:
+        """Create interactive treemap visualization of feature importances.
+
+        Args:
+            quantile: Which quantile of importance scores to display.
+                Defaults to median (0.5).
+
+        Returns:
+            Plotly Figure containing treemap with feature importance scores.
+            Color intensity indicates relative importance of each feature.
+        """
+        return FeatureImportancePlotter().plot(scores=self.feature_importances, quantile=quantile)
+
+
+class ContributionsMixin(ABC):
+    """Mixin for forecasters that can explain per-sample feature contributions.
+
+    Unlike ``ExplainableForecaster`` which provides aggregate feature importance,
+    this mixin provides per-sample decomposition of predictions — i.e., how
+    much each feature contributed to the prediction for each individual sample.
+
+    For tree-based models (XGBoost), this corresponds to SHAP TreeExplainer values.
+    For linear models (GBLinear), this is the coefficient x feature value decomposition.
+    For ensembles, this shows each base model's contribution weight.
+    """
+
+    @abstractmethod
+    def predict_contributions(self, data: ForecastInputDataset) -> TimeSeriesDataset:
+        """Compute per-sample feature contributions for the given input data.
+
+        Returns a TimeSeriesDataset where columns are feature names (or model
+        names for ensemble contributions) and rows correspond to the same time
+        index as the input.  Values represent the additive contribution of each
+        feature to the prediction at that timestep.
+
+        Args:
+            data: Preprocessed input data (same format as ``predict()`` takes).
+
+        Returns:
+            TimeSeriesDataset with feature contributions.  Columns are features,
+            rows are timesteps.  A ``bias`` column may be included for the
+            model intercept/base value.
+        """
+
+    def plot_contributions(
+        self,
+        data: ForecastInputDataset,
+        kind: Literal["heatmap", "waterfall", "bar"] = "heatmap",
+        **kwargs: Any,
+    ) -> go.Figure:
+        """Plot per-sample feature contributions.
+
+        Calls ``predict_contributions()`` and visualizes the result using the
+        requested chart type.
+
+        Args:
+            data: Preprocessed input data.
+            kind: Chart type — ``"heatmap"``, ``"waterfall"``, or ``"bar"``.
+            **kwargs: Forwarded to the corresponding plotter method
+                (e.g. ``top_n``, ``timestep``).
+
+        Returns:
+            Plotly Figure.
+
+        Raises:
+            ValueError: If *kind* is not one of the supported chart types.
+        """
+        contributions = self.predict_contributions(data)
+        plotters = {
+            "heatmap": ContributionsPlotter.plot_heatmap,
+            "waterfall": ContributionsPlotter.plot_waterfall,
+            "bar": ContributionsPlotter.plot_bar,
+        }
+        if kind not in plotters:
+            msg = f"Unknown plot kind {kind!r}. Choose from {list(plotters)}"
+            raise ValueError(msg)
+        return plotters[kind](contributions=contributions, **kwargs)

openstef_models/explainability/plotters/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Visualization tools for model explainability.
+
+Provides plotters for creating interactive visualizations of feature importance
+scores and other model explanation outputs.
+"""
+
+from .contributions_plotter import ContributionsPlotter
+from .feature_importance_plotter import FeatureImportancePlotter
+
+__all__ = [
+    "ContributionsPlotter",
+    "FeatureImportancePlotter",
+]

openstef_models/explainability/plotters/contributions_plotter.py

@@ -0,0 +1,228 @@
+# SPDX-FileCopyrightText: 2026 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Visualizations for per-sample feature contributions (SHAP values)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots  # pyright: ignore[reportUnknownVariableType]
+
+from openstef_core.datasets import TimeSeriesDataset  # noqa: TC001  # runtime needed for pyright
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+
+class ContributionsPlotter:
+    """Visualizations for per-timestep feature contributions."""
+
+    @staticmethod
+    def plot_heatmap(
+        contributions: TimeSeriesDataset,
+        top_n: int = 10,
+        target_column: str = "load",
+        bias_column: str = "bias",
+        *,
+        show_prediction: bool = True,
+    ) -> go.Figure:
+        """Create an interactive heatmap of feature contributions over time.
+
+        X-axis is the prediction datetime, Y-axis shows feature names ranked by mean absolute contribution
+        (most important at top). Color ranges from blue (negative) through white (zero) to red (positive).
+        When ``show_prediction`` is True a line plot of the model prediction (sum of contributions + bias)
+        is shown above the heatmap.
+
+        Args:
+            contributions: Output of ``predict_contributions()``.
+            top_n: Number of top features to show (ranked by mean absolute contribution).
+            target_column: Name of the target column to exclude. Default "load".
+            bias_column: Name of the bias column. Default "bias".
+            show_prediction: If True, add a prediction line subplot above the heatmap. Default True.
+
+        Returns:
+            Plotly Figure with a diverging heatmap centered at zero (and optional prediction line).
+        """
+        bias = contributions.data[bias_column] if bias_column in contributions.data.columns else None
+        cols_to_drop = [c for c in [target_column, bias_column] if c in contributions.data.columns]
+        df = contributions.data.drop(columns=cols_to_drop)
+        ranked: list[str] = df.abs().mean().sort_values(ascending=False).head(top_n).index.tolist()
+
+        # Most-important feature at top of Y-axis
+        y_labels = list(reversed(ranked))
+
+        heatmap = go.Heatmap(
+            z=df[y_labels].T.values,
+            x=df.index,
+            y=y_labels,
+            colorscale="RdBu_r",
+            zmid=0,
+            colorbar={"title": "Contribution"},
+            showlegend=False,
+        )
+
+        if show_prediction:
+            prediction = df.sum(axis=1)
+            if bias is not None:
+                prediction += bias
+
+            fig = make_subplots(
+                rows=2,
+                cols=1,
+                shared_xaxes=True,
+                row_heights=[0.2, 0.8],
+                vertical_spacing=0.03,
+            )
+
+            fig.add_trace(  # pyright: ignore[reportUnknownMemberType]
+                go.Scatter(
+                    x=df.index,
+                    y=prediction,
+                    mode="lines",
+                    name="Prediction",
+                    line={"color": "black", "width": 1.5},
+                    showlegend=False,
+                ),
+                row=1,
+                col=1,
+            )
+            fig.add_trace(heatmap, row=2, col=1)  # pyright: ignore[reportUnknownMemberType]
+
+            fig.update_layout(  # pyright: ignore[reportUnknownMemberType]
+                yaxis_title="Prediction",
+                yaxis2_title="Feature",
+                xaxis2_title="Time",
+                margin={"t": 30, "r": 10, "b": 40, "l": 120},
+            )
+        else:
+            fig = go.Figure(
+                data=heatmap,
+                layout={
+                    "xaxis_title": "Time",
+                    "yaxis_title": "Feature",
+                    "margin": {"t": 30, "r": 10, "b": 40, "l": 120},
+                },
+            )
+
+        return fig
+
+    @staticmethod
+    def plot_waterfall(
+        contributions: TimeSeriesDataset,
+        timestep: int = 0,
+        top_n: int = 10,
+        target_column: str = "load",
+        bias_column: str = "bias",
+    ) -> go.Figure:
+        """Create a waterfall chart decomposing a single timestep's prediction.
+
+        Shows how the bias (base value) is pushed up or down by each feature's
+        contribution to arrive at the final prediction.
+
+        Args:
+            contributions: Output of ``predict_contributions()``.
+            timestep: Row index (0-based) of the timestep to explain.
+            top_n: Number of top features to show. Remaining features are
+                aggregated into an "other" bar.
+            target_column: Name of the target column to exclude. Default "load".
+            bias_column: Name of the bias column used as base value. Default "bias".
+
+        Returns:
+            Plotly Figure with waterfall chart.
+        """
+        bias = contributions.data[bias_column] if bias_column in contributions.data.columns else None
+        cols_to_drop = [c for c in [target_column, bias_column] if c in contributions.data.columns]
+        df = contributions.data.drop(columns=cols_to_drop)
+        row = df.iloc[timestep]
+        base_value = float(bias.iloc[timestep]) if bias is not None else 0.0
+
+        # Rank by |contribution| for this specific timestep
+        abs_sorted = row.abs().sort_values(ascending=False)
+        top = abs_sorted.head(top_n).index.tolist()
+        remaining = [c for c in abs_sorted.index if c not in top]
+
+        names: list[str] = [bias_column]
+        values: list[float] = [base_value]
+        measures: list[str] = ["absolute"]
+
+        for feat in top:
+            names.append(feat)
+            values.append(float(row[feat]))  # pyright: ignore[reportArgumentType]
+            measures.append("relative")
+
+        if len(remaining) > 0:
+            other_sum = float(row[remaining].sum())
+            names.append(f"other ({len(remaining)})")
+            values.append(other_sum)
+            measures.append("relative")
+
+        names.append("Prediction")
+        values.append(base_value + float(row.sum()))
+        measures.append("total")
+
+        timestamp = contributions.data.index[timestep]
+        return go.Figure(
+            go.Waterfall(
+                x=names,
+                y=values,
+                measure=measures,
+                connector={"line": {"color": "grey", "width": 0.5}},
+                increasing={"marker": {"color": "#ff4136"}},
+                decreasing={"marker": {"color": "#0074d9"}},
+                totals={"marker": {"color": "#2ecc40"}},
+                textposition="outside",
+                text=[f"{v:+.4f}" if m == "relative" else f"{v:.4f}" for v, m in zip(values, measures, strict=True)],
+            ),
+            layout={
+                "title": f"Contributions at {timestamp}",
+                "yaxis_title": "Contribution",
+                "margin": {"t": 50, "r": 10, "b": 40, "l": 60},
+                "showlegend": False,
+            },
+        )
+
+    @staticmethod
+    def plot_bar(
+        contributions: TimeSeriesDataset,
+        top_n: int = 10,
+        target_column: str = "load",
+        bias_column: str = "bias",
+    ) -> go.Figure:
+        """Create a horizontal bar chart of mean absolute contributions per feature.
+
+        Features are ranked from most to least important (top to bottom).
+
+        Args:
+            contributions: Output of ``predict_contributions()``.
+            top_n: Number of top features to show.
+            target_column: Name of the target column to exclude. Default "load".
+            bias_column: Name of the bias column to exclude. Default "bias".
+
+        Returns:
+            Plotly Figure with horizontal bar chart.
+        """
+        cols_to_drop = [c for c in [target_column, bias_column] if c in contributions.data.columns]
+        df = contributions.data.drop(columns=cols_to_drop)
+        mean_abs: pd.Series = df.abs().mean().sort_values(ascending=False).head(top_n)
+
+        # Reverse for plotly (bottom-to-top rendering)
+        mean_abs = mean_abs.iloc[::-1]
+
+        return go.Figure(
+            go.Bar(
+                x=mean_abs.values,  # pyright: ignore[reportArgumentType]
+                y=mean_abs.index.tolist(),
+                orientation="h",
+                marker_color="#1f77b4",
+                hovertemplate="<b>%{y}</b><br>mean |SHAP|: %{x:.4f}<extra></extra>",
+            ),
+            layout={
+                "xaxis_title": "mean |SHAP value|",
+                "yaxis_title": "Feature",
+                "margin": {"t": 30, "r": 10, "b": 40, "l": 120},
+                "showlegend": False,
+            },
+        )

openstef_models/explainability/plotters/feature_importance_plotter.py

@@ -0,0 +1,59 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Interactive treemap visualization for feature importance scores.
+
+Creates color-coded treemaps showing relative importance of features in
+forecasting models.
+"""
+
+import pandas as pd
+import plotly.graph_objects as go
+
+from openstef_core.base_model import BaseConfig
+from openstef_core.datasets.validation import validate_required_columns
+from openstef_core.types import Q, Quantile
+
+
+class FeatureImportancePlotter(BaseConfig):
+    """Creates treemap visualizations of feature importance scores."""
+
+    @staticmethod
+    def plot(scores: pd.DataFrame, quantile: Quantile = Q(0.5)) -> go.Figure:
+        """Generate interactive treemap showing feature importance.
+
+        Creates a color-coded treemap where each box size and color intensity
+        represents the relative importance of a feature. Useful for quickly
+        identifying which features contribute most to model predictions.
+
+        Args:
+            scores: Feature importance scores with feature names as index and
+                quantiles as columns (e.g., 'q0.5', 'q0.95'). Values should be
+                normalized to sum to 1.0.
+            quantile: Which quantile column to visualize. Defaults to median (0.5).
+
+        Returns:
+            Plotly Figure containing interactive treemap with hover information.
+            Larger boxes and darker green colors indicate higher importance.
+        """
+        quantile_column = quantile.format()
+        validate_required_columns(scores, required_columns=[quantile_column])
+
+        return go.Figure(
+            go.Treemap(
+                labels=scores.index,
+                parents=pd.Series(data=["Feature importance"] * len(scores), index=scores.index),
+                values=scores[quantile_column],
+                marker={"colors": scores[quantile_column], "colorscale": "greens"},
+                hovertemplate=("<b>%{label}</b><br>importance: %{value:.1%}<extra></extra>"),
+            ),
+            layout={
+                "margin": {
+                    "t": 0,
+                    "r": 0,
+                    "b": 0,
+                    "l": 0,
+                }
+            },
+        )

openstef_models/integrations/__init__.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Integration components for extending OpenSTEF functionality.
+
+Contains implementations for callbacks and storage systems that hook into and
+extend OpenSTEF functionality by integrating with external systems such as
+monitoring tools, databases, cloud storage, and custom processing pipelines.
+"""
+
+__all__ = ["joblib", "mlflow", "optuna"]  # noqa: F822  # pyright: ignore[reportUnsupportedDunderAll]  # Sub-packages with optional deps; not imported to avoid missing-extra errors at import time

openstef_models/integrations/joblib/__init__.py

@@ -0,0 +1,15 @@
+"""Joblib-based model storage integration.
+
+Provides local file-based model persistence using joblib for serialization.
+This integration allows storing and loading ForecastingModel instances on
+the local filesystem, making it suitable for development, testing, and
+single-machine deployments.
+"""
+
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+from .joblib_model_serializer import JoblibModelSerializer
+
+__all__ = ["JoblibModelSerializer"]

openstef_models/integrations/joblib/joblib_model_serializer.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+"""Local model storage implementation using joblib serialization.
+
+Provides file-based persistence for ForecastingModel instances using joblib's
+pickle-based serialization. This storage backend is suitable for development,
+testing, and single-machine deployments where models need to be persisted
+to the local filesystem.
+"""
+
+from typing import BinaryIO, ClassVar, override
+
+from openstef_core.exceptions import MissingExtraError
+from openstef_models.mixins.model_serializer import ModelSerializer
+
+try:
+    import joblib
+except ImportError as e:
+    raise MissingExtraError("joblib", package="openstef-models") from e
+
+
+class JoblibModelSerializer(ModelSerializer):
+    """File-based model storage using joblib serialization.
+
+    Provides persistent storage for ForecastingModel instances on the local
+    filesystem. Models are serialized using joblib and stored as pickle files
+    in the specified directory.
+
+    This storage implementation is suitable for development, testing, and
+    single-machine deployments where simple file-based persistence is sufficient.
+
+    Note:
+        joblib.dump() and joblib.load() are based on the Python pickle serialization model,
+        which means that arbitrary Python code can be executed when loading a serialized object
+        with joblib.load().
+
+        joblib.load() should therefore never be used to load objects from an untrusted source
+        or otherwise you will introduce a security vulnerability in your program.
+
+    Invariants:
+        - Models are stored as .pkl files in the configured storage directory
+        - Model files use the pattern: {model_id}.pkl
+        - Storage directory is created automatically if it doesn't exist
+        - Load operations fail with ModelNotFoundError if model file doesn't exist
+
+    Example:
+        Basic usage with model persistence
+
+        >>> from pathlib import Path
+        >>> from openstef_models.models.forecasting_model import ForecastingModel
+        >>> storage = LocalModelStorage(storage_dir=Path("./models"))  # doctest: +SKIP
+        >>> storage.save_model("my_model", my_forecasting_model)  # doctest: +SKIP
+        >>> loaded_model = storage.load_model("my_model")  # doctest: +SKIP
+    """
+
+    extension: ClassVar[str] = "joblib"
+
+    @override
+    def serialize(self, model: object, file: BinaryIO) -> None:
+        joblib.dump(model, file)  # type: ignore[reportUnknownMemberType]
+
+    @override
+    def deserialize(self, file: BinaryIO) -> object:
+        return joblib.load(file)  # type: ignore[reportUnknownMemberType]
+
+
+__all__ = ["JoblibModelSerializer"]

openstef_models/integrations/mlflow/__init__.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""MLflow integration for model tracking and storage.
+
+Provides integration with MLflow for model lifecycle management, experiment
+tracking, and model registry functionality. This package enables OpenSTEF
+models to be stored, versioned, and tracked using MLflow's
+model registry.
+
+Note:
+    This package requires MLflow to be installed as an optional dependency.
+    MLflow integration is particularly useful for production deployments
+    requiring model versioning, experiment tracking, and centralized storage.
+"""
+
+from .mlflow_storage import MLFlowStorage
+from .mlflow_storage_callback import (
+    MLFlowStorageCallback,
+)
+
+__all__ = [
+    "MLFlowStorage",
+    "MLFlowStorageCallback",
+]