openstef-models 4.0.0__tar.gz

openstef_models-4.0.0/.gitignore

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <openstef@lfenergy.org> # noqa E501>
+# SPDX-License-Identifier: MPL-2.0
+
+# Core
+config.user.yaml
+git-template
+.DS_Store
+tmp
+
+# Python bytecode
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Packaging and build artifacts
+.Python
+build/
+dist/
+downloads/
+wheels/
+share/python-wheels/
+sdist/
+.eggs/
+*.egg-info/
+*.egg
+.installed.cfg
+MANIFEST
+
+# uv
+.uv/
+
+# Ruff
+.ruff_cache/
+
+# Pyright
+.pyright/
+# pyright-report/
+
+# Test, coverage, tox
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.cover/
+cover/
+pytest-report.xml
+.tox/
+.nox/
+coverage.xml
+
+# Hypothesis
+.hypothesis/
+
+# Cython
+cython_debug/
+
+# Type checkers (misc)
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+.pyre/
+
+# Sphinx
+docs/_build/
+docs/source/api/generated/
+docs/source/tutorials/
+docs/source/benchmarks/
+docs/source/user_guide/**/quick_start_tutorial.py
+docs/source/user_guide/**/feature_engineering_tutorial.py
+docs/source/user_guide/**/datasets_tutorial.py
+docs/source/user_guide/**/backtesting_tutorial.py
+
+# docs/_doctrees/
+# docs/_static_gen/
+
+# Editors: JetBrains
+.idea/
+
+# Editors: VS Code (allow shared configs)
+.vscode/*
+
+# Jupyter / IPython
+.ipynb_checkpoints
+profile_default/
+ipython_config.py
+
+# Spyder / Rope
+.spyderproject
+.spyproject
+.ropeproject
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PEP 582
+__pypackages__/
+
+# web frameworks / services
+*.sqlite
+*.sqlite3
+*.log
+instance/
+.webassets-cache
+celerybeat-schedule
+celerybeat.pid
+*.sage.py
+tmp/
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Project outputs
+output/
+prof/
+certificates/
+
+# Output artifacts
+*.html
+*.pkl
+
+# Benchmark outputs
+benchmark_results*/
+
+# Local dataset files
+liander_dataset/
+
+# Mlflow
+/mlflow
+/mlflow_artifacts_local
+
+.github/instructions
+
+# Jupyter notebook cache (myst-nb execution outputs)
+.jupyter_cache/
+docs/build.zip

openstef_models-4.0.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: openstef-models
+Version: 4.0.0
+Summary: Core models for OpenSTEF
+Project-URL: Documentation, https://openstef.github.io/openstef/index.html
+Project-URL: Homepage, https://lfenergy.org/projects/openstef/
+Project-URL: Issues, https://github.com/OpenSTEF/openstef/issues
+Project-URL: Repository, https://github.com/OpenSTEF/openstef
+Author-email: "Alliander N.V" <openstef@lfenergy.org>
+License-Expression: MPL-2.0
+Keywords: energy,forecasting,machinelearning
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: <4.0,>=3.12
+Requires-Dist: holidays>=0.79
+Requires-Dist: mlflow-skinny<4,>=3
+Requires-Dist: openstef-beam<5,>=4.0.0.dev0
+Requires-Dist: openstef-core<5,>=4.0.0.dev0
+Requires-Dist: pvlib>=0.13
+Requires-Dist: pycountry>=24.6.1
+Requires-Dist: scikit-learn<1.8,>=1.7.1
+Requires-Dist: scipy<2,>=1.16.3
+Provides-Extra: lgbm
+Requires-Dist: lightgbm>=4.6; extra == 'lgbm'
+Provides-Extra: tuning
+Requires-Dist: optuna>=4.7; extra == 'tuning'
+Provides-Extra: xgb-cpu
+Requires-Dist: xgboost-cpu<4,>=3; (sys_platform == 'linux' or sys_platform == 'win32') and extra == 'xgb-cpu'
+Requires-Dist: xgboost<4,>=3; (sys_platform == 'darwin') and extra == 'xgb-cpu'
+Provides-Extra: xgb-gpu
+Requires-Dist: xgboost<4,>=3; extra == 'xgb-gpu'
+Description-Content-Type: text/markdown
+
+<!--
+SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+
+SPDX-License-Identifier: MPL-2.0
+-->
+
+# openstef-model

openstef_models-4.0.0/README.md

@@ -0,0 +1,7 @@
+<!--
+SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+
+SPDX-License-Identifier: MPL-2.0
+-->
+
+# openstef-model

openstef_models-4.0.0/pyproject.toml

@@ -0,0 +1,59 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+[build-system]
+build-backend = "hatchling.build"
+
+requires = [ "hatchling" ]
+
+[project]
+name = "openstef-models"
+version = "4.0.0"
+description = "Core models for OpenSTEF"
+readme = "README.md"
+keywords = [ "energy", "forecasting", "machinelearning" ]
+license = "MPL-2.0"
+authors = [
+  { name = "Alliander N.V", email = "openstef@lfenergy.org" },
+]
+requires-python = ">=3.12,<4.0"
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+]
+
+dependencies = [
+  "holidays>=0.79",
+  "mlflow-skinny>=3,<4",
+  "openstef-beam>=4.0.0.dev0,<5",
+  "openstef-core>=4.0.0.dev0,<5",
+  "pvlib>=0.13",
+  "pycountry>=24.6.1",
+  "scikit-learn>=1.7.1,<1.8",
+  "scipy>=1.16.3,<2",
+]
+
+optional-dependencies.lgbm = [
+  "lightgbm>=4.6",
+]
+
+optional-dependencies.tuning = [ "optuna>=4.7" ]
+
+optional-dependencies.xgb-cpu = [
+  "xgboost>=3,<4; sys_platform=='darwin'",
+  "xgboost-cpu>=3,<4; sys_platform=='linux' or sys_platform=='win32'",
+]
+
+optional-dependencies.xgb-gpu = [ "xgboost>=3,<4" ]
+urls.Documentation = "https://openstef.github.io/openstef/index.html"
+urls.Homepage = "https://lfenergy.org/projects/openstef/"
+urls.Issues = "https://github.com/OpenSTEF/openstef/issues"
+urls.Repository = "https://github.com/OpenSTEF/openstef"
+
+[tool.hatch.build.targets.wheel]
+packages = [ "src/openstef_models" ]

openstef_models-4.0.0/src/openstef_models/__init__.py

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+"""Core models for OpenSTEF."""
+
+import logging
+
+# Set up logging configuration
+root_logger = logging.getLogger(name=__name__)
+if not root_logger.handlers:
+    root_logger.addHandler(logging.NullHandler())
+
+__all__ = []

openstef_models-4.0.0/src/openstef_models/explainability/__init__.py

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Explainability utilities for OpenSTEF.
+
+Tools for feature importance, attribution and model interpretation.
+"""
+
+from .mixins import ContributionsMixin, ExplainableForecaster
+from .plotters import ContributionsPlotter, FeatureImportancePlotter
+
+__all__ = [
+    "ContributionsMixin",
+    "ContributionsPlotter",
+    "ExplainableForecaster",
+    "FeatureImportancePlotter",
+]

openstef_models-4.0.0/src/openstef_models/explainability/mixins.py

@@ -0,0 +1,126 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Mixins for adding explainability features to forecasting models.
+
+Provides base classes that enable models to expose feature importance scores
+and generate visualization plots.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Literal
+
+import pandas as pd
+import plotly.graph_objects as go
+
+from openstef_core.datasets import ForecastInputDataset, TimeSeriesDataset
+from openstef_core.types import Q, Quantile
+from openstef_models.explainability.plotters.contributions_plotter import ContributionsPlotter
+from openstef_models.explainability.plotters.feature_importance_plotter import FeatureImportancePlotter
+
+
+class ExplainableForecaster(ABC):
+    """Mixin for forecasters that can explain feature importance.
+
+    Provides a standardized interface for accessing and visualizing feature
+    importance scores across different forecasting models.
+    """
+
+    @property
+    @abstractmethod
+    def feature_importances(self) -> pd.DataFrame:
+        """Get feature importance scores for this model.
+
+        Returns DataFrame with feature names as index and quantiles as columns.
+        Each quantile represents the importance distribution across multiple
+        model training runs or folds.
+
+        Returns:
+            DataFrame with feature names as index and quantile columns.
+            Values represent normalized importance scores summing to 1.0.
+
+        Note:
+            The returned DataFrame must have feature names as index and quantile
+            columns in format 'quantile_PXX' (e.g., 'quantile_P50', 'quantile_P95').
+            All quantile values must be between 0 and 1.
+        """
+        raise NotImplementedError
+
+    def plot_feature_importances(self, quantile: Quantile = Q(0.5)) -> go.Figure:
+        """Create interactive treemap visualization of feature importances.
+
+        Args:
+            quantile: Which quantile of importance scores to display.
+                Defaults to median (0.5).
+
+        Returns:
+            Plotly Figure containing treemap with feature importance scores.
+            Color intensity indicates relative importance of each feature.
+        """
+        return FeatureImportancePlotter().plot(scores=self.feature_importances, quantile=quantile)
+
+
+class ContributionsMixin(ABC):
+    """Mixin for forecasters that can explain per-sample feature contributions.
+
+    Unlike ``ExplainableForecaster`` which provides aggregate feature importance,
+    this mixin provides per-sample decomposition of predictions — i.e., how
+    much each feature contributed to the prediction for each individual sample.
+
+    For tree-based models (XGBoost), this corresponds to SHAP TreeExplainer values.
+    For linear models (GBLinear), this is the coefficient x feature value decomposition.
+    For ensembles, this shows each base model's contribution weight.
+    """
+
+    @abstractmethod
+    def predict_contributions(self, data: ForecastInputDataset) -> TimeSeriesDataset:
+        """Compute per-sample feature contributions for the given input data.
+
+        Returns a TimeSeriesDataset where columns are feature names (or model
+        names for ensemble contributions) and rows correspond to the same time
+        index as the input.  Values represent the additive contribution of each
+        feature to the prediction at that timestep.
+
+        Args:
+            data: Preprocessed input data (same format as ``predict()`` takes).
+
+        Returns:
+            TimeSeriesDataset with feature contributions.  Columns are features,
+            rows are timesteps.  A ``bias`` column may be included for the
+            model intercept/base value.
+        """
+
+    def plot_contributions(
+        self,
+        data: ForecastInputDataset,
+        kind: Literal["heatmap", "waterfall", "bar"] = "heatmap",
+        **kwargs: Any,
+    ) -> go.Figure:
+        """Plot per-sample feature contributions.
+
+        Calls ``predict_contributions()`` and visualizes the result using the
+        requested chart type.
+
+        Args:
+            data: Preprocessed input data.
+            kind: Chart type — ``"heatmap"``, ``"waterfall"``, or ``"bar"``.
+            **kwargs: Forwarded to the corresponding plotter method
+                (e.g. ``top_n``, ``timestep``).
+
+        Returns:
+            Plotly Figure.
+
+        Raises:
+            ValueError: If *kind* is not one of the supported chart types.
+        """
+        contributions = self.predict_contributions(data)
+        plotters = {
+            "heatmap": ContributionsPlotter.plot_heatmap,
+            "waterfall": ContributionsPlotter.plot_waterfall,
+            "bar": ContributionsPlotter.plot_bar,
+        }
+        if kind not in plotters:
+            msg = f"Unknown plot kind {kind!r}. Choose from {list(plotters)}"
+            raise ValueError(msg)
+        return plotters[kind](contributions=contributions, **kwargs)

openstef_models-4.0.0/src/openstef_models/explainability/plotters/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Visualization tools for model explainability.
+
+Provides plotters for creating interactive visualizations of feature importance
+scores and other model explanation outputs.
+"""
+
+from .contributions_plotter import ContributionsPlotter
+from .feature_importance_plotter import FeatureImportancePlotter
+
+__all__ = [
+    "ContributionsPlotter",
+    "FeatureImportancePlotter",
+]

openstef_models-4.0.0/src/openstef_models/explainability/plotters/contributions_plotter.py

@@ -0,0 +1,228 @@
+# SPDX-FileCopyrightText: 2026 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Visualizations for per-sample feature contributions (SHAP values)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots  # pyright: ignore[reportUnknownVariableType]
+
+from openstef_core.datasets import TimeSeriesDataset  # noqa: TC001  # runtime needed for pyright
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+
+class ContributionsPlotter:
+    """Visualizations for per-timestep feature contributions."""
+
+    @staticmethod
+    def plot_heatmap(
+        contributions: TimeSeriesDataset,
+        top_n: int = 10,
+        target_column: str = "load",
+        bias_column: str = "bias",
+        *,
+        show_prediction: bool = True,
+    ) -> go.Figure:
+        """Create an interactive heatmap of feature contributions over time.
+
+        X-axis is the prediction datetime, Y-axis shows feature names ranked by mean absolute contribution
+        (most important at top). Color ranges from blue (negative) through white (zero) to red (positive).
+        When ``show_prediction`` is True a line plot of the model prediction (sum of contributions + bias)
+        is shown above the heatmap.
+
+        Args:
+            contributions: Output of ``predict_contributions()``.
+            top_n: Number of top features to show (ranked by mean absolute contribution).
+            target_column: Name of the target column to exclude. Default "load".
+            bias_column: Name of the bias column. Default "bias".
+            show_prediction: If True, add a prediction line subplot above the heatmap. Default True.
+
+        Returns:
+            Plotly Figure with a diverging heatmap centered at zero (and optional prediction line).
+        """
+        bias = contributions.data[bias_column] if bias_column in contributions.data.columns else None
+        cols_to_drop = [c for c in [target_column, bias_column] if c in contributions.data.columns]
+        df = contributions.data.drop(columns=cols_to_drop)
+        ranked: list[str] = df.abs().mean().sort_values(ascending=False).head(top_n).index.tolist()
+
+        # Most-important feature at top of Y-axis
+        y_labels = list(reversed(ranked))
+
+        heatmap = go.Heatmap(
+            z=df[y_labels].T.values,
+            x=df.index,
+            y=y_labels,
+            colorscale="RdBu_r",
+            zmid=0,
+            colorbar={"title": "Contribution"},
+            showlegend=False,
+        )
+
+        if show_prediction:
+            prediction = df.sum(axis=1)
+            if bias is not None:
+                prediction += bias
+
+            fig = make_subplots(
+                rows=2,
+                cols=1,
+                shared_xaxes=True,
+                row_heights=[0.2, 0.8],
+                vertical_spacing=0.03,
+            )
+
+            fig.add_trace(  # pyright: ignore[reportUnknownMemberType]
+                go.Scatter(
+                    x=df.index,
+                    y=prediction,
+                    mode="lines",
+                    name="Prediction",
+                    line={"color": "black", "width": 1.5},
+                    showlegend=False,
+                ),
+                row=1,
+                col=1,
+            )
+            fig.add_trace(heatmap, row=2, col=1)  # pyright: ignore[reportUnknownMemberType]
+
+            fig.update_layout(  # pyright: ignore[reportUnknownMemberType]
+                yaxis_title="Prediction",
+                yaxis2_title="Feature",
+                xaxis2_title="Time",
+                margin={"t": 30, "r": 10, "b": 40, "l": 120},
+            )
+        else:
+            fig = go.Figure(
+                data=heatmap,
+                layout={
+                    "xaxis_title": "Time",
+                    "yaxis_title": "Feature",
+                    "margin": {"t": 30, "r": 10, "b": 40, "l": 120},
+                },
+            )
+
+        return fig
+
+    @staticmethod
+    def plot_waterfall(
+        contributions: TimeSeriesDataset,
+        timestep: int = 0,
+        top_n: int = 10,
+        target_column: str = "load",
+        bias_column: str = "bias",
+    ) -> go.Figure:
+        """Create a waterfall chart decomposing a single timestep's prediction.
+
+        Shows how the bias (base value) is pushed up or down by each feature's
+        contribution to arrive at the final prediction.
+
+        Args:
+            contributions: Output of ``predict_contributions()``.
+            timestep: Row index (0-based) of the timestep to explain.
+            top_n: Number of top features to show. Remaining features are
+                aggregated into an "other" bar.
+            target_column: Name of the target column to exclude. Default "load".
+            bias_column: Name of the bias column used as base value. Default "bias".
+
+        Returns:
+            Plotly Figure with waterfall chart.
+        """
+        bias = contributions.data[bias_column] if bias_column in contributions.data.columns else None
+        cols_to_drop = [c for c in [target_column, bias_column] if c in contributions.data.columns]
+        df = contributions.data.drop(columns=cols_to_drop)
+        row = df.iloc[timestep]
+        base_value = float(bias.iloc[timestep]) if bias is not None else 0.0
+
+        # Rank by |contribution| for this specific timestep
+        abs_sorted = row.abs().sort_values(ascending=False)
+        top = abs_sorted.head(top_n).index.tolist()
+        remaining = [c for c in abs_sorted.index if c not in top]
+
+        names: list[str] = [bias_column]
+        values: list[float] = [base_value]
+        measures: list[str] = ["absolute"]
+
+        for feat in top:
+            names.append(feat)
+            values.append(float(row[feat]))  # pyright: ignore[reportArgumentType]
+            measures.append("relative")
+
+        if len(remaining) > 0:
+            other_sum = float(row[remaining].sum())
+            names.append(f"other ({len(remaining)})")
+            values.append(other_sum)
+            measures.append("relative")
+
+        names.append("Prediction")
+        values.append(base_value + float(row.sum()))
+        measures.append("total")
+
+        timestamp = contributions.data.index[timestep]
+        return go.Figure(
+            go.Waterfall(
+                x=names,
+                y=values,
+                measure=measures,
+                connector={"line": {"color": "grey", "width": 0.5}},
+                increasing={"marker": {"color": "#ff4136"}},
+                decreasing={"marker": {"color": "#0074d9"}},
+                totals={"marker": {"color": "#2ecc40"}},
+                textposition="outside",
+                text=[f"{v:+.4f}" if m == "relative" else f"{v:.4f}" for v, m in zip(values, measures, strict=True)],
+            ),
+            layout={
+                "title": f"Contributions at {timestamp}",
+                "yaxis_title": "Contribution",
+                "margin": {"t": 50, "r": 10, "b": 40, "l": 60},
+                "showlegend": False,
+            },
+        )
+
+    @staticmethod
+    def plot_bar(
+        contributions: TimeSeriesDataset,
+        top_n: int = 10,
+        target_column: str = "load",
+        bias_column: str = "bias",
+    ) -> go.Figure:
+        """Create a horizontal bar chart of mean absolute contributions per feature.
+
+        Features are ranked from most to least important (top to bottom).
+
+        Args:
+            contributions: Output of ``predict_contributions()``.
+            top_n: Number of top features to show.
+            target_column: Name of the target column to exclude. Default "load".
+            bias_column: Name of the bias column to exclude. Default "bias".
+
+        Returns:
+            Plotly Figure with horizontal bar chart.
+        """
+        cols_to_drop = [c for c in [target_column, bias_column] if c in contributions.data.columns]
+        df = contributions.data.drop(columns=cols_to_drop)
+        mean_abs: pd.Series = df.abs().mean().sort_values(ascending=False).head(top_n)
+
+        # Reverse for plotly (bottom-to-top rendering)
+        mean_abs = mean_abs.iloc[::-1]
+
+        return go.Figure(
+            go.Bar(
+                x=mean_abs.values,  # pyright: ignore[reportArgumentType]
+                y=mean_abs.index.tolist(),
+                orientation="h",
+                marker_color="#1f77b4",
+                hovertemplate="<b>%{y}</b><br>mean |SHAP|: %{x:.4f}<extra></extra>",
+            ),
+            layout={
+                "xaxis_title": "mean |SHAP value|",
+                "yaxis_title": "Feature",
+                "margin": {"t": 30, "r": 10, "b": 40, "l": 120},
+                "showlegend": False,
+            },
+        )