openstef-beam 4.0.0a0__tar.gz

openstef_beam-4.0.0a0/.gitignore

@@ -0,0 +1,127 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com> # noqa E501>
+# SPDX-License-Identifier: MPL-2.0
+
+# Core
+config.user.yaml
+git-template
+.DS_Store
+tmp
+
+# Python bytecode
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Packaging and build artifacts
+.Python
+build/
+dist/
+downloads/
+wheels/
+share/python-wheels/
+sdist/
+.eggs/
+*.egg-info/
+*.egg
+.installed.cfg
+MANIFEST
+
+# uv
+.uv/
+
+# Ruff
+.ruff_cache/
+
+# Pyright
+.pyright/
+# pyright-report/
+
+# Test, coverage, tox
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.cover/
+cover/
+pytest-report.xml
+.tox/
+.nox/
+coverage.xml
+
+# Hypothesis
+.hypothesis/
+
+# Cython
+cython_debug/
+
+# Type checkers (misc)
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+.pyre/
+
+# Sphinx
+docs/_build/
+docs/source/api/generated/
+
+# docs/_doctrees/
+# docs/_static_gen/
+
+# Editors: JetBrains
+.idea/
+
+# Editors: VS Code (allow shared configs)
+.vscode/*
+
+# Jupyter / IPython
+.ipynb_checkpoints
+profile_default/
+ipython_config.py
+
+# Spyder / Rope
+.spyderproject
+.spyproject
+.ropeproject
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PEP 582
+__pypackages__/
+
+# web frameworks / services
+*.sqlite
+*.sqlite3
+*.log
+instance/
+.webassets-cache
+celerybeat-schedule
+celerybeat.pid
+*.sage.py
+tmp/
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Project outputs
+output/
+prof/
+certificates/
+
+# Output artifacts
+*.html
+*.pkl
+
+# Benchmark outputs
+benchmark_results*/

openstef_beam-4.0.0a0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: openstef-beam
+Version: 4.0.0a0
+Summary: Backtesting, Evaluation, Analysis and Metrics (BEAM) library for OpenSTEF
+Project-URL: Documentation, https://openstef.github.io/openstef/index.html
+Project-URL: Homepage, https://lfenergy.org/projects/openstef/
+Project-URL: Issues, https://github.com/OpenSTEF/openstef/issues
+Project-URL: Repository, https://github.com/OpenSTEF/openstef
+Author-email: "Alliander N.V" <short.term.energy.forecasts@alliander.com>
+License-Expression: MPL-2.0
+Keywords: energy,forecasting,machinelearning
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: <4.0,>=3.12
+Requires-Dist: openstef-core<5,>=4.0.0.dev0
+Requires-Dist: plotly>=6.3
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: scoringrules>=0.8
+Requires-Dist: tqdm>=4.67.1
+Provides-Extra: all
+Requires-Dist: s3fs>=2025.5.1; extra == 'all'
+Description-Content-Type: text/markdown
+
+<!--
+SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+
+SPDX-License-Identifier: MPL-2.0
+-->
+
+# openstef-beam

openstef_beam-4.0.0a0/README.md

@@ -0,0 +1,7 @@
+<!--
+SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+
+SPDX-License-Identifier: MPL-2.0
+-->
+
+# openstef-beam

openstef_beam-4.0.0a0/pyproject.toml

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+[build-system]
+build-backend = "hatchling.build"
+
+requires = [ "hatchling" ]
+
+[project]
+name = "openstef-beam"
+version = "4.0.0a0"
+description = "Backtesting, Evaluation, Analysis and Metrics (BEAM) library for OpenSTEF"
+readme = "README.md"
+keywords = [ "energy", "forecasting", "machinelearning" ]
+license = "MPL-2.0"
+authors = [
+  { name = "Alliander N.V", email = "short.term.energy.forecasts@alliander.com" },
+]
+requires-python = ">=3.12,<4.0"
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+]
+
+dependencies = [
+  "openstef-core>=4.0.0.dev0,<5",
+  "plotly>=6.3",
+  "pyyaml>=6.0.2",
+  "scoringrules>=0.8",
+  "tqdm>=4.67.1",
+]
+
+optional-dependencies.all = [
+  "s3fs>=2025.5.1",
+]
+urls.Documentation = "https://openstef.github.io/openstef/index.html"
+urls.Homepage = "https://lfenergy.org/projects/openstef/"
+urls.Issues = "https://github.com/OpenSTEF/openstef/issues"
+urls.Repository = "https://github.com/OpenSTEF/openstef"
+
+[tool.hatch.build.targets.wheel]
+packages = [ "src/openstef_beam" ]
+
+[tool.uv.sources]
+openstef-core = { workspace = true }

openstef_beam-4.0.0a0/src/openstef_beam/__init__.py

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""OpenSTEF BEAM: Backtesting, Evaluation, Analysis and Metrics framework.
+
+BEAM provides all the tools to test energy forecasting models under realistic conditions.
+Instead of simple validation that can mislead, BEAM simulates real-world scenarios using
+versioned data - ensuring models only use information available at prediction time.
+
+Key features:
+
+**Real-world simulation**: Uses versioned data to prevent data leakage. Models are
+retrained periodically and can only access historical information, just like in production.
+
+**Flexible integration**: Plug in your own forecasting models, create custom metrics,
+design specific visualizations, and select relevant lead times for your use case.
+
+**Complete workflow**: Backtesting → Evaluation → Analysis → Benchmarking. From testing
+individual models to comparing multiple approaches across different energy targets.
+
+**Lead time analysis**: Evaluate how forecast quality changes from 1-hour to 48-hour
+predictions, critical for operational planning in energy systems.
+
+Use BEAM to:
+    - Test models under realistic operational constraints
+    - Compare forecasting approaches with versioned data integrity
+    - Generate flexible reports tailored to your metrics and visualizations
+    - Ensure evaluation results match real-world deployment performance
+
+BEAM's versioned data approach and flexible architecture make it the reliable choice
+for energy forecasting model validation that translates to production success.
+"""
+
+import logging
+
+# Set up logging configuration
+root_logger = logging.getLogger(name=__name__)
+if not root_logger.handlers:
+    root_logger.addHandler(logging.NullHandler())
+
+__all__ = []

openstef_beam-4.0.0a0/src/openstef_beam/analysis/__init__.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Turns evaluation results into visualizations and reports for decision making.
+
+Numbers and metrics are hard to interpret. This module creates charts, plots, and
+summary reports that help you understand model performance and make decisions about
+which models to deploy. It automatically generates the visualizations most relevant
+for energy forecasting operations.
+
+What you get:
+    - Performance charts: See how models compare across different metrics
+    - Time series plots: Visualize predictions vs actual load over time
+    - Error analysis: Understand when and why models make mistakes
+    - Comparison reports: Side-by-side model performance analysis
+"""
+
+from openstef_beam.analysis import plots, visualizations
+from openstef_beam.analysis.analysis_pipeline import AnalysisConfig, AnalysisPipeline
+from openstef_beam.analysis.models import AnalysisOutput, AnalysisScope, VisualizationOutput
+
+__all__ = [
+    "AnalysisConfig",
+    "AnalysisOutput",
+    "AnalysisPipeline",
+    "AnalysisScope",
+    "VisualizationOutput",
+    "plots",
+    "visualizations",
+]

openstef_beam-4.0.0a0/src/openstef_beam/analysis/analysis_pipeline.py

@@ -0,0 +1,187 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Analysis pipeline for generating visualizations from evaluation reports.
+
+This module provides the core pipeline that orchestrates visualization generation
+from evaluation reports at different aggregation levels. It integrates with the
+benchmarking framework to provide consistent analysis outputs across benchmark runs.
+"""
+
+from collections import defaultdict
+from collections.abc import Sequence
+
+from pydantic import Field
+
+from openstef_beam.analysis.models import (
+    AnalysisAggregation,
+    AnalysisOutput,
+    AnalysisScope,
+    GroupName,
+    TargetMetadata,
+    VisualizationOutput,
+)
+from openstef_beam.analysis.visualizations import ReportTuple, VisualizationProvider
+from openstef_beam.evaluation import EvaluationReport
+from openstef_beam.evaluation.models import Filtering
+from openstef_core.base_model import BaseConfig
+from openstef_core.utils.itertools import groupby
+
+
+class AnalysisConfig(BaseConfig):
+    """Configuration for the analytics pipeline."""
+
+    visualization_providers: list[VisualizationProvider] = Field(
+        default=[], description="List of visualization providers to use for generating analysis outputs"
+    )
+
+
+class AnalysisPipeline:
+    """Orchestrates the generation of visualizations from evaluation reports.
+
+    The pipeline processes evaluation reports at different aggregation levels:
+    - Individual targets: Creates detailed visualizations for single targets
+    - Multiple targets: Creates comparative visualizations across target groups
+
+    It integrates with the benchmarking framework by being called from BenchmarkPipeline
+    after evaluation is complete, ensuring consistent visualization generation across
+    all benchmark runs.
+    """
+
+    def __init__(
+        self,
+        config: AnalysisConfig,
+    ) -> None:
+        """Initialize the analysis pipeline with configuration.
+
+        Args:
+            config: Analysis configuration containing visualization providers.
+        """
+        super().__init__()
+        self.config = config
+
+    @staticmethod
+    def _group_by_filtering(
+        reports: Sequence[tuple[TargetMetadata, EvaluationReport]],
+    ) -> dict[Filtering, list[ReportTuple]]:
+        """Group reports by their lead time filtering conditions.
+
+        Organizes evaluation reports based on their lead time criteria (e.g.,
+        1-hour ahead vs 24-hour ahead forecasts), enabling comparison of model
+        performance across different forecasting horizons.
+
+        Returns:
+            Dictionary mapping lead time filtering conditions to lists of report tuples.
+        """
+        return groupby(
+            (subset.filtering, (base_metadata.with_filtering(subset.filtering), subset))
+            for base_metadata, report in reports
+            for subset in report.subset_reports
+        )
+
+    def run_for_subsets(
+        self,
+        reports: list[ReportTuple],
+        aggregation: AnalysisAggregation,
+    ) -> list[VisualizationOutput]:
+        """Generate visualizations for a set of evaluation reports at a specific aggregation level.
+
+        Processes the provided evaluation reports through all configured visualization
+        providers that support the requested aggregation level. Only providers that
+        declare support for the aggregation are used.
+
+        Args:
+            reports: List of (metadata, evaluation_subset_report) tuples to visualize.
+                The metadata provides context about the target and run, while the
+                evaluation report contains the metrics and predictions to visualize.
+            aggregation: The aggregation level determining how reports are grouped
+                and compared in visualizations.
+
+        Returns:
+            List of visualization outputs from all applicable providers. Empty if
+            no providers support the requested aggregation level.
+        """
+        return [
+            provider.create(
+                reports=reports,
+                aggregation=aggregation,
+            )
+            for provider in self.config.visualization_providers
+            if aggregation in provider.supported_aggregations
+        ]
+
+    def run_for_reports(
+        self,
+        reports: Sequence[tuple[TargetMetadata, EvaluationReport]],
+        scope: AnalysisScope,
+    ) -> AnalysisOutput:
+        """Generate visualizations for evaluation reports within a specific scope.
+
+        Groups reports by lead time filtering conditions and generates visualizations
+        for each group using all configured visualization providers that support the
+        scope's aggregation level. This enables comparing model performance across
+        different forecasting horizons (e.g., short-term vs long-term predictions).
+
+        Args:
+            reports: List of (metadata, evaluation_report) tuples to visualize.
+            scope: Analysis scope defining aggregation level and context.
+
+        Returns:
+            Analysis output containing all generated visualizations grouped by
+            lead time filtering conditions.
+        """
+        grouped = self._group_by_filtering(reports)
+
+        result: dict[Filtering, list[VisualizationOutput]] = defaultdict(list)
+        for filtering, subset_reports in grouped.items():
+            visualizations = self.run_for_subsets(
+                reports=subset_reports,
+                aggregation=scope.aggregation,
+            )
+            result[filtering].extend(visualizations)
+
+        return AnalysisOutput(
+            scope=scope,
+            visualizations=result,
+        )
+
+    def run_for_groups(
+        self,
+        reports: Sequence[tuple[TargetMetadata, EvaluationReport]],
+        scope: AnalysisScope,
+    ) -> dict[GroupName, AnalysisOutput]:
+        """Generate visualizations for multiple target groups at a specific aggregation level.
+
+        This method processes all evaluation reports, grouping them by their target
+        group names and generating visualizations for each group.
+
+        Args:
+            reports: List of (metadata, evaluation_report) tuples to visualize.
+                The metadata provides context about the target and run, while the
+                evaluation report contains the metrics and predictions to visualize.
+            scope: The analytics scope defining how reports are grouped and aggregated.
+
+        Returns:
+            Dictionary mapping group names to their corresponding AnalyticsOutput.
+        """
+        reports_by_group: dict[GroupName, list[tuple[TargetMetadata, EvaluationReport]]] = groupby(
+            (report[0].group_name, report) for report in reports
+        )
+
+        result: dict[GroupName, AnalysisOutput] = {}
+        for group_name, group_reports in reports_by_group.items():
+            if not group_reports:
+                continue
+
+            result[group_name] = self.run_for_reports(
+                reports=group_reports,
+                scope=AnalysisScope(
+                    aggregation=scope.aggregation,
+                    target_name=scope.target_name,
+                    run_name=scope.run_name,
+                    group_name=group_name,
+                ),
+            )
+
+        return result

openstef_beam-4.0.0a0/src/openstef_beam/analysis/models/__init__.py

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Data models for analysis pipeline components.
+
+This package provides data models and type definitions for the analysis pipeline,
+including aggregation types, output structures, and metadata containers.
+"""
+
+from openstef_beam.analysis.models.target_metadata import GroupName, RunName, TargetMetadata, TargetName
+from openstef_beam.analysis.models.visualization_aggregation import AnalysisAggregation
+from openstef_beam.analysis.models.visualization_output import AnalysisOutput, AnalysisScope, VisualizationOutput
+
+__all__ = [
+    "AnalysisAggregation",
+    "AnalysisOutput",
+    "AnalysisScope",
+    "GroupName",
+    "RunName",
+    "TargetMetadata",
+    "TargetName",
+    "VisualizationOutput",
+]

openstef_beam-4.0.0a0/src/openstef_beam/analysis/models/target_metadata.py

@@ -0,0 +1,97 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Target metadata models for analysis pipeline.
+
+This module defines the metadata structure that carries target information
+through the analysis pipeline, including grouping and filtering context.
+"""
+
+from typing import Self
+
+from pydantic import Field, model_validator
+
+from openstef_beam.evaluation.models import Filtering
+from openstef_core.base_model import BaseModel
+
+type TargetName = str
+type GroupName = str
+type RunName = str
+
+
+class TargetMetadata(BaseModel):
+    """Metadata for a forecasting target in the analysis pipeline.
+
+    Contains essential information about a target including its grouping context
+    and lead time filtering criteria. Lead time filtering determines which
+    predictions are included based on how far ahead they were made (e.g.,
+    1-hour ahead vs 24-hour ahead forecasts).
+
+    Raises:
+        ValueError: When limit constraints are not met (either 'limit' alone or both
+                   'upper_limit' and 'lower_limit' must be specified).
+    """
+
+    name: TargetName = Field(description="Name of the target")
+    group_name: GroupName = Field(description="Name of the group this target belongs to, used for grouping in reports")
+    filtering: Filtering | None = Field(
+        description="Lead time filtering criteria - either AvailableAt (data availability time) "
+        "or LeadTime (forecast horizon)"
+    )
+    limit: float | None = Field(default=None, description="Capacity limit of the target in appropriate units")
+    upper_limit: float | None = Field(
+        default=None, description="Upper capacity limit of the target in appropriate units"
+    )
+    lower_limit: float | None = Field(
+        default=None, description="Lower capacity limit of the target in appropriate units"
+    )
+    run_name: RunName = Field(description="Name of the run associated with this target")
+
+    @model_validator(mode="after")
+    def validate_limits(self) -> "TargetMetadata":
+        """Validate that either limit or both upper_limit and lower_limit are provided.
+
+        Returns:
+            The validated TargetMetadata instance.
+
+        Raises:
+            ValueError: If neither limit nor (upper_limit and lower_limit) are provided,
+                       or if both limit and (upper_limit or lower_limit) are provided.
+        """
+        has_limit = self.limit is not None
+        has_upper = self.upper_limit is not None
+        has_lower = self.lower_limit is not None
+
+        if has_limit and (has_upper or has_lower):
+            raise ValueError(
+                "Cannot specify both 'limit' and 'upper_limit'/'lower_limit'. "
+                "Use either 'limit' alone or both 'upper_limit' and 'lower_limit'."
+            )
+
+        if not has_limit and not (has_upper and has_lower):
+            raise ValueError("Must specify either 'limit' or both 'upper_limit' and 'lower_limit'.")
+
+        return self
+
+    def with_filtering(self, filtering: Filtering) -> Self:
+        """Returns a copy of the target metadata with different lead time filtering applied.
+
+        Args:
+            filtering: New lead time filtering criteria to apply
+
+        Returns:
+            New TargetMetadata instance with updated lead time filtering
+        """
+        return type(self)(
+            name=self.name,
+            group_name=self.group_name,
+            filtering=filtering,
+            limit=self.limit,
+            upper_limit=self.upper_limit,
+            lower_limit=self.lower_limit,
+            run_name=self.run_name,
+        )
+
+
+__all__ = ["GroupName", "RunName", "TargetMetadata", "TargetName"]

openstef_beam-4.0.0a0/src/openstef_beam/analysis/models/visualization_aggregation.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <short.term.energy.forecasts@alliander.com>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Analysis aggregation models for visualization grouping.
+
+This module defines aggregation levels and scopes that control how evaluation
+reports are grouped and compared in visualizations.
+"""
+
+from enum import StrEnum
+
+from openstef_beam.analysis.models.target_metadata import GroupName, RunName, TargetName
+from openstef_core.base_model import BaseConfig
+
+
+class AnalysisAggregation(StrEnum):
+    """Defines the aggregation levels for visualizations.
+
+    Each aggregation level determines how evaluation reports are grouped and
+    compared in visualizations, enabling different analytical perspectives.
+
+    Members:
+    - NONE ("none"): Single run, single target - individual performance analysis
+    - TARGET ("target"): Single run, per target - cross-target comparison (e.g., RMAE per target)
+    - GROUP ("group"): Single run, multiple targets - cross-group comparison (e.g., RMAE per group)
+    - RUN ("run"): Multiple runs, per target - model comparison on same target for all targets
+    - RUN_AND_GROUP ("run_and_group"): Multiple runs, multiple targets - comparison matrix
+    """
+
+    NONE = "none"
+    TARGET = "target"
+    GROUP = "group"
+    RUN_AND_NONE = "run_and_none"
+    RUN_AND_TARGET = "run_and_target"
+    RUN_AND_GROUP = "run_and_group"
+
+
+class AnalysisScope(BaseConfig):
+    """Defines the scope context for analysis operations.
+
+    Specifies which targets, groups, and runs are included in an analysis,
+    along with the aggregation level for grouping data.
+    """
+
+    target_name: TargetName | None = None
+    group_name: GroupName | None = None
+    run_name: RunName | None = None
+    aggregation: AnalysisAggregation
+
+    def __hash__(self) -> int:
+        """Enable using AnalysisScope as a dictionary key.
+
+        Returns:
+            Hash value based on all scope fields.
+        """
+        return hash((self.target_name, self.group_name, self.run_name, self.aggregation))
+
+
+__all__ = ["AnalysisAggregation", "AnalysisScope"]