openstef-beam 4.0.0__py3-none-any.whl

openstef_beam/__init__.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""OpenSTEF BEAM: Backtesting, Evaluation, Analysis and Metrics framework.
+
+BEAM provides all the tools to test energy forecasting models under realistic conditions.
+Instead of simple validation that can mislead, BEAM simulates real-world scenarios using
+versioned data - ensuring models only use information available at prediction time.
+
+Key features:
+
+**Real-world simulation**: Uses versioned data to prevent data leakage. Models are
+retrained periodically and can only access historical information, just like in production.
+
+**Flexible integration**: Plug in your own forecasting models, create custom metrics,
+design specific visualizations, and select relevant lead times for your use case.
+
+**Complete workflow**: Backtesting → Evaluation → Analysis → Benchmarking. From testing
+individual models to comparing multiple approaches across different energy targets.
+
+**Lead time analysis**: Evaluate how forecast quality changes from 1-hour to 48-hour
+predictions, critical for operational planning in energy systems.
+
+Use BEAM to:
+
+    - Test models under realistic operational constraints
+    - Compare forecasting approaches with versioned data integrity
+    - Generate flexible reports tailored to your metrics and visualizations
+    - Ensure evaluation results match real-world deployment performance
+
+BEAM's versioned data approach and flexible architecture make it the reliable choice
+for energy forecasting model validation that translates to production success.
+"""
+
+import logging
+
+# Set up logging configuration
+root_logger = logging.getLogger(name=__name__)
+if not root_logger.handlers:
+    root_logger.addHandler(logging.NullHandler())
+
+__all__ = []

openstef_beam/analysis/__init__.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Turns evaluation results into visualizations and reports for decision making.
+
+Numbers and metrics are hard to interpret. This module creates charts, plots, and
+summary reports that help you understand model performance and make decisions about
+which models to deploy. It automatically generates the visualizations most relevant
+for energy forecasting operations.
+
+What you get:
+
+    - Performance charts: See how models compare across different metrics
+    - Time series plots: Visualize predictions vs actual load over time
+    - Error analysis: Understand when and why models make mistakes
+    - Comparison reports: Side-by-side model performance analysis
+"""
+
+from openstef_beam.analysis import plots, visualizations
+from openstef_beam.analysis.analysis_pipeline import AnalysisConfig, AnalysisPipeline
+from openstef_beam.analysis.models import AnalysisOutput, AnalysisScope, VisualizationOutput
+
+__all__ = [
+    "AnalysisConfig",
+    "AnalysisOutput",
+    "AnalysisPipeline",
+    "AnalysisScope",
+    "VisualizationOutput",
+    "plots",
+    "visualizations",
+]

openstef_beam/analysis/analysis_pipeline.py

@@ -0,0 +1,194 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Analysis pipeline for generating visualizations from evaluation reports.
+
+This module provides the core pipeline that orchestrates visualization generation
+from evaluation reports at different aggregation levels. It integrates with the
+benchmarking framework to provide consistent analysis outputs across benchmark runs.
+"""
+
+from collections import defaultdict
+from collections.abc import Sequence
+
+from pydantic import Field
+
+from openstef_beam.analysis.models import (
+    AnalysisAggregation,
+    AnalysisOutput,
+    AnalysisScope,
+    GroupName,
+    TargetMetadata,
+    VisualizationOutput,
+)
+from openstef_beam.analysis.visualizations import ReportTuple, VisualizationProvider
+from openstef_beam.evaluation import EvaluationReport
+from openstef_beam.evaluation.models import Filtering
+from openstef_core.base_model import BaseConfig
+from openstef_core.utils.itertools import groupby
+
+
+class AnalysisConfig(BaseConfig):
+    """Configuration for the analytics pipeline."""
+
+    visualization_providers: list[VisualizationProvider] = Field(
+        default=[], description="List of visualization providers to use for generating analysis outputs"
+    )
+    filterings: list[Filtering] | None = Field(
+        default=None,
+        description="When set, only include these filterings (e.g. LeadTime, AvailableAt) in analysis. "
+        "None means use all filterings found in the evaluation data.",
+    )
+
+
+class AnalysisPipeline:
+    """Orchestrates the generation of visualizations from evaluation reports.
+
+    The pipeline processes evaluation reports at different aggregation levels:
+
+    - Individual targets: Creates detailed visualizations for single targets
+    - Multiple targets: Creates comparative visualizations across target groups
+
+    It integrates with the benchmarking framework by being called from BenchmarkPipeline
+    after evaluation is complete, ensuring consistent visualization generation across
+    all benchmark runs.
+    """
+
+    def __init__(
+        self,
+        config: AnalysisConfig,
+    ) -> None:
+        """Initialize the analysis pipeline with configuration.
+
+        Args:
+            config: Analysis configuration containing visualization providers.
+        """
+        super().__init__()
+        self.config = config
+
+    def _group_by_filtering(
+        self,
+        reports: Sequence[tuple[TargetMetadata, EvaluationReport]],
+    ) -> dict[Filtering, list[ReportTuple]]:
+        """Group reports by their lead time filtering conditions.
+
+        Organizes evaluation reports based on their lead time criteria (e.g.,
+        1-hour ahead vs 24-hour ahead forecasts), enabling comparison of model
+        performance across different forecasting horizons.
+
+        When ``config.filterings`` is set, only subsets matching those filterings are included.
+
+        Returns:
+            Dictionary mapping lead time filtering conditions to lists of report tuples.
+        """
+        allowed = set(self.config.filterings) if self.config.filterings is not None else None
+        return groupby(
+            (subset.filtering, (base_metadata.with_filtering(subset.filtering), subset))
+            for base_metadata, report in reports
+            for subset in report.subset_reports
+            if allowed is None or subset.filtering in allowed
+        )
+
+    def run_for_subsets(
+        self,
+        reports: list[ReportTuple],
+        aggregation: AnalysisAggregation,
+    ) -> list[VisualizationOutput]:
+        """Generate visualizations for a set of evaluation reports at a specific aggregation level.
+
+        Processes the provided evaluation reports through all configured visualization
+        providers that support the requested aggregation level. Only providers that
+        declare support for the aggregation are used.
+
+        Args:
+            reports: List of (metadata, evaluation_subset_report) tuples to visualize.
+                The metadata provides context about the target and run, while the
+                evaluation report contains the metrics and predictions to visualize.
+            aggregation: The aggregation level determining how reports are grouped
+                and compared in visualizations.
+
+        Returns:
+            List of visualization outputs from all applicable providers. Empty if
+            no providers support the requested aggregation level.
+        """
+        return [
+            provider.create(reports=reports, aggregation=aggregation)
+            for provider in self.config.visualization_providers
+            if aggregation in provider.supported_aggregations
+        ]
+
+    def run_for_reports(
+        self,
+        reports: Sequence[tuple[TargetMetadata, EvaluationReport]],
+        scope: AnalysisScope,
+    ) -> AnalysisOutput:
+        """Generate visualizations for evaluation reports within a specific scope.
+
+        Groups reports by lead time filtering conditions and generates visualizations
+        for each group using all configured visualization providers that support the
+        scope's aggregation level. This enables comparing model performance across
+        different forecasting horizons (e.g., short-term vs long-term predictions).
+
+        Args:
+            reports: List of (metadata, evaluation_report) tuples to visualize.
+            scope: Analysis scope defining aggregation level and context.
+
+        Returns:
+            Analysis output containing all generated visualizations grouped by
+            lead time filtering conditions.
+        """
+        grouped = self._group_by_filtering(reports)
+
+        result: dict[Filtering, list[VisualizationOutput]] = defaultdict(list)
+        for filtering, subset_reports in grouped.items():
+            visualizations = self.run_for_subsets(
+                reports=subset_reports,
+                aggregation=scope.aggregation,
+            )
+            result[filtering].extend(visualizations)
+
+        return AnalysisOutput(
+            scope=scope,
+            visualizations=result,
+        )
+
+    def run_for_groups(
+        self,
+        reports: Sequence[tuple[TargetMetadata, EvaluationReport]],
+        scope: AnalysisScope,
+    ) -> dict[GroupName, AnalysisOutput]:
+        """Generate visualizations for multiple target groups at a specific aggregation level.
+
+        This method processes all evaluation reports, grouping them by their target
+        group names and generating visualizations for each group.
+
+        Args:
+            reports: List of (metadata, evaluation_report) tuples to visualize.
+                The metadata provides context about the target and run, while the
+                evaluation report contains the metrics and predictions to visualize.
+            scope: The analytics scope defining how reports are grouped and aggregated.
+
+        Returns:
+            Dictionary mapping group names to their corresponding AnalyticsOutput.
+        """
+        reports_by_group: dict[GroupName, list[tuple[TargetMetadata, EvaluationReport]]] = groupby(
+            (report[0].group_name, report) for report in reports
+        )
+
+        result: dict[GroupName, AnalysisOutput] = {}
+        for group_name, group_reports in reports_by_group.items():
+            if not group_reports:
+                continue
+
+            result[group_name] = self.run_for_reports(
+                reports=group_reports,
+                scope=AnalysisScope(
+                    aggregation=scope.aggregation,
+                    target_name=scope.target_name,
+                    run_name=scope.run_name,
+                    group_name=group_name,
+                ),
+            )
+
+        return result

openstef_beam/analysis/models/__init__.py

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Data models for analysis pipeline components.
+
+This package provides data models and type definitions for the analysis pipeline,
+including aggregation types, output structures, and metadata containers.
+"""
+
+from openstef_beam.analysis.models.target_metadata import GroupName, RunName, TargetMetadata, TargetName
+from openstef_beam.analysis.models.visualization_aggregation import AnalysisAggregation
+from openstef_beam.analysis.models.visualization_output import AnalysisOutput, AnalysisScope, VisualizationOutput
+
+__all__ = [
+    "AnalysisAggregation",
+    "AnalysisOutput",
+    "AnalysisScope",
+    "GroupName",
+    "RunName",
+    "TargetMetadata",
+    "TargetName",
+    "VisualizationOutput",
+]

openstef_beam/analysis/models/target_metadata.py

@@ -0,0 +1,97 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Target metadata models for analysis pipeline.
+
+This module defines the metadata structure that carries target information
+through the analysis pipeline, including grouping and filtering context.
+"""
+
+from typing import Self
+
+from pydantic import Field, model_validator
+
+from openstef_beam.evaluation.models import Filtering
+from openstef_core.base_model import BaseModel
+
+type TargetName = str
+type GroupName = str
+type RunName = str
+
+
+class TargetMetadata(BaseModel):
+    """Metadata for a forecasting target in the analysis pipeline.
+
+    Contains essential information about a target including its grouping context
+    and lead time filtering criteria. Lead time filtering determines which
+    predictions are included based on how far ahead they were made (e.g.,
+    1-hour ahead vs 24-hour ahead forecasts).
+
+    Raises:
+        ValueError: When limit constraints are not met (either 'limit' alone or both
+                   'upper_limit' and 'lower_limit' must be specified).
+    """
+
+    name: TargetName = Field(description="Name of the target")
+    group_name: GroupName = Field(description="Name of the group this target belongs to, used for grouping in reports")
+    filtering: Filtering | None = Field(
+        description="Lead time filtering criteria - either AvailableAt (data availability time) "
+        "or LeadTime (forecast horizon)"
+    )
+    limit: float | None = Field(default=None, description="Capacity limit of the target in appropriate units")
+    upper_limit: float | None = Field(
+        default=None, description="Upper capacity limit of the target in appropriate units"
+    )
+    lower_limit: float | None = Field(
+        default=None, description="Lower capacity limit of the target in appropriate units"
+    )
+    run_name: RunName = Field(description="Name of the run associated with this target")
+
+    @model_validator(mode="after")
+    def validate_limits(self) -> "TargetMetadata":
+        """Validate that either limit or both upper_limit and lower_limit are provided.
+
+        Returns:
+            The validated TargetMetadata instance.
+
+        Raises:
+            ValueError: If neither limit nor (upper_limit and lower_limit) are provided,
+                       or if both limit and (upper_limit or lower_limit) are provided.
+        """
+        has_limit = self.limit is not None
+        has_upper = self.upper_limit is not None
+        has_lower = self.lower_limit is not None
+
+        if has_limit and (has_upper or has_lower):
+            raise ValueError(
+                "Cannot specify both 'limit' and 'upper_limit'/'lower_limit'. "
+                "Use either 'limit' alone or both 'upper_limit' and 'lower_limit'."
+            )
+
+        if not has_limit and not (has_upper and has_lower):
+            raise ValueError("Must specify either 'limit' or both 'upper_limit' and 'lower_limit'.")
+
+        return self
+
+    def with_filtering(self, filtering: Filtering) -> Self:
+        """Returns a copy of the target metadata with different lead time filtering applied.
+
+        Args:
+            filtering: New lead time filtering criteria to apply
+
+        Returns:
+            New TargetMetadata instance with updated lead time filtering
+        """
+        return type(self)(
+            name=self.name,
+            group_name=self.group_name,
+            filtering=filtering,
+            limit=self.limit,
+            upper_limit=self.upper_limit,
+            lower_limit=self.lower_limit,
+            run_name=self.run_name,
+        )
+
+
+__all__ = ["GroupName", "RunName", "TargetMetadata", "TargetName"]

openstef_beam/analysis/models/visualization_aggregation.py

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Analysis aggregation models for visualization grouping.
+
+This module defines aggregation levels and scopes that control how evaluation
+reports are grouped and compared in visualizations.
+"""
+
+from enum import StrEnum
+
+from openstef_beam.analysis.models.target_metadata import GroupName, RunName, TargetName
+from openstef_core.base_model import BaseConfig
+
+
+class AnalysisAggregation(StrEnum):
+    """Defines the aggregation levels for visualizations.
+
+    Each aggregation level determines how evaluation reports are grouped and
+    compared in visualizations, enabling different analytical perspectives.
+
+    Members:
+
+    - NONE ("none"): Single run, single target - individual performance analysis
+    - TARGET ("target"): Single run, per target - cross-target comparison (e.g., RMAE per target)
+    - GROUP ("group"): Single run, multiple targets - cross-group comparison (e.g., RMAE per group)
+    - RUN ("run"): Multiple runs, per target - model comparison on same target for all targets
+    - RUN_AND_GROUP ("run_and_group"): Multiple runs, multiple targets - comparison matrix
+    """
+
+    NONE = "none"
+    TARGET = "target"
+    GROUP = "group"
+    RUN_AND_NONE = "run_and_none"
+    RUN_AND_TARGET = "run_and_target"
+    RUN_AND_GROUP = "run_and_group"
+
+
+class AnalysisScope(BaseConfig):
+    """Defines the scope context for analysis operations.
+
+    Specifies which targets, groups, and runs are included in an analysis,
+    along with the aggregation level for grouping data.
+    """
+
+    target_name: TargetName | None = None
+    group_name: GroupName | None = None
+    run_name: RunName | None = None
+    aggregation: AnalysisAggregation
+
+    def __hash__(self) -> int:
+        """Enable using AnalysisScope as a dictionary key.
+
+        Returns:
+            Hash value based on all scope fields.
+        """
+        return hash((self.target_name, self.group_name, self.run_name, self.aggregation))
+
+
+__all__ = ["AnalysisAggregation", "AnalysisScope"]

openstef_beam/analysis/models/visualization_output.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Output models for analysis visualizations.
+
+This module defines the output structures for generated visualizations,
+including individual visualization outputs and aggregated analysis results.
+"""
+
+from pathlib import Path
+from typing import Any
+
+import plotly.graph_objects as go
+
+from openstef_beam.analysis.models.visualization_aggregation import AnalysisScope
+from openstef_beam.evaluation.models import Filtering
+from openstef_core.base_model import BaseModel
+
+
+class VisualizationOutput:
+    """A generated visualization from evaluation data.
+
+    Represents a single chart, plot, or visual analysis that can be saved as HTML.
+    Contains either a Plotly figure object for interactive visualizations or raw
+    HTML content for static displays.
+
+    Example:
+        Creating a visualization from a Plotly figure
+
+        >>> import plotly.graph_objects as go
+        >>> fig = go.Figure(data=go.Scatter(x=[1, 2, 3], y=[4, 5, 6]))
+        >>> viz = VisualizationOutput("my_chart", figure=fig)
+        >>> viz.get_file_name()
+        'my_chart.html'
+    """
+
+    def __init__(self, name: str, figure: go.Figure | None = None, html: str | None = None):
+        """Initialize visualization output with either a figure or HTML content.
+
+        Args:
+            name: Name identifier for the visualization.
+            figure: Plotly figure object, if available.
+            html: Raw HTML string content, if available.
+
+        Raises:
+            ValueError: If neither figure nor html is provided.
+        """
+        if figure is None and html is None:
+            raise ValueError("Either a plotly figure or an HTML string must be provided.")
+        self.name = name
+        self.figure = figure
+        self.html = html
+
+    def write_html(self, file_path: Path, **kwargs: dict[str, Any]) -> None:
+        """Write the plotly figure or HTML string to an HTML file.
+
+        Raises:
+            ValueError: If neither figure nor HTML content is available.
+        """
+        if self.figure is not None:
+            self.figure.write_html(file_path, include_plotlyjs="cdn", **kwargs)  # type: ignore[reportUnknownMemberType]
+        elif self.html is not None:
+            file_path.write_text(self.html, encoding="utf-8")
+        else:
+            raise ValueError("No figure or HTML to write.")
+
+    def get_file_name(self) -> str:
+        """Return the file name for the visualization."""
+        return f"{self.name}.html"
+
+
+class AnalysisOutput(BaseModel):
+    """Container for analysis results from the benchmarking pipeline.
+
+    Holds all visualizations generated for a specific analysis scope, organized
+    by lead time filtering conditions. This allows comparing model performance
+    across different forecasting horizons (e.g., 1-hour vs 24-hour ahead predictions).
+
+    The output structure enables systematic organization of results from benchmark
+    runs, making it easy to generate reports that compare multiple models across
+    various lead times and targets.
+
+    Attributes:
+        scope: Analysis context defining what was analyzed (targets, runs, aggregation)
+        visualizations: Generated charts and plots grouped by lead time filtering
+    """
+
+    scope: AnalysisScope
+    visualizations: dict[Filtering, list[VisualizationOutput]]
+
+
+__all__ = ["AnalysisOutput", "VisualizationOutput"]

openstef_beam/analysis/plots/__init__.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Plotting components for generating visualizations from evaluation data.
+
+This package provides specialized plotters for different types of analysis
+visualizations, including time series, metrics, and statistical plots.
+"""
+
+from .forecast_time_series_plotter import ForecastTimeSeriesPlotter
+from .grouped_target_metric_plotter import GroupedTargetMetricPlotter
+from .precision_recall_curve_plotter import PrecisionRecallCurvePlotter
+from .quantile_calibration_box_plotter import QuantileCalibrationBoxPlotter
+from .quantile_probability_plotter import QuantileProbabilityPlotter
+from .summary_table_plotter import SummaryTablePlotter
+from .windowed_metric_plotter import WindowedMetricPlotter
+
+__all__ = [
+    "ForecastTimeSeriesPlotter",
+    "GroupedTargetMetricPlotter",
+    "PrecisionRecallCurvePlotter",
+    "QuantileCalibrationBoxPlotter",
+    "QuantileProbabilityPlotter",
+    "SummaryTablePlotter",
+    "WindowedMetricPlotter",
+]