openstef-core 4.0.0__py3-none-any.whl

openstef_core/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2017-2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+"""Core functionality for OpenSTEF, a framework for short-term energy forecasting."""

openstef_core/base_model.py

@@ -0,0 +1,205 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Configuration utilities for OpenSTEF Beam.
+
+This module provides a `BaseConfig` class extending Pydantic's `BaseModel`
+with convenience helpers for reading from and writing to YAML files. It also
+exposes two helper functions `write_yaml_config` and `read_yaml_config` that
+operate on arbitrary config instances or Pydantic models / adapters.
+"""
+
+from pathlib import Path
+from typing import Annotated, Any, Self
+
+import yaml
+from pydantic import BaseModel as PydanticBaseModel
+from pydantic import (
+    BeforeValidator,
+    ConfigDict,
+    GetCoreSchemaHandler,
+    GetJsonSchemaHandler,
+    TypeAdapter,
+    ValidationInfo,
+)
+from pydantic_core import core_schema
+
+
+class BaseModel(PydanticBaseModel):
+    """Base model class for OpenSTEF components."""
+
+    model_config = ConfigDict(protected_namespaces=(), arbitrary_types_allowed=True, ser_json_inf_nan="null")
+
+
+class BaseConfig(PydanticBaseModel):
+    """Base configuration model.
+
+    It configures Pydantic model for safe YAML serialization / deserialization.
+    """
+
+    model_config = ConfigDict(
+        protected_namespaces=(),
+        extra="ignore",
+        arbitrary_types_allowed=False,
+    )
+
+    @classmethod
+    def read_yaml(cls, path: Path) -> Self:
+        """Create an instance from a YAML file.
+
+        Args:
+            path: Path to the YAML file to read.
+
+        Returns:
+            An instance of the config class populated with the file contents.
+        """
+        return read_yaml_config(path, class_type=cls)
+
+    def write_yaml(self, path: Path) -> None:
+        """Write this configuration to a YAML file.
+
+        Args:
+            path: Destination path for the YAML file (will be overwritten).
+        """
+        write_yaml_config(self, path)
+
+
+def write_yaml_config(config: BaseConfig, path: Path) -> None:
+    """Write the config to a YAML file.
+
+    Args:
+        config: The configuration object to serialize.
+        path: Destination path for the YAML file (will be overwritten).
+
+    Example:
+        >>> from pathlib import Path
+        >>> from pydantic import BaseModel
+        >>> class MyConfig(BaseModel):
+        ...     foo: int
+        >>> cfg = MyConfig(foo=123)
+        >>> write_yaml_config(cfg, Path("/tmp/test.yaml"))
+    """
+    with path.open("w", encoding="utf-8") as f:
+        yaml.dump(config.model_dump(mode="json"), f, allow_unicode=True)
+
+
+def read_yaml_config[T: BaseConfig, U](path: Path, class_type: type[T] | TypeAdapter[U]) -> T | U:
+    """Read a configuration object from a YAML file.
+
+    This function supports two kinds of targets:
+
+    * A subclass of `BaseConfig`, in which case Pydantic's `model_validate` is used.
+    * A `TypeAdapter` instance for more advanced / non-`BaseModel` schema validation.
+
+    Args:
+        path: Path to the YAML file to read.
+        class_type: The target type (a `BaseConfig` subclass) or a `TypeAdapter`.
+
+    Returns:
+        A validated configuration instance (either ``T`` or ``U`` depending on
+        the provided ``class_type``).
+    """
+    with path.open("r", encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+
+    if isinstance(class_type, TypeAdapter):
+        return class_type.validate_python(data)
+
+    return class_type.model_validate(data)
+
+
+class PydanticStringPrimitive:
+    """Base class for Pydantic-compatible types with string serialization."""
+
+    def __str__(self) -> str:
+        """Convert to string representation."""
+        raise NotImplementedError("Subclasses must implement __str__")
+
+    @classmethod
+    def from_string(cls, s: str) -> Self:
+        """Create an instance from string representation."""
+        raise NotImplementedError("Subclasses must implement from_string")
+
+    @classmethod
+    def validate(cls, v: Any, _info: ValidationInfo | None = None) -> Self:  # noqa: ANN401
+        """Validate and convert input to this type.
+
+        Args:
+            v: Input value to validate.
+            _info: Additional validation info (unused).
+
+        Returns:
+            Validated instance of this type.
+
+        Raises:
+            ValueError: If input cannot be converted to this type.
+        """
+        if isinstance(v, cls):
+            return v
+        if isinstance(v, str):
+            return cls.from_string(v)
+
+        # Subclasses should handle their specific types
+        error_message = f"Cannot convert {v} to {cls.__name__}"
+        raise ValueError(error_message)
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, _source_type: type[Any], _handler: GetCoreSchemaHandler
+    ) -> core_schema.CoreSchema:
+        """Define Pydantic validation and serialization behavior.
+
+        Returns:
+            Core schema for Pydantic validation and serialization.
+        """
+        return core_schema.with_info_plain_validator_function(
+            function=cls.validate, serialization=core_schema.plain_serializer_function_ser_schema(cls.__str__)
+        )
+
+    @classmethod
+    def __get_pydantic_json_schema__(  # noqa: PLW3201
+        cls,
+        _schema: core_schema.CoreSchema,
+        handler: GetJsonSchemaHandler,
+    ) -> dict[str, Any]:
+        """Generate JSON schema for OpenAPI / FastAPI compatibility.
+
+        All string-primitive types serialise as plain strings.
+
+        Returns:
+            JSON schema describing the type as a string.
+        """
+        return {"type": "string"}
+
+    def __eq__(self, other: object) -> bool:
+        """Check equality based on string representation.
+
+        Returns:
+            True if both objects have the same string representation, False otherwise.
+        """
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return str(self) == str(other)
+
+    def __hash__(self) -> int:
+        """Return hash based on string representation."""
+        return hash(str(self))
+
+
+def _convert_none_to_nan(v: float | None) -> float:
+    if v is None:
+        return float("nan")
+    return v
+
+
+FloatOrNan = Annotated[float, BeforeValidator(_convert_none_to_nan)]
+
+__all__ = [
+    "BaseConfig",
+    "BaseModel",
+    "FloatOrNan",
+    "PydanticStringPrimitive",
+    "read_yaml_config",
+    "write_yaml_config",
+]

openstef_core/constants.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Shared constants for the openstef_core package."""
+
+LIANDER_DATASET_REPO_ID = "OpenSTEF/liander2024-energy-forecasting-benchmark"
+
+
+__all__ = ["LIANDER_DATASET_REPO_ID"]

openstef_core/datasets/__init__.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Time series datasets and versioned data access.
+
+This module provides core data structures for handling time series data in OpenSTEF forecasting.
+It includes both simple time series datasets and versioned datasets that track data availability
+over time, enabling realistic backtesting and training and forecasting.
+
+The module supports:
+
+    - Regular time series with consistent sampling intervals
+    - Versioned time series that track when data became available
+    - Validated datasets with domain-specific constraints
+    - Data transformations and validation utilities
+    - Feature concatenation and horizon restriction operations
+"""
+
+from openstef_core.datasets.timeseries_dataset import TimeSeriesDataset, validate_horizons_present
+from openstef_core.datasets.validated_datasets import (
+    EnergyComponentDataset,
+    EnsembleForecastDataset,
+    ForecastDataset,
+    ForecastInputDataset,
+)
+from openstef_core.datasets.versioned_timeseries_dataset import VersionedTimeSeriesDataset
+
+__all__ = [
+    "EnergyComponentDataset",
+    "EnsembleForecastDataset",
+    "ForecastDataset",
+    "ForecastInputDataset",
+    "TimeSeriesDataset",
+    "VersionedTimeSeriesDataset",
+    "validate_horizons_present",
+]

openstef_core/datasets/mixins.py

@@ -0,0 +1,221 @@
+# SPDX-FileCopyrightText: 2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+"""Protocol definitions for time series dataset interfaces.
+
+This module provides protocol classes that define the core interfaces for time series
+datasets in OpenSTEF. Protocols enable type checking and documentation of expected
+behavior without requiring inheritance.
+
+Key protocols:
+
+    - TimeSeries: Core interface for all time series datasets with filtering and versioning
+    - DatasetMixin: Interface for dataset persistence operations
+"""
+
+from abc import abstractmethod
+from collections.abc import Callable
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Concatenate, Protocol, Self
+
+from pydantic import FilePath
+
+from openstef_core.types import AvailableAt, LeadTime
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+    from openstef_core.datasets.timeseries_dataset import TimeSeriesDataset
+
+
+class TimeSeriesMixin(Protocol):
+    """Interface defining the interface for time series datasets.
+
+    This interface defines the essential operations that all time series datasets
+    must implement. It provides access to feature metadata, temporal properties,
+    the dataset's temporal index, and filtering/versioning capabilities.
+
+    Classes implementing this interface must provide:
+
+        - Access to the datetime index
+        - Sample interval information
+        - Feature names list
+        - Versioning status indicator
+        - Filtering methods for time ranges, availability, and lead times
+        - Version selection for point-in-time data reconstruction
+    """
+
+    @property
+    @abstractmethod
+    def index(self) -> "pd.DatetimeIndex":
+        """Get the datetime index of the dataset.
+
+        Returns:
+            DatetimeIndex representing all timestamps in the dataset.
+        """
+
+    @property
+    @abstractmethod
+    def sample_interval(self) -> timedelta:
+        """Get the fixed time interval between consecutive data points.
+
+        Returns:
+            The sample interval as a timedelta.
+        """
+
+    @property
+    @abstractmethod
+    def feature_names(self) -> list[str]:
+        """Get the names of all available features in the dataset.
+
+        Returns:
+            List of feature names, excluding metadata columns like timestamp,
+            available_at, or horizon.
+        """
+
+    @property
+    @abstractmethod
+    def is_versioned(self) -> bool:
+        """Check if the dataset tracks data availability over time.
+
+        Returns:
+            True if the dataset is versioned (tracks availability via horizon
+            or available_at columns), False for regular time series.
+        """
+
+    @abstractmethod
+    def filter_by_range(self, start: datetime | None = None, end: datetime | None = None) -> Self:
+        """Filter the dataset to include only data within the specified time range.
+
+        Args:
+            start: Inclusive start time of the range. If None, no start boundary applied.
+            end: Exclusive end time of the range. If None, no end boundary applied.
+
+        Returns:
+            New instance containing only data within [start, end).
+        """
+
+    @abstractmethod
+    def filter_by_available_before(self, available_before: datetime) -> Self:
+        """Filter to include only data available before the specified timestamp.
+
+        Args:
+            available_before: Cutoff time for data availability.
+
+        Returns:
+            New instance containing only data available before the cutoff.
+        """
+
+    @abstractmethod
+    def filter_by_available_at(self, available_at: AvailableAt) -> Self:
+        """Filter based on realistic daily data availability constraints.
+
+        Args:
+            available_at: Specification defining when data becomes available.
+
+        Returns:
+            New instance with data filtered by availability pattern.
+        """
+
+    @abstractmethod
+    def filter_by_lead_time(self, lead_time: LeadTime) -> Self:
+        """Filter to include only data available at or longer than the specified lead time.
+
+        Args:
+            lead_time: Minimum time gap required between data availability and timestamp.
+
+        Returns:
+            New instance containing only data available with the required lead time.
+        """
+
+    @abstractmethod
+    def select_version(self) -> "TimeSeriesDataset":
+        """Select a specific version of the dataset based on data availability.
+
+        Creates a point-in-time snapshot by selecting the latest available version
+        for each timestamp. Essential for preventing lookahead bias in backtesting.
+
+        Returns:
+            TimeSeriesDataset containing the selected version of the data.
+        """
+
+    def calculate_time_coverage(self) -> timedelta:
+        """Calculates the total time span covered by the dataset.
+
+        This method computes the total duration represented by the dataset
+        based on its unique timestamps and sample interval.
+
+        Returns:
+            timedelta: Total time coverage of the dataset.
+        """
+        return len(self.index.unique()) * self.sample_interval
+
+
+class DatasetMixin(Protocol):
+    """Abstract base class for dataset persistence operations.
+
+    This mixin defines the interface for saving and loading datasets to/from
+    parquet files. It ensures datasets can be persisted with all their metadata
+    and reconstructed exactly as they were saved.
+
+    Classes implementing this mixin must:
+
+    - Save all data and metadata necessary for complete reconstruction
+    - Store metadata in parquet file attributes using attrs
+    - Handle missing metadata gracefully with sensible defaults when loading
+
+    See Also:
+        TimeSeriesDataset: Implementation for standard time series datasets.
+        VersionedTimeSeriesDataset: Implementation for versioned dataset segments.
+    """
+
+    @abstractmethod
+    def to_parquet(self, path: FilePath) -> None:
+        """Save the dataset to a parquet file.
+
+        Stores both the dataset's data and all necessary metadata for complete
+        reconstruction. Metadata should be stored in the parquet file's attrs
+        dictionary.
+
+        Args:
+            path: File path where the dataset should be saved.
+
+        See Also:
+            read_parquet: Counterpart method for loading datasets.
+        """
+
+    @classmethod
+    @abstractmethod
+    def read_parquet(cls, path: FilePath) -> Self:
+        """Load a dataset from a parquet file.
+
+        Reconstructs a dataset from a parquet file created with to_parquet,
+        including all data and metadata. Should handle missing metadata
+        gracefully with sensible defaults.
+
+        Args:
+            path: Path to the parquet file to load.
+
+        Returns:
+            New dataset instance reconstructed from the file.
+
+        See Also:
+            to_parquet: Counterpart method for saving datasets.
+        """
+
+    def pipe[T, **P](self, func: Callable[Concatenate[Self, P], T], *args: P.args, **kwargs: P.kwargs) -> T:
+        """Applies a function to the dataset and returns the result.
+
+        This method allows for functional-style transformations and operations
+        on the dataset, enabling method chaining and cleaner code.
+
+        Args:
+            func: A callable that takes the dataset instance and returns a value of type T.
+            *args: Positional arguments to pass to the function.
+            **kwargs: Keyword arguments to pass to the function.
+
+        Returns:
+            The result of applying the function to the dataset.
+        """
+        return func(self, *args, **kwargs)