ocf-data-sampler 0.0.18__py3-none-any.whl → 0.0.42__py3-none-any.whl

ocf_data_sampler/config/__init__.py

@@ -0,0 +1,5 @@
+"""Configuration model"""
+
+from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.config.save import save_yaml_configuration
+from ocf_data_sampler.config.load import load_yaml_configuration

ocf_data_sampler/config/load.py

@@ -0,0 +1,33 @@
+"""Loading configuration functions.
+
+Example:
+
+    from ocf_data_sampler.config import load_yaml_configuration
+    configuration = load_yaml_configuration(filename)
+"""
+
+import fsspec
+from pathy import Pathy
+from pyaml_env import parse_config
+
+from ocf_data_sampler.config import Configuration
+
+
+def load_yaml_configuration(filename: str | Pathy) -> Configuration:
+    """
+    Load a yaml file which has a configuration in it
+
+    Args:
+        filename: the file name that you want to load.  Will load from local, AWS, or GCP
+            depending on the protocol suffix (e.g. 's3://bucket/config.yaml').
+
+    Returns:pydantic class
+
+    """
+    # load the file to a dictionary
+    with fsspec.open(filename, mode="r") as stream:
+        configuration = parse_config(data=stream)
+        # this means we can load ENVs in the yaml file
+    # turn into pydantic class
+    configuration = Configuration(**configuration)
+    return configuration

ocf_data_sampler/config/model.py

@@ -0,0 +1,246 @@
+"""Configuration model for the dataset.
+
+All paths must include the protocol prefix.  For local files,
+it's sufficient to just start with a '/'.  For aws, start with 's3://',
+for gcp start with 'gs://'.
+
+Example:
+
+    from ocf_data_sampler.config import Configuration
+    config = Configuration(**config_dict)
+"""
+
+import logging
+from typing import Dict, List, Optional
+from typing_extensions import Self
+
+from pydantic import BaseModel, Field, RootModel, field_validator, ValidationInfo, model_validator
+
+from ocf_data_sampler.constants import NWP_PROVIDERS
+
+logger = logging.getLogger(__name__)
+
+providers = ["pvoutput.org", "solar_sheffield_passiv"]
+
+
+class Base(BaseModel):
+    """Pydantic Base model where no extras can be added"""
+
+    class Config:
+        """config class"""
+
+        extra = "forbid"  # forbid use of extra kwargs
+
+
+class General(Base):
+    """General pydantic model"""
+
+    name: str = Field("example", description="The name of this configuration file")
+    description: str = Field(
+        "example configuration", description="Description of this configuration file"
+    )
+
+
+class TimeWindowMixin(Base):
+    """Mixin class, to add interval start, end and resolution minutes"""
+
+    time_resolution_minutes: int = Field(
+        ...,
+        gt=0,
+        description="The temporal resolution of the data in minutes",
+    )
+    
+    interval_start_minutes: int = Field(
+        ...,
+        description="Data interval starts at `t0 + interval_start_minutes`",
+    )
+
+    interval_end_minutes: int = Field(
+        ...,
+        description="Data interval ends at `t0 + interval_end_minutes`",
+    )
+    
+    @model_validator(mode='after')
+    def check_interval_range(cls, values):
+        if values.interval_start_minutes > values.interval_end_minutes:
+            raise ValueError('interval_start_minutes must be <= interval_end_minutes')
+        return values
+
+    @field_validator("interval_start_minutes")
+    def interval_start_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            raise ValueError("interval_start_minutes must be divisible by time_resolution_minutes")
+        return v
+
+    @field_validator("interval_end_minutes")
+    def interval_end_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            raise ValueError("interval_end_minutes must be divisible by time_resolution_minutes")
+        return v
+
+
+
+# noinspection PyMethodParameters
+class DropoutMixin(Base):
+    """Mixin class, to add dropout minutes"""
+
+    dropout_timedeltas_minutes: Optional[List[int]] = Field(
+        default=None,
+        description="List of possible minutes before t0 where data availability may start. Must be "
+        "negative or zero.",
+    )
+
+    dropout_fraction: float = Field(
+        default=0,
+        description="Chance of dropout being applied to each sample",
+        ge=0,
+        le=1,
+    )
+
+    @field_validator("dropout_timedeltas_minutes")
+    def dropout_timedeltas_minutes_negative(cls, v: List[int]) -> List[int]:
+        """Validate 'dropout_timedeltas_minutes'"""
+        if v is not None:
+            for m in v:
+                assert m <= 0, "Dropout timedeltas must be negative"
+        return v
+
+    @model_validator(mode="after")
+    def dropout_instructions_consistent(self) -> Self:
+        if self.dropout_fraction == 0:
+            if self.dropout_timedeltas_minutes is not None:
+                raise ValueError("To use dropout timedeltas dropout fraction should be > 0")
+        else:
+            if self.dropout_timedeltas_minutes is None:
+                raise ValueError("To dropout fraction > 0 requires a list of dropout timedeltas")
+        return self
+
+
+class SpatialWindowMixin(Base):
+    """Mixin class, to add path and image size"""
+
+    image_size_pixels_height: int = Field(
+        ...,
+        ge=0,
+        description="The number of pixels of the height of the region of interest",
+    )
+
+    image_size_pixels_width: int = Field(
+        ...,
+        ge=0,
+        description="The number of pixels of the width of the region of interest",
+    )
+
+
+class Satellite(TimeWindowMixin, DropoutMixin, SpatialWindowMixin):
+    """Satellite configuration model"""
+    
+    zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path or list of paths which hold the data zarr",
+    )
+
+    channels: list[str] = Field(
+        ..., description="the satellite channels that are used"
+    )
+
+
+# noinspection PyMethodParameters
+class NWP(TimeWindowMixin, DropoutMixin, SpatialWindowMixin):
+    """NWP configuration model"""
+    
+    zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path or list of paths which hold the data zarr",
+    )
+    
+    channels: list[str] = Field(
+        ..., description="the channels used in the nwp data"
+    )
+
+    provider: str = Field(..., description="The provider of the NWP data")
+
+    accum_channels: list[str] = Field([], description="the nwp channels which need to be diffed")
+
+    max_staleness_minutes: Optional[int] = Field(
+        None,
+        description="Sets a limit on how stale an NWP init time is allowed to be whilst still being"
+        " used to construct an example. If set to None, then the max staleness is set according to"
+        " the maximum forecast horizon of the NWP and the requested forecast length.",
+    )
+
+
+    @field_validator("provider")
+    def validate_provider(cls, v: str) -> str:
+        """Validate 'provider'"""
+        if v.lower() not in NWP_PROVIDERS:
+            message = f"NWP provider {v} is not in {NWP_PROVIDERS}"
+            logger.warning(message)
+            raise Exception(message)
+        return v
+
+
+class MultiNWP(RootModel):
+    """Configuration for multiple NWPs"""
+
+    root: Dict[str, NWP]
+
+    def __getattr__(self, item):
+        return self.root[item]
+
+    def __getitem__(self, item):
+        return self.root[item]
+
+    def __len__(self):
+        return len(self.root)
+
+    def __iter__(self):
+        return iter(self.root)
+
+    def keys(self):
+        """Returns dictionary-like keys"""
+        return self.root.keys()
+
+    def items(self):
+        """Returns dictionary-like items"""
+        return self.root.items()
+
+
+class GSP(TimeWindowMixin, DropoutMixin):
+    """GSP configuration model"""
+
+    zarr_path: str = Field(..., description="The path which holds the GSP zarr")
+
+
+class Site(TimeWindowMixin, DropoutMixin):
+    """Site configuration model"""
+
+    file_path: str = Field(
+        ...,
+        description="The NetCDF files holding the power timeseries.",
+    )
+    metadata_file_path: str = Field(
+        ...,
+        description="The CSV files describing power system",
+    )
+
+    # TODO validate the netcdf for sites
+    # TODO validate the csv for metadata
+
+
+
+# noinspection PyPep8Naming
+class InputData(Base):
+    """Input data model"""
+
+    satellite: Optional[Satellite] = None
+    nwp: Optional[MultiNWP] = None
+    gsp: Optional[GSP] = None
+    site: Optional[Site] = None
+
+
+class Configuration(Base):
+    """Configuration model for the dataset"""
+
+    general: General = General()
+    input_data: InputData = InputData()

ocf_data_sampler/config/save.py

@@ -0,0 +1,73 @@
+"""Save functions for the configuration model.
+
+This module provides functionality to save configuration objects to YAML files,
+supporting local and cloud storage locations.
+
+Example:
+    from ocf_data_sampler.config import save_yaml_configuration
+    saved_path = save_yaml_configuration(config, "config.yaml")
+"""
+
+import json
+
+from pathlib import Path
+from typing import Union
+
+import fsspec
+import yaml
+
+from ocf_data_sampler.config import Configuration
+
+
+def save_yaml_configuration(
+    configuration: Configuration,
+    filename: Union[str, Path],
+) -> Path:
+    """Save a configuration object to a YAML file.
+
+    Args:
+        configuration: Configuration object containing the settings to save
+        filename: Destination path for the YAML file. Can be a local path or
+                 cloud storage URL (e.g., 'gs://', 's3://'). For local paths,
+                 absolute paths are recommended.
+
+    Returns:
+        Path: The path where the configuration was saved
+
+    Raises:
+        ValueError: If filename is None or if writing to the specified path fails
+        TypeError: If the configuration cannot be serialized
+    """
+    if filename is None:
+        raise ValueError("filename cannot be None")
+
+    try:
+        # Convert to absolute path if it's a relative path
+        if isinstance(filename, (str, Path)) and not any(
+            str(filename).startswith(prefix) for prefix in ('gs://', 's3://', '/')
+        ):
+            filename = Path.cwd() / filename
+
+        filepath = Path(filename)
+
+        # For local files, check if directory exists before proceeding
+        if filepath.is_absolute():
+            directory = filepath.parent
+            if not directory.exists():
+                raise ValueError("Directory does not exist")
+
+        # Serialize configuration to JSON-compatible dictionary
+        config_dict = json.loads(configuration.model_dump_json())
+
+        # Save to YAML file using fsspec
+        with fsspec.open(str(filepath), mode='w') as yaml_file:
+            yaml.safe_dump(config_dict, yaml_file, default_flow_style=False)
+
+        return filepath
+
+    except json.JSONDecodeError as e:
+        raise TypeError(f"Failed to serialize configuration: {str(e)}") from e
+    except PermissionError as e:
+        raise ValueError(f"Permission denied when writing to {filename}") from e
+    except (IOError, OSError) as e:
+        raise ValueError(f"Failed to write configuration to {filename}: {str(e)}") from e

ocf_data_sampler/constants.py

@@ -0,0 +1,173 @@
+import xarray as xr
+import numpy as np
+
+
+NWP_PROVIDERS = [
+    "ukv",
+    "ecmwf",
+]
+
+
+def _to_data_array(d):
+    return xr.DataArray(
+        [d[k] for k in d.keys()],
+        coords={"channel": [k for k in d.keys()]},
+    ).astype(np.float32)
+
+
+class NWPStatDict(dict):
+    """Custom dictionary class to hold NWP normalization stats"""
+
+    def __getitem__(self, key):
+        if key not in NWP_PROVIDERS:
+            raise KeyError(f"{key} is not a supported NWP provider - {NWP_PROVIDERS}")
+        elif key in self.keys():
+            return super().__getitem__(key)
+        else:
+            raise KeyError(
+                f"Values for {key} not yet available in ocf-data-sampler {list(self.keys())}"
+            )
+
+
+# ------ UKV
+# Means and std computed WITH version_7 and higher, MetOffice values
+UKV_STD = {
+    "cdcb": 2126.99350113,
+    "lcc": 39.33210726,
+    "mcc": 41.91144559,
+    "hcc": 38.07184418,
+    "sde": 0.1029753,
+    "hcct": 18382.63958991,
+    "dswrf": 190.47216887,
+    "dlwrf": 39.45988077,
+    "h": 1075.77812282,
+    "t": 4.38818501,
+    "r": 11.45012499,
+    "dpt": 4.57250482,
+    "vis": 21578.97975625,
+    "si10": 3.94718813,
+    "wdir10": 94.08407495,
+    "prmsl": 1252.71790539,
+    "prate": 0.00021497,
+}
+
+UKV_MEAN = {
+    "cdcb": 1412.26599062,
+    "lcc": 50.08362643,
+    "mcc": 40.88984494,
+    "hcc": 29.11949682,
+    "sde": 0.00289545,
+    "hcct": -18345.97478167,
+    "dswrf": 111.28265039,
+    "dlwrf": 325.03130139,
+    "h": 2096.51991356,
+    "t": 283.64913206,
+    "r": 81.79229501,
+    "dpt": 280.54379901,
+    "vis": 32262.03285118,
+    "si10": 6.88348448,
+    "wdir10": 199.41891636,
+    "prmsl": 101321.61574029,
+    "prate": 3.45793433e-05,
+}
+
+UKV_STD = _to_data_array(UKV_STD)
+UKV_MEAN = _to_data_array(UKV_MEAN)
+
+# ------ ECMWF
+# These were calculated from 100 random init times of UK data from 2020-2023
+ECMWF_STD = {
+    "dlwrf": 15855867.0,
+    "dswrf": 13025427.0,
+    "duvrs": 1445635.25,
+    "hcc": 0.42244860529899597,
+    "lcc": 0.3791404366493225,
+    "mcc": 0.38039860129356384,
+    "prate": 9.81039775069803e-05,
+    "sde": 0.000913831521756947,
+    "sr": 16294988.0,
+    "t2m": 3.692270040512085,
+    "tcc": 0.37487083673477173,
+    "u10": 5.531515598297119,
+    "u100": 7.2320556640625,
+    "u200": 8.049470901489258,
+    "v10": 5.411230564117432,
+    "v100": 6.944501876831055,
+    "v200": 7.561611652374268,
+    "diff_dlwrf": 131942.03125,
+    "diff_dswrf": 715366.3125,
+    "diff_duvrs": 81605.25,
+    "diff_sr": 818950.6875,
+}
+
+ECMWF_MEAN = {
+    "dlwrf": 27187026.0,
+    "dswrf": 11458988.0,
+    "duvrs": 1305651.25,
+    "hcc": 0.3961029052734375,
+    "lcc": 0.44901806116104126,
+    "mcc": 0.3288780450820923,
+    "prate": 3.108070450252853e-05,
+    "sde": 8.107526082312688e-05,
+    "sr": 12905302.0,
+    "t2m": 283.48333740234375,
+    "tcc": 0.7049227356910706,
+    "u10": 1.7677178382873535,
+    "u100": 2.393547296524048,
+    "u200": 2.7963004112243652,
+    "v10": 0.985887885093689,
+    "v100": 1.4244288206100464,
+    "v200": 1.6010299921035767,
+    "diff_dlwrf": 1136464.0,
+    "diff_dswrf": 420584.6875,
+    "diff_duvrs": 48265.4765625,
+    "diff_sr": 469169.5,
+}
+
+ECMWF_STD = _to_data_array(ECMWF_STD)
+ECMWF_MEAN = _to_data_array(ECMWF_MEAN)
+
+NWP_STDS = NWPStatDict(
+    ukv=UKV_STD,
+    ecmwf=ECMWF_STD,
+)
+NWP_MEANS = NWPStatDict(
+    ukv=UKV_MEAN,
+    ecmwf=ECMWF_MEAN,
+)
+
+# ------ Satellite
+# RSS Mean and std values from randomised 20% of 2020 imagery
+
+RSS_STD = {
+    "HRV": 0.11405209,
+    "IR_016": 0.21462157,
+    "IR_039": 0.04618041,
+    "IR_087": 0.06687243,
+    "IR_097": 0.0468558,
+    "IR_108": 0.17482725,
+    "IR_120": 0.06115861,
+    "IR_134": 0.04492306,
+    "VIS006": 0.12184761,
+    "VIS008": 0.13090034,
+    "WV_062": 0.16111417,
+    "WV_073": 0.12924142,
+}
+
+RSS_MEAN = {
+    "HRV": 0.09298719,
+    "IR_016": 0.17594202,
+    "IR_039": 0.86167645,
+    "IR_087": 0.7719318,
+    "IR_097": 0.8014212,
+    "IR_108": 0.71254843,
+    "IR_120": 0.89058584,
+    "IR_134": 0.944365,
+    "VIS006": 0.09633306,
+    "VIS008": 0.11426069,
+    "WV_062": 0.7359355,
+    "WV_073": 0.62479186,
+}
+
+RSS_STD = _to_data_array(RSS_STD)
+RSS_MEAN = _to_data_array(RSS_MEAN)

ocf_data_sampler/load/load_dataset.py

@@ -0,0 +1,55 @@
+""" Loads all data sources """
+import xarray as xr
+
+from ocf_data_sampler.config import Configuration
+from ocf_data_sampler.load.gsp import open_gsp
+from ocf_data_sampler.load.nwp import open_nwp
+from ocf_data_sampler.load.satellite import open_sat_data
+from ocf_data_sampler.load.site import open_site
+
+
+def get_dataset_dict(config: Configuration) -> dict[str, dict[xr.DataArray]]:
+    """Construct dictionary of all of the input data sources
+
+    Args:
+        config: Configuration file
+    """
+
+    in_config = config.input_data
+
+    datasets_dict = {}
+
+    # Load GSP data unless the path is None
+    if in_config.gsp and in_config.gsp.zarr_path:
+        da_gsp = open_gsp(zarr_path=in_config.gsp.zarr_path).compute()
+
+        # Remove national GSP
+        datasets_dict["gsp"] = da_gsp.sel(gsp_id=slice(1, None))
+
+    # Load NWP data if in config
+    if in_config.nwp:
+
+        datasets_dict["nwp"] = {}
+        for nwp_source, nwp_config in in_config.nwp.items():
+
+            da_nwp = open_nwp(nwp_config.zarr_path, provider=nwp_config.provider)
+
+            da_nwp = da_nwp.sel(channel=list(nwp_config.channels))
+
+            datasets_dict["nwp"][nwp_source] = da_nwp
+
+    # Load satellite data if in config
+    if in_config.satellite:
+        sat_config = config.input_data.satellite
+
+        da_sat = open_sat_data(sat_config.zarr_path)
+
+        da_sat = da_sat.sel(channel=list(sat_config.channels))
+
+        datasets_dict["sat"] = da_sat
+
+    if in_config.site:
+        da_sites = open_site(in_config.site)
+        datasets_dict["site"] = da_sites
+
+    return datasets_dict

ocf_data_sampler/load/nwp/providers/ecmwf.py

@@ -9,7 +9,6 @@ from ocf_data_sampler.load.utils import (
 )
 
 
-
 def open_ifs(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArray:
     """
     Opens the ECMWF IFS NWP data
@@ -27,10 +26,14 @@ def open_ifs(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArray:
     ds = ds.rename(
         {
             "init_time": "init_time_utc",
-            "variable": "channel",
         }
     )
 
+    # LEGACY SUPPORT
+    # rename variable to channel if it exists
+    if "variable" in ds:
+        ds = ds.rename({"variable": "channel"})
+
     # Check the timestamps are unique and increasing
     check_time_unique_increasing(ds.init_time_utc)
 

ocf_data_sampler/load/site.py

@@ -0,0 +1,30 @@
+import pandas as pd
+import xarray as xr
+import numpy as np
+
+from ocf_data_sampler.config.model import Site
+
+
+def open_site(sites_config: Site) -> xr.DataArray:
+
+    # Load site generation xr.Dataset
+    site_generation_ds = xr.open_dataset(sites_config.file_path)
+
+    # Load site generation data
+    metadata_df = pd.read_csv(sites_config.metadata_file_path, index_col="site_id")
+
+    # Ensure metadata aligns with the site_id dimension in data_ds
+    metadata_df = metadata_df.reindex(site_generation_ds.site_id.values)
+
+    # Assign coordinates to the Dataset using the aligned metadata
+    site_generation_ds = site_generation_ds.assign_coords(
+        latitude=("site_id", metadata_df["latitude"].values),
+        longitude=("site_id", metadata_df["longitude"].values),
+        capacity_kwp=("site_id", metadata_df["capacity_kwp"].values),
+    )
+
+    # Sanity checks
+    assert np.isfinite(site_generation_ds.capacity_kwp.values).all()
+    assert (site_generation_ds.capacity_kwp.values > 0).all()
+    assert metadata_df.index.is_unique
+    return site_generation_ds.generation_kw

ocf_data_sampler/numpy_sample/__init__.py

@@ -0,0 +1,8 @@
+"""Conversion from Xarray to NumpySample"""
+
+from .gsp import convert_gsp_to_numpy_sample, GSPSampleKey
+from .nwp import convert_nwp_to_numpy_sample, NWPSampleKey
+from .satellite import convert_satellite_to_numpy_sample, SatelliteSampleKey
+from .sun_position import make_sun_position_numpy_sample
+from .site import convert_site_to_numpy_sample
+