ocf-data-sampler 0.0.19__py3-none-any.whl → 0.0.22__py3-none-any.whl

ocf_data_sampler/config/__init__.py

@@ -0,0 +1,5 @@
+"""Configuration model"""
+
+from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.config.save import save_yaml_configuration
+from ocf_data_sampler.config.load import load_yaml_configuration

ocf_data_sampler/config/load.py

@@ -0,0 +1,33 @@
+"""Loading configuration functions.
+
+Example:
+
+    from ocf_data_sampler.config import load_yaml_configuration
+    configuration = load_yaml_configuration(filename)
+"""
+
+import fsspec
+from pathy import Pathy
+from pyaml_env import parse_config
+
+from ocf_data_sampler.config import Configuration
+
+
+def load_yaml_configuration(filename: str | Pathy) -> Configuration:
+    """
+    Load a yaml file which has a configuration in it
+
+    Args:
+        filename: the file name that you want to load.  Will load from local, AWS, or GCP
+            depending on the protocol suffix (e.g. 's3://bucket/config.yaml').
+
+    Returns:pydantic class
+
+    """
+    # load the file to a dictionary
+    with fsspec.open(filename, mode="r") as stream:
+        configuration = parse_config(data=stream)
+        # this means we can load ENVs in the yaml file
+    # turn into pydantic class
+    configuration = Configuration(**configuration)
+    return configuration

ocf_data_sampler/config/model.py

@@ -0,0 +1,249 @@
+"""Configuration model for the dataset.
+
+All paths must include the protocol prefix.  For local files,
+it's sufficient to just start with a '/'.  For aws, start with 's3://',
+for gcp start with 'gs://'.
+
+Example:
+
+    from ocf_data_sampler.config import Configuration
+    config = Configuration(**config_dict)
+"""
+
+import logging
+from typing import Dict, List, Optional
+from typing_extensions import Self
+
+from pydantic import BaseModel, Field, RootModel, field_validator, ValidationInfo, model_validator
+from ocf_data_sampler.constants import NWP_PROVIDERS
+
+logger = logging.getLogger(__name__)
+
+providers = ["pvoutput.org", "solar_sheffield_passiv"]
+
+
+class Base(BaseModel):
+    """Pydantic Base model where no extras can be added"""
+
+    class Config:
+        """config class"""
+
+        extra = "forbid"  # forbid use of extra kwargs
+
+
+class General(Base):
+    """General pydantic model"""
+
+    name: str = Field("example", description="The name of this configuration file.")
+    description: str = Field(
+        "example configuration", description="Description of this configuration file"
+    )
+
+
+class DataSourceMixin(Base):
+    """Mixin class, to add forecast and history minutes"""
+
+    forecast_minutes: int = Field(
+        ...,
+        ge=0,
+        description="how many minutes to forecast in the future. ",
+    )
+    history_minutes: int = Field(
+        ...,
+        ge=0,
+        description="how many historic minutes to use. ",
+    )
+
+
+# noinspection PyMethodParameters
+class DropoutMixin(Base):
+    """Mixin class, to add dropout minutes"""
+
+    dropout_timedeltas_minutes: Optional[List[int]] = Field(
+        default=None,
+        description="List of possible minutes before t0 where data availability may start. Must be "
+        "negative or zero.",
+    )
+
+    dropout_fraction: float = Field(0, description="Chance of dropout being applied to each sample")
+
+    @field_validator("dropout_timedeltas_minutes")
+    def dropout_timedeltas_minutes_negative(cls, v: List[int]) -> List[int]:
+        """Validate 'dropout_timedeltas_minutes'"""
+        if v is not None:
+            for m in v:
+                assert m <= 0, "Dropout timedeltas must be negative"
+        return v
+
+    @field_validator("dropout_fraction")
+    def dropout_fraction_valid(cls, v: float) -> float:
+        """Validate 'dropout_fraction'"""
+        assert 0 <= v <= 1, "Dropout fraction must be between 0 and 1"
+        return v
+
+    @model_validator(mode="after")
+    def dropout_instructions_consistent(self) -> Self:
+        if self.dropout_fraction == 0:
+            if self.dropout_timedeltas_minutes is not None:
+                raise ValueError("To use dropout timedeltas dropout fraction should be > 0")
+        else:
+            if self.dropout_timedeltas_minutes is None:
+                raise ValueError("To dropout fraction > 0 requires a list of dropout timedeltas")
+        return self
+
+
+# noinspection PyMethodParameters
+class TimeResolutionMixin(Base):
+    """Time resolution mix in"""
+
+    time_resolution_minutes: int = Field(
+        ...,
+        description="The temporal resolution of the data in minutes",
+    )
+
+
+class Satellite(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """Satellite configuration model"""
+
+    # Todo: remove 'satellite' from names
+    satellite_zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path or list of paths which hold the satellite zarr",
+    )
+    satellite_channels: list[str] = Field(
+        ..., description="the satellite channels that are used"
+    )
+    satellite_image_size_pixels_height: int = Field(
+        ...,
+        description="The number of pixels of the height of the region of interest"
+        " for non-HRV satellite channels.",
+    )
+
+    satellite_image_size_pixels_width: int = Field(
+        ...,
+        description="The number of pixels of the width of the region "
+        "of interest for non-HRV satellite channels.",
+    )
+
+    live_delay_minutes: int = Field(
+        ..., description="The expected delay in minutes of the satellite data"
+    )
+
+
+# noinspection PyMethodParameters
+class NWP(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """NWP configuration model"""
+
+    nwp_zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path which holds the NWP zarr",
+    )
+    nwp_channels: list[str] = Field(
+        ..., description="the channels used in the nwp data"
+    )
+    nwp_accum_channels: list[str] = Field([], description="the nwp channels which need to be diffed")
+    nwp_image_size_pixels_height: int = Field(..., description="The size of NWP spacial crop in pixels")
+    nwp_image_size_pixels_width: int = Field(..., description="The size of NWP spacial crop in pixels")
+
+    nwp_provider: str = Field(..., description="The provider of the NWP data")
+
+    max_staleness_minutes: Optional[int] = Field(
+        None,
+        description="Sets a limit on how stale an NWP init time is allowed to be whilst still being"
+        " used to construct an example. If set to None, then the max staleness is set according to"
+        " the maximum forecast horizon of the NWP and the requested forecast length.",
+    )
+
+
+    @field_validator("nwp_provider")
+    def validate_nwp_provider(cls, v: str) -> str:
+        """Validate 'nwp_provider'"""
+        if v.lower() not in NWP_PROVIDERS:
+            message = f"NWP provider {v} is not in {NWP_PROVIDERS}"
+            logger.warning(message)
+            raise Exception(message)
+        return v
+
+    # Todo: put into time mixin when moving intervals there
+    @field_validator("forecast_minutes")
+    def forecast_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "Forecast duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+    @field_validator("history_minutes")
+    def history_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "History duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+
+class MultiNWP(RootModel):
+    """Configuration for multiple NWPs"""
+
+    root: Dict[str, NWP]
+
+    def __getattr__(self, item):
+        return self.root[item]
+
+    def __getitem__(self, item):
+        return self.root[item]
+
+    def __len__(self):
+        return len(self.root)
+
+    def __iter__(self):
+        return iter(self.root)
+
+    def keys(self):
+        """Returns dictionary-like keys"""
+        return self.root.keys()
+
+    def items(self):
+        """Returns dictionary-like items"""
+        return self.root.items()
+
+
+# noinspection PyMethodParameters
+class GSP(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """GSP configuration model"""
+
+    gsp_zarr_path: str = Field(..., description="The path which holds the GSP zarr")
+
+    @field_validator("forecast_minutes")
+    def forecast_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "Forecast duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+    @field_validator("history_minutes")
+    def history_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "History duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+
+# noinspection PyPep8Naming
+class InputData(Base):
+    """
+    Input data model.
+    """
+
+    satellite: Optional[Satellite] = None
+    nwp: Optional[MultiNWP] = None
+    gsp: Optional[GSP] = None
+
+
+class Configuration(Base):
+    """Configuration model for the dataset"""
+
+    general: General = General()
+    input_data: InputData = InputData()

ocf_data_sampler/config/save.py

@@ -0,0 +1,36 @@
+"""Save functions for the configuration model.
+
+Example:
+
+    from ocf_data_sampler.config import save_yaml_configuration
+    configuration = save_yaml_configuration(config, filename)
+"""
+
+import json
+
+import fsspec
+import yaml
+from pathy import Pathy
+
+from ocf_data_sampler.config import Configuration
+
+
+def save_yaml_configuration(
+    configuration: Configuration, filename: str | Pathy
+):
+    """
+    Save a local yaml file which has the configuration in it.
+
+    If `filename` is None then saves to configuration.output_data.filepath / configuration.yaml.
+
+    Will save to GCP, AWS, or local, depending on the protocol suffix of filepath.
+    """
+    # make a dictionary from the configuration,
+    # Note that we make the object json'able first, so that it can be saved to a yaml file
+    d = json.loads(configuration.model_dump_json())
+    if filename is None:
+        filename = Pathy(configuration.output_data.filepath) / "configuration.yaml"
+
+    # save to a yaml file
+    with fsspec.open(filename, "w") as yaml_file:
+        yaml.safe_dump(d, yaml_file, default_flow_style=False)

ocf_data_sampler/constants.py

@@ -0,0 +1,135 @@
+import xarray as xr
+import numpy as np
+
+
+NWP_PROVIDERS = [
+    "ukv",
+    "ecmwf",
+]
+
+
+def _to_data_array(d):
+    return xr.DataArray(
+        [d[k] for k in d.keys()],
+        coords={"channel": [k for k in d.keys()]},
+    ).astype(np.float32)
+
+
+class NWPStatDict(dict):
+    """Custom dictionary class to hold NWP normalization stats"""
+
+    def __getitem__(self, key):
+        if key not in NWP_PROVIDERS:
+            raise KeyError(f"{key} is not a supported NWP provider - {NWP_PROVIDERS}")
+        elif key in self.keys():
+            return super().__getitem__(key)
+        else:
+            raise KeyError(
+                f"Values for {key} not yet available in ocf-data-sampler {list(self.keys())}"
+            )
+
+# ------ UKV
+# Means and std computed WITH version_7 and higher, MetOffice values
+UKV_STD = {
+    "cdcb": 2126.99350113,
+    "lcc": 39.33210726,
+    "mcc": 41.91144559,
+    "hcc": 38.07184418,
+    "sde": 0.1029753,
+    "hcct": 18382.63958991,
+    "dswrf": 190.47216887,
+    "dlwrf": 39.45988077,
+    "h": 1075.77812282,
+    "t": 4.38818501,
+    "r": 11.45012499,
+    "dpt": 4.57250482,
+    "vis": 21578.97975625,
+    "si10": 3.94718813,
+    "wdir10": 94.08407495,
+    "prmsl": 1252.71790539,
+    "prate": 0.00021497,
+}
+UKV_MEAN = {
+    "cdcb": 1412.26599062,
+    "lcc": 50.08362643,
+    "mcc": 40.88984494,
+    "hcc": 29.11949682,
+    "sde": 0.00289545,
+    "hcct": -18345.97478167,
+    "dswrf": 111.28265039,
+    "dlwrf": 325.03130139,
+    "h": 2096.51991356,
+    "t": 283.64913206,
+    "r": 81.79229501,
+    "dpt": 280.54379901,
+    "vis": 32262.03285118,
+    "si10": 6.88348448,
+    "wdir10": 199.41891636,
+    "prmsl": 101321.61574029,
+    "prate": 3.45793433e-05,
+}
+
+UKV_STD = _to_data_array(UKV_STD)
+UKV_MEAN = _to_data_array(UKV_MEAN)
+
+# ------ ECMWF
+# These were calculated from 100 random init times of UK data from 2020-2023
+ECMWF_STD = {
+    "dlwrf": 15855867.0,
+    "dswrf": 13025427.0,
+    "duvrs": 1445635.25,
+    "hcc": 0.42244860529899597,
+    "lcc": 0.3791404366493225,
+    "mcc": 0.38039860129356384,
+    "prate": 9.81039775069803e-05,
+    "sde": 0.000913831521756947,
+    "sr": 16294988.0,
+    "t2m": 3.692270040512085,
+    "tcc": 0.37487083673477173,
+    "u10": 5.531515598297119,
+    "u100": 7.2320556640625,
+    "u200": 8.049470901489258,
+    "v10": 5.411230564117432,
+    "v100": 6.944501876831055,
+    "v200": 7.561611652374268,
+    "diff_dlwrf": 131942.03125,
+    "diff_dswrf": 715366.3125,
+    "diff_duvrs": 81605.25,
+    "diff_sr": 818950.6875,
+}
+ECMWF_MEAN = {
+    "dlwrf": 27187026.0,
+    "dswrf": 11458988.0,
+    "duvrs": 1305651.25,
+    "hcc": 0.3961029052734375,
+    "lcc": 0.44901806116104126,
+    "mcc": 0.3288780450820923,
+    "prate": 3.108070450252853e-05,
+    "sde": 8.107526082312688e-05,
+    "sr": 12905302.0,
+    "t2m": 283.48333740234375,
+    "tcc": 0.7049227356910706,
+    "u10": 1.7677178382873535,
+    "u100": 2.393547296524048,
+    "u200": 2.7963004112243652,
+    "v10": 0.985887885093689,
+    "v100": 1.4244288206100464,
+    "v200": 1.6010299921035767,
+    "diff_dlwrf": 1136464.0,
+    "diff_dswrf": 420584.6875,
+    "diff_duvrs": 48265.4765625,
+    "diff_sr": 469169.5,
+}
+
+ECMWF_STD = _to_data_array(ECMWF_STD)
+ECMWF_MEAN = _to_data_array(ECMWF_MEAN)
+
+NWP_STDS = NWPStatDict(
+    ukv=UKV_STD,
+    ecmwf=ECMWF_STD,
+)
+NWP_MEANS = NWPStatDict(
+    ukv=UKV_MEAN,
+    ecmwf=ECMWF_MEAN,
+)
+

ocf_data_sampler/numpy_batch/gsp.py

@@ -1,20 +1,33 @@
 """Convert GSP to Numpy Batch"""
 
 import xarray as xr
-from ocf_datapipes.batch import BatchKey, NumpyBatch
 
 
-def convert_gsp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> NumpyBatch:
+class GSPBatchKey:
+
+    gsp = 'gsp'
+    gsp_nominal_capacity_mwp = 'gsp_nominal_capacity_mwp'
+    gsp_effective_capacity_mwp = 'gsp_effective_capacity_mwp'
+    gsp_time_utc = 'gsp_time_utc'
+    gsp_t0_idx = 'gsp_t0_idx'
+    gsp_solar_azimuth = 'gsp_solar_azimuth'
+    gsp_solar_elevation = 'gsp_solar_elevation'
+    gsp_id = 'gsp_id'
+    gsp_x_osgb = 'gsp_x_osgb'
+    gsp_y_osgb = 'gsp_y_osgb'
+
+
+def convert_gsp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
     """Convert from Xarray to NumpyBatch"""
 
-    example: NumpyBatch = {
-        BatchKey.gsp: da.values,
-        BatchKey.gsp_nominal_capacity_mwp: da.isel(time_utc=0)["nominal_capacity_mwp"].values,
-        BatchKey.gsp_effective_capacity_mwp: da.isel(time_utc=0)["effective_capacity_mwp"].values,
-        BatchKey.gsp_time_utc: da["time_utc"].values.astype(float),
+    example = {
+        GSPBatchKey.gsp: da.values,
+        GSPBatchKey.gsp_nominal_capacity_mwp: da.isel(time_utc=0)["nominal_capacity_mwp"].values,
+        GSPBatchKey.gsp_effective_capacity_mwp: da.isel(time_utc=0)["effective_capacity_mwp"].values,
+        GSPBatchKey.gsp_time_utc: da["time_utc"].values.astype(float),
     }
 
     if t0_idx is not None:
-        example[BatchKey.gsp_t0_idx] = t0_idx
+        example[GSPBatchKey.gsp_t0_idx] = t0_idx
 
     return example

ocf_data_sampler/numpy_batch/nwp.py

@@ -3,13 +3,23 @@
 import pandas as pd
 import xarray as xr
 
-from ocf_datapipes.batch import NWPBatchKey, NWPNumpyBatch
 
+class NWPBatchKey:
 
-def convert_nwp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> NWPNumpyBatch:
+    nwp = 'nwp'
+    nwp_channel_names = 'nwp_channel_names'
+    nwp_init_time_utc = 'nwp_init_time_utc'
+    nwp_step = 'nwp_step'
+    nwp_target_time_utc = 'nwp_target_time_utc'
+    nwp_t0_idx = 'nwp_t0_idx'
+    nwp_y_osgb = 'nwp_y_osgb'
+    nwp_x_osgb = 'nwp_x_osgb'
+
+
+def convert_nwp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
     """Convert from Xarray to NWP NumpyBatch"""
 
-    example: NWPNumpyBatch = {
+    example = {
         NWPBatchKey.nwp: da.values,
         NWPBatchKey.nwp_channel_names: da.channel.values,
         NWPBatchKey.nwp_init_time_utc: da.init_time_utc.values.astype(float),

ocf_data_sampler/numpy_batch/satellite.py

@@ -1,23 +1,30 @@
 """Convert Satellite to NumpyBatch"""
 import xarray as xr
 
-from ocf_datapipes.batch import BatchKey, NumpyBatch
 
+class SatelliteBatchKey:
 
-def convert_satellite_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> NumpyBatch:
+    satellite_actual = 'satellite_actual'
+    satellite_time_utc = 'satellite_time_utc'
+    satellite_x_geostationary = 'satellite_x_geostationary'
+    satellite_y_geostationary = 'satellite_y_geostationary'
+    satellite_t0_idx = 'satellite_t0_idx'
+
+
+def convert_satellite_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
     """Convert from Xarray to NumpyBatch"""
-    example: NumpyBatch = {
-        BatchKey.satellite_actual: da.values,
-        BatchKey.satellite_time_utc: da.time_utc.values.astype(float),
+    example = {
+        SatelliteBatchKey.satellite_actual: da.values,
+        SatelliteBatchKey.satellite_time_utc: da.time_utc.values.astype(float),
     }
 
     for batch_key, dataset_key in (
-         (BatchKey.satellite_x_geostationary, "x_geostationary"),
-        (BatchKey.satellite_y_geostationary, "y_geostationary"),
+         (SatelliteBatchKey.satellite_x_geostationary, "x_geostationary"),
+        (SatelliteBatchKey.satellite_y_geostationary, "y_geostationary"),
     ):
         example[batch_key] = da[dataset_key].values
 
     if t0_idx is not None:
-        example[BatchKey.satellite_t0_idx] = t0_idx
+        example[SatelliteBatchKey.satellite_t0_idx] = t0_idx
 
     return example

ocf_data_sampler/numpy_batch/sun_position.py

@@ -2,7 +2,6 @@
 import pvlib
 import numpy as np
 import pandas as pd
-from ocf_datapipes.batch import BatchKey, NumpyBatch
 
 
 def calculate_azimuth_and_elevation(
@@ -37,8 +36,8 @@ def make_sun_position_numpy_batch(
         datetimes: pd.DatetimeIndex, 
         lon: float, 
         lat: float, 
-        key_preffix: str = "gsp"
-) -> NumpyBatch:
+        key_prefix: str = "gsp"
+) -> dict:
     """Creates NumpyBatch with standardized solar coordinates
 
     Args:
@@ -58,9 +57,9 @@ def make_sun_position_numpy_batch(
     elevation = elevation / 180 + 0.5
     
     # Make NumpyBatch
-    sun_numpy_batch: NumpyBatch = {
-        BatchKey[key_preffix + "_solar_azimuth"]: azimuth,
-        BatchKey[key_preffix + "_solar_elevation"]: elevation,
+    sun_numpy_batch = {
+        key_prefix + "_solar_azimuth": azimuth,
+        key_prefix + "_solar_elevation": elevation,
     }
 
     return sun_numpy_batch

ocf_data_sampler/select/dropout.py

@@ -12,7 +12,7 @@ def draw_dropout_time(
     if dropout_timedeltas is not None:
         assert len(dropout_timedeltas) >= 1, "Must include list of relative dropout timedeltas"
         assert all(
-            [t < pd.Timedelta("0min") for t in dropout_timedeltas]
+            [t <= pd.Timedelta("0min") for t in dropout_timedeltas]
         ), "dropout timedeltas must be negative"
     assert 0 <= dropout_frac <= 1
 
@@ -35,4 +35,4 @@ def apply_dropout_time(
         return ds
     else:
         # This replaces the times after the dropout with NaNs
-        return ds.where(ds.time_utc <= dropout_time)
+        return ds.where(ds.time_utc <= dropout_time)