ocf-data-sampler 0.0.18__py3-none-any.whl → 0.0.21__py3-none-any.whl

ocf_data_sampler/config/__init__.py

@@ -0,0 +1,5 @@
+"""Configuration model"""
+
+from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.config.save import save_yaml_configuration
+from ocf_data_sampler.config.load import load_yaml_configuration

ocf_data_sampler/config/load.py

@@ -0,0 +1,33 @@
+"""Loading configuration functions.
+
+Example:
+
+    from ocf_data_sampler.config import load_yaml_configuration
+    configuration = load_yaml_configuration(filename)
+"""
+
+import fsspec
+from pathy import Pathy
+from pyaml_env import parse_config
+
+from ocf_data_sampler.config import Configuration
+
+
+def load_yaml_configuration(filename: str | Pathy) -> Configuration:
+    """
+    Load a yaml file which has a configuration in it
+
+    Args:
+        filename: the file name that you want to load.  Will load from local, AWS, or GCP
+            depending on the protocol suffix (e.g. 's3://bucket/config.yaml').
+
+    Returns:pydantic class
+
+    """
+    # load the file to a dictionary
+    with fsspec.open(filename, mode="r") as stream:
+        configuration = parse_config(data=stream)
+        # this means we can load ENVs in the yaml file
+    # turn into pydantic class
+    configuration = Configuration(**configuration)
+    return configuration

ocf_data_sampler/config/model.py

@@ -0,0 +1,249 @@
+"""Configuration model for the dataset.
+
+All paths must include the protocol prefix.  For local files,
+it's sufficient to just start with a '/'.  For aws, start with 's3://',
+for gcp start with 'gs://'.
+
+Example:
+
+    from ocf_data_sampler.config import Configuration
+    config = Configuration(**config_dict)
+"""
+
+import logging
+from typing import Dict, List, Optional
+from typing_extensions import Self
+
+from pydantic import BaseModel, Field, RootModel, field_validator, ValidationInfo, model_validator
+from ocf_datapipes.utils.consts import NWP_PROVIDERS
+
+logger = logging.getLogger(__name__)
+
+providers = ["pvoutput.org", "solar_sheffield_passiv"]
+
+
+class Base(BaseModel):
+    """Pydantic Base model where no extras can be added"""
+
+    class Config:
+        """config class"""
+
+        extra = "forbid"  # forbid use of extra kwargs
+
+
+class General(Base):
+    """General pydantic model"""
+
+    name: str = Field("example", description="The name of this configuration file.")
+    description: str = Field(
+        "example configuration", description="Description of this configuration file"
+    )
+
+
+class DataSourceMixin(Base):
+    """Mixin class, to add forecast and history minutes"""
+
+    forecast_minutes: int = Field(
+        ...,
+        ge=0,
+        description="how many minutes to forecast in the future. ",
+    )
+    history_minutes: int = Field(
+        ...,
+        ge=0,
+        description="how many historic minutes to use. ",
+    )
+
+
+# noinspection PyMethodParameters
+class DropoutMixin(Base):
+    """Mixin class, to add dropout minutes"""
+
+    dropout_timedeltas_minutes: Optional[List[int]] = Field(
+        default=None,
+        description="List of possible minutes before t0 where data availability may start. Must be "
+        "negative or zero.",
+    )
+
+    dropout_fraction: float = Field(0, description="Chance of dropout being applied to each sample")
+
+    @field_validator("dropout_timedeltas_minutes")
+    def dropout_timedeltas_minutes_negative(cls, v: List[int]) -> List[int]:
+        """Validate 'dropout_timedeltas_minutes'"""
+        if v is not None:
+            for m in v:
+                assert m <= 0, "Dropout timedeltas must be negative"
+        return v
+
+    @field_validator("dropout_fraction")
+    def dropout_fraction_valid(cls, v: float) -> float:
+        """Validate 'dropout_fraction'"""
+        assert 0 <= v <= 1, "Dropout fraction must be between 0 and 1"
+        return v
+
+    @model_validator(mode="after")
+    def dropout_instructions_consistent(self) -> Self:
+        if self.dropout_fraction == 0:
+            if self.dropout_timedeltas_minutes is not None:
+                raise ValueError("To use dropout timedeltas dropout fraction should be > 0")
+        else:
+            if self.dropout_timedeltas_minutes is None:
+                raise ValueError("To dropout fraction > 0 requires a list of dropout timedeltas")
+        return self
+
+
+# noinspection PyMethodParameters
+class TimeResolutionMixin(Base):
+    """Time resolution mix in"""
+
+    time_resolution_minutes: int = Field(
+        ...,
+        description="The temporal resolution of the data in minutes",
+    )
+
+
+class Satellite(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """Satellite configuration model"""
+
+    # Todo: remove 'satellite' from names
+    satellite_zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path or list of paths which hold the satellite zarr",
+    )
+    satellite_channels: list[str] = Field(
+        ..., description="the satellite channels that are used"
+    )
+    satellite_image_size_pixels_height: int = Field(
+        ...,
+        description="The number of pixels of the height of the region of interest"
+        " for non-HRV satellite channels.",
+    )
+
+    satellite_image_size_pixels_width: int = Field(
+        ...,
+        description="The number of pixels of the width of the region "
+        "of interest for non-HRV satellite channels.",
+    )
+
+    live_delay_minutes: int = Field(
+        ..., description="The expected delay in minutes of the satellite data"
+    )
+
+
+# noinspection PyMethodParameters
+class NWP(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """NWP configuration model"""
+
+    nwp_zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path which holds the NWP zarr",
+    )
+    nwp_channels: list[str] = Field(
+        ..., description="the channels used in the nwp data"
+    )
+    nwp_accum_channels: list[str] = Field([], description="the nwp channels which need to be diffed")
+    nwp_image_size_pixels_height: int = Field(..., description="The size of NWP spacial crop in pixels")
+    nwp_image_size_pixels_width: int = Field(..., description="The size of NWP spacial crop in pixels")
+
+    nwp_provider: str = Field(..., description="The provider of the NWP data")
+
+    max_staleness_minutes: Optional[int] = Field(
+        None,
+        description="Sets a limit on how stale an NWP init time is allowed to be whilst still being"
+        " used to construct an example. If set to None, then the max staleness is set according to"
+        " the maximum forecast horizon of the NWP and the requested forecast length.",
+    )
+
+
+    @field_validator("nwp_provider")
+    def validate_nwp_provider(cls, v: str) -> str:
+        """Validate 'nwp_provider'"""
+        if v.lower() not in NWP_PROVIDERS:
+            message = f"NWP provider {v} is not in {NWP_PROVIDERS}"
+            logger.warning(message)
+            raise Exception(message)
+        return v
+
+    # Todo: put into time mixin when moving intervals there
+    @field_validator("forecast_minutes")
+    def forecast_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "Forecast duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+    @field_validator("history_minutes")
+    def history_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "History duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+
+class MultiNWP(RootModel):
+    """Configuration for multiple NWPs"""
+
+    root: Dict[str, NWP]
+
+    def __getattr__(self, item):
+        return self.root[item]
+
+    def __getitem__(self, item):
+        return self.root[item]
+
+    def __len__(self):
+        return len(self.root)
+
+    def __iter__(self):
+        return iter(self.root)
+
+    def keys(self):
+        """Returns dictionary-like keys"""
+        return self.root.keys()
+
+    def items(self):
+        """Returns dictionary-like items"""
+        return self.root.items()
+
+
+# noinspection PyMethodParameters
+class GSP(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """GSP configuration model"""
+
+    gsp_zarr_path: str = Field(..., description="The path which holds the GSP zarr")
+
+    @field_validator("forecast_minutes")
+    def forecast_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "Forecast duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+    @field_validator("history_minutes")
+    def history_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "History duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+
+# noinspection PyPep8Naming
+class InputData(Base):
+    """
+    Input data model.
+    """
+
+    satellite: Optional[Satellite] = None
+    nwp: Optional[MultiNWP] = None
+    gsp: Optional[GSP] = None
+
+
+class Configuration(Base):
+    """Configuration model for the dataset"""
+
+    general: General = General()
+    input_data: InputData = InputData()

ocf_data_sampler/config/save.py

@@ -0,0 +1,36 @@
+"""Save functions for the configuration model.
+
+Example:
+
+    from ocf_data_sampler.config import save_yaml_configuration
+    configuration = save_yaml_configuration(config, filename)
+"""
+
+import json
+
+import fsspec
+import yaml
+from pathy import Pathy
+
+from ocf_data_sampler.config import Configuration
+
+
+def save_yaml_configuration(
+    configuration: Configuration, filename: str | Pathy
+):
+    """
+    Save a local yaml file which has the configuration in it.
+
+    If `filename` is None then saves to configuration.output_data.filepath / configuration.yaml.
+
+    Will save to GCP, AWS, or local, depending on the protocol suffix of filepath.
+    """
+    # make a dictionary from the configuration,
+    # Note that we make the object json'able first, so that it can be saved to a yaml file
+    d = json.loads(configuration.model_dump_json())
+    if filename is None:
+        filename = Pathy(configuration.output_data.filepath) / "configuration.yaml"
+
+    # save to a yaml file
+    with fsspec.open(filename, "w") as yaml_file:
+        yaml.safe_dump(d, yaml_file, default_flow_style=False)

ocf_data_sampler/select/dropout.py

@@ -12,7 +12,7 @@ def draw_dropout_time(
     if dropout_timedeltas is not None:
         assert len(dropout_timedeltas) >= 1, "Must include list of relative dropout timedeltas"
         assert all(
-            [t < pd.Timedelta("0min") for t in dropout_timedeltas]
+            [t <= pd.Timedelta("0min") for t in dropout_timedeltas]
         ), "dropout timedeltas must be negative"
     assert 0 <= dropout_frac <= 1
 
@@ -35,4 +35,4 @@ def apply_dropout_time(
         return ds
     else:
         # This replaces the times after the dropout with NaNs
-        return ds.where(ds.time_utc <= dropout_time)
+        return ds.where(ds.time_utc <= dropout_time)

ocf_data_sampler/select/geospatial.py

@@ -0,0 +1,118 @@
+"""Geospatial functions"""
+
+from numbers import Number
+from typing import Union
+
+import numpy as np
+import pyproj
+import xarray as xr
+
+# OSGB is also called "OSGB 1936 / British National Grid -- United
+# Kingdom Ordnance Survey".  OSGB is used in many UK electricity
+# system maps, and is used by the UK Met Office UKV model.  OSGB is a
+# Transverse Mercator projection, using 'easting' and 'northing'
+# coordinates which are in meters.  See https://epsg.io/27700
+OSGB36 = 27700
+
+# WGS84 is short for "World Geodetic System 1984", used in GPS. Uses
+# latitude and longitude.
+WGS84 = 4326
+
+
+_osgb_to_lon_lat = pyproj.Transformer.from_crs(
+    crs_from=OSGB36, crs_to=WGS84, always_xy=True
+).transform
+_lon_lat_to_osgb = pyproj.Transformer.from_crs(
+    crs_from=WGS84, crs_to=OSGB36, always_xy=True
+).transform
+
+
+def osgb_to_lon_lat(
+    x: Union[Number, np.ndarray], y: Union[Number, np.ndarray]
+) -> tuple[Union[Number, np.ndarray], Union[Number, np.ndarray]]:
+    """Change OSGB coordinates to lon, lat.
+
+    Args:
+        x: osgb east-west
+        y: osgb north-south
+    Return: 2-tuple of longitude (east-west), latitude (north-south)
+    """
+    return _osgb_to_lon_lat(xx=x, yy=y)
+
+
+def lon_lat_to_osgb(
+    x: Union[Number, np.ndarray],
+    y: Union[Number, np.ndarray],
+) -> tuple[Union[Number, np.ndarray], Union[Number, np.ndarray]]:
+    """Change lon-lat coordinates to OSGB.
+
+    Args:
+        x: longitude east-west
+        y: latitude north-south
+
+    Return: 2-tuple of OSGB x, y
+    """
+    return _lon_lat_to_osgb(xx=x, yy=y)
+
+
+def osgb_to_geostationary_area_coords(
+    x: Union[Number, np.ndarray],
+    y: Union[Number, np.ndarray],
+    xr_data: xr.DataArray,
+) -> tuple[Union[Number, np.ndarray], Union[Number, np.ndarray]]:
+    """Loads geostationary area and transformation from OSGB to geostationary coords
+
+    Args:
+        x: osgb east-west
+        y: osgb north-south
+        xr_data: xarray object with geostationary area
+
+    Returns:
+        Geostationary coords: x, y
+    """
+    # Only load these if using geostationary projection
+    import pyresample
+
+    area_definition_yaml = xr_data.attrs["area"]
+
+    geostationary_area_definition = pyresample.area_config.load_area_from_string(
+        area_definition_yaml
+    )
+    geostationary_crs = geostationary_area_definition.crs
+    osgb_to_geostationary = pyproj.Transformer.from_crs(
+        crs_from=OSGB36, crs_to=geostationary_crs, always_xy=True
+    ).transform
+    return osgb_to_geostationary(xx=x, yy=y)
+
+
+def _coord_priority(available_coords):
+    if "longitude" in available_coords:
+        return "lon_lat", "longitude", "latitude"
+    elif "x_geostationary" in available_coords:
+        return "geostationary", "x_geostationary", "y_geostationary"
+    elif "x_osgb" in available_coords:
+        return "osgb", "x_osgb", "y_osgb"
+    else:
+        raise ValueError(f"Unrecognized coordinate system: {available_coords}")
+
+
+def spatial_coord_type(ds: xr.DataArray):
+    """Searches the data array to determine the kind of spatial coordinates present.
+
+    This search has a preference for the dimension coordinates of the xarray object.
+
+    Args:
+        ds: Dataset with spatial coords
+
+    Returns:
+        str: The kind of the coordinate system
+        x_coord: Name of the x-coordinate
+        y_coord: Name of the y-coordinate
+    """
+    if isinstance(ds, xr.DataArray):
+        # Search dimension coords of dataarray
+        coords = _coord_priority(ds.xindexes)
+    else:
+        raise ValueError(f"Unrecognized input type: {type(ds)}")
+
+    return coords

ocf_data_sampler/select/location.py

@@ -0,0 +1,62 @@
+"""location"""
+
+from typing import Optional
+
+import numpy as np
+from pydantic import BaseModel, Field, model_validator
+
+
+allowed_coordinate_systems =["osgb", "lon_lat", "geostationary", "idx"]
+
+class Location(BaseModel):
+    """Represent a spatial location."""
+
+    coordinate_system: Optional[str] = "osgb"  # ["osgb", "lon_lat", "geostationary", "idx"]
+    x: float
+    y: float
+    id: Optional[int] = Field(None)
+
+    @model_validator(mode='after')
+    def validate_coordinate_system(self):
+        """Validate 'coordinate_system'"""
+        if self.coordinate_system not in allowed_coordinate_systems:
+            raise ValueError(f"coordinate_system = {self.coordinate_system} is not in {allowed_coordinate_systems}")
+        return self
+
+    @model_validator(mode='after')
+    def validate_x(self):
+        """Validate 'x'"""
+        min_x: float
+        max_x: float
+
+        co = self.coordinate_system
+        if co == "osgb":
+            min_x, max_x = -103976.3, 652897.98
+        if co == "lon_lat":
+            min_x, max_x = -180, 180
+        if co == "geostationary":
+            min_x, max_x = -5568748.275756836, 5567248.074173927
+        if co == "idx":
+            min_x, max_x = 0, np.inf
+        if self.x < min_x or self.x > max_x:
+            raise ValueError(f"x = {self.x} must be within {[min_x, max_x]} for {co} coordinate system")
+        return self
+
+    @model_validator(mode='after')
+    def validate_y(self):
+        """Validate 'y'"""
+        min_y: float
+        max_y: float
+
+        co = self.coordinate_system
+        if co == "osgb":
+            min_y, max_y = -16703.87, 1199851.44
+        if co == "lon_lat":
+            min_y, max_y = -90, 90
+        if co == "geostationary":
+            min_y, max_y = 1393687.2151494026, 5570748.323202133
+        if co == "idx":
+            min_y, max_y = 0, np.inf
+        if self.y < min_y or self.y > max_y:
+            raise ValueError(f"y = {self.y} must be within {[min_y, max_y]} for {co} coordinate system")
+        return self

ocf_data_sampler/select/select_spatial_slice.py

@@ -5,15 +5,14 @@ import logging
 import numpy as np
 import xarray as xr
 
-from ocf_datapipes.utils import Location
-from ocf_datapipes.utils.geospatial import (
-    lon_lat_to_geostationary_area_coords,
+from ocf_data_sampler.select.location import Location
+from ocf_data_sampler.select.geospatial import (
     lon_lat_to_osgb,
     osgb_to_geostationary_area_coords,
     osgb_to_lon_lat,
     spatial_coord_type,
 )
-from ocf_datapipes.utils.utils import searchsorted
+
 
 logger = logging.getLogger(__name__)
 
@@ -45,9 +44,6 @@ def convert_coords_to_match_xarray(
         if from_coords == "osgb":
             x, y = osgb_to_geostationary_area_coords(x, y, da)
 
-        elif from_coords == "lon_lat":
-            x, y = lon_lat_to_geostationary_area_coords(x, y, da)
-
     elif target_coords == "lon_lat":
         if from_coords == "osgb":
             x, y = osgb_to_lon_lat(x, y)
@@ -130,13 +126,8 @@ def _get_idx_of_pixel_closest_to_poi_geostationary(
         f"{y} is not in the interval {da[y_dim].min().values}: {da[y_dim].max().values}"
 
     # Get the index into x and y nearest to x_center_geostationary and y_center_geostationary:
-    x_index_at_center = searchsorted(
-        da[x_dim].values, center_geostationary.x, assume_ascending=True
-    )
-
-    y_index_at_center = searchsorted(
-        da[y_dim].values, center_geostationary.y, assume_ascending=True
-    )
+    x_index_at_center = np.searchsorted(da[x_dim].values, center_geostationary.x)
+    y_index_at_center = np.searchsorted(da[y_dim].values, center_geostationary.y)
 
     return Location(x=x_index_at_center, y=y_index_at_center, coordinate_system="idx")
 

ocf_data_sampler/torch_datasets/pvnet_uk_regional.py

@@ -27,8 +27,7 @@ from ocf_data_sampler.numpy_batch import (
 )
 
 
-from ocf_datapipes.config.model import Configuration
-from ocf_datapipes.config.load import load_yaml_configuration
+from ocf_data_sampler.config import Configuration, load_yaml_configuration
 from ocf_datapipes.batch import BatchKey, NumpyBatch
 
 from ocf_datapipes.utils.location import Location
@@ -451,8 +450,12 @@ def compute(xarray_dict: dict) -> dict:
     return xarray_dict
 
 
-def get_gsp_locations() -> list[Location]:
+def get_gsp_locations(gsp_ids: list[int] | None = None) -> list[Location]:
     """Get list of locations of all GSPs"""
+    
+    if gsp_ids is None:
+        gsp_ids = [i for i in range(1, 318)]
+    
     locations = []
 
     # Load UK GSP locations
@@ -461,7 +464,7 @@ def get_gsp_locations() -> list[Location]:
         index_col="gsp_id",
     )
 
-    for gsp_id in np.arange(1, 318):
+    for gsp_id in gsp_ids:
         locations.append(
             Location(
                 coordinate_system = "osgb",
@@ -480,6 +483,7 @@ class PVNetUKRegionalDataset(Dataset):
         config_filename: str, 
         start_time: str | None = None,
         end_time: str| None = None,
+        gsp_ids: list[int] | None = None,
     ):
         """A torch Dataset for creating PVNet UK GSP samples
 
@@ -487,6 +491,7 @@ class PVNetUKRegionalDataset(Dataset):
             config_filename: Path to the configuration file
             start_time: Limit the init-times to be after this
             end_time: Limit the init-times to be before this
+            gsp_ids: List of GSP IDs to create samples for. Defaults to all
         """
         
         config = load_yaml_configuration(config_filename)
@@ -504,7 +509,7 @@ class PVNetUKRegionalDataset(Dataset):
             valid_t0_times = valid_t0_times[valid_t0_times<=pd.Timestamp(end_time)]
 
         # Construct list of locations to sample from
-        locations = get_gsp_locations()
+        locations = get_gsp_locations(gsp_ids)
 
         # Construct a lookup for locations - useful for users to construct sample by GSP ID
         location_lookup = {loc.id: loc for loc in locations}