ocf-data-sampler 0.0.19__py3-none-any.whl → 0.0.22__py3-none-any.whl

ocf_data_sampler/select/geospatial.py

@@ -0,0 +1,118 @@
+"""Geospatial functions"""
+
+from numbers import Number
+from typing import Union
+
+import numpy as np
+import pyproj
+import xarray as xr
+
+# OSGB is also called "OSGB 1936 / British National Grid -- United
+# Kingdom Ordnance Survey".  OSGB is used in many UK electricity
+# system maps, and is used by the UK Met Office UKV model.  OSGB is a
+# Transverse Mercator projection, using 'easting' and 'northing'
+# coordinates which are in meters.  See https://epsg.io/27700
+OSGB36 = 27700
+
+# WGS84 is short for "World Geodetic System 1984", used in GPS. Uses
+# latitude and longitude.
+WGS84 = 4326
+
+
+_osgb_to_lon_lat = pyproj.Transformer.from_crs(
+    crs_from=OSGB36, crs_to=WGS84, always_xy=True
+).transform
+_lon_lat_to_osgb = pyproj.Transformer.from_crs(
+    crs_from=WGS84, crs_to=OSGB36, always_xy=True
+).transform
+
+
+def osgb_to_lon_lat(
+    x: Union[Number, np.ndarray], y: Union[Number, np.ndarray]
+) -> tuple[Union[Number, np.ndarray], Union[Number, np.ndarray]]:
+    """Change OSGB coordinates to lon, lat.
+
+    Args:
+        x: osgb east-west
+        y: osgb north-south
+    Return: 2-tuple of longitude (east-west), latitude (north-south)
+    """
+    return _osgb_to_lon_lat(xx=x, yy=y)
+
+
+def lon_lat_to_osgb(
+    x: Union[Number, np.ndarray],
+    y: Union[Number, np.ndarray],
+) -> tuple[Union[Number, np.ndarray], Union[Number, np.ndarray]]:
+    """Change lon-lat coordinates to OSGB.
+
+    Args:
+        x: longitude east-west
+        y: latitude north-south
+
+    Return: 2-tuple of OSGB x, y
+    """
+    return _lon_lat_to_osgb(xx=x, yy=y)
+
+
+def osgb_to_geostationary_area_coords(
+    x: Union[Number, np.ndarray],
+    y: Union[Number, np.ndarray],
+    xr_data: xr.DataArray,
+) -> tuple[Union[Number, np.ndarray], Union[Number, np.ndarray]]:
+    """Loads geostationary area and transformation from OSGB to geostationary coords
+
+    Args:
+        x: osgb east-west
+        y: osgb north-south
+        xr_data: xarray object with geostationary area
+
+    Returns:
+        Geostationary coords: x, y
+    """
+    # Only load these if using geostationary projection
+    import pyresample
+
+    area_definition_yaml = xr_data.attrs["area"]
+
+    geostationary_area_definition = pyresample.area_config.load_area_from_string(
+        area_definition_yaml
+    )
+    geostationary_crs = geostationary_area_definition.crs
+    osgb_to_geostationary = pyproj.Transformer.from_crs(
+        crs_from=OSGB36, crs_to=geostationary_crs, always_xy=True
+    ).transform
+    return osgb_to_geostationary(xx=x, yy=y)
+
+
+def _coord_priority(available_coords):
+    if "longitude" in available_coords:
+        return "lon_lat", "longitude", "latitude"
+    elif "x_geostationary" in available_coords:
+        return "geostationary", "x_geostationary", "y_geostationary"
+    elif "x_osgb" in available_coords:
+        return "osgb", "x_osgb", "y_osgb"
+    else:
+        raise ValueError(f"Unrecognized coordinate system: {available_coords}")
+
+
+def spatial_coord_type(ds: xr.DataArray):
+    """Searches the data array to determine the kind of spatial coordinates present.
+
+    This search has a preference for the dimension coordinates of the xarray object.
+
+    Args:
+        ds: Dataset with spatial coords
+
+    Returns:
+        str: The kind of the coordinate system
+        x_coord: Name of the x-coordinate
+        y_coord: Name of the y-coordinate
+    """
+    if isinstance(ds, xr.DataArray):
+        # Search dimension coords of dataarray
+        coords = _coord_priority(ds.xindexes)
+    else:
+        raise ValueError(f"Unrecognized input type: {type(ds)}")
+
+    return coords

ocf_data_sampler/select/location.py

@@ -0,0 +1,62 @@
+"""location"""
+
+from typing import Optional
+
+import numpy as np
+from pydantic import BaseModel, Field, model_validator
+
+
+allowed_coordinate_systems =["osgb", "lon_lat", "geostationary", "idx"]
+
+class Location(BaseModel):
+    """Represent a spatial location."""
+
+    coordinate_system: Optional[str] = "osgb"  # ["osgb", "lon_lat", "geostationary", "idx"]
+    x: float
+    y: float
+    id: Optional[int] = Field(None)
+
+    @model_validator(mode='after')
+    def validate_coordinate_system(self):
+        """Validate 'coordinate_system'"""
+        if self.coordinate_system not in allowed_coordinate_systems:
+            raise ValueError(f"coordinate_system = {self.coordinate_system} is not in {allowed_coordinate_systems}")
+        return self
+
+    @model_validator(mode='after')
+    def validate_x(self):
+        """Validate 'x'"""
+        min_x: float
+        max_x: float
+
+        co = self.coordinate_system
+        if co == "osgb":
+            min_x, max_x = -103976.3, 652897.98
+        if co == "lon_lat":
+            min_x, max_x = -180, 180
+        if co == "geostationary":
+            min_x, max_x = -5568748.275756836, 5567248.074173927
+        if co == "idx":
+            min_x, max_x = 0, np.inf
+        if self.x < min_x or self.x > max_x:
+            raise ValueError(f"x = {self.x} must be within {[min_x, max_x]} for {co} coordinate system")
+        return self
+
+    @model_validator(mode='after')
+    def validate_y(self):
+        """Validate 'y'"""
+        min_y: float
+        max_y: float
+
+        co = self.coordinate_system
+        if co == "osgb":
+            min_y, max_y = -16703.87, 1199851.44
+        if co == "lon_lat":
+            min_y, max_y = -90, 90
+        if co == "geostationary":
+            min_y, max_y = 1393687.2151494026, 5570748.323202133
+        if co == "idx":
+            min_y, max_y = 0, np.inf
+        if self.y < min_y or self.y > max_y:
+            raise ValueError(f"y = {self.y} must be within {[min_y, max_y]} for {co} coordinate system")
+        return self

ocf_data_sampler/select/select_spatial_slice.py

@@ -5,15 +5,14 @@ import logging
 import numpy as np
 import xarray as xr
 
-from ocf_datapipes.utils import Location
-from ocf_datapipes.utils.geospatial import (
-    lon_lat_to_geostationary_area_coords,
+from ocf_data_sampler.select.location import Location
+from ocf_data_sampler.select.geospatial import (
     lon_lat_to_osgb,
     osgb_to_geostationary_area_coords,
     osgb_to_lon_lat,
     spatial_coord_type,
 )
-from ocf_datapipes.utils.utils import searchsorted
+
 
 logger = logging.getLogger(__name__)
 
@@ -45,9 +44,6 @@ def convert_coords_to_match_xarray(
         if from_coords == "osgb":
             x, y = osgb_to_geostationary_area_coords(x, y, da)
 
-        elif from_coords == "lon_lat":
-            x, y = lon_lat_to_geostationary_area_coords(x, y, da)
-
     elif target_coords == "lon_lat":
         if from_coords == "osgb":
             x, y = osgb_to_lon_lat(x, y)
@@ -130,13 +126,8 @@ def _get_idx_of_pixel_closest_to_poi_geostationary(
         f"{y} is not in the interval {da[y_dim].min().values}: {da[y_dim].max().values}"
 
     # Get the index into x and y nearest to x_center_geostationary and y_center_geostationary:
-    x_index_at_center = searchsorted(
-        da[x_dim].values, center_geostationary.x, assume_ascending=True
-    )
-
-    y_index_at_center = searchsorted(
-        da[y_dim].values, center_geostationary.y, assume_ascending=True
-    )
+    x_index_at_center = np.searchsorted(da[x_dim].values, center_geostationary.x)
+    y_index_at_center = np.searchsorted(da[y_dim].values, center_geostationary.y)
 
     return Location(x=x_index_at_center, y=y_index_at_center, coordinate_system="idx")
 

ocf_data_sampler/torch_datasets/pvnet_uk_regional.py

@@ -27,19 +27,15 @@ from ocf_data_sampler.numpy_batch import (
 )
 
 
-from ocf_datapipes.config.model import Configuration
-from ocf_datapipes.config.load import load_yaml_configuration
-from ocf_datapipes.batch import BatchKey, NumpyBatch
+from ocf_data_sampler.config import Configuration, load_yaml_configuration
+from ocf_data_sampler.numpy_batch.nwp import NWPBatchKey
+from ocf_data_sampler.numpy_batch.gsp import GSPBatchKey
 
-from ocf_datapipes.utils.location import Location
-from ocf_datapipes.utils.geospatial import osgb_to_lon_lat
+from ocf_data_sampler.select.location import Location
+from ocf_data_sampler.select.geospatial import osgb_to_lon_lat
 
-from ocf_datapipes.utils.consts import (
-    NWP_MEANS,
-    NWP_STDS,
-)
+from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS
 
-from ocf_datapipes.training.common import concat_xr_time_utc, normalize_gsp
 
 
 
@@ -344,7 +340,7 @@ def slice_datasets_by_time(
     return sliced_datasets_dict
 
 
-def fill_nans_in_arrays(batch: NumpyBatch) -> NumpyBatch:
+def fill_nans_in_arrays(batch: dict) -> dict:
     """Fills all NaN values in each np.ndarray in the batch dictionary with zeros.
 
     Operation is performed in-place on the batch.
@@ -376,7 +372,7 @@ def process_and_combine_datasets(
         config: Configuration,
         t0: pd.Timedelta,
         location: Location,
-    ) -> NumpyBatch:
+    ) -> dict:
     """Normalize and convert data to numpy arrays"""    
 
     numpy_modalities = []
@@ -393,7 +389,7 @@ def process_and_combine_datasets(
             nwp_numpy_modalities[nwp_key] = convert_nwp_to_numpy_batch(da_nwp)
         
         # Combine the NWPs into NumpyBatch
-        numpy_modalities.append({BatchKey.nwp: nwp_numpy_modalities})
+        numpy_modalities.append({NWPBatchKey.nwp: nwp_numpy_modalities})
 
     if "sat" in dataset_dict:
         # Satellite is already in the range [0-1] so no need to standardise
@@ -405,8 +401,8 @@ def process_and_combine_datasets(
     gsp_config = config.input_data.gsp
     
     if "gsp" in dataset_dict:
-        da_gsp = concat_xr_time_utc([dataset_dict["gsp"], dataset_dict["gsp_future"]])
-        da_gsp = normalize_gsp(da_gsp)
+        da_gsp = xr.concat([dataset_dict["gsp"], dataset_dict["gsp_future"]], dim="time_utc")
+        da_gsp = da_gsp / da_gsp.effective_capacity_mwp
 
         numpy_modalities.append(
             convert_gsp_to_numpy_batch(
@@ -429,9 +425,9 @@ def process_and_combine_datasets(
     # Add coordinate data
     # TODO: Do we need all of these?
     numpy_modalities.append({
-        BatchKey.gsp_id: location.id,
-        BatchKey.gsp_x_osgb: location.x,
-        BatchKey.gsp_y_osgb: location.y,
+        GSPBatchKey.gsp_id: location.id,
+        GSPBatchKey.gsp_x_osgb: location.x,
+        GSPBatchKey.gsp_y_osgb: location.y,
     })
 
     # Combine all the modalities and fill NaNs
@@ -539,7 +535,7 @@ class PVNetUKRegionalDataset(Dataset):
         return len(self.index_pairs)
     
     
-    def _get_sample(self, t0: pd.Timestamp, location: Location) -> NumpyBatch:
+    def _get_sample(self, t0: pd.Timestamp, location: Location) -> dict:
         """Generate the PVNet sample for given coordinates
         
         Args:
@@ -566,7 +562,7 @@ class PVNetUKRegionalDataset(Dataset):
         return self._get_sample(t0, location)
     
 
-    def get_sample(self, t0: pd.Timestamp, gsp_id: int) -> NumpyBatch:
+    def get_sample(self, t0: pd.Timestamp, gsp_id: int) -> dict:
         """Generate a sample for the given coordinates. 
         
         Useful for users to generate samples by GSP ID.

ocf_data_sampler-0.0.22.dist-info/METADATA

@@ -0,0 +1,88 @@
+Metadata-Version: 2.1
+Name: ocf_data_sampler
+Version: 0.0.22
+Summary: Sample from weather data for renewable energy prediction
+Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
+Author-email: info@openclimatefix.org
+Maintainer: Open Climate Fix Ltd
+License: MIT License
+        
+        Copyright (c) 2023 Open Climate Fix
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: homepage, https://github.com/openclimatefix
+Project-URL: repository, https://github.com/openclimatefix/ocf-data-sampler
+Keywords: weather data,renewable energy prediction,sample weather data
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: xarray
+Requires-Dist: zarr
+Requires-Dist: dask
+Requires-Dist: ocf-blosc2
+Requires-Dist: pvlib
+Requires-Dist: pydantic
+Requires-Dist: pyproj
+Requires-Dist: pathy
+Requires-Dist: pyaml-env
+Requires-Dist: pyresample
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.2; extra == "docs"
+Requires-Dist: mkdocs-material>=8.0; extra == "docs"
+
+# OCF Data Sampler
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-5-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+[![ease of contribution: easy](https://img.shields.io/badge/ease%20of%20contribution:%20easy-32bd50)](https://github.com/openclimatefix/ocf-meta-repo?tab=readme-ov-file#overview-of-ocfs-nowcasting-repositories) 
+
+A repo for sampling from weather data for renewable energy prediction
+
+## Contributors ✨
+
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dfulu"><img src="https://avatars.githubusercontent.com/u/41546094?v=4?s=100" width="100px;" alt="James Fulton"/><br /><sub><b>James Fulton</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=dfulu" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/AUdaltsova"><img src="https://avatars.githubusercontent.com/u/43303448?v=4?s=100" width="100px;" alt="Alexandra Udaltsova"/><br /><sub><b>Alexandra Udaltsova</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=AUdaltsova" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sukh-P"><img src="https://avatars.githubusercontent.com/u/42407101?v=4?s=100" width="100px;" alt="Sukhil Patel"/><br /><sub><b>Sukhil Patel</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sukh-P" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/peterdudfield"><img src="https://avatars.githubusercontent.com/u/34686298?v=4?s=100" width="100px;" alt="Peter Dudfield"/><br /><sub><b>Peter Dudfield</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=peterdudfield" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/VikramsDataScience"><img src="https://avatars.githubusercontent.com/u/45002417?v=4?s=100" width="100px;" alt="Vikram Pande"/><br /><sub><b>Vikram Pande</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=VikramsDataScience" title="Code">💻</a></td>
+    </tr>
+  </tbody>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

ocf_data_sampler-0.0.22.dist-info/RECORD

@@ -0,0 +1,54 @@
+ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+ocf_data_sampler/constants.py,sha256=tUwHrsGShqIn5Izze4i32_xB6X0v67rvQwIYB-P5PJQ,3355
+ocf_data_sampler/config/__init__.py,sha256=YXnAkgHViHB26hSsjiv32b6EbpG-A1kKTkARJf0_RkY,212
+ocf_data_sampler/config/load.py,sha256=4f7vPHAIAmd-55tPxoIzn7F_TI_ue4NxkDcLPoVWl0g,943
+ocf_data_sampler/config/model.py,sha256=bvU3BEMtcUh-N17fMVLTYtN-J2GcTM9Qq-CI5AfbE4Q,8128
+ocf_data_sampler/config/save.py,sha256=wKdctbv0dxIIiQtcRHLRxpWQVhEFQ_FCWg-oNaRLIps,1093
+ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
+ocf_data_sampler/load/__init__.py,sha256=MjgfxilTzyz1RYFoBEeAXmE9hyjknLvdmlHPmlAoiQY,44
+ocf_data_sampler/load/gsp.py,sha256=Gcr1JVUOPKhFRDCSHtfPDjxx0BtyyEhXrZvGEKLPJ5I,759
+ocf_data_sampler/load/satellite.py,sha256=3KlA1fx4SwxdzM-jC1WRaONXO0D6m0WxORnEnwUnZrA,2967
+ocf_data_sampler/load/utils.py,sha256=EQGvVWlGMoSOdbDYuMfVAa0v6wmAOPmHIAemdrTB5v4,1406
+ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
+ocf_data_sampler/load/nwp/nwp.py,sha256=O4QnajEZem8BvBgTcYYDBhRhgqPYuJkolHmpMRmrXEA,610
+ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=vW-p3vCyQ-CofKo555-gE7VDi5hlpjtjTLfHqWF0HEE,1175
+ocf_data_sampler/load/nwp/providers/ukv.py,sha256=79Bm7q-K_GJPYMy62SUIZbRWRF4-tIaB1dYPEgLD9vo,1207
+ocf_data_sampler/load/nwp/providers/utils.py,sha256=Sy2exG1wpXLLhMXYdsfR-DZMR3txG1_bBmBdchlc-yA,848
+ocf_data_sampler/numpy_batch/__init__.py,sha256=mrtqwbGik5Zc9MYP5byfCTBm08wMtS2XnTsypC4fPMo,245
+ocf_data_sampler/numpy_batch/gsp.py,sha256=FQg5infqOZ8QAsRCXq00HPplN8XnsbKoWUeV3N7TGK8,1008
+ocf_data_sampler/numpy_batch/nwp.py,sha256=dt4gXZonrdkyoVmMnwkKyJ_xFnJ4M5xscr-pdNoRsc4,1311
+ocf_data_sampler/numpy_batch/satellite.py,sha256=yWYmq6hEvX4LTJ95K_e0BRJJkBmRurQT2mI2KibmwCI,983
+ocf_data_sampler/numpy_batch/sun_position.py,sha256=zw2bjtcjsm_tvKk0r_MZmgfYUJLHuLjLly2sMjwP3XI,1606
+ocf_data_sampler/select/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+ocf_data_sampler/select/dropout.py,sha256=zDpVLMjGb70RRyYKN-WI2Kp3x9SznstT4cMcZ4dsvJg,1066
+ocf_data_sampler/select/fill_time_periods.py,sha256=iTtMjIPFYG5xtUYYedAFBLjTWWUa7t7WQ0-yksWf0-E,440
+ocf_data_sampler/select/find_contiguous_time_periods.py,sha256=6ioB8LeFpFNBMgKDxrgG3zqzNjkBF_jlV9yye2ZYT2E,11925
+ocf_data_sampler/select/geospatial.py,sha256=oHJoKEKubn3v3yKCVeuiPxuGroVA4RyrpNi6ARq5woE,3558
+ocf_data_sampler/select/location.py,sha256=26Y5ZjfFngShBwXieuWSoOA-RLaRzci4TTmcDk3Wg7U,2015
+ocf_data_sampler/select/select_spatial_slice.py,sha256=hWIJe4_VzuQ2iiiQh7V17AXwTILT5kIkUvzG458J_Gw,11220
+ocf_data_sampler/select/select_time_slice.py,sha256=41cch1fQr59fZgv7UHsNGc3OvoynrixT3bmr3_1d7cU,6628
+ocf_data_sampler/torch_datasets/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+ocf_data_sampler/torch_datasets/pvnet_uk_regional.py,sha256=5AV_eFtQ842-E7kNeQPLvEvcmV4LDkJKYbV2rvU_wkE,19113
+tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/conftest.py,sha256=O77dmow8mGpGPbZ6Pz7ma7cLaiV1k8mxW1eYg37Avrw,5585
+tests/config/test_config.py,sha256=G_PD_pXib0zdRBPUIn0jjwJ9VyoKaO_TanLN1Mh5Ca4,5055
+tests/load/test_load_gsp.py,sha256=aT_nqaSXmUTcdHzuTT7AmXJr3R31k4OEN-Fv3eLxlQE,424
+tests/load/test_load_nwp.py,sha256=3qyyDkB1q9t3tyAwogfotNrxqUOpXXimco1CImoEWGg,753
+tests/load/test_load_satellite.py,sha256=STX5AqqmOAgUgE9R1xyq_sM3P1b8NKdGjO-hDhayfxM,524
+tests/numpy_batch/test_gsp.py,sha256=ke4CsFn9ZKRgrciT5-OHS8jmd8nu9gGKKr0T2WzZK6M,592
+tests/numpy_batch/test_nwp.py,sha256=yk77jZfba-dd5ImjOTWNTMNnjJR8SF0kSpZ-513vPTw,1490
+tests/numpy_batch/test_satellite.py,sha256=DIVnVq7JYgoC6Y6xMtLCNUnk3ADzakCi0bZ44QTdPgQ,1230
+tests/numpy_batch/test_sun_position.py,sha256=zw7ErTKARkW8NrpXJ9MeGp-dkNBJsCscxQx0dnZHg2c,2513
+tests/select/test_dropout.py,sha256=kiycl7RxAQYMCZJlokmx6Da5h_oBpSs8Is8pmSW4gOU,2413
+tests/select/test_fill_time_periods.py,sha256=o59f2YRe5b0vJrG3B0aYZkYeHnpNk4s6EJxdXZluNQg,907
+tests/select/test_find_contiguous_time_periods.py,sha256=G6tJRJd0DMfH9EdfzlKWsmfTbtMwOf3w-2filjJzuIQ,5998
+tests/select/test_location.py,sha256=_WZk2FPYeJ-nIfCJS6Sp_yaVEEo7m31DmMFoZzgyCts,2712
+tests/select/test_select_spatial_slice.py,sha256=7EX9b6g-pMdACQx3yefjs5do2s-Rho2UmKevV4oglsU,5147
+tests/select/test_select_time_slice.py,sha256=rH4h90HdQCoWE7vV7ivMEKhiCStQDEcMBCPamiDuO0k,10147
+tests/torch_datasets/test_pvnet_uk_regional.py,sha256=r1SHtwaXQrOYU3EOH1OEp_Bamo338IMv-9Q_gKsUOa4,2789
+ocf_data_sampler-0.0.22.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
+ocf_data_sampler-0.0.22.dist-info/METADATA,sha256=iB2KIy-7AQjWsOQzQoINItbAlKFWv_3TIRcOJ1-f_Uw,5269
+ocf_data_sampler-0.0.22.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+ocf_data_sampler-0.0.22.dist-info/top_level.txt,sha256=KaQn5qzkJGJP6hKWqsVAc9t0cMLjVvSTk8-kTrW79SA,23
+ocf_data_sampler-0.0.22.dist-info/RECORD,,

{ocf_data_sampler-0.0.19.dist-info → ocf_data_sampler-0.0.22.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/config/test_config.py

@@ -0,0 +1,152 @@
+import tempfile
+
+import pytest
+from pydantic import ValidationError
+
+from ocf_data_sampler.config import (
+    load_yaml_configuration,
+    Configuration,
+    save_yaml_configuration
+)
+
+
+def test_default():
+    """Test default pydantic class"""
+
+    _ = Configuration()
+
+
+def test_yaml_load_test_config(test_config_filename):
+    """
+    Test that yaml loading works for 'test_config.yaml'
+    and fails for an empty .yaml file
+    """
+
+    # check we get an error if loading a file with no config
+    with tempfile.NamedTemporaryFile(suffix=".yaml") as fp:
+        filename = fp.name
+
+        # check that temp file can't be loaded
+        with pytest.raises(TypeError):
+            _ = load_yaml_configuration(filename)
+
+    # test can load test_config.yaml
+    config = load_yaml_configuration(test_config_filename)
+
+    assert isinstance(config, Configuration)
+
+
+def test_yaml_save(test_config_filename):
+    """
+    Check configuration can be saved to a .yaml file
+    """
+
+    test_config = load_yaml_configuration(test_config_filename)
+
+    with tempfile.NamedTemporaryFile(suffix=".yaml") as fp:
+        filename = fp.name
+
+        # save default config to file
+        save_yaml_configuration(test_config, filename)
+
+        # check the file can be loaded back
+        tmp_config = load_yaml_configuration(filename)
+
+        # check loaded configuration is the same as the one passed to save
+        assert test_config == tmp_config
+
+
+def test_extra_field():
+    """
+    Check an extra parameters in config causes error
+    """
+
+    configuration = Configuration()
+    configuration_dict = configuration.model_dump()
+    configuration_dict["extra_field"] = "extra_value"
+    with pytest.raises(ValidationError, match="Extra inputs are not permitted"):
+        _ = Configuration(**configuration_dict)
+
+
+def test_incorrect_forecast_minutes(test_config_filename):
+    """
+    Check a forecast length not divisible by time resolution causes error
+    """
+
+    configuration = load_yaml_configuration(test_config_filename)
+
+    configuration.input_data.nwp['ukv'].forecast_minutes = 1111
+    with pytest.raises(Exception, match="duration must be divisible by time resolution"):
+        _ = Configuration(**configuration.model_dump())
+
+
+def test_incorrect_history_minutes(test_config_filename):
+    """
+    Check a history length not divisible by time resolution causes error
+    """
+
+    configuration = load_yaml_configuration(test_config_filename)
+
+    configuration.input_data.nwp['ukv'].history_minutes = 1111
+    with pytest.raises(Exception, match="duration must be divisible by time resolution"):
+        _ = Configuration(**configuration.model_dump())
+
+
+def test_incorrect_nwp_provider(test_config_filename):
+    """
+    Check an unexpected nwp provider causes error
+    """
+
+    configuration = load_yaml_configuration(test_config_filename)
+
+    configuration.input_data.nwp['ukv'].nwp_provider = "unexpected_provider"
+    with pytest.raises(Exception, match="NWP provider"):
+        _ = Configuration(**configuration.model_dump())
+
+def test_incorrect_dropout(test_config_filename):
+    """
+    Check a dropout timedelta over 0 causes error and 0 doesn't
+    """
+
+    configuration = load_yaml_configuration(test_config_filename)
+
+    # check a positive number is not allowed
+    configuration.input_data.nwp['ukv'].dropout_timedeltas_minutes = [120]
+    with pytest.raises(Exception, match="Dropout timedeltas must be negative"):
+        _ = Configuration(**configuration.model_dump())
+
+    # check 0 is allowed
+    configuration.input_data.nwp['ukv'].dropout_timedeltas_minutes = [0]
+    _ = Configuration(**configuration.model_dump())
+
+def test_incorrect_dropout_fraction(test_config_filename):
+    """
+    Check dropout fraction outside of range causes error
+    """
+
+    configuration = load_yaml_configuration(test_config_filename)
+
+    configuration.input_data.nwp['ukv'].dropout_fraction= 1.1
+    with pytest.raises(Exception, match="Dropout fraction must be between 0 and 1"):
+        _ = Configuration(**configuration.model_dump())
+
+    configuration.input_data.nwp['ukv'].dropout_fraction= -0.1
+    with pytest.raises(Exception, match="Dropout fraction must be between 0 and 1"):
+        _ = Configuration(**configuration.model_dump())
+
+
+def test_inconsistent_dropout_use(test_config_filename):
+    """
+    Check dropout fraction outside of range causes error
+    """
+
+    configuration = load_yaml_configuration(test_config_filename)
+    configuration.input_data.satellite.dropout_fraction= 1.0
+    configuration.input_data.satellite.dropout_timedeltas_minutes = None
+
+    with pytest.raises(ValueError, match="To dropout fraction > 0 requires a list of dropout timedeltas"):
+        _ = Configuration(**configuration.model_dump())
+    configuration.input_data.satellite.dropout_fraction= 0.0
+    configuration.input_data.satellite.dropout_timedeltas_minutes = [-120, -60]
+    with pytest.raises(ValueError, match="To use dropout timedeltas dropout fraction should be > 0"):
+        _ = Configuration(**configuration.model_dump())

tests/conftest.py

@@ -6,11 +6,16 @@ import pytest
 import xarray as xr
 import tempfile
 
+_top_test_directory = os.path.dirname(os.path.realpath(__file__))
+
+@pytest.fixture()
+def test_config_filename():
+    return f"{_top_test_directory}/test_data/configs/test_config.yaml"
 
 
 @pytest.fixture(scope="session")
 def config_filename():
-    return f"{os.path.dirname(os.path.abspath(__file__))}/test_data/pvnet_test_config.yaml"
+    return f"{os.path.dirname(os.path.abspath(__file__))}/test_data/configs/pvnet_test_config.yaml"
 
 
 @pytest.fixture(scope="session")

tests/load/test_load_gsp.py

@@ -0,0 +1,15 @@
+from ocf_data_sampler.load.gsp import open_gsp
+import xarray as xr
+
+
+def test_open_gsp(uk_gsp_zarr_path):
+    da = open_gsp(uk_gsp_zarr_path)
+
+    assert isinstance(da, xr.DataArray)
+    assert da.dims == ("time_utc", "gsp_id")
+
+    assert "nominal_capacity_mwp" in da.coords
+    assert "effective_capacity_mwp" in da.coords
+    assert "x_osgb" in da.coords
+    assert "y_osgb" in da.coords
+    assert da.shape == (49, 318)