ocf-data-sampler 0.2.12__py3-none-any.whl → 0.2.14__py3-none-any.whl

ocf_data_sampler/load/__init__.py

@@ -1,4 +1,3 @@
-import ocf_blosc2
 from ocf_data_sampler.load.gsp import open_gsp
 from ocf_data_sampler.load.nwp import open_nwp
 from ocf_data_sampler.load.satellite import open_sat_data

ocf_data_sampler/torch_datasets/sample/uk_regional.py

@@ -3,14 +3,18 @@
 import torch
 from typing_extensions import override
 
+from ocf_data_sampler.config import Configuration
 from ocf_data_sampler.numpy_sample import (
     GSPSampleKey,
     NWPSampleKey,
     SatelliteSampleKey,
 )
 from ocf_data_sampler.numpy_sample.common_types import NumpySample
-
-from .base import SampleBase
+from ocf_data_sampler.torch_datasets.sample.base import SampleBase
+from ocf_data_sampler.torch_datasets.utils.validation_utils import (
+    calculate_expected_shapes,
+    check_dimensions,
+)
 
 
 class UKRegionalSample(SampleBase):
@@ -22,22 +26,123 @@ class UKRegionalSample(SampleBase):
 
     @override
     def to_numpy(self) -> NumpySample:
+        """Returns the data as a NumPy sample."""
         return self._data
 
     @override
     def save(self, path: str) -> None:
+        """Saves sample to the specified path in pickle format."""
         # Saves to pickle format
         torch.save(self._data, path)
 
     @classmethod
     @override
     def load(cls, path: str) -> "UKRegionalSample":
+        """Loads sample from the specified path.
+
+        Args:
+            path: Path to the saved sample file.
+
+        Returns:
+            A UKRegionalSample instance with the loaded data.
+        """
         # Loads from .pt format
         # TODO: We should move away from using torch.load(..., weights_only=False)
         return cls(torch.load(path, weights_only=False))
 
+    def validate_sample(self, config: Configuration) -> bool:
+        """Validates that the sample has the expected structure and data shapes.
+
+        Args:
+            config: Configuration dict with expected shapes and required fields.
+
+        Returns:
+            bool: True if validation passes, otherwise raises an exception.
+        """
+        if not isinstance(config, Configuration):
+            raise TypeError("config must be Configuration object")
+
+        # Calculate expected shapes from configuration
+        expected_shapes = calculate_expected_shapes(config)
+
+        # Check GSP shape if specified
+        gsp_key = GSPSampleKey.gsp
+        # Check if GSP data is expected but missing
+        if gsp_key in expected_shapes and gsp_key not in self._data:
+            raise ValueError(f"Configuration expects GSP data ('{gsp_key}') but is missing.")
+
+        # Check GSP shape if data exists and is expected
+        if gsp_key in self._data and gsp_key in expected_shapes:
+            gsp_data = self._data[gsp_key]
+            check_dimensions(
+                actual_shape=gsp_data.shape,
+                expected_shape=expected_shapes[gsp_key],
+                name="GSP",
+            )
+
+        # Checks for NWP data - nested structure
+        nwp_key = NWPSampleKey.nwp
+        if nwp_key in expected_shapes and nwp_key not in self._data:
+            raise ValueError(f"Configuration expects NWP data ('{nwp_key}') but is missing.")
+
+        # Check NWP structure and shapes if data exists
+        if nwp_key in self._data:
+            nwp_data_all_providers = self._data[nwp_key]
+            if not isinstance(nwp_data_all_providers, dict):
+                raise ValueError(f"NWP data ('{nwp_key}') should be a dictionary.")
+
+            # Loop through providers present in actual data
+            for provider, provider_data in nwp_data_all_providers.items():
+                if "nwp" not in provider_data:
+                    raise ValueError(f"Missing array key in NWP data for provider '{provider}'.")
+
+                if nwp_key in expected_shapes and provider in expected_shapes[nwp_key]:
+                    nwp_array = provider_data["nwp"]
+                    actual_shape = nwp_array.shape
+                    expected_shape = expected_shapes[nwp_key][provider]
+
+                    check_dimensions(
+                        actual_shape=actual_shape,
+                        expected_shape=expected_shape,
+                        name=f"NWP data ({provider})",
+                    )
+
+        # Validate satellite data
+        sat_key = SatelliteSampleKey.satellite_actual
+        # Check if Satellite data is expected but missing
+        if sat_key in expected_shapes and sat_key not in self._data:
+            raise ValueError(f"Configuration expects Satellite data ('{sat_key}') but is missing.")
+
+        # Check satellite shape if data exists and is expected
+        if sat_key in self._data and sat_key in expected_shapes:
+            sat_data = self._data[sat_key]
+            check_dimensions(
+                actual_shape=sat_data.shape,
+                expected_shape=expected_shapes[sat_key],
+                name="Satellite data",
+            )
+
+        # Validate solar coordinates data
+        solar_keys = ["solar_azimuth", "solar_elevation"]
+        # Check if solar coordinate is expected but missing
+        for solar_key in solar_keys:
+            if solar_key in expected_shapes and solar_key not in self._data:
+                raise ValueError(f"Configuration expects {solar_key} data but is missing.")
+
+            # Check solar coordinate shape if data exists and is expected
+            if solar_key in self._data and solar_key in expected_shapes:
+                solar_data = self._data[solar_key]
+                check_dimensions(
+                    actual_shape=solar_data.shape,
+                    expected_shape=expected_shapes[solar_key],
+                    name=f"{solar_key.replace('_', ' ').title()} data",
+                )
+
+        return True
+
     @override
     def plot(self) -> None:
+        """Plots the sample data for visualization."""
         from matplotlib import pyplot as plt
 
         fig, axes = plt.subplots(2, 2, figsize=(12, 8))
@@ -58,10 +163,10 @@ class UKRegionalSample(SampleBase):
             axes[0, 0].set_title("GSP Generation")
 
         if "solar_azimuth" in self._data and "solar_elevation" in self._data:
-                    axes[1, 1].plot(self._data["solar_azimuth"], label="Azimuth")
-                    axes[1, 1].plot(self._data["solar_elevation"], label="Elevation")
-                    axes[1, 1].set_title("Solar Position")
-                    axes[1, 1].legend()
+            axes[1, 1].plot(self._data["solar_azimuth"], label="Azimuth")
+            axes[1, 1].plot(self._data["solar_elevation"], label="Elevation")
+            axes[1, 1].set_title("Solar Position")
+            axes[1, 1].legend()
 
         if SatelliteSampleKey.satellite_actual in self._data:
             axes[1, 0].imshow(self._data[SatelliteSampleKey.satellite_actual])

ocf_data_sampler/torch_datasets/utils/validation_utils.py

@@ -0,0 +1,108 @@
+"""Validate sample shape against expected shape - utility function."""
+
+from ocf_data_sampler.config import Configuration
+from ocf_data_sampler.numpy_sample import GSPSampleKey, NWPSampleKey, SatelliteSampleKey
+
+
+def check_dimensions(
+    actual_shape: tuple[int, ...],
+    expected_shape: tuple[int, ...],
+    name: str,
+) -> None:
+    """Check if dimensions match between actual and expected shapes.
+
+    Args:
+        actual_shape: The actual shape of the data (e.g., array.shape).
+        expected_shape: The expected shape.
+        name: Name of the data component for clear error messages.
+
+    Raises:
+        ValueError: If dimensions don't match.
+    """
+    if actual_shape != expected_shape:
+        raise ValueError(
+            f"'{name}' shape mismatch: "
+            f"Actual shape: {actual_shape}, Expected shape: {expected_shape}",
+        )
+
+
+def calculate_expected_shapes(
+    config: Configuration,
+) -> dict[str, tuple[int, ...]]:
+    """Calculate expected shapes from configuration.
+
+    Args:
+        config: Configuration object with shape information.
+
+    Returns:
+        Dictionary mapping data keys to their expected shapes.
+    """
+    expected_shapes = {}
+    input_data = config.input_data
+
+    # Calculate GSP shape
+    gsp_config = input_data.gsp
+    expected_shapes[GSPSampleKey.gsp] = (
+        _calculate_time_steps(
+            gsp_config.interval_start_minutes,
+            gsp_config.interval_end_minutes,
+            gsp_config.time_resolution_minutes,
+        ),
+    )
+
+    # Calculate NWP shape for multiple providers
+    expected_shapes[NWPSampleKey.nwp] = {}
+    for provider_key, provider_config in input_data.nwp.items():
+        expected_shapes[NWPSampleKey.nwp][provider_key] = (
+            _calculate_time_steps(
+                provider_config.interval_start_minutes,
+                provider_config.interval_end_minutes,
+                provider_config.time_resolution_minutes,
+            ),
+            len(provider_config.channels),
+            provider_config.image_size_pixels_height,
+            provider_config.image_size_pixels_width,
+        )
+
+    # Calculate satellite shape
+    sat_config = input_data.satellite
+    expected_shapes[SatelliteSampleKey.satellite_actual] = (
+        _calculate_time_steps(
+            sat_config.interval_start_minutes,
+            sat_config.interval_end_minutes,
+            sat_config.time_resolution_minutes,
+        ),
+        len(sat_config.channels),
+        sat_config.image_size_pixels_height,
+        sat_config.image_size_pixels_width,
+    )
+
+    # Calculate solar coordinates shapes
+    solar_config = input_data.solar_position
+    # For solar azimuth
+    expected_shapes["solar_azimuth"] = (
+        _calculate_time_steps(
+            solar_config.interval_start_minutes,
+            solar_config.interval_end_minutes,
+            solar_config.time_resolution_minutes,
+        ),
+    )
+    # For solar elevation
+    expected_shapes["solar_elevation"] = expected_shapes["solar_azimuth"]
+
+    return expected_shapes
+
+
+def _calculate_time_steps(start_minutes: int, end_minutes: int, resolution_minutes: int) -> int:
+    """Calculate number of time steps based on interval and resolution.
+
+    Args:
+        start_minutes: Start of interval in minutes
+        end_minutes: End of interval in minutes
+        resolution_minutes: Time resolution in minutes
+
+    Returns:
+        Number of time steps
+    """
+    time_span = end_minutes - start_minutes
+    return (time_span // resolution_minutes) + 1

{ocf_data_sampler-0.2.12.dist-info → ocf_data_sampler-0.2.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.12
+Version: 0.2.14
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License
@@ -38,11 +38,9 @@ Requires-Dist: zarr==2.18.3
 Requires-Dist: numcodecs==0.13.1
 Requires-Dist: dask
 Requires-Dist: matplotlib
-Requires-Dist: ocf_blosc2
 Requires-Dist: pvlib
 Requires-Dist: pydantic
 Requires-Dist: pyproj
-Requires-Dist: pathy
 Requires-Dist: pyaml_env
 Requires-Dist: pyresample
 Requires-Dist: h5netcdf

{ocf_data_sampler-0.2.12.dist-info → ocf_data_sampler-0.2.14.dist-info}/RECORD

@@ -5,7 +5,7 @@ ocf_data_sampler/config/load.py,sha256=LL-7wemI8o4KPkx35j-wQ3HjsMvDgqXr7G46IcASf
 ocf_data_sampler/config/model.py,sha256=pb02qtCmWhJhrU3_T_gUzC7i2_JcO8xGwwhKGd8yMuk,10209
 ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdRaI,1064
 ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
-ocf_data_sampler/load/__init__.py,sha256=T5Zj1PGt0aiiNEN7Ra1Ac-cBsNKhphmmHy_8g7XU_w0,219
+ocf_data_sampler/load/__init__.py,sha256=-vQP9g0UOWdVbjEGyVX_ipa7R1btmiETIKAf6aw4d78,201
 ocf_data_sampler/load/gsp.py,sha256=keB3Nv_CNK1P6pS9Kdfc8PoZXTI1_YFN-spsvEv_Ewc,899
 ocf_data_sampler/load/load_dataset.py,sha256=0NyDxCDfgE_esKVW3s-rZEe16WB30FQ74ClWlrIo72M,1602
 ocf_data_sampler/load/satellite.py,sha256=E7Ln7Y60Qr1RTV-_R71YoxXQM-Ca7Y1faIo3oKB2eFk,2292
@@ -43,16 +43,17 @@ ocf_data_sampler/torch_datasets/datasets/site.py,sha256=nRUlhXQQGVrTuBmE1QnwXAUs
 ocf_data_sampler/torch_datasets/sample/__init__.py,sha256=GL84vdZl_SjHDGVyh9Uekx2XhPYuZ0dnO3l6f6KXnHI,100
 ocf_data_sampler/torch_datasets/sample/base.py,sha256=cQ1oIyhdmlotejZK8B3Cw6MNvpdnBPD8G_o2h7Ye4Vc,2206
 ocf_data_sampler/torch_datasets/sample/site.py,sha256=ZUEgn50g-GmqujOEtezNILF7wjokF80sDAA4OOldcRI,1268
-ocf_data_sampler/torch_datasets/sample/uk_regional.py,sha256=zpCeUw3eljOnoJTSUYW2R4kiWrY6hbuXjK8igJrXgPg,2441
+ocf_data_sampler/torch_datasets/sample/uk_regional.py,sha256=8hDgaMg5Vb6eYitqYiljpAeTeTemwsYaRpZn7_3_XjI,7013
 ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=N7i_hHtWUDiJqsiJoDx4T_QuiYOuvIyulPrn6xEA4TY,309
 ocf_data_sampler/torch_datasets/utils/channel_dict_to_dataarray.py,sha256=un2IiyoAmTDIymdeMiPU899_86iCDMD-oIifjHlNyqw,555
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=we7BTxRH7B7jKayDT7YfNyfI3zZClz2Bk-HXKQIokgU,956
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py,sha256=Hvz0wHSWMYYamf2oHNiGlzJcM4cAH6pL_7ZEvIBL2dE,1882
 ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=1DN6VsWWdLvkpJxodZtBRDUgC4vJE2td_RP5J3ZqPNw,4268
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0WrglnX5YUAFjXXlO2GwEBHWyqo8TDuiOA,4714
+ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=2fwW-kpsMM2a-FWBG0YBT_r2LDIhhn7WokQ7GWvgx6U,3504
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
 utils/compute_icon_mean_stddev.py,sha256=a1oWMRMnny39rV-dvu8rcx85sb4bXzPFrR1gkUr4Jpg,2296
-ocf_data_sampler-0.2.12.dist-info/METADATA,sha256=G0rtK1M1DqIZrvtV0dOlbfumxxmOB8P96pXr61QTEfU,11628
-ocf_data_sampler-0.2.12.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ocf_data_sampler-0.2.12.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
-ocf_data_sampler-0.2.12.dist-info/RECORD,,
+ocf_data_sampler-0.2.14.dist-info/METADATA,sha256=fb2tvDSt9FrsJBD8mZvjJ8YwTgp3OfUjavuZda3cblA,11581
+ocf_data_sampler-0.2.14.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ocf_data_sampler-0.2.14.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
+ocf_data_sampler-0.2.14.dist-info/RECORD,,