ocf-data-sampler 0.2.13__tar.gz → 0.2.14__tar.gz

{ocf_data_sampler-0.2.13 → ocf_data_sampler-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.13
+Version: 0.2.14
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

ocf_data_sampler-0.2.14/ocf_data_sampler/torch_datasets/sample/uk_regional.py

@@ -0,0 +1,176 @@
+"""PVNet UK Regional sample implementation for dataset handling and visualisation."""
+
+import torch
+from typing_extensions import override
+
+from ocf_data_sampler.config import Configuration
+from ocf_data_sampler.numpy_sample import (
+    GSPSampleKey,
+    NWPSampleKey,
+    SatelliteSampleKey,
+)
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+from ocf_data_sampler.torch_datasets.sample.base import SampleBase
+from ocf_data_sampler.torch_datasets.utils.validation_utils import (
+    calculate_expected_shapes,
+    check_dimensions,
+)
+
+
+class UKRegionalSample(SampleBase):
+    """Handles UK Regional PVNet data operations."""
+
+    def __init__(self, data: NumpySample) -> None:
+        """Initialises UK Regional sample with data."""
+        self._data = data
+
+    @override
+    def to_numpy(self) -> NumpySample:
+        """Returns the data as a NumPy sample."""
+        return self._data
+
+    @override
+    def save(self, path: str) -> None:
+        """Saves sample to the specified path in pickle format."""
+        # Saves to pickle format
+        torch.save(self._data, path)
+
+    @classmethod
+    @override
+    def load(cls, path: str) -> "UKRegionalSample":
+        """Loads sample from the specified path.
+
+        Args:
+            path: Path to the saved sample file.
+
+        Returns:
+            A UKRegionalSample instance with the loaded data.
+        """
+        # Loads from .pt format
+        # TODO: We should move away from using torch.load(..., weights_only=False)
+        return cls(torch.load(path, weights_only=False))
+
+    def validate_sample(self, config: Configuration) -> bool:
+        """Validates that the sample has the expected structure and data shapes.
+
+        Args:
+            config: Configuration dict with expected shapes and required fields.
+
+        Returns:
+            bool: True if validation passes, otherwise raises an exception.
+        """
+        if not isinstance(config, Configuration):
+            raise TypeError("config must be Configuration object")
+
+        # Calculate expected shapes from configuration
+        expected_shapes = calculate_expected_shapes(config)
+
+        # Check GSP shape if specified
+        gsp_key = GSPSampleKey.gsp
+        # Check if GSP data is expected but missing
+        if gsp_key in expected_shapes and gsp_key not in self._data:
+            raise ValueError(f"Configuration expects GSP data ('{gsp_key}') but is missing.")
+
+        # Check GSP shape if data exists and is expected
+        if gsp_key in self._data and gsp_key in expected_shapes:
+            gsp_data = self._data[gsp_key]
+            check_dimensions(
+                actual_shape=gsp_data.shape,
+                expected_shape=expected_shapes[gsp_key],
+                name="GSP",
+            )
+
+        # Checks for NWP data - nested structure
+        nwp_key = NWPSampleKey.nwp
+        if nwp_key in expected_shapes and nwp_key not in self._data:
+            raise ValueError(f"Configuration expects NWP data ('{nwp_key}') but is missing.")
+
+        # Check NWP structure and shapes if data exists
+        if nwp_key in self._data:
+            nwp_data_all_providers = self._data[nwp_key]
+            if not isinstance(nwp_data_all_providers, dict):
+                raise ValueError(f"NWP data ('{nwp_key}') should be a dictionary.")
+
+            # Loop through providers present in actual data
+            for provider, provider_data in nwp_data_all_providers.items():
+                if "nwp" not in provider_data:
+                    raise ValueError(f"Missing array key in NWP data for provider '{provider}'.")
+
+                if nwp_key in expected_shapes and provider in expected_shapes[nwp_key]:
+                    nwp_array = provider_data["nwp"]
+                    actual_shape = nwp_array.shape
+                    expected_shape = expected_shapes[nwp_key][provider]
+
+                    check_dimensions(
+                        actual_shape=actual_shape,
+                        expected_shape=expected_shape,
+                        name=f"NWP data ({provider})",
+                    )
+
+        # Validate satellite data
+        sat_key = SatelliteSampleKey.satellite_actual
+        # Check if Satellite data is expected but missing
+        if sat_key in expected_shapes and sat_key not in self._data:
+            raise ValueError(f"Configuration expects Satellite data ('{sat_key}') but is missing.")
+
+        # Check satellite shape if data exists and is expected
+        if sat_key in self._data and sat_key in expected_shapes:
+            sat_data = self._data[sat_key]
+            check_dimensions(
+                actual_shape=sat_data.shape,
+                expected_shape=expected_shapes[sat_key],
+                name="Satellite data",
+            )
+
+        # Validate solar coordinates data
+        solar_keys = ["solar_azimuth", "solar_elevation"]
+        # Check if solar coordinate is expected but missing
+        for solar_key in solar_keys:
+            if solar_key in expected_shapes and solar_key not in self._data:
+                raise ValueError(f"Configuration expects {solar_key} data but is missing.")
+
+            # Check solar coordinate shape if data exists and is expected
+            if solar_key in self._data and solar_key in expected_shapes:
+                solar_data = self._data[solar_key]
+                check_dimensions(
+                    actual_shape=solar_data.shape,
+                    expected_shape=expected_shapes[solar_key],
+                    name=f"{solar_key.replace('_', ' ').title()} data",
+                )
+
+        return True
+
+    @override
+    def plot(self) -> None:
+        """Plots the sample data for visualization."""
+        from matplotlib import pyplot as plt
+
+        fig, axes = plt.subplots(2, 2, figsize=(12, 8))
+
+        if NWPSampleKey.nwp in self._data:
+            first_nwp = next(iter(self._data[NWPSampleKey.nwp].values()))
+            if "nwp" in first_nwp:
+                axes[0, 1].imshow(first_nwp["nwp"][0])
+                title = "NWP (First Channel)"
+                if NWPSampleKey.channel_names in first_nwp:
+                    channel_names = first_nwp[NWPSampleKey.channel_names]
+                    if channel_names:
+                        title = f"NWP: {channel_names[0]}"
+                axes[0, 1].set_title(title)
+
+        if GSPSampleKey.gsp in self._data:
+            axes[0, 0].plot(self._data[GSPSampleKey.gsp])
+            axes[0, 0].set_title("GSP Generation")
+
+        if "solar_azimuth" in self._data and "solar_elevation" in self._data:
+            axes[1, 1].plot(self._data["solar_azimuth"], label="Azimuth")
+            axes[1, 1].plot(self._data["solar_elevation"], label="Elevation")
+            axes[1, 1].set_title("Solar Position")
+            axes[1, 1].legend()
+
+        if SatelliteSampleKey.satellite_actual in self._data:
+            axes[1, 0].imshow(self._data[SatelliteSampleKey.satellite_actual])
+            axes[1, 0].set_title("Satellite Data")
+
+        plt.tight_layout()
+        plt.show()

ocf_data_sampler-0.2.14/ocf_data_sampler/torch_datasets/utils/validation_utils.py

@@ -0,0 +1,108 @@
+"""Validate sample shape against expected shape - utility function."""
+
+from ocf_data_sampler.config import Configuration
+from ocf_data_sampler.numpy_sample import GSPSampleKey, NWPSampleKey, SatelliteSampleKey
+
+
+def check_dimensions(
+    actual_shape: tuple[int, ...],
+    expected_shape: tuple[int, ...],
+    name: str,
+) -> None:
+    """Check if dimensions match between actual and expected shapes.
+
+    Args:
+        actual_shape: The actual shape of the data (e.g., array.shape).
+        expected_shape: The expected shape.
+        name: Name of the data component for clear error messages.
+
+    Raises:
+        ValueError: If dimensions don't match.
+    """
+    if actual_shape != expected_shape:
+        raise ValueError(
+            f"'{name}' shape mismatch: "
+            f"Actual shape: {actual_shape}, Expected shape: {expected_shape}",
+        )
+
+
+def calculate_expected_shapes(
+    config: Configuration,
+) -> dict[str, tuple[int, ...]]:
+    """Calculate expected shapes from configuration.
+
+    Args:
+        config: Configuration object with shape information.
+
+    Returns:
+        Dictionary mapping data keys to their expected shapes.
+    """
+    expected_shapes = {}
+    input_data = config.input_data
+
+    # Calculate GSP shape
+    gsp_config = input_data.gsp
+    expected_shapes[GSPSampleKey.gsp] = (
+        _calculate_time_steps(
+            gsp_config.interval_start_minutes,
+            gsp_config.interval_end_minutes,
+            gsp_config.time_resolution_minutes,
+        ),
+    )
+
+    # Calculate NWP shape for multiple providers
+    expected_shapes[NWPSampleKey.nwp] = {}
+    for provider_key, provider_config in input_data.nwp.items():
+        expected_shapes[NWPSampleKey.nwp][provider_key] = (
+            _calculate_time_steps(
+                provider_config.interval_start_minutes,
+                provider_config.interval_end_minutes,
+                provider_config.time_resolution_minutes,
+            ),
+            len(provider_config.channels),
+            provider_config.image_size_pixels_height,
+            provider_config.image_size_pixels_width,
+        )
+
+    # Calculate satellite shape
+    sat_config = input_data.satellite
+    expected_shapes[SatelliteSampleKey.satellite_actual] = (
+        _calculate_time_steps(
+            sat_config.interval_start_minutes,
+            sat_config.interval_end_minutes,
+            sat_config.time_resolution_minutes,
+        ),
+        len(sat_config.channels),
+        sat_config.image_size_pixels_height,
+        sat_config.image_size_pixels_width,
+    )
+
+    # Calculate solar coordinates shapes
+    solar_config = input_data.solar_position
+    # For solar azimuth
+    expected_shapes["solar_azimuth"] = (
+        _calculate_time_steps(
+            solar_config.interval_start_minutes,
+            solar_config.interval_end_minutes,
+            solar_config.time_resolution_minutes,
+        ),
+    )
+    # For solar elevation
+    expected_shapes["solar_elevation"] = expected_shapes["solar_azimuth"]
+
+    return expected_shapes
+
+
+def _calculate_time_steps(start_minutes: int, end_minutes: int, resolution_minutes: int) -> int:
+    """Calculate number of time steps based on interval and resolution.
+
+    Args:
+        start_minutes: Start of interval in minutes
+        end_minutes: End of interval in minutes
+        resolution_minutes: Time resolution in minutes
+
+    Returns:
+        Number of time steps
+    """
+    time_span = end_minutes - start_minutes
+    return (time_span // resolution_minutes) + 1

{ocf_data_sampler-0.2.13 → ocf_data_sampler-0.2.14}/ocf_data_sampler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.13
+Version: 0.2.14
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.13 → ocf_data_sampler-0.2.14}/ocf_data_sampler.egg-info/SOURCES.txt

@@ -58,5 +58,6 @@ ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py
 ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py
+ocf_data_sampler/torch_datasets/utils/validation_utils.py
 scripts/refactor_site.py
 utils/compute_icon_mean_stddev.py

ocf_data_sampler-0.2.13/ocf_data_sampler/torch_datasets/sample/uk_regional.py

@@ -1,71 +0,0 @@
-"""PVNet UK Regional sample implementation for dataset handling and visualisation."""
-
-import torch
-from typing_extensions import override
-
-from ocf_data_sampler.numpy_sample import (
-    GSPSampleKey,
-    NWPSampleKey,
-    SatelliteSampleKey,
-)
-from ocf_data_sampler.numpy_sample.common_types import NumpySample
-
-from .base import SampleBase
-
-
-class UKRegionalSample(SampleBase):
-    """Handles UK Regional PVNet data operations."""
-
-    def __init__(self, data: NumpySample) -> None:
-        """Initialises UK Regional sample with data."""
-        self._data = data
-
-    @override
-    def to_numpy(self) -> NumpySample:
-        return self._data
-
-    @override
-    def save(self, path: str) -> None:
-        # Saves to pickle format
-        torch.save(self._data, path)
-
-    @classmethod
-    @override
-    def load(cls, path: str) -> "UKRegionalSample":
-        # Loads from .pt format
-        # TODO: We should move away from using torch.load(..., weights_only=False)
-        return cls(torch.load(path, weights_only=False))
-
-    @override
-    def plot(self) -> None:
-        from matplotlib import pyplot as plt
-
-        fig, axes = plt.subplots(2, 2, figsize=(12, 8))
-
-        if NWPSampleKey.nwp in self._data:
-            first_nwp = next(iter(self._data[NWPSampleKey.nwp].values()))
-            if "nwp" in first_nwp:
-                axes[0, 1].imshow(first_nwp["nwp"][0])
-                title = "NWP (First Channel)"
-                if NWPSampleKey.channel_names in first_nwp:
-                    channel_names = first_nwp[NWPSampleKey.channel_names]
-                    if channel_names:
-                        title = f"NWP: {channel_names[0]}"
-                axes[0, 1].set_title(title)
-
-        if GSPSampleKey.gsp in self._data:
-            axes[0, 0].plot(self._data[GSPSampleKey.gsp])
-            axes[0, 0].set_title("GSP Generation")
-
-        if "solar_azimuth" in self._data and "solar_elevation" in self._data:
-                    axes[1, 1].plot(self._data["solar_azimuth"], label="Azimuth")
-                    axes[1, 1].plot(self._data["solar_elevation"], label="Elevation")
-                    axes[1, 1].set_title("Solar Position")
-                    axes[1, 1].legend()
-
-        if SatelliteSampleKey.satellite_actual in self._data:
-            axes[1, 0].imshow(self._data[SatelliteSampleKey.satellite_actual])
-            axes[1, 0].set_title("Satellite Data")
-
-        plt.tight_layout()
-        plt.show()