ocf-data-sampler 0.2.14__tar.gz → 0.2.16__tar.gz

{ocf_data_sampler-0.2.14 → ocf_data_sampler-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.14
+Version: 0.2.16
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.14 → ocf_data_sampler-0.2.16}/ocf_data_sampler/load/load_dataset.py

@@ -6,20 +6,28 @@ from ocf_data_sampler.config import InputData
 from ocf_data_sampler.load import open_gsp, open_nwp, open_sat_data, open_site
 
 
-def get_dataset_dict(input_config: InputData) -> dict[str, dict[xr.DataArray] | xr.DataArray]:
+def get_dataset_dict(input_config: InputData, gsp_ids: list[int] | None = None)\
+        -> dict[str, dict[xr.DataArray] | xr.DataArray]:
     """Construct dictionary of all of the input data sources.
 
     Args:
         input_config: InputData configuration object
+        gsp_ids: List of GSP IDs to load. If None, all GSPs are loaded (not National).
     """
     datasets_dict = {}
 
     # Load GSP data unless the path is None
     if input_config.gsp and input_config.gsp.zarr_path:
+
         da_gsp = open_gsp(zarr_path=input_config.gsp.zarr_path).compute()
 
-        # Remove national GSP
-        datasets_dict["gsp"] = da_gsp.sel(gsp_id=slice(1, None))
+        if gsp_ids is None:
+            # Remove national (gsp_id=0)
+            da_gsp = da_gsp.sel(gsp_id=slice(1, None))
+        else:
+            da_gsp = da_gsp.sel(gsp_id=gsp_ids)
+
+        datasets_dict["gsp"] = da_gsp
 
     # Load NWP data if in config
     if input_config.nwp:

{ocf_data_sampler-0.2.14 → ocf_data_sampler-0.2.16}/ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -95,7 +95,7 @@ class AbstractPVNetUKDataset(Dataset):
             gsp_ids: List of GSP IDs to create samples for. Defaults to all
         """
         config = load_yaml_configuration(config_filename)
-        datasets_dict = get_dataset_dict(config.input_data)
+        datasets_dict = get_dataset_dict(config.input_data, gsp_ids=gsp_ids)
 
         # Get t0 times where all input data is available
         valid_t0_times = self.find_valid_t0_times(datasets_dict, config)

ocf_data_sampler-0.2.16/ocf_data_sampler/torch_datasets/sample/uk_regional.py

@@ -0,0 +1,262 @@
+"""PVNet UK Regional sample implementation for dataset handling and visualisation."""
+
+import logging
+
+import torch
+from typing_extensions import override
+
+from ocf_data_sampler.config import Configuration
+from ocf_data_sampler.numpy_sample import (
+    GSPSampleKey,
+    NWPSampleKey,
+    SatelliteSampleKey,
+)
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+from ocf_data_sampler.torch_datasets.sample.base import SampleBase
+from ocf_data_sampler.torch_datasets.utils.validation_utils import (
+    calculate_expected_shapes,
+    check_dimensions,
+    validation_warning,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class UKRegionalSample(SampleBase):
+    """Handles UK Regional PVNet data operations."""
+
+    def __init__(self, data: NumpySample) -> None:
+        """Initialises UK Regional sample with data."""
+        self._data = data
+
+    @override
+    def to_numpy(self) -> NumpySample:
+        """Returns the data as a NumPy sample."""
+        return self._data
+
+    @override
+    def save(self, path: str) -> None:
+        """Saves sample to the specified path in pickle format."""
+        # Saves to pickle format
+        torch.save(self._data, path)
+
+    @classmethod
+    @override
+    def load(cls, path: str) -> "UKRegionalSample":
+        """Loads sample from the specified path.
+
+        Args:
+            path: Path to the saved sample file.
+
+        Returns:
+            A UKRegionalSample instance with the loaded data.
+        """
+        # Loads from .pt format
+        # TODO: We should move away from using torch.load(..., weights_only=False)
+        return cls(torch.load(path, weights_only=False))
+
+    def validate_sample(self, config: Configuration) -> dict:
+        """Validates the sample, logging warnings and raising errors.
+
+        Checks that the sample has the expected structure and data shapes based
+        on the provided configuration. Critical issues (missing required data,
+        shape mismatches) will raise a ValueError. Non-critical issues (e.g.,
+        unexpected data components found) will be logged as warnings using
+        the standard Python logging module.
+
+        Args:
+            config: Configuration object defining expected shapes and required fields.
+
+        Returns:
+            dict: A dictionary indicating success: `{"valid": True}`.
+                  If validation fails due to a critical issue, an exception is raised
+                  instead of returning. Warnings encountered are logged.
+
+        Raises:
+            TypeError: If `config` is not a Configuration object.
+            ValueError: For critical validation failures like missing expected data,
+                        incorrect data shapes, or missing required NWP providers.
+        """
+        if not isinstance(config, Configuration):
+            raise TypeError("config must be Configuration object")
+
+        # Calculate expected shapes from configuration
+        expected_shapes = calculate_expected_shapes(config)
+
+        # Check GSP shape if specified
+        gsp_key = GSPSampleKey.gsp
+        if gsp_key in expected_shapes and gsp_key not in self._data:
+            raise ValueError(f"Configuration expects GSP data ('{gsp_key}') but is missing.")
+
+        if gsp_key in self._data:
+            if gsp_key in expected_shapes:
+                gsp_data = self._data[gsp_key]
+                check_dimensions(
+                    actual_shape=gsp_data.shape,
+                    expected_shape=expected_shapes[gsp_key],
+                    name="GSP",
+                )
+            else:
+                validation_warning(
+                    message=f"GSP data ('{gsp_key}') is present but not expected in configuration.",
+                    warning_type="unexpected_component",
+                    component=str(gsp_key),
+                )
+
+        # Checks for NWP data
+        nwp_key = NWPSampleKey.nwp
+        if nwp_key in expected_shapes and nwp_key not in self._data:
+            raise ValueError(f"Configuration expects NWP data ('{nwp_key}') but is missing.")
+
+        if nwp_key in self._data:
+            nwp_data_all_providers = self._data[nwp_key]
+            if not isinstance(nwp_data_all_providers, dict):
+                raise ValueError(f"NWP data ('{nwp_key}') should be a dictionary.")
+
+            if nwp_key in expected_shapes:
+                expected_providers = set(expected_shapes[nwp_key].keys())
+                actual_providers = set(nwp_data_all_providers.keys())
+
+                unexpected_providers = actual_providers - expected_providers
+                if unexpected_providers:
+                    validation_warning(
+                        message=f"Unexpected NWP providers found: {list(unexpected_providers)}",
+                        warning_type="unexpected_provider",
+                        providers=list(unexpected_providers),
+                    )
+
+                missing_expected_providers = expected_providers - actual_providers
+                if missing_expected_providers:
+                    raise ValueError(
+                        f"Expected NWP providers are missing from the data: "
+                        f"{list(missing_expected_providers)}",
+                    )
+
+                for provider in expected_shapes[nwp_key]:
+                    provider_data = nwp_data_all_providers[provider]
+
+                    if "nwp" not in provider_data:
+                        error_msg = (
+                            f"Missing array key 'nwp' in NWP data for provider '{provider}'."
+                        )
+                        raise ValueError(error_msg)
+
+                    nwp_array = provider_data["nwp"]
+                    check_dimensions(
+                        actual_shape=nwp_array.shape,
+                        expected_shape=expected_shapes[nwp_key][provider],
+                        name=f"NWP data ({provider})",
+                    )
+            else:
+                validation_warning(
+                    message=(
+                        f"NWP data ('{nwp_key}') is present but not expected "
+                        "in configuration."
+                    ),
+                    warning_type="unexpected_component",
+                    component=str(nwp_key),
+                )
+
+        # Validate satellite data
+        sat_key = SatelliteSampleKey.satellite_actual
+        if sat_key in expected_shapes and sat_key not in self._data:
+            raise ValueError(f"Configuration expects Satellite data ('{sat_key}') but is missing.")
+
+        if sat_key in self._data:
+            if sat_key in expected_shapes:
+                sat_data = self._data[sat_key]
+                check_dimensions(
+                    actual_shape=sat_data.shape,
+                    expected_shape=expected_shapes[sat_key],
+                    name="Satellite data",
+                )
+            else:
+                validation_warning(
+                    message=(
+                        f"Satellite data ('{sat_key}') is present but not expected "
+                        "in configuration."
+                    ),
+                    warning_type="unexpected_component",
+                    component=str(sat_key),
+                )
+
+        # Validate solar coordinates data
+        solar_keys = ["solar_azimuth", "solar_elevation"]
+        for solar_key in solar_keys:
+            solar_name = solar_key.replace("_", " ").title()
+            if solar_key in expected_shapes and solar_key not in self._data:
+                raise ValueError(f"Configuration expects {solar_key} data but is missing.")
+
+            if solar_key in self._data:
+                if solar_key in expected_shapes:
+                    solar_data = self._data[solar_key]
+                    check_dimensions(
+                        actual_shape=solar_data.shape,
+                        expected_shape=expected_shapes[solar_key],
+                        name=f"{solar_name} data",
+                    )
+                else:
+                    validation_warning(
+                        message=(
+                            f"{solar_name} data is present but not expected "
+                            "in configuration."
+                        ),
+                        warning_type="unexpected_component",
+                        component=solar_key,
+                    )
+
+        # Check for potentially unexpected components
+        checked_keys = {gsp_key, nwp_key, sat_key} | set(solar_keys)
+        all_present_keys = set(self._data.keys())
+        unexpected_present_keys = all_present_keys - set(expected_shapes.keys())
+
+        for key in unexpected_present_keys:
+            if key not in checked_keys:
+                validation_warning(
+                    message=(
+                        f"Unexpected component '{key}' is present in data but not defined "
+                        "in configuration's expected shapes."
+                    ),
+                    warning_type="unexpected_component",
+                    component=str(key),
+                )
+
+        return {
+            "valid": True,
+        }
+
+
+    @override
+    def plot(self) -> None:
+        """Plots the sample data for visualization."""
+        from matplotlib import pyplot as plt
+
+        fig, axes = plt.subplots(2, 2, figsize=(12, 8))
+
+        if NWPSampleKey.nwp in self._data:
+            first_nwp = next(iter(self._data[NWPSampleKey.nwp].values()))
+            if "nwp" in first_nwp:
+                axes[0, 1].imshow(first_nwp["nwp"][0])
+                title = "NWP (First Channel)"
+                if NWPSampleKey.channel_names in first_nwp:
+                    channel_names = first_nwp[NWPSampleKey.channel_names]
+                    if channel_names:
+                        title = f"NWP: {channel_names[0]}"
+                axes[0, 1].set_title(title)
+
+        if GSPSampleKey.gsp in self._data:
+            axes[0, 0].plot(self._data[GSPSampleKey.gsp])
+            axes[0, 0].set_title("GSP Generation")
+
+        if "solar_azimuth" in self._data and "solar_elevation" in self._data:
+            axes[1, 1].plot(self._data["solar_azimuth"], label="Azimuth")
+            axes[1, 1].plot(self._data["solar_elevation"], label="Elevation")
+            axes[1, 1].set_title("Solar Position")
+            axes[1, 1].legend()
+
+        if SatelliteSampleKey.satellite_actual in self._data:
+            axes[1, 0].imshow(self._data[SatelliteSampleKey.satellite_actual])
+            axes[1, 0].set_title("Satellite Data")
+
+        plt.tight_layout()
+        plt.show()

{ocf_data_sampler-0.2.14 → ocf_data_sampler-0.2.16}/ocf_data_sampler/torch_datasets/utils/validation_utils.py

@@ -1,8 +1,13 @@
 """Validate sample shape against expected shape - utility function."""
 
+import logging
+from typing import Any
+
 from ocf_data_sampler.config import Configuration
 from ocf_data_sampler.numpy_sample import GSPSampleKey, NWPSampleKey, SatelliteSampleKey
 
+logger = logging.getLogger(__name__)
+
 
 def check_dimensions(
     actual_shape: tuple[int, ...],
@@ -93,6 +98,40 @@ def calculate_expected_shapes(
     return expected_shapes
 
 
+def validation_warning(
+    message: str,
+    warning_type: str,
+    *,
+    component: str | None = None,
+    providers: list[str] | None = None,
+) -> dict[str, Any]:
+    """Constructs warning details and logs a standard warning message.
+
+    Args:
+        message: The base warning message string.
+        warning_type: The category of the warning (e.g., 'unexpected_component').
+        component: Optional component identifier (e.g., 'gsp').
+        providers: Optional list of provider names (e.g., ['ukv']).
+
+    Returns:
+        None - This function now directly logs the warning.
+    """
+    warning_info: dict[str, Any] = {"type": warning_type, "message": message}
+    log_message_parts = [message]
+    log_message_parts.append(f"(Type: {warning_type}")
+
+    if component is not None:
+        warning_info["component"] = component
+        log_message_parts.append(f", Component: {component}")
+    if providers is not None:
+        warning_info["providers"] = providers
+        log_message_parts.append(f", Providers: {providers}")
+
+    log_message_parts.append(")")
+    log_message = " ".join(log_message_parts)
+    logger.warning(log_message)
+
+
 def _calculate_time_steps(start_minutes: int, end_minutes: int, resolution_minutes: int) -> int:
     """Calculate number of time steps based on interval and resolution.
 

{ocf_data_sampler-0.2.14 → ocf_data_sampler-0.2.16}/ocf_data_sampler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.14
+Version: 0.2.16
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

ocf_data_sampler-0.2.14/ocf_data_sampler/torch_datasets/sample/uk_regional.py

@@ -1,176 +0,0 @@
-"""PVNet UK Regional sample implementation for dataset handling and visualisation."""
-
-import torch
-from typing_extensions import override
-
-from ocf_data_sampler.config import Configuration
-from ocf_data_sampler.numpy_sample import (
-    GSPSampleKey,
-    NWPSampleKey,
-    SatelliteSampleKey,
-)
-from ocf_data_sampler.numpy_sample.common_types import NumpySample
-from ocf_data_sampler.torch_datasets.sample.base import SampleBase
-from ocf_data_sampler.torch_datasets.utils.validation_utils import (
-    calculate_expected_shapes,
-    check_dimensions,
-)
-
-
-class UKRegionalSample(SampleBase):
-    """Handles UK Regional PVNet data operations."""
-
-    def __init__(self, data: NumpySample) -> None:
-        """Initialises UK Regional sample with data."""
-        self._data = data
-
-    @override
-    def to_numpy(self) -> NumpySample:
-        """Returns the data as a NumPy sample."""
-        return self._data
-
-    @override
-    def save(self, path: str) -> None:
-        """Saves sample to the specified path in pickle format."""
-        # Saves to pickle format
-        torch.save(self._data, path)
-
-    @classmethod
-    @override
-    def load(cls, path: str) -> "UKRegionalSample":
-        """Loads sample from the specified path.
-
-        Args:
-            path: Path to the saved sample file.
-
-        Returns:
-            A UKRegionalSample instance with the loaded data.
-        """
-        # Loads from .pt format
-        # TODO: We should move away from using torch.load(..., weights_only=False)
-        return cls(torch.load(path, weights_only=False))
-
-    def validate_sample(self, config: Configuration) -> bool:
-        """Validates that the sample has the expected structure and data shapes.
-
-        Args:
-            config: Configuration dict with expected shapes and required fields.
-
-        Returns:
-            bool: True if validation passes, otherwise raises an exception.
-        """
-        if not isinstance(config, Configuration):
-            raise TypeError("config must be Configuration object")
-
-        # Calculate expected shapes from configuration
-        expected_shapes = calculate_expected_shapes(config)
-
-        # Check GSP shape if specified
-        gsp_key = GSPSampleKey.gsp
-        # Check if GSP data is expected but missing
-        if gsp_key in expected_shapes and gsp_key not in self._data:
-            raise ValueError(f"Configuration expects GSP data ('{gsp_key}') but is missing.")
-
-        # Check GSP shape if data exists and is expected
-        if gsp_key in self._data and gsp_key in expected_shapes:
-            gsp_data = self._data[gsp_key]
-            check_dimensions(
-                actual_shape=gsp_data.shape,
-                expected_shape=expected_shapes[gsp_key],
-                name="GSP",
-            )
-
-        # Checks for NWP data - nested structure
-        nwp_key = NWPSampleKey.nwp
-        if nwp_key in expected_shapes and nwp_key not in self._data:
-            raise ValueError(f"Configuration expects NWP data ('{nwp_key}') but is missing.")
-
-        # Check NWP structure and shapes if data exists
-        if nwp_key in self._data:
-            nwp_data_all_providers = self._data[nwp_key]
-            if not isinstance(nwp_data_all_providers, dict):
-                raise ValueError(f"NWP data ('{nwp_key}') should be a dictionary.")
-
-            # Loop through providers present in actual data
-            for provider, provider_data in nwp_data_all_providers.items():
-                if "nwp" not in provider_data:
-                    raise ValueError(f"Missing array key in NWP data for provider '{provider}'.")
-
-                if nwp_key in expected_shapes and provider in expected_shapes[nwp_key]:
-                    nwp_array = provider_data["nwp"]
-                    actual_shape = nwp_array.shape
-                    expected_shape = expected_shapes[nwp_key][provider]
-
-                    check_dimensions(
-                        actual_shape=actual_shape,
-                        expected_shape=expected_shape,
-                        name=f"NWP data ({provider})",
-                    )
-
-        # Validate satellite data
-        sat_key = SatelliteSampleKey.satellite_actual
-        # Check if Satellite data is expected but missing
-        if sat_key in expected_shapes and sat_key not in self._data:
-            raise ValueError(f"Configuration expects Satellite data ('{sat_key}') but is missing.")
-
-        # Check satellite shape if data exists and is expected
-        if sat_key in self._data and sat_key in expected_shapes:
-            sat_data = self._data[sat_key]
-            check_dimensions(
-                actual_shape=sat_data.shape,
-                expected_shape=expected_shapes[sat_key],
-                name="Satellite data",
-            )
-
-        # Validate solar coordinates data
-        solar_keys = ["solar_azimuth", "solar_elevation"]
-        # Check if solar coordinate is expected but missing
-        for solar_key in solar_keys:
-            if solar_key in expected_shapes and solar_key not in self._data:
-                raise ValueError(f"Configuration expects {solar_key} data but is missing.")
-
-            # Check solar coordinate shape if data exists and is expected
-            if solar_key in self._data and solar_key in expected_shapes:
-                solar_data = self._data[solar_key]
-                check_dimensions(
-                    actual_shape=solar_data.shape,
-                    expected_shape=expected_shapes[solar_key],
-                    name=f"{solar_key.replace('_', ' ').title()} data",
-                )
-
-        return True
-
-    @override
-    def plot(self) -> None:
-        """Plots the sample data for visualization."""
-        from matplotlib import pyplot as plt
-
-        fig, axes = plt.subplots(2, 2, figsize=(12, 8))
-
-        if NWPSampleKey.nwp in self._data:
-            first_nwp = next(iter(self._data[NWPSampleKey.nwp].values()))
-            if "nwp" in first_nwp:
-                axes[0, 1].imshow(first_nwp["nwp"][0])
-                title = "NWP (First Channel)"
-                if NWPSampleKey.channel_names in first_nwp:
-                    channel_names = first_nwp[NWPSampleKey.channel_names]
-                    if channel_names:
-                        title = f"NWP: {channel_names[0]}"
-                axes[0, 1].set_title(title)
-
-        if GSPSampleKey.gsp in self._data:
-            axes[0, 0].plot(self._data[GSPSampleKey.gsp])
-            axes[0, 0].set_title("GSP Generation")
-
-        if "solar_azimuth" in self._data and "solar_elevation" in self._data:
-            axes[1, 1].plot(self._data["solar_azimuth"], label="Azimuth")
-            axes[1, 1].plot(self._data["solar_elevation"], label="Elevation")
-            axes[1, 1].set_title("Solar Position")
-            axes[1, 1].legend()
-
-        if SatelliteSampleKey.satellite_actual in self._data:
-            axes[1, 0].imshow(self._data[SatelliteSampleKey.satellite_actual])
-            axes[1, 0].set_title("Satellite Data")
-
-        plt.tight_layout()
-        plt.show()