ocf-data-sampler 0.2.14__py3-none-any.whl → 0.2.16__py3-none-any.whl

ocf_data_sampler/load/load_dataset.py

@@ -6,20 +6,28 @@ from ocf_data_sampler.config import InputData
 from ocf_data_sampler.load import open_gsp, open_nwp, open_sat_data, open_site
 
 
-def get_dataset_dict(input_config: InputData) -> dict[str, dict[xr.DataArray] | xr.DataArray]:
+def get_dataset_dict(input_config: InputData, gsp_ids: list[int] | None = None)\
+        -> dict[str, dict[xr.DataArray] | xr.DataArray]:
     """Construct dictionary of all of the input data sources.
 
     Args:
         input_config: InputData configuration object
+        gsp_ids: List of GSP IDs to load. If None, all GSPs are loaded (not National).
     """
     datasets_dict = {}
 
     # Load GSP data unless the path is None
     if input_config.gsp and input_config.gsp.zarr_path:
+
         da_gsp = open_gsp(zarr_path=input_config.gsp.zarr_path).compute()
 
-        # Remove national GSP
-        datasets_dict["gsp"] = da_gsp.sel(gsp_id=slice(1, None))
+        if gsp_ids is None:
+            # Remove national (gsp_id=0)
+            da_gsp = da_gsp.sel(gsp_id=slice(1, None))
+        else:
+            da_gsp = da_gsp.sel(gsp_id=gsp_ids)
+
+        datasets_dict["gsp"] = da_gsp
 
     # Load NWP data if in config
     if input_config.nwp:

ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -95,7 +95,7 @@ class AbstractPVNetUKDataset(Dataset):
             gsp_ids: List of GSP IDs to create samples for. Defaults to all
         """
         config = load_yaml_configuration(config_filename)
-        datasets_dict = get_dataset_dict(config.input_data)
+        datasets_dict = get_dataset_dict(config.input_data, gsp_ids=gsp_ids)
 
         # Get t0 times where all input data is available
         valid_t0_times = self.find_valid_t0_times(datasets_dict, config)

ocf_data_sampler/torch_datasets/sample/uk_regional.py

@@ -1,5 +1,7 @@
 """PVNet UK Regional sample implementation for dataset handling and visualisation."""
 
+import logging
+
 import torch
 from typing_extensions import override
 
@@ -14,8 +16,11 @@ from ocf_data_sampler.torch_datasets.sample.base import SampleBase
 from ocf_data_sampler.torch_datasets.utils.validation_utils import (
     calculate_expected_shapes,
     check_dimensions,
+    validation_warning,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class UKRegionalSample(SampleBase):
     """Handles UK Regional PVNet data operations."""
@@ -50,14 +55,27 @@ class UKRegionalSample(SampleBase):
         # TODO: We should move away from using torch.load(..., weights_only=False)
         return cls(torch.load(path, weights_only=False))
 
-    def validate_sample(self, config: Configuration) -> bool:
-        """Validates that the sample has the expected structure and data shapes.
+    def validate_sample(self, config: Configuration) -> dict:
+        """Validates the sample, logging warnings and raising errors.
+
+        Checks that the sample has the expected structure and data shapes based
+        on the provided configuration. Critical issues (missing required data,
+        shape mismatches) will raise a ValueError. Non-critical issues (e.g.,
+        unexpected data components found) will be logged as warnings using
+        the standard Python logging module.
 
         Args:
-            config: Configuration dict with expected shapes and required fields.
+            config: Configuration object defining expected shapes and required fields.
 
         Returns:
-            bool: True if validation passes, otherwise raises an exception.
+            dict: A dictionary indicating success: `{"valid": True}`.
+                  If validation fails due to a critical issue, an exception is raised
+                  instead of returning. Warnings encountered are logged.
+
+        Raises:
+            TypeError: If `config` is not a Configuration object.
+            ValueError: For critical validation failures like missing expected data,
+                        incorrect data shapes, or missing required NWP providers.
         """
         if not isinstance(config, Configuration):
             raise TypeError("config must be Configuration object")
@@ -67,78 +85,146 @@ class UKRegionalSample(SampleBase):
 
         # Check GSP shape if specified
         gsp_key = GSPSampleKey.gsp
-        # Check if GSP data is expected but missing
         if gsp_key in expected_shapes and gsp_key not in self._data:
             raise ValueError(f"Configuration expects GSP data ('{gsp_key}') but is missing.")
 
-        # Check GSP shape if data exists and is expected
-        if gsp_key in self._data and gsp_key in expected_shapes:
-            gsp_data = self._data[gsp_key]
-            check_dimensions(
-                actual_shape=gsp_data.shape,
-                expected_shape=expected_shapes[gsp_key],
-                name="GSP",
-            )
+        if gsp_key in self._data:
+            if gsp_key in expected_shapes:
+                gsp_data = self._data[gsp_key]
+                check_dimensions(
+                    actual_shape=gsp_data.shape,
+                    expected_shape=expected_shapes[gsp_key],
+                    name="GSP",
+                )
+            else:
+                validation_warning(
+                    message=f"GSP data ('{gsp_key}') is present but not expected in configuration.",
+                    warning_type="unexpected_component",
+                    component=str(gsp_key),
+                )
 
-        # Checks for NWP data - nested structure
+        # Checks for NWP data
         nwp_key = NWPSampleKey.nwp
         if nwp_key in expected_shapes and nwp_key not in self._data:
             raise ValueError(f"Configuration expects NWP data ('{nwp_key}') but is missing.")
 
-        # Check NWP structure and shapes if data exists
         if nwp_key in self._data:
             nwp_data_all_providers = self._data[nwp_key]
             if not isinstance(nwp_data_all_providers, dict):
                 raise ValueError(f"NWP data ('{nwp_key}') should be a dictionary.")
 
-            # Loop through providers present in actual data
-            for provider, provider_data in nwp_data_all_providers.items():
-                if "nwp" not in provider_data:
-                    raise ValueError(f"Missing array key in NWP data for provider '{provider}'.")
+            if nwp_key in expected_shapes:
+                expected_providers = set(expected_shapes[nwp_key].keys())
+                actual_providers = set(nwp_data_all_providers.keys())
 
-                if nwp_key in expected_shapes and provider in expected_shapes[nwp_key]:
-                    nwp_array = provider_data["nwp"]
-                    actual_shape = nwp_array.shape
-                    expected_shape = expected_shapes[nwp_key][provider]
+                unexpected_providers = actual_providers - expected_providers
+                if unexpected_providers:
+                    validation_warning(
+                        message=f"Unexpected NWP providers found: {list(unexpected_providers)}",
+                        warning_type="unexpected_provider",
+                        providers=list(unexpected_providers),
+                    )
+
+                missing_expected_providers = expected_providers - actual_providers
+                if missing_expected_providers:
+                    raise ValueError(
+                        f"Expected NWP providers are missing from the data: "
+                        f"{list(missing_expected_providers)}",
+                    )
+
+                for provider in expected_shapes[nwp_key]:
+                    provider_data = nwp_data_all_providers[provider]
+
+                    if "nwp" not in provider_data:
+                        error_msg = (
+                            f"Missing array key 'nwp' in NWP data for provider '{provider}'."
+                        )
+                        raise ValueError(error_msg)
 
+                    nwp_array = provider_data["nwp"]
                     check_dimensions(
-                        actual_shape=actual_shape,
-                        expected_shape=expected_shape,
+                        actual_shape=nwp_array.shape,
+                        expected_shape=expected_shapes[nwp_key][provider],
                         name=f"NWP data ({provider})",
                     )
+            else:
+                validation_warning(
+                    message=(
+                        f"NWP data ('{nwp_key}') is present but not expected "
+                        "in configuration."
+                    ),
+                    warning_type="unexpected_component",
+                    component=str(nwp_key),
+                )
 
         # Validate satellite data
         sat_key = SatelliteSampleKey.satellite_actual
-        # Check if Satellite data is expected but missing
         if sat_key in expected_shapes and sat_key not in self._data:
             raise ValueError(f"Configuration expects Satellite data ('{sat_key}') but is missing.")
 
-        # Check satellite shape if data exists and is expected
-        if sat_key in self._data and sat_key in expected_shapes:
-            sat_data = self._data[sat_key]
-            check_dimensions(
-                actual_shape=sat_data.shape,
-                expected_shape=expected_shapes[sat_key],
-                name="Satellite data",
-            )
+        if sat_key in self._data:
+            if sat_key in expected_shapes:
+                sat_data = self._data[sat_key]
+                check_dimensions(
+                    actual_shape=sat_data.shape,
+                    expected_shape=expected_shapes[sat_key],
+                    name="Satellite data",
+                )
+            else:
+                validation_warning(
+                    message=(
+                        f"Satellite data ('{sat_key}') is present but not expected "
+                        "in configuration."
+                    ),
+                    warning_type="unexpected_component",
+                    component=str(sat_key),
+                )
 
         # Validate solar coordinates data
         solar_keys = ["solar_azimuth", "solar_elevation"]
-        # Check if solar coordinate is expected but missing
         for solar_key in solar_keys:
+            solar_name = solar_key.replace("_", " ").title()
             if solar_key in expected_shapes and solar_key not in self._data:
                 raise ValueError(f"Configuration expects {solar_key} data but is missing.")
 
-            # Check solar coordinate shape if data exists and is expected
-            if solar_key in self._data and solar_key in expected_shapes:
-                solar_data = self._data[solar_key]
-                check_dimensions(
-                    actual_shape=solar_data.shape,
-                    expected_shape=expected_shapes[solar_key],
-                    name=f"{solar_key.replace('_', ' ').title()} data",
+            if solar_key in self._data:
+                if solar_key in expected_shapes:
+                    solar_data = self._data[solar_key]
+                    check_dimensions(
+                        actual_shape=solar_data.shape,
+                        expected_shape=expected_shapes[solar_key],
+                        name=f"{solar_name} data",
+                    )
+                else:
+                    validation_warning(
+                        message=(
+                            f"{solar_name} data is present but not expected "
+                            "in configuration."
+                        ),
+                        warning_type="unexpected_component",
+                        component=solar_key,
+                    )
+
+        # Check for potentially unexpected components
+        checked_keys = {gsp_key, nwp_key, sat_key} | set(solar_keys)
+        all_present_keys = set(self._data.keys())
+        unexpected_present_keys = all_present_keys - set(expected_shapes.keys())
+
+        for key in unexpected_present_keys:
+            if key not in checked_keys:
+                validation_warning(
+                    message=(
+                        f"Unexpected component '{key}' is present in data but not defined "
+                        "in configuration's expected shapes."
+                    ),
+                    warning_type="unexpected_component",
+                    component=str(key),
                 )
 
-        return True
+        return {
+            "valid": True,
+        }
+
 
     @override
     def plot(self) -> None:

ocf_data_sampler/torch_datasets/utils/validation_utils.py

@@ -1,8 +1,13 @@
 """Validate sample shape against expected shape - utility function."""
 
+import logging
+from typing import Any
+
 from ocf_data_sampler.config import Configuration
 from ocf_data_sampler.numpy_sample import GSPSampleKey, NWPSampleKey, SatelliteSampleKey
 
+logger = logging.getLogger(__name__)
+
 
 def check_dimensions(
     actual_shape: tuple[int, ...],
@@ -93,6 +98,40 @@ def calculate_expected_shapes(
     return expected_shapes
 
 
+def validation_warning(
+    message: str,
+    warning_type: str,
+    *,
+    component: str | None = None,
+    providers: list[str] | None = None,
+) -> dict[str, Any]:
+    """Constructs warning details and logs a standard warning message.
+
+    Args:
+        message: The base warning message string.
+        warning_type: The category of the warning (e.g., 'unexpected_component').
+        component: Optional component identifier (e.g., 'gsp').
+        providers: Optional list of provider names (e.g., ['ukv']).
+
+    Returns:
+        None - This function now directly logs the warning.
+    """
+    warning_info: dict[str, Any] = {"type": warning_type, "message": message}
+    log_message_parts = [message]
+    log_message_parts.append(f"(Type: {warning_type}")
+
+    if component is not None:
+        warning_info["component"] = component
+        log_message_parts.append(f", Component: {component}")
+    if providers is not None:
+        warning_info["providers"] = providers
+        log_message_parts.append(f", Providers: {providers}")
+
+    log_message_parts.append(")")
+    log_message = " ".join(log_message_parts)
+    logger.warning(log_message)
+
+
 def _calculate_time_steps(start_minutes: int, end_minutes: int, resolution_minutes: int) -> int:
     """Calculate number of time steps based on interval and resolution.
 

{ocf_data_sampler-0.2.14.dist-info → ocf_data_sampler-0.2.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.14
+Version: 0.2.16
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.14.dist-info → ocf_data_sampler-0.2.16.dist-info}/RECORD

@@ -7,7 +7,7 @@ ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdR
 ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
 ocf_data_sampler/load/__init__.py,sha256=-vQP9g0UOWdVbjEGyVX_ipa7R1btmiETIKAf6aw4d78,201
 ocf_data_sampler/load/gsp.py,sha256=keB3Nv_CNK1P6pS9Kdfc8PoZXTI1_YFN-spsvEv_Ewc,899
-ocf_data_sampler/load/load_dataset.py,sha256=0NyDxCDfgE_esKVW3s-rZEe16WB30FQ74ClWlrIo72M,1602
+ocf_data_sampler/load/load_dataset.py,sha256=Cn-yz7RgHR2HkH3xQM1njivVEkp8rZC3KXXgcidwuME,1863
 ocf_data_sampler/load/satellite.py,sha256=E7Ln7Y60Qr1RTV-_R71YoxXQM-Ca7Y1faIo3oKB2eFk,2292
 ocf_data_sampler/load/site.py,sha256=zOzlWk6pYZBB5daqG8URGksmDXWKrkutUvN8uALAIh8,1468
 ocf_data_sampler/load/utils.py,sha256=sZ0-zzconcLkVQwAkCYrqKDo98Hrh5ChdiQJv5Bh91g,2040
@@ -38,22 +38,22 @@ ocf_data_sampler/select/location.py,sha256=AZvGR8y62opiW7zACGXjoOtBEWRfSLOZIA73O
 ocf_data_sampler/select/select_spatial_slice.py,sha256=liAqIa-Amj58pOqx5r16i99HURj9oQ41j7gnPgRDQP4,8201
 ocf_data_sampler/select/select_time_slice.py,sha256=HeHbwZ0CP03x0-LaJtpbSdtpLufwVTR73p6wH6O_PS8,5513
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=jfJSFcR0eO1AqeH7S3KnGjsBqVZT5w3oyi784PUR6Q0,146
-ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=tx5Sg64eknhU6VIcONiAaG2PurN6Y8Te6rE3AaWg8t4,12338
+ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=9BJ4wVcZUMEzStVCbbWrf2eK8WPpV9SoeOQviZktHAc,12355
 ocf_data_sampler/torch_datasets/datasets/site.py,sha256=nRUlhXQQGVrTuBmE1QnwXAUsPTXz0dsezlQjwK71jIQ,17641
 ocf_data_sampler/torch_datasets/sample/__init__.py,sha256=GL84vdZl_SjHDGVyh9Uekx2XhPYuZ0dnO3l6f6KXnHI,100
 ocf_data_sampler/torch_datasets/sample/base.py,sha256=cQ1oIyhdmlotejZK8B3Cw6MNvpdnBPD8G_o2h7Ye4Vc,2206
 ocf_data_sampler/torch_datasets/sample/site.py,sha256=ZUEgn50g-GmqujOEtezNILF7wjokF80sDAA4OOldcRI,1268
-ocf_data_sampler/torch_datasets/sample/uk_regional.py,sha256=8hDgaMg5Vb6eYitqYiljpAeTeTemwsYaRpZn7_3_XjI,7013
+ocf_data_sampler/torch_datasets/sample/uk_regional.py,sha256=Xx5cBYUyaM6PGUWQ76MHT9hwj6IJ7WAOxbpmYFbJGhc,10483
 ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=N7i_hHtWUDiJqsiJoDx4T_QuiYOuvIyulPrn6xEA4TY,309
 ocf_data_sampler/torch_datasets/utils/channel_dict_to_dataarray.py,sha256=un2IiyoAmTDIymdeMiPU899_86iCDMD-oIifjHlNyqw,555
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=we7BTxRH7B7jKayDT7YfNyfI3zZClz2Bk-HXKQIokgU,956
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py,sha256=Hvz0wHSWMYYamf2oHNiGlzJcM4cAH6pL_7ZEvIBL2dE,1882
 ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=1DN6VsWWdLvkpJxodZtBRDUgC4vJE2td_RP5J3ZqPNw,4268
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0WrglnX5YUAFjXXlO2GwEBHWyqo8TDuiOA,4714
-ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=2fwW-kpsMM2a-FWBG0YBT_r2LDIhhn7WokQ7GWvgx6U,3504
+ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul3l0EP73Ik002fStr_bhsZh9mQqEU,4735
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
 utils/compute_icon_mean_stddev.py,sha256=a1oWMRMnny39rV-dvu8rcx85sb4bXzPFrR1gkUr4Jpg,2296
-ocf_data_sampler-0.2.14.dist-info/METADATA,sha256=fb2tvDSt9FrsJBD8mZvjJ8YwTgp3OfUjavuZda3cblA,11581
-ocf_data_sampler-0.2.14.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ocf_data_sampler-0.2.14.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
-ocf_data_sampler-0.2.14.dist-info/RECORD,,
+ocf_data_sampler-0.2.16.dist-info/METADATA,sha256=JplbQHR2wlMPCQEDXYk5GPDgu15wYKV3SYZmL0kH2Ho,11581
+ocf_data_sampler-0.2.16.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+ocf_data_sampler-0.2.16.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
+ocf_data_sampler-0.2.16.dist-info/RECORD,,

{ocf_data_sampler-0.2.14.dist-info → ocf_data_sampler-0.2.16.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any