ocf-data-sampler 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

ocf_data_sampler/constants.py

@@ -86,7 +86,7 @@ ECMWF_STD = {
     "lcc": 0.3791404366493225,
     "mcc": 0.38039860129356384,
     "prate": 9.81039775069803e-05,
-    "sde": 0.000913831521756947,
+    "sd": 0.000913831521756947,
     "sr": 16294988.0,
     "t2m": 3.692270040512085,
     "tcc": 0.37487083673477173,
@@ -110,7 +110,7 @@ ECMWF_MEAN = {
     "lcc": 0.44901806116104126,
     "mcc": 0.3288780450820923,
     "prate": 3.108070450252853e-05,
-    "sde": 8.107526082312688e-05,
+    "sd": 8.107526082312688e-05,
     "sr": 12905302.0,
     "t2m": 283.48333740234375,
     "tcc": 0.7049227356910706,

ocf_data_sampler/sample/base.py

@@ -73,3 +73,26 @@ def batch_to_tensor(batch: NumpyBatch) -> TensorBatch:
             elif np.issubdtype(v.dtype, np.number):
                 batch[k] = torch.as_tensor(v)
     return batch
+
+
+def copy_batch_to_device(batch: dict, device: torch.device) -> dict:
+    """
+    Moves tensor leaves in a nested dict to a new device.
+
+    Args:
+        batch: Nested dict with tensors to move.
+        device: Device to move tensors to.
+
+    Returns:
+        A dict with tensors moved to the new device.
+    """
+    batch_copy = {}
+
+    for k, v in batch.items():
+        if isinstance(v, dict):
+            batch_copy[k] = copy_batch_to_device(v, device)  
+        elif isinstance(v, torch.Tensor):
+            batch_copy[k] = v.to(device)
+        else:
+            batch_copy[k] = v
+    return batch_copy

ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -31,10 +31,15 @@ from ocf_data_sampler.torch_datasets.utils.merge_and_fill_utils import (
     merge_dicts,
     fill_nans_in_arrays,
 )
+from ocf_data_sampler.torch_datasets.utils.validate_channels import (
+    validate_nwp_channels,
+    validate_satellite_channels,
+)
 
 
 xr.set_options(keep_attrs=True)
 
+
 def process_and_combine_datasets(
     dataset_dict: dict,
     config: Configuration,
@@ -47,27 +52,23 @@ def process_and_combine_datasets(
     numpy_modalities = []
 
     if "nwp" in dataset_dict:
-
         nwp_numpy_modalities = dict()
 
         for nwp_key, da_nwp in dataset_dict["nwp"].items():
-            # Standardise
             provider = config.input_data.nwp[nwp_key].provider
-            da_nwp = (da_nwp - NWP_MEANS[provider]) / NWP_STDS[provider]
 
-            # Convert to NumpyBatch
+            # Standardise and convert to NumpyBatch
+            da_nwp = (da_nwp - NWP_MEANS[provider]) / NWP_STDS[provider]
             nwp_numpy_modalities[nwp_key] = convert_nwp_to_numpy_sample(da_nwp)
 
         # Combine the NWPs into NumpyBatch
         numpy_modalities.append({NWPSampleKey.nwp: nwp_numpy_modalities})
 
-
     if "sat" in dataset_dict:
-        # Standardise
         da_sat = dataset_dict["sat"]
-        da_sat = (da_sat - RSS_MEAN) / RSS_STD
 
-        # Convert to NumpyBatch
+        # Standardise and convert to NumpyBatch
+        da_sat = (da_sat - RSS_MEAN) / RSS_STD
         numpy_modalities.append(convert_satellite_to_numpy_sample(da_sat))
 
     gsp_config = config.input_data.gsp
@@ -186,9 +187,13 @@ class PVNetUKRegionalDataset(Dataset):
         """
         
         config = load_yaml_configuration(config_filename)
-        
+
+        # Validate channels for NWP and satellite data
+        validate_nwp_channels(config)
+        validate_satellite_channels(config)
+
         datasets_dict = get_dataset_dict(config.input_data)
-        
+    
         # Get t0 times where all input data is available
         valid_t0_times = find_valid_t0_times(datasets_dict, config)
 
@@ -294,7 +299,11 @@ class PVNetUKConcurrentDataset(Dataset):
         """
         
         config = load_yaml_configuration(config_filename)
-        
+
+        # Validate channels for NWP and satellite data
+        validate_nwp_channels(config)
+        validate_satellite_channels(config)
+
         datasets_dict = get_dataset_dict(config.input_data)
         
         # Get t0 times where all input data is available
@@ -361,4 +370,4 @@ class PVNetUKConcurrentDataset(Dataset):
         """
         # Check data is availablle for init-time t0
         assert t0 in self.valid_t0_times        
-        return self._get_sample(t0)
+        return self._get_sample(t0)

ocf_data_sampler/torch_datasets/datasets/site.py

@@ -1,4 +1,5 @@
 """Torch dataset for sites"""
+
 import logging
 import numpy as np
 import pandas as pd
@@ -19,6 +20,8 @@ from ocf_data_sampler.select import (
 from ocf_data_sampler.utils import minutes
 from ocf_data_sampler.torch_datasets.utils.valid_time_periods import find_valid_time_periods
 from ocf_data_sampler.torch_datasets.utils.merge_and_fill_utils import merge_dicts, fill_nans_in_arrays
+from ocf_data_sampler.torch_datasets.utils.validate_channels import validate_nwp_channels
+
 from ocf_data_sampler.numpy_sample import (
     convert_site_to_numpy_sample, 
     convert_satellite_to_numpy_sample, 
@@ -29,8 +32,10 @@ from ocf_data_sampler.numpy_sample import (
 from ocf_data_sampler.numpy_sample import NWPSampleKey
 from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS
 
+
 xr.set_options(keep_attrs=True)
 
+
 class SitesDataset(Dataset):
     def __init__(
         self,
@@ -47,6 +52,10 @@ class SitesDataset(Dataset):
         """
 
         config: Configuration = load_yaml_configuration(config_filename)
+
+        # Validate NWP channels 
+        validate_nwp_channels(config)
+
         datasets_dict = get_dataset_dict(config.input_data)
 
         # Assign config and input data to self
@@ -221,8 +230,9 @@ class SitesDataset(Dataset):
 
         if "nwp" in dataset_dict:
             for nwp_key, da_nwp in dataset_dict["nwp"].items():
-                # Standardise
                 provider = self.config.input_data.nwp[nwp_key].provider
+                
+                # Standardise
                 da_nwp = (da_nwp - NWP_MEANS[provider]) / NWP_STDS[provider]
                 data_arrays.append((f"nwp-{provider}", da_nwp))
             

ocf_data_sampler/torch_datasets/utils/validate_channels.py

@@ -0,0 +1,82 @@
+import xarray as xr
+
+from ocf_data_sampler.config import Configuration
+from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS, RSS_MEAN, RSS_STD
+
+
+def validate_channels(
+    data_channels: list,
+    means_channels: list,
+    stds_channels: list,
+    source_name: str | None = None
+) -> None:
+    """
+    Validates that all channels in data have corresponding normalisation constants.
+
+    Args:
+        data_channels: Set of channels from the data
+        means_channels: Set of channels from means constants
+        stds_channels: Set of channels from stds constants
+        source_name: Name of data source (e.g., 'ecmwf', 'satellite') for error messages
+
+    Raises:
+        ValueError: If there's a mismatch between data channels and normalisation constants
+    """
+
+    data_set = set(data_channels)
+    means_set = set(means_channels)
+    stds_set = set(stds_channels)
+
+    # Find missing channels in means
+    missing_in_means = data_set - means_set
+    if missing_in_means:
+        raise ValueError(
+            f"The following channels for {source_name} are missing in normalisation means: "
+            f"{missing_in_means}"
+        )
+            
+    # Find missing channels in stds
+    missing_in_stds = data_set - stds_set
+    if missing_in_stds:
+        raise ValueError(
+            f"The following channels for {source_name} are missing in normalisation stds: "
+            f"{missing_in_stds}"
+        )
+
+
+def validate_nwp_channels(config: Configuration) -> None:
+    """Validate that NWP channels in config have corresponding normalisation constants.
+    
+    Args:
+        config: Configuration object containing NWP channel information
+        
+    Raises:
+        ValueError: If there's a mismatch between configured NWP channels and normalisation constants
+    """
+    if hasattr(config.input_data, "nwp"):
+        for nwp_key, nwp_config in config.input_data.nwp.items():
+            provider = nwp_config.provider
+            validate_channels(
+                data_channels=nwp_config.channels,
+                means_channels=NWP_MEANS[provider].channel.values,
+                stds_channels=NWP_STDS[provider].channel.values,
+                source_name=provider
+            )
+
+
+def validate_satellite_channels(config: Configuration) -> None:
+    """Validate that satellite channels in config have corresponding normalisation constants.
+    
+    Args:
+        config: Configuration object containing satellite channel information
+        
+    Raises:
+        ValueError: If there's a mismatch between configured satellite channels and normalisation constants
+    """
+    if hasattr(config.input_data, "satellite"):
+        validate_channels(
+            data_channels=config.input_data.satellite.channels,
+            means_channels=RSS_MEAN.channel.values,
+            stds_channels=RSS_STD.channel.values,
+            source_name="satellite"
+        )

{ocf_data_sampler-0.1.7.dist-info → ocf_data_sampler-0.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.7
+Version: 0.1.9
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.7.dist-info → ocf_data_sampler-0.1.9.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/constants.py,sha256=ClteRIgp7EPlUPqIbkel83BfIaD7_VIDjUeHzUfyhnM,5079
+ocf_data_sampler/constants.py,sha256=0HYNmqwBaHVTAEEx9qzk6WD9YInh0gSKLeI3pyq7aNs,5077
 ocf_data_sampler/utils.py,sha256=rKA0BHAyAG4f90zEcgxp25EEYrXS-aOVNzttZ6Mzv2k,250
 ocf_data_sampler/config/__init__.py,sha256=O29mbH0XG2gIY1g3BaveGCnpBO2SFqdu-qzJ7a6evl0,223
 ocf_data_sampler/config/load.py,sha256=sKCKmhkkeFvvkNL5xmnFvdAulaCtV4-rigPsFvVDPDc,634
@@ -27,7 +27,7 @@ ocf_data_sampler/numpy_sample/satellite.py,sha256=8OaTvkPjzSjotcdKsa6BKmmlBKDBun
 ocf_data_sampler/numpy_sample/site.py,sha256=I-cAXCOF0SDdm5Hx43lFqYZ3jh61kltLQK-fc4_nNu0,1314
 ocf_data_sampler/numpy_sample/sun_position.py,sha256=UklhucCxCT6GMlAhCWL6c4cfWrdc1cWgegrYaqUoHOY,1611
 ocf_data_sampler/sample/__init__.py,sha256=02CM7E5nKkGiYbVW-kvzjNd4RaqGuHCkDChtmDBDUoA,248
-ocf_data_sampler/sample/base.py,sha256=qeKuWyyO8M4QX6QDbItioeCiss0fG05NXRtf0TCMQSc,2246
+ocf_data_sampler/sample/base.py,sha256=q3wpqoW4JXRmzfar6ed7UMn1nxBxSJXNvMLJmHXy1dw,2856
 ocf_data_sampler/sample/site.py,sha256=0BvDXs0kxTjUq7kWpeoITK_uN4uE0w1IvEFXZUoKOb0,2507
 ocf_data_sampler/sample/uk_regional.py,sha256=D1A6nQB1PYCmxb3FzU9gqbNufQfx__wcprcDm50jCJw,4381
 ocf_data_sampler/select/__init__.py,sha256=E4AJulEbO2K-o0UlG1fgaEteuf_1ZFjHTvrotXSb4YU,332
@@ -41,11 +41,12 @@ ocf_data_sampler/select/select_time_slice.py,sha256=9M-yvDv9K77XfEys_OIR31_aVB56
 ocf_data_sampler/select/spatial_slice_for_dataset.py,sha256=3tRrMBXr7s4CnClbVSIq7hpls3H4Y3qYTDwswcxCCCE,1763
 ocf_data_sampler/select/time_slice_for_dataset.py,sha256=Z7pOiilSHScxmBKZNG18K5J-S4ifdXXAYGZoHRHD3AY,4324
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=jfJSFcR0eO1AqeH7S3KnGjsBqVZT5w3oyi784PUR6Q0,146
-ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=xuNJyCXZ4dZ9UldX1lqOoRSRNP39Vcy0DR77Vr7dxlk,11895
-ocf_data_sampler/torch_datasets/datasets/site.py,sha256=ZjvJS0mWUyQE7ZcrhS1TdMHaPrEZXVbBAv2vDwBvQwA,16044
+ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=N85duDyEm6LIYgYIpLhrpxHddMIcvFosuZg8rzIztwE,12267
+ocf_data_sampler/torch_datasets/datasets/site.py,sha256=L_4w967ZxPjd7vHRkPtj7ZSmamEShKRT28j9_f-enJY,16228
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=hIbekql64eXsNDFIoEc--GWxwdVWrh2qKegdOi70Bow,874
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=Qo65qUHtle_bW5tLTYr7empHTRv-lpjvfx_6GNJj3Xg,4371
-scripts/refactor_site.py,sha256=asZ27hQ4IyXgCCUaFJqcz1ObBNcV2W3ywqHBpSXA_fc,1728
+ocf_data_sampler/torch_datasets/utils/validate_channels.py,sha256=u2EpiFAKAOHpmvINhOUJCT8Vbc-cle6qJ3YNVse4yLs,2884
+scripts/refactor_site.py,sha256=xaJGxt2_WObIPrPAnRiOMMB68r-5Q51jWRx409AcscM,1747
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/conftest.py,sha256=RlC7YYtBLipUzFS1tQxela1SgHCxSpReUKEJ4429PwQ,7689
 tests/config/test_config.py,sha256=VQjNiucIk5VnPQdGA6Mr-RNd9CwGI06AiikChTHrcnY,3969
@@ -67,14 +68,15 @@ tests/select/test_find_contiguous_time_periods.py,sha256=kOga_V7er5We7ewMARXaKdM
 tests/select/test_location.py,sha256=_WZk2FPYeJ-nIfCJS6Sp_yaVEEo7m31DmMFoZzgyCts,2712
 tests/select/test_select_spatial_slice.py,sha256=7EX9b6g-pMdACQx3yefjs5do2s-Rho2UmKevV4oglsU,5147
 tests/select/test_select_time_slice.py,sha256=nYrdlmZlGEygJKiE26bADiluNPN1qt5kD4FrI2vtxUw,9686
-tests/test_sample/test_base.py,sha256=CkqKCZbrq3Vb4T7bOwPh3_0p8OTl0LfSLNBctYC_jag,4199
+tests/test_sample/test_base.py,sha256=sD9NZghYQWbkAcQP9YXypWZowqYkO3xeNMH-_mEoD5I,4833
 tests/test_sample/test_site_sample.py,sha256=Gln-Or060cUWvA7Q7c1vsthgCttOAM2z9yBI9zUIrDw,6238
 tests/test_sample/test_uk_regional_sample.py,sha256=gkeQWC2wC757jKJz_QBmDMFQjn3R54q_tEo948yyxCY,4840
 tests/torch_datasets/test_merge_and_fill_utils.py,sha256=GtuQg82BM1eHQjT7Ik1x1zaVcuc7KJO4_NC9stXsd4s,1123
-tests/torch_datasets/test_pvnet_uk.py,sha256=loueo7PUUYJVda3-vBn3bQIC_zgrTAThfx-GTDcBOZg,5596
+tests/torch_datasets/test_pvnet_uk.py,sha256=F0D-DugFgVtt8G1q7lylmPLrOZj6H6YPNd9s_6Wn_yM,5594
 tests/torch_datasets/test_site.py,sha256=t57vAR_RRWcbG_kEFk6VrFCYzVxwFG6qJKBnRHF02fM,7000
-ocf_data_sampler-0.1.7.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.1.7.dist-info/METADATA,sha256=8SbL1qjkmeFDYdv1_hHBL9jxbSpt4aFCpx70rEEPeb0,12173
-ocf_data_sampler-0.1.7.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ocf_data_sampler-0.1.7.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
-ocf_data_sampler-0.1.7.dist-info/RECORD,,
+tests/torch_datasets/test_validate_channels_utils.py,sha256=Rzdweu98j1of45jCOUrSiBtyPlf-dDaCceulf0H7ml8,2921
+ocf_data_sampler-0.1.9.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
+ocf_data_sampler-0.1.9.dist-info/METADATA,sha256=Lfu8Yrj4CSlqPzGhk0iDy5r5zCLd5REnGAlVcFuKuow,12173
+ocf_data_sampler-0.1.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ocf_data_sampler-0.1.9.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
+ocf_data_sampler-0.1.9.dist-info/RECORD,,

scripts/refactor_site.py

@@ -1,5 +1,5 @@
 """ Helper functions for refactoring legacy site data """
-
+import xarray as xr
 
 def legacy_format(data_ds, metadata_df):
     """This formats old legacy data to the new format.

tests/test_sample/test_base.py

@@ -9,7 +9,8 @@ import numpy as np
 from pathlib import Path
 from ocf_data_sampler.sample.base import (
     SampleBase,
-    batch_to_tensor
+    batch_to_tensor,
+    copy_batch_to_device
 )
 
 class TestSample(SampleBase):
@@ -145,3 +146,19 @@ def test_batch_to_tensor_multidimensional():
     assert tensor_batch['matrix'].shape == (2, 2)
     assert tensor_batch['tensor'].shape == (2, 2, 2)
     assert torch.equal(tensor_batch['matrix'], torch.tensor([[1, 2], [3, 4]]))
+
+
+def test_copy_batch_to_device():
+    """ Test moving tensors to a different device """
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    batch = {
+        'tensor_data': torch.tensor([1, 2, 3]),
+        'nested': {
+            'matrix': torch.tensor([[1, 2], [3, 4]])
+        },
+        'non_tensor': 'unchanged'
+    }
+    moved_batch = copy_batch_to_device(batch, device)
+    assert moved_batch['tensor_data'].device == device
+    assert moved_batch['nested']['matrix'].device == device
+    assert moved_batch['non_tensor'] == 'unchanged'  # Non-tensors should remain unchanged

tests/torch_datasets/test_pvnet_uk.py

@@ -24,7 +24,7 @@ def test_process_and_combine_datasets(pvnet_config_filename):
         dims=["time_utc", "channel", "y", "x"],
         coords={
             "time_utc": pd.date_range("2024-01-01 00:00", periods=4, freq="h"),
-            "channel": ["t2m", "dswrf"],
+            "channel": ["t", "dswrf"],
             "step": ("time_utc", pd.timedelta_range(start='0h', periods=4, freq='h')),
             "init_time_utc": pd.Timestamp("2024-01-01 00:00")
         }
@@ -54,7 +54,7 @@ def test_process_and_combine_datasets(pvnet_config_filename):
     assert isinstance(sample, dict)
     assert "nwp" in sample
     assert sample["satellite_actual"].shape == (7, 1, 2, 2)
-    assert sample["nwp"]["ukv"]["nwp"].shape == (4, 1, 2, 2)
+    assert sample["nwp"]["ukv"]["nwp"].shape == (4, 2, 2, 2)
     assert "gsp_id" in sample
 
 

tests/torch_datasets/test_validate_channels_utils.py

@@ -0,0 +1,78 @@
+"""Tests for channel validation utility functions"""
+
+import pytest
+from ocf_data_sampler.torch_datasets.utils.validate_channels import (
+    validate_channels,
+    validate_nwp_channels,
+    validate_satellite_channels,
+)
+
+
+class TestChannelValidation:
+    """Tests for channel validation functions"""
+    
+    @pytest.mark.parametrize("test_case", [
+        # Base validation - success case
+        {
+            "data_channels": ["channel1", "channel2"],
+            "norm_channels": ["channel1", "channel2", "extra"],
+            "source_name": "test_source",
+            "expect_error": False
+        },
+        # Base validation - error case
+        {
+            "data_channels": ["channel1", "missing_channel"],
+            "norm_channels": ["channel1"],
+            "source_name": "test_source",
+            "expect_error": True,
+            "error_match": "following channels for test_source are missing in normalisation means"
+        },
+        # NWP case - success
+        {
+            "data_channels": ["t2m", "dswrf"],
+            "norm_channels": ["t2m", "dswrf", "extra"],
+            "source_name": "ecmwf",
+            "expect_error": False
+        },
+        # NWP case - error
+        {
+            "data_channels": ["t2m", "missing_channel"],
+            "norm_channels": ["t2m"],
+            "source_name": "ecmwf",
+            "expect_error": True,
+            "error_match": "following channels for ecmwf are missing in normalisation means"
+        },
+        # Satellite case - success
+        {
+            "data_channels": ["IR_016", "VIS006"],
+            "norm_channels": ["IR_016", "VIS006", "extra"],
+            "source_name": "satellite",
+            "expect_error": False
+        },
+        # Satellite case - error
+        {
+            "data_channels": ["IR_016", "missing_channel"],
+            "norm_channels": ["IR_016"],
+            "source_name": "satellite",
+            "expect_error": True,
+            "error_match": "following channels for satellite are missing in normalisation means"
+        }
+    ])
+    def test_channel_validation(self, test_case):
+        """Test channel validation for both base, NWP and satellite data"""
+        if test_case["expect_error"]:
+            with pytest.raises(ValueError, match=test_case["error_match"]):
+                validate_channels(
+                    data_channels=test_case["data_channels"],
+                    means_channels=test_case["norm_channels"],
+                    stds_channels=test_case["norm_channels"],
+                    source_name=test_case["source_name"]
+                )
+        else:
+            # Should not raise any exceptions
+            validate_channels(
+                data_channels=test_case["data_channels"],
+                means_channels=test_case["norm_channels"],
+                stds_channels=test_case["norm_channels"],
+                source_name=test_case["source_name"]
+            )