ocf-data-sampler 0.1.6__tar.gz → 0.1.8__tar.gz

{ocf_data_sampler-0.1.6/ocf_data_sampler.egg-info → ocf_data_sampler-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.6
+Version: 0.1.8
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8}/ocf_data_sampler/constants.py

@@ -86,7 +86,7 @@ ECMWF_STD = {
     "lcc": 0.3791404366493225,
     "mcc": 0.38039860129356384,
     "prate": 9.81039775069803e-05,
-    "sde": 0.000913831521756947,
+    "sd": 0.000913831521756947,
     "sr": 16294988.0,
     "t2m": 3.692270040512085,
     "tcc": 0.37487083673477173,
@@ -110,7 +110,7 @@ ECMWF_MEAN = {
     "lcc": 0.44901806116104126,
     "mcc": 0.3288780450820923,
     "prate": 3.108070450252853e-05,
-    "sde": 8.107526082312688e-05,
+    "sd": 8.107526082312688e-05,
     "sr": 12905302.0,
     "t2m": 283.48333740234375,
     "tcc": 0.7049227356910706,

ocf_data_sampler-0.1.8/ocf_data_sampler/sample/base.py

@@ -0,0 +1,75 @@
+"""
+Base class definition - abstract
+Handling of both flat and nested structures - consideration for NWP
+"""
+
+import logging
+import numpy as np
+import torch
+import xarray as xr
+
+from pathlib import Path
+from typing import Any, Dict, Optional, Union, TypeAlias
+from abc import ABC, abstractmethod
+
+
+logger = logging.getLogger(__name__)
+
+NumpySample: TypeAlias = Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
+NumpyBatch: TypeAlias = Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
+TensorBatch: TypeAlias = Dict[str, Union[torch.Tensor, Dict[str, torch.Tensor]]]
+
+
+class SampleBase(ABC):
+    """ 
+    Abstract base class for all sample types 
+    Provides core data storage functionality
+    """
+
+    def __init__(self, data: Optional[Union[NumpySample, xr.Dataset]] = None):
+        """ Initialise data container """
+        logger.debug("Initialising SampleBase instance")
+        self._data = data
+
+    @abstractmethod
+    def to_numpy(self) -> NumpySample:
+        """ Convert data to a numpy array representation """
+        raise NotImplementedError
+
+    @abstractmethod
+    def plot(self, **kwargs) -> None:
+        """ Abstract method for plotting """
+        raise NotImplementedError
+
+    @abstractmethod
+    def save(self, path: Union[str, Path]) -> None:
+        """ Abstract method for saving sample data """
+        raise NotImplementedError
+
+    @classmethod
+    @abstractmethod
+    def load(cls, path: Union[str, Path]) -> 'SampleBase':
+        """ Abstract class method for loading sample data """
+        raise NotImplementedError
+
+
+def batch_to_tensor(batch: NumpyBatch) -> TensorBatch:
+    """
+    Moves ndarrays in a nested dict to torch tensors
+    Args:
+        batch: NumpyBatch with data in numpy arrays
+    Returns:
+        TensorBatch with data in torch tensors
+    """
+    if not batch:
+        raise ValueError("Cannot convert empty batch to tensors")
+
+    for k, v in batch.items():
+        if isinstance(v, dict):
+            batch[k] = batch_to_tensor(v)
+        elif isinstance(v, np.ndarray):
+            if v.dtype == np.bool_:
+                batch[k] = torch.tensor(v, dtype=torch.bool)
+            elif np.issubdtype(v.dtype, np.number):
+                batch[k] = torch.as_tensor(v)
+    return batch

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8}/ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -31,10 +31,15 @@ from ocf_data_sampler.torch_datasets.utils.merge_and_fill_utils import (
     merge_dicts,
     fill_nans_in_arrays,
 )
+from ocf_data_sampler.torch_datasets.utils.validate_channels import (
+    validate_nwp_channels,
+    validate_satellite_channels,
+)
 
 
 xr.set_options(keep_attrs=True)
 
+
 def process_and_combine_datasets(
     dataset_dict: dict,
     config: Configuration,
@@ -47,27 +52,23 @@ def process_and_combine_datasets(
     numpy_modalities = []
 
     if "nwp" in dataset_dict:
-
         nwp_numpy_modalities = dict()
 
         for nwp_key, da_nwp in dataset_dict["nwp"].items():
-            # Standardise
             provider = config.input_data.nwp[nwp_key].provider
-            da_nwp = (da_nwp - NWP_MEANS[provider]) / NWP_STDS[provider]
 
-            # Convert to NumpyBatch
+            # Standardise and convert to NumpyBatch
+            da_nwp = (da_nwp - NWP_MEANS[provider]) / NWP_STDS[provider]
             nwp_numpy_modalities[nwp_key] = convert_nwp_to_numpy_sample(da_nwp)
 
         # Combine the NWPs into NumpyBatch
         numpy_modalities.append({NWPSampleKey.nwp: nwp_numpy_modalities})
 
-
     if "sat" in dataset_dict:
-        # Standardise
         da_sat = dataset_dict["sat"]
-        da_sat = (da_sat - RSS_MEAN) / RSS_STD
 
-        # Convert to NumpyBatch
+        # Standardise and convert to NumpyBatch
+        da_sat = (da_sat - RSS_MEAN) / RSS_STD
         numpy_modalities.append(convert_satellite_to_numpy_sample(da_sat))
 
     gsp_config = config.input_data.gsp
@@ -186,9 +187,13 @@ class PVNetUKRegionalDataset(Dataset):
         """
         
         config = load_yaml_configuration(config_filename)
-        
+
+        # Validate channels for NWP and satellite data
+        validate_nwp_channels(config)
+        validate_satellite_channels(config)
+
         datasets_dict = get_dataset_dict(config.input_data)
-        
+    
         # Get t0 times where all input data is available
         valid_t0_times = find_valid_t0_times(datasets_dict, config)
 
@@ -294,7 +299,11 @@ class PVNetUKConcurrentDataset(Dataset):
         """
         
         config = load_yaml_configuration(config_filename)
-        
+
+        # Validate channels for NWP and satellite data
+        validate_nwp_channels(config)
+        validate_satellite_channels(config)
+
         datasets_dict = get_dataset_dict(config.input_data)
         
         # Get t0 times where all input data is available
@@ -361,4 +370,4 @@ class PVNetUKConcurrentDataset(Dataset):
         """
         # Check data is availablle for init-time t0
         assert t0 in self.valid_t0_times        
-        return self._get_sample(t0)
+        return self._get_sample(t0)

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8}/ocf_data_sampler/torch_datasets/datasets/site.py

@@ -1,4 +1,5 @@
 """Torch dataset for sites"""
+
 import logging
 import numpy as np
 import pandas as pd
@@ -19,6 +20,8 @@ from ocf_data_sampler.select import (
 from ocf_data_sampler.utils import minutes
 from ocf_data_sampler.torch_datasets.utils.valid_time_periods import find_valid_time_periods
 from ocf_data_sampler.torch_datasets.utils.merge_and_fill_utils import merge_dicts, fill_nans_in_arrays
+from ocf_data_sampler.torch_datasets.utils.validate_channels import validate_nwp_channels
+
 from ocf_data_sampler.numpy_sample import (
     convert_site_to_numpy_sample, 
     convert_satellite_to_numpy_sample, 
@@ -29,8 +32,10 @@ from ocf_data_sampler.numpy_sample import (
 from ocf_data_sampler.numpy_sample import NWPSampleKey
 from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS
 
+
 xr.set_options(keep_attrs=True)
 
+
 class SitesDataset(Dataset):
     def __init__(
         self,
@@ -47,6 +52,10 @@ class SitesDataset(Dataset):
         """
 
         config: Configuration = load_yaml_configuration(config_filename)
+
+        # Validate NWP channels 
+        validate_nwp_channels(config)
+
         datasets_dict = get_dataset_dict(config.input_data)
 
         # Assign config and input data to self
@@ -221,8 +230,9 @@ class SitesDataset(Dataset):
 
         if "nwp" in dataset_dict:
             for nwp_key, da_nwp in dataset_dict["nwp"].items():
-                # Standardise
                 provider = self.config.input_data.nwp[nwp_key].provider
+                
+                # Standardise
                 da_nwp = (da_nwp - NWP_MEANS[provider]) / NWP_STDS[provider]
                 data_arrays.append((f"nwp-{provider}", da_nwp))
             

ocf_data_sampler-0.1.8/ocf_data_sampler/torch_datasets/utils/validate_channels.py

@@ -0,0 +1,82 @@
+import xarray as xr
+
+from ocf_data_sampler.config import Configuration
+from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS, RSS_MEAN, RSS_STD
+
+
+def validate_channels(
+    data_channels: list,
+    means_channels: list,
+    stds_channels: list,
+    source_name: str | None = None
+) -> None:
+    """
+    Validates that all channels in data have corresponding normalisation constants.
+
+    Args:
+        data_channels: Set of channels from the data
+        means_channels: Set of channels from means constants
+        stds_channels: Set of channels from stds constants
+        source_name: Name of data source (e.g., 'ecmwf', 'satellite') for error messages
+
+    Raises:
+        ValueError: If there's a mismatch between data channels and normalisation constants
+    """
+
+    data_set = set(data_channels)
+    means_set = set(means_channels)
+    stds_set = set(stds_channels)
+
+    # Find missing channels in means
+    missing_in_means = data_set - means_set
+    if missing_in_means:
+        raise ValueError(
+            f"The following channels for {source_name} are missing in normalisation means: "
+            f"{missing_in_means}"
+        )
+            
+    # Find missing channels in stds
+    missing_in_stds = data_set - stds_set
+    if missing_in_stds:
+        raise ValueError(
+            f"The following channels for {source_name} are missing in normalisation stds: "
+            f"{missing_in_stds}"
+        )
+
+
+def validate_nwp_channels(config: Configuration) -> None:
+    """Validate that NWP channels in config have corresponding normalisation constants.
+    
+    Args:
+        config: Configuration object containing NWP channel information
+        
+    Raises:
+        ValueError: If there's a mismatch between configured NWP channels and normalisation constants
+    """
+    if hasattr(config.input_data, "nwp"):
+        for nwp_key, nwp_config in config.input_data.nwp.items():
+            provider = nwp_config.provider
+            validate_channels(
+                data_channels=nwp_config.channels,
+                means_channels=NWP_MEANS[provider].channel.values,
+                stds_channels=NWP_STDS[provider].channel.values,
+                source_name=provider
+            )
+
+
+def validate_satellite_channels(config: Configuration) -> None:
+    """Validate that satellite channels in config have corresponding normalisation constants.
+    
+    Args:
+        config: Configuration object containing satellite channel information
+        
+    Raises:
+        ValueError: If there's a mismatch between configured satellite channels and normalisation constants
+    """
+    if hasattr(config.input_data, "satellite"):
+        validate_channels(
+            data_channels=config.input_data.satellite.channels,
+            means_channels=RSS_MEAN.channel.values,
+            stds_channels=RSS_STD.channel.values,
+            source_name="satellite"
+        )

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8/ocf_data_sampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.6
+Version: 0.1.8
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8}/ocf_data_sampler.egg-info/SOURCES.txt

@@ -54,6 +54,7 @@ ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py
 ocf_data_sampler/torch_datasets/datasets/site.py
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py
+ocf_data_sampler/torch_datasets/utils/validate_channels.py
 scripts/refactor_site.py
 tests/__init__.py
 tests/conftest.py
@@ -81,4 +82,5 @@ tests/test_sample/test_site_sample.py
 tests/test_sample/test_uk_regional_sample.py
 tests/torch_datasets/test_merge_and_fill_utils.py
 tests/torch_datasets/test_pvnet_uk.py
-tests/torch_datasets/test_site.py
+tests/torch_datasets/test_site.py
+tests/torch_datasets/test_validate_channels_utils.py

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ocf_data_sampler"
-version = "0.1.6"
+version = "0.1.8"
 license = { file = "LICENSE" }
 readme = "README.md"
 description = "Sample from weather data for renewable energy prediction"

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8}/tests/test_sample/test_base.py

@@ -3,11 +3,14 @@ Base class testing - SampleBase
 """
 
 import pytest
+import torch
 import numpy as np
 
 from pathlib import Path
-from ocf_data_sampler.sample.base import SampleBase
-
+from ocf_data_sampler.sample.base import (
+    SampleBase,
+    batch_to_tensor
+)
 
 class TestSample(SampleBase):
     """
@@ -84,3 +87,61 @@ def test_sample_base_to_numpy():
     assert isinstance(numpy_data, dict)
     assert all(isinstance(value, np.ndarray) for value in numpy_data.values())
     assert np.array_equal(numpy_data['list_data'], np.array([1, 2, 3]))
+
+
+def test_batch_to_tensor_nested():
+    """ Test nested dictionary conversion """
+    batch = {
+        'outer': {
+            'inner': np.array([1, 2, 3])
+        }
+    }
+    tensor_batch = batch_to_tensor(batch)
+    
+    assert torch.equal(tensor_batch['outer']['inner'], torch.tensor([1, 2, 3]))
+
+
+def test_batch_to_tensor_mixed_types():
+    """ Test handling of mixed data types """
+    batch = {
+        'tensor_data': np.array([1, 2, 3]),
+        'string_data': 'not_a_tensor',
+        'nested': {
+            'numbers': np.array([4, 5, 6]),
+            'text': 'still_not_a_tensor'
+        }
+    }
+    tensor_batch = batch_to_tensor(batch)
+    
+    assert isinstance(tensor_batch['tensor_data'], torch.Tensor)
+    assert isinstance(tensor_batch['string_data'], str)
+    assert isinstance(tensor_batch['nested']['numbers'], torch.Tensor)
+    assert isinstance(tensor_batch['nested']['text'], str)
+
+
+def test_batch_to_tensor_different_dtypes():
+    """ Test conversion of arrays with different dtypes """
+    batch = {
+        'float_data': np.array([1.0, 2.0, 3.0], dtype=np.float32),
+        'int_data': np.array([1, 2, 3], dtype=np.int64),
+        'bool_data': np.array([True, False, True], dtype=np.bool_)
+    }
+    tensor_batch = batch_to_tensor(batch)
+    
+    assert isinstance(tensor_batch['bool_data'], torch.Tensor)
+    assert tensor_batch['float_data'].dtype == torch.float32
+    assert tensor_batch['int_data'].dtype == torch.int64
+    assert tensor_batch['bool_data'].dtype == torch.bool
+
+
+def test_batch_to_tensor_multidimensional():
+    """ Test conversion of multidimensional arrays """
+    batch = {
+        'matrix': np.array([[1, 2], [3, 4]]),
+        'tensor': np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+    }
+    tensor_batch = batch_to_tensor(batch)
+    
+    assert tensor_batch['matrix'].shape == (2, 2)
+    assert tensor_batch['tensor'].shape == (2, 2, 2)
+    assert torch.equal(tensor_batch['matrix'], torch.tensor([[1, 2], [3, 4]]))

{ocf_data_sampler-0.1.6 → ocf_data_sampler-0.1.8}/tests/torch_datasets/test_pvnet_uk.py

@@ -24,7 +24,7 @@ def test_process_and_combine_datasets(pvnet_config_filename):
         dims=["time_utc", "channel", "y", "x"],
         coords={
             "time_utc": pd.date_range("2024-01-01 00:00", periods=4, freq="h"),
-            "channel": ["t2m", "dswrf"],
+            "channel": ["t", "dswrf"],
             "step": ("time_utc", pd.timedelta_range(start='0h', periods=4, freq='h')),
             "init_time_utc": pd.Timestamp("2024-01-01 00:00")
         }
@@ -54,7 +54,7 @@ def test_process_and_combine_datasets(pvnet_config_filename):
     assert isinstance(sample, dict)
     assert "nwp" in sample
     assert sample["satellite_actual"].shape == (7, 1, 2, 2)
-    assert sample["nwp"]["ukv"]["nwp"].shape == (4, 1, 2, 2)
+    assert sample["nwp"]["ukv"]["nwp"].shape == (4, 2, 2, 2)
     assert "gsp_id" in sample
 
 

ocf_data_sampler-0.1.8/tests/torch_datasets/test_validate_channels_utils.py

@@ -0,0 +1,78 @@
+"""Tests for channel validation utility functions"""
+
+import pytest
+from ocf_data_sampler.torch_datasets.utils.validate_channels import (
+    validate_channels,
+    validate_nwp_channels,
+    validate_satellite_channels,
+)
+
+
+class TestChannelValidation:
+    """Tests for channel validation functions"""
+    
+    @pytest.mark.parametrize("test_case", [
+        # Base validation - success case
+        {
+            "data_channels": ["channel1", "channel2"],
+            "norm_channels": ["channel1", "channel2", "extra"],
+            "source_name": "test_source",
+            "expect_error": False
+        },
+        # Base validation - error case
+        {
+            "data_channels": ["channel1", "missing_channel"],
+            "norm_channels": ["channel1"],
+            "source_name": "test_source",
+            "expect_error": True,
+            "error_match": "following channels for test_source are missing in normalisation means"
+        },
+        # NWP case - success
+        {
+            "data_channels": ["t2m", "dswrf"],
+            "norm_channels": ["t2m", "dswrf", "extra"],
+            "source_name": "ecmwf",
+            "expect_error": False
+        },
+        # NWP case - error
+        {
+            "data_channels": ["t2m", "missing_channel"],
+            "norm_channels": ["t2m"],
+            "source_name": "ecmwf",
+            "expect_error": True,
+            "error_match": "following channels for ecmwf are missing in normalisation means"
+        },
+        # Satellite case - success
+        {
+            "data_channels": ["IR_016", "VIS006"],
+            "norm_channels": ["IR_016", "VIS006", "extra"],
+            "source_name": "satellite",
+            "expect_error": False
+        },
+        # Satellite case - error
+        {
+            "data_channels": ["IR_016", "missing_channel"],
+            "norm_channels": ["IR_016"],
+            "source_name": "satellite",
+            "expect_error": True,
+            "error_match": "following channels for satellite are missing in normalisation means"
+        }
+    ])
+    def test_channel_validation(self, test_case):
+        """Test channel validation for both base, NWP and satellite data"""
+        if test_case["expect_error"]:
+            with pytest.raises(ValueError, match=test_case["error_match"]):
+                validate_channels(
+                    data_channels=test_case["data_channels"],
+                    means_channels=test_case["norm_channels"],
+                    stds_channels=test_case["norm_channels"],
+                    source_name=test_case["source_name"]
+                )
+        else:
+            # Should not raise any exceptions
+            validate_channels(
+                data_channels=test_case["data_channels"],
+                means_channels=test_case["norm_channels"],
+                stds_channels=test_case["norm_channels"],
+                source_name=test_case["source_name"]
+            )

ocf_data_sampler-0.1.6/ocf_data_sampler/sample/base.py

@@ -1,44 +0,0 @@
-"""
-Base class definition - abstract
-Handling of both flat and nested structures - consideration for NWP
-"""
-
-import logging
-import numpy as np
-
-from pathlib import Path
-from typing import Any, Dict, Optional, Union
-from abc import ABC, abstractmethod
-
-logger = logging.getLogger(__name__)
-
-class SampleBase(ABC):
-    """ 
-    Abstract base class for all sample types 
-    Provides core data storage functionality
-    """
-
-    def __init__(self):
-        """ Initialise data container """
-        logger.debug("Initialising SampleBase instance")
-
-    @abstractmethod
-    def to_numpy(self) -> Dict[str, Any]:
-        """ Convert data to a numpy array representation """
-        raise NotImplementedError
-
-    @abstractmethod
-    def plot(self, **kwargs) -> None:
-        """ Abstract method for plotting """
-        raise NotImplementedError
-
-    @abstractmethod
-    def save(self, path: Union[str, Path]) -> None:
-        """ Abstract method for saving sample data """
-        raise NotImplementedError
-
-    @classmethod
-    @abstractmethod
-    def load(cls, path: Union[str, Path]) -> 'SampleBase':
-        """ Abstract class method for loading sample data """
-        raise NotImplementedError