ocf-data-sampler 0.1.8__tar.gz → 0.1.10__tar.gz

{ocf_data_sampler-0.1.8/ocf_data_sampler.egg-info → ocf_data_sampler-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.8
+Version: 0.1.10
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.8 → ocf_data_sampler-0.1.10}/ocf_data_sampler/sample/base.py

@@ -73,3 +73,26 @@ def batch_to_tensor(batch: NumpyBatch) -> TensorBatch:
             elif np.issubdtype(v.dtype, np.number):
                 batch[k] = torch.as_tensor(v)
     return batch
+
+
+def copy_batch_to_device(batch: dict, device: torch.device) -> dict:
+    """
+    Moves tensor leaves in a nested dict to a new device.
+
+    Args:
+        batch: Nested dict with tensors to move.
+        device: Device to move tensors to.
+
+    Returns:
+        A dict with tensors moved to the new device.
+    """
+    batch_copy = {}
+
+    for k, v in batch.items():
+        if isinstance(v, dict):
+            batch_copy[k] = copy_batch_to_device(v, device)  
+        elif isinstance(v, torch.Tensor):
+            batch_copy[k] = v.to(device)
+        else:
+            batch_copy[k] = v
+    return batch_copy

{ocf_data_sampler-0.1.8 → ocf_data_sampler-0.1.10}/ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -186,9 +186,8 @@ class PVNetUKRegionalDataset(Dataset):
             gsp_ids: List of GSP IDs to create samples for. Defaults to all
         """
         
-        config = load_yaml_configuration(config_filename)
-
-        # Validate channels for NWP and satellite data
+        # config = load_yaml_configuration(config_filename)
+        config: Configuration = load_yaml_configuration(config_filename)
         validate_nwp_channels(config)
         validate_satellite_channels(config)
 

{ocf_data_sampler-0.1.8 → ocf_data_sampler-0.1.10}/ocf_data_sampler/torch_datasets/datasets/site.py

@@ -20,7 +20,6 @@ from ocf_data_sampler.select import (
 from ocf_data_sampler.utils import minutes
 from ocf_data_sampler.torch_datasets.utils.valid_time_periods import find_valid_time_periods
 from ocf_data_sampler.torch_datasets.utils.merge_and_fill_utils import merge_dicts, fill_nans_in_arrays
-from ocf_data_sampler.torch_datasets.utils.validate_channels import validate_nwp_channels
 
 from ocf_data_sampler.numpy_sample import (
     convert_site_to_numpy_sample, 
@@ -30,8 +29,12 @@ from ocf_data_sampler.numpy_sample import (
     make_sun_position_numpy_sample,
 )
 from ocf_data_sampler.numpy_sample import NWPSampleKey
-from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS
+from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS, RSS_MEAN, RSS_STD
 
+from ocf_data_sampler.torch_datasets.utils.validate_channels import (
+    validate_nwp_channels,
+    validate_satellite_channels,
+)
 
 xr.set_options(keep_attrs=True)
 
@@ -52,9 +55,8 @@ class SitesDataset(Dataset):
         """
 
         config: Configuration = load_yaml_configuration(config_filename)
-
-        # Validate NWP channels 
         validate_nwp_channels(config)
+        validate_satellite_channels(config)
 
         datasets_dict = get_dataset_dict(config.input_data)
 
@@ -237,8 +239,10 @@ class SitesDataset(Dataset):
                 data_arrays.append((f"nwp-{provider}", da_nwp))
             
         if "sat" in dataset_dict:
-            # TODO add some satellite normalisation
             da_sat = dataset_dict["sat"]
+
+            # Standardise
+            da_sat = (da_sat - RSS_MEAN) / RSS_STD
             data_arrays.append(("satellite", da_sat))
 
         if "site" in dataset_dict:

{ocf_data_sampler-0.1.8 → ocf_data_sampler-0.1.10/ocf_data_sampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.8
+Version: 0.1.10
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.8 → ocf_data_sampler-0.1.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ocf_data_sampler"
-version = "0.1.8"
+version = "0.1.10"
 license = { file = "LICENSE" }
 readme = "README.md"
 description = "Sample from weather data for renewable energy prediction"

{ocf_data_sampler-0.1.8 → ocf_data_sampler-0.1.10}/scripts/refactor_site.py

@@ -1,5 +1,5 @@
 """ Helper functions for refactoring legacy site data """
-
+import xarray as xr
 
 def legacy_format(data_ds, metadata_df):
     """This formats old legacy data to the new format.

{ocf_data_sampler-0.1.8 → ocf_data_sampler-0.1.10}/tests/test_sample/test_base.py

@@ -9,7 +9,8 @@ import numpy as np
 from pathlib import Path
 from ocf_data_sampler.sample.base import (
     SampleBase,
-    batch_to_tensor
+    batch_to_tensor,
+    copy_batch_to_device
 )
 
 class TestSample(SampleBase):
@@ -145,3 +146,19 @@ def test_batch_to_tensor_multidimensional():
     assert tensor_batch['matrix'].shape == (2, 2)
     assert tensor_batch['tensor'].shape == (2, 2, 2)
     assert torch.equal(tensor_batch['matrix'], torch.tensor([[1, 2], [3, 4]]))
+
+
+def test_copy_batch_to_device():
+    """ Test moving tensors to a different device """
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    batch = {
+        'tensor_data': torch.tensor([1, 2, 3]),
+        'nested': {
+            'matrix': torch.tensor([[1, 2], [3, 4]])
+        },
+        'non_tensor': 'unchanged'
+    }
+    moved_batch = copy_batch_to_device(batch, device)
+    assert moved_batch['tensor_data'].device == device
+    assert moved_batch['nested']['matrix'].device == device
+    assert moved_batch['non_tensor'] == 'unchanged'  # Non-tensors should remain unchanged