ocf-data-sampler 0.0.35__tar.gz → 0.0.37__tar.gz

{ocf_data_sampler-0.0.35/ocf_data_sampler.egg-info → ocf_data_sampler-0.0.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.35
+Version: 0.0.37
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org
@@ -56,7 +56,7 @@ Requires-Dist: mkdocs-material>=8.0; extra == "docs"
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-6-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-7-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -129,6 +129,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/peterdudfield"><img src="https://avatars.githubusercontent.com/u/34686298?v=4?s=100" width="100px;" alt="Peter Dudfield"/><br /><sub><b>Peter Dudfield</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=peterdudfield" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/VikramsDataScience"><img src="https://avatars.githubusercontent.com/u/45002417?v=4?s=100" width="100px;" alt="Vikram Pande"/><br /><sub><b>Vikram Pande</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=VikramsDataScience" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/SophiaLi20"><img src="https://avatars.githubusercontent.com/u/163532536?v=4?s=100" width="100px;" alt="Unnati Bhardwaj"/><br /><sub><b>Unnati Bhardwaj</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=SophiaLi20" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/alirashidAR"><img src="https://avatars.githubusercontent.com/u/110668489?v=4?s=100" width="100px;" alt="Ali Rashid"/><br /><sub><b>Ali Rashid</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=alirashidAR" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.0.35 → ocf_data_sampler-0.0.37}/README.md

@@ -1,7 +1,7 @@
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-6-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-7-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -74,6 +74,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/peterdudfield"><img src="https://avatars.githubusercontent.com/u/34686298?v=4?s=100" width="100px;" alt="Peter Dudfield"/><br /><sub><b>Peter Dudfield</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=peterdudfield" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/VikramsDataScience"><img src="https://avatars.githubusercontent.com/u/45002417?v=4?s=100" width="100px;" alt="Vikram Pande"/><br /><sub><b>Vikram Pande</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=VikramsDataScience" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/SophiaLi20"><img src="https://avatars.githubusercontent.com/u/163532536?v=4?s=100" width="100px;" alt="Unnati Bhardwaj"/><br /><sub><b>Unnati Bhardwaj</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=SophiaLi20" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/alirashidAR"><img src="https://avatars.githubusercontent.com/u/110668489?v=4?s=100" width="100px;" alt="Ali Rashid"/><br /><sub><b>Ali Rashid</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=alirashidAR" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

ocf_data_sampler-0.0.37/ocf_data_sampler/numpy_batch/collate.py

@@ -0,0 +1,79 @@
+from ocf_data_sampler.numpy_batch import NWPBatchKey
+
+import numpy as np
+import logging
+from typing import Union
+
+logger = logging.getLogger(__name__)
+
+
+def stack_np_examples_into_batch(dict_list):
+    """
+    Stacks Numpy examples into a batch
+
+    See also: `unstack_np_batch_into_examples()` for opposite
+
+    Args:
+        dict_list: A list of dict-like Numpy examples to stack
+
+    Returns:
+        The stacked NumpyBatch object
+    """
+    batch = {}
+
+    batch_keys = list(dict_list[0].keys())
+
+    for batch_key in batch_keys:
+        # NWP is nested so treat separately
+        if batch_key == "nwp":
+            nwp_batch: dict[str, NWPBatchKey] = {}
+
+            # Unpack source keys
+            nwp_sources = list(dict_list[0]["nwp"].keys())
+
+            for nwp_source in nwp_sources:
+                # Keys can be different for different NWPs
+                nwp_batch_keys = list(dict_list[0]["nwp"][nwp_source].keys())
+
+                nwp_source_batch = {}
+                for nwp_batch_key in nwp_batch_keys:
+                    nwp_source_batch[nwp_batch_key] = stack_data_list(
+                        [d["nwp"][nwp_source][nwp_batch_key] for d in dict_list],
+                        nwp_batch_key,
+                    )
+
+                nwp_batch[nwp_source] = nwp_source_batch
+
+            batch["nwp"] = nwp_batch
+
+        else:
+            batch[batch_key] = stack_data_list(
+                [d[batch_key] for d in dict_list],
+                batch_key,
+            )
+
+    return batch
+
+
+def _key_is_constant(batch_key):
+    is_constant = batch_key.endswith("t0_idx") or batch_key == NWPBatchKey.channel_names
+    return is_constant
+
+
+def stack_data_list(
+    data_list: list,
+    batch_key: Union[str, NWPBatchKey],
+):
+    """How to combine data entries for each key
+    """
+    if _key_is_constant(batch_key):
+        # These are always the same for all examples.
+        return data_list[0]
+    try:
+        return np.stack(data_list)
+    except Exception as e:
+        logger.debug(f"Could not stack the following shapes together, ({batch_key})")
+        shapes = [example.shape for example in data_list]
+        logger.debug(shapes)
+        logger.error(e)
+        raise e

ocf_data_sampler-0.0.37/ocf_data_sampler/numpy_batch/site.py

@@ -0,0 +1,29 @@
+"""Convert site to Numpy Batch"""
+
+import xarray as xr
+
+
+class SiteBatchKey:
+
+    generation = "site"
+    capacity_kwp = "site_capacity_kwp"
+    time_utc = "site_time_utc"
+    t0_idx = "site_t0_idx"
+    solar_azimuth = "site_solar_azimuth"
+    solar_elevation = "site_solar_elevation"
+    id = "site_id"
+
+
+def convert_site_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
+    """Convert from Xarray to NumpyBatch"""
+
+    example = {
+        SiteBatchKey.generation: da.values,
+        SiteBatchKey.capacity_kwp: da.isel(time_utc=0)["capacity_kwp"].values,
+        SiteBatchKey.time_utc: da["time_utc"].values.astype(float),
+    }
+
+    if t0_idx is not None:
+        example[SiteBatchKey.t0_idx] = t0_idx
+
+    return example

{ocf_data_sampler-0.0.35 → ocf_data_sampler-0.0.37/ocf_data_sampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.35
+Version: 0.0.37
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org
@@ -56,7 +56,7 @@ Requires-Dist: mkdocs-material>=8.0; extra == "docs"
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-6-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-7-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -129,6 +129,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/peterdudfield"><img src="https://avatars.githubusercontent.com/u/34686298?v=4?s=100" width="100px;" alt="Peter Dudfield"/><br /><sub><b>Peter Dudfield</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=peterdudfield" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/VikramsDataScience"><img src="https://avatars.githubusercontent.com/u/45002417?v=4?s=100" width="100px;" alt="Vikram Pande"/><br /><sub><b>Vikram Pande</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=VikramsDataScience" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/SophiaLi20"><img src="https://avatars.githubusercontent.com/u/163532536?v=4?s=100" width="100px;" alt="Unnati Bhardwaj"/><br /><sub><b>Unnati Bhardwaj</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=SophiaLi20" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/alirashidAR"><img src="https://avatars.githubusercontent.com/u/110668489?v=4?s=100" width="100px;" alt="Ali Rashid"/><br /><sub><b>Ali Rashid</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=alirashidAR" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.0.35 → ocf_data_sampler-0.0.37}/ocf_data_sampler.egg-info/SOURCES.txt

@@ -28,6 +28,7 @@ ocf_data_sampler/load/nwp/providers/ecmwf.py
 ocf_data_sampler/load/nwp/providers/ukv.py
 ocf_data_sampler/load/nwp/providers/utils.py
 ocf_data_sampler/numpy_batch/__init__.py
+ocf_data_sampler/numpy_batch/collate.py
 ocf_data_sampler/numpy_batch/gsp.py
 ocf_data_sampler/numpy_batch/nwp.py
 ocf_data_sampler/numpy_batch/satellite.py
@@ -56,6 +57,7 @@ tests/load/test_load_gsp.py
 tests/load/test_load_nwp.py
 tests/load/test_load_satellite.py
 tests/load/test_load_sites.py
+tests/numpy_batch/test_collate.py
 tests/numpy_batch/test_gsp.py
 tests/numpy_batch/test_nwp.py
 tests/numpy_batch/test_satellite.py

{ocf_data_sampler-0.0.35 → ocf_data_sampler-0.0.37}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ocf_data_sampler"
-version = "0.0.35"
+version = "0.0.37"
 license = { file = "LICENSE" }
 readme = "README.md"
 description = "Sample from weather data for renewable energy prediction"

{ocf_data_sampler-0.0.35 → ocf_data_sampler-0.0.37}/tests/conftest.py

@@ -7,6 +7,7 @@ import xarray as xr
 import tempfile
 
 from ocf_data_sampler.config.model import Site
+from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configuration
 
 _top_test_directory = os.path.dirname(os.path.realpath(__file__))
 
@@ -269,3 +270,18 @@ def uk_gsp_zarr_path(ds_uk_gsp):
         ds_uk_gsp.to_zarr(filename)
         yield filename
 
+
+@pytest.fixture()
+def pvnet_config_filename(
+    tmp_path, config_filename, nwp_ukv_zarr_path, uk_gsp_zarr_path, sat_zarr_path
+):
+
+    # adjust config to point to the zarr file
+    config = load_yaml_configuration(config_filename)
+    config.input_data.nwp["ukv"].zarr_path = nwp_ukv_zarr_path
+    config.input_data.satellite.zarr_path = sat_zarr_path
+    config.input_data.gsp.zarr_path = uk_gsp_zarr_path
+
+    filename = f"{tmp_path}/configuration.yaml"
+    save_yaml_configuration(config, filename)
+    return filename

ocf_data_sampler-0.0.37/tests/numpy_batch/test_collate.py

@@ -0,0 +1,26 @@
+from ocf_data_sampler.numpy_batch import GSPBatchKey, SatelliteBatchKey
+from ocf_data_sampler.numpy_batch.collate import stack_np_examples_into_batch
+from ocf_data_sampler.torch_datasets import PVNetUKRegionalDataset
+
+
+def test_pvnet(pvnet_config_filename):
+
+    # Create dataset object
+    dataset = PVNetUKRegionalDataset(pvnet_config_filename)
+
+    assert len(dataset.locations) == 317
+    assert len(dataset.valid_t0_times) == 39
+    assert len(dataset) == 317 * 39
+
+    # Generate 2 samples
+    sample1 = dataset[0]
+    sample2 = dataset[1]
+
+    batch = stack_np_examples_into_batch([sample1, sample2])
+
+    assert isinstance(batch, dict)
+    assert "nwp" in batch
+    assert isinstance(batch["nwp"], dict)
+    assert "ukv" in batch["nwp"]
+    assert GSPBatchKey.gsp in batch
+    assert SatelliteBatchKey.satellite_actual in batch

{ocf_data_sampler-0.0.35 → ocf_data_sampler-0.0.37}/tests/torch_datasets/test_pvnet_uk_regional.py

@@ -6,19 +6,6 @@ from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configura
 from ocf_data_sampler.numpy_batch import NWPBatchKey, GSPBatchKey, SatelliteBatchKey
 
 
-@pytest.fixture()
-def pvnet_config_filename(tmp_path, config_filename, nwp_ukv_zarr_path, uk_gsp_zarr_path, sat_zarr_path):
-
-    # adjust config to point to the zarr file
-    config = load_yaml_configuration(config_filename)
-    config.input_data.nwp['ukv'].zarr_path = nwp_ukv_zarr_path
-    config.input_data.satellite.zarr_path = sat_zarr_path
-    config.input_data.gsp.zarr_path = uk_gsp_zarr_path
-
-    filename = f"{tmp_path}/configuration.yaml"
-    save_yaml_configuration(config, filename)
-    return filename
-
 
 def test_pvnet(pvnet_config_filename):
 

ocf_data_sampler-0.0.35/ocf_data_sampler/numpy_batch/site.py

@@ -1,29 +0,0 @@
-"""Convert site to Numpy Batch"""
-
-import xarray as xr
-
-
-class SiteBatchKey:
-
-    generation = "site"
-    site_capacity_kwp = "site_capacity_kwp"
-    site_time_utc = "site_time_utc"
-    site_t0_idx = "site_t0_idx"
-    site_solar_azimuth = "site_solar_azimuth"
-    site_solar_elevation = "site_solar_elevation"
-    site_id = "site_id"
-
-
-def convert_site_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
-    """Convert from Xarray to NumpyBatch"""
-
-    example = {
-        SiteBatchKey.generation: da.values,
-        SiteBatchKey.site_capacity_kwp: da.isel(time_utc=0)["capacity_kwp"].values,
-        SiteBatchKey.site_time_utc: da["time_utc"].values.astype(float),
-    }
-
-    if t0_idx is not None:
-        example[SiteBatchKey.site_t0_idx] = t0_idx
-
-    return example