ocf-data-sampler 0.0.51__tar.gz → 0.0.53__tar.gz

{ocf_data_sampler-0.0.51/ocf_data_sampler.egg-info → ocf_data_sampler-0.0.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.0.51
+Version: 0.0.53
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org
@@ -56,7 +56,7 @@ Requires-Dist: mkdocs-material>=8.0; extra == "docs"
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -136,6 +136,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://timothyajaniportfolio-b6v3zq29k-timthegreat.vercel.app/"><img src="https://avatars.githubusercontent.com/u/60073728?v=4?s=100" width="100px;" alt="Ajani Timothy"/><br /><sub><b>Ajani Timothy</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Tim1119" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://rupeshmangalam.vercel.app/"><img src="https://avatars.githubusercontent.com/u/91172425?v=4?s=100" width="100px;" alt="Rupesh Mangalam"/><br /><sub><b>Rupesh Mangalam</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=RupeshMangalam21" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://siddharth7113.github.io"><img src="https://avatars.githubusercontent.com/u/114160268?v=4?s=100" width="100px;" alt="Siddharth"/><br /><sub><b>Siddharth</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=siddharth7113" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sachin-G13"><img src="https://avatars.githubusercontent.com/u/190184500?v=4?s=100" width="100px;" alt="Sachin-G13"/><br /><sub><b>Sachin-G13</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sachin-G13" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.0.51 → ocf_data_sampler-0.0.53}/README.md

@@ -1,7 +1,7 @@
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -81,6 +81,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://timothyajaniportfolio-b6v3zq29k-timthegreat.vercel.app/"><img src="https://avatars.githubusercontent.com/u/60073728?v=4?s=100" width="100px;" alt="Ajani Timothy"/><br /><sub><b>Ajani Timothy</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Tim1119" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://rupeshmangalam.vercel.app/"><img src="https://avatars.githubusercontent.com/u/91172425?v=4?s=100" width="100px;" alt="Rupesh Mangalam"/><br /><sub><b>Rupesh Mangalam</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=RupeshMangalam21" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://siddharth7113.github.io"><img src="https://avatars.githubusercontent.com/u/114160268?v=4?s=100" width="100px;" alt="Siddharth"/><br /><sub><b>Siddharth</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=siddharth7113" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sachin-G13"><img src="https://avatars.githubusercontent.com/u/190184500?v=4?s=100" width="100px;" alt="Sachin-G13"/><br /><sub><b>Sachin-G13</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sachin-G13" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.0.51 → ocf_data_sampler-0.0.53}/ocf_data_sampler/constants.py

@@ -5,6 +5,7 @@ import numpy as np
 NWP_PROVIDERS = [
     "ukv",
     "ecmwf",
+    "gfs"
 ]
 
 
@@ -127,13 +128,60 @@ ECMWF_MEAN = {
 ECMWF_STD = _to_data_array(ECMWF_STD)
 ECMWF_MEAN = _to_data_array(ECMWF_MEAN)
 
+# ------ GFS
+GFS_STD = {
+    "dlwrf": 96.305916,
+    "dswrf": 246.18533,
+    "hcc": 42.525383,
+    "lcc": 44.3732,
+    "mcc": 43.150745,
+    "prate": 0.00010159573,
+    "r": 25.440672,
+    "sde": 0.43345627,
+    "t": 22.825893,
+    "tcc": 41.030598,
+    "u10": 5.470838,
+    "u100": 6.8899174,
+    "v10": 4.7401133,
+    "v100": 6.076132,
+    "vis": 8294.022,
+    "u": 10.614556,
+    "v": 7.176398,
+}
+
+GFS_MEAN = {
+    "dlwrf": 298.342,
+    "dswrf": 168.12321,
+    "hcc": 35.272,
+    "lcc": 43.578342,
+    "mcc": 33.738823,
+    "prate": 2.8190969e-05,
+    "r": 18.359747,
+    "sde": 0.36937004,
+    "t": 278.5223,
+    "tcc": 66.841606,
+    "u10": -0.0022310058,
+    "u100": 0.0823025,
+    "v10": 0.06219831,
+    "v100": 0.0797807,
+    "vis": 19628.32,
+    "u": 11.645444,
+    "v": 0.12330122,
+}
+
+GFS_STD = _to_data_array(GFS_STD)
+GFS_MEAN = _to_data_array(GFS_MEAN)
+
+
 NWP_STDS = NWPStatDict(
     ukv=UKV_STD,
     ecmwf=ECMWF_STD,
+    gfs=GFS_STD
 )
 NWP_MEANS = NWPStatDict(
     ukv=UKV_MEAN,
     ecmwf=ECMWF_MEAN,
+    gfs=GFS_MEAN
 )
 
 # ------ Satellite

{ocf_data_sampler-0.0.51 → ocf_data_sampler-0.0.53}/ocf_data_sampler/load/utils.py

@@ -40,4 +40,4 @@ def get_xr_data_array_from_xr_dataset(ds: xr.Dataset) -> xr.DataArray:
 
     datavars = list(ds.var())
     assert len(datavars) == 1, "Cannot open as xr.DataArray: dataset contains multiple variables"
-    return ds[datavars[0]]
+    return ds[datavars[0]]

{ocf_data_sampler-0.0.51 → ocf_data_sampler-0.0.53}/ocf_data_sampler/torch_datasets/datasets/site.py

@@ -1,6 +1,6 @@
 """Torch dataset for sites"""
 import logging
-
+import numpy as np
 import pandas as pd
 import xarray as xr
 from typing import Tuple
@@ -421,3 +421,27 @@ def convert_to_numpy_and_combine(
     combined_sample = fill_nans_in_arrays(combined_sample)
 
     return combined_sample
+
+
+def coarsen_data(xr_data: xr.Dataset, coarsen_to_deg: float=0.1):
+    """
+    Coarsen the data to a specified resolution in degrees.
+    
+    Args:
+        xr_data: xarray dataset to coarsen
+        coarsen_to_deg: resolution to coarsen to in degrees
+    """
+
+    if "latitude" in xr_data.coords and "longitude" in xr_data.coords:
+        step = np.abs(xr_data.latitude.values[1]-xr_data.latitude.values[0])
+        step = np.round(step,4)
+        coarsen_factor = int(coarsen_to_deg/step)
+        if coarsen_factor > 1:
+            xr_data = xr_data.coarsen(
+                latitude=coarsen_factor,
+                longitude=coarsen_factor,
+                boundary="pad",
+                coord_func="min"
+            ).mean()
+        
+    return xr_data

{ocf_data_sampler-0.0.51 → ocf_data_sampler-0.0.53/ocf_data_sampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.0.51
+Version: 0.0.53
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org
@@ -56,7 +56,7 @@ Requires-Dist: mkdocs-material>=8.0; extra == "docs"
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -136,6 +136,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://timothyajaniportfolio-b6v3zq29k-timthegreat.vercel.app/"><img src="https://avatars.githubusercontent.com/u/60073728?v=4?s=100" width="100px;" alt="Ajani Timothy"/><br /><sub><b>Ajani Timothy</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Tim1119" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://rupeshmangalam.vercel.app/"><img src="https://avatars.githubusercontent.com/u/91172425?v=4?s=100" width="100px;" alt="Rupesh Mangalam"/><br /><sub><b>Rupesh Mangalam</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=RupeshMangalam21" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://siddharth7113.github.io"><img src="https://avatars.githubusercontent.com/u/114160268?v=4?s=100" width="100px;" alt="Siddharth"/><br /><sub><b>Siddharth</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=siddharth7113" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sachin-G13"><img src="https://avatars.githubusercontent.com/u/190184500?v=4?s=100" width="100px;" alt="Sachin-G13"/><br /><sub><b>Sachin-G13</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sachin-G13" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.0.51 → ocf_data_sampler-0.0.53}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ocf_data_sampler"
-version = "0.0.51"
+version = "0.0.53"
 license = { file = "LICENSE" }
 readme = "README.md"
 description = "Sample from weather data for renewable energy prediction"

{ocf_data_sampler-0.0.51 → ocf_data_sampler-0.0.53}/tests/torch_datasets/test_site.py

@@ -1,7 +1,8 @@
 import pandas as pd
 import numpy as np
-from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset, convert_from_dataset_to_dict_datasets
+from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset, convert_from_dataset_to_dict_datasets, coarsen_data
 from xarray import Dataset, DataArray
+import xarray as xr
 
 from torch.utils.data import DataLoader
 
@@ -43,7 +44,6 @@ def test_site(site_config_filename):
 
     expected_data_vars = {"nwp-ukv", "satellite", "site"}
 
-    import xarray as xr
 
     sample.to_netcdf("sample.nc")
     sample = xr.open_dataset("sample.nc")
@@ -198,3 +198,18 @@ def test_process_and_combine_site_sample_dict(site_config_filename):
     assert nwp_result.shape == (4, 1, 2, 2), f"Unexpected shape for nwp-ukv : {nwp_result.shape}"
     site_result = result["site"]
     assert site_result.shape == (197,), f"Unexpected shape for site: {site_result.shape}"
+
+
+def test_potentially_coarsen(ds_nwp_ecmwf):
+    """Test potentially_coarsen function with ECMWF_UK data."""
+    nwp_data = ds_nwp_ecmwf
+    assert nwp_data.ECMWF_UK.shape[3:] == (15, 12)  # Check initial shape (lon, lat)
+
+    data = coarsen_data(xr_data=nwp_data, coarsen_to_deg=2)
+    assert data.ECMWF_UK.shape[3:] == (8, 6)  # Coarsen to every 2 degrees
+
+    data = coarsen_data(xr_data=nwp_data, coarsen_to_deg=3)
+    assert data.ECMWF_UK.shape[3:] == (5, 4)  # Coarsen to every 3 degrees
+
+    data = coarsen_data(xr_data=nwp_data, coarsen_to_deg=1)
+    assert data.ECMWF_UK.shape[3:] == (15, 12)  # No coarsening (same shape)