ocf-data-sampler 0.0.51__py3-none-any.whl → 0.0.53__py3-none-any.whl

ocf_data_sampler/constants.py

@@ -5,6 +5,7 @@ import numpy as np
 NWP_PROVIDERS = [
     "ukv",
     "ecmwf",
+    "gfs"
 ]
 
 
@@ -127,13 +128,60 @@ ECMWF_MEAN = {
 ECMWF_STD = _to_data_array(ECMWF_STD)
 ECMWF_MEAN = _to_data_array(ECMWF_MEAN)
 
+# ------ GFS
+GFS_STD = {
+    "dlwrf": 96.305916,
+    "dswrf": 246.18533,
+    "hcc": 42.525383,
+    "lcc": 44.3732,
+    "mcc": 43.150745,
+    "prate": 0.00010159573,
+    "r": 25.440672,
+    "sde": 0.43345627,
+    "t": 22.825893,
+    "tcc": 41.030598,
+    "u10": 5.470838,
+    "u100": 6.8899174,
+    "v10": 4.7401133,
+    "v100": 6.076132,
+    "vis": 8294.022,
+    "u": 10.614556,
+    "v": 7.176398,
+}
+
+GFS_MEAN = {
+    "dlwrf": 298.342,
+    "dswrf": 168.12321,
+    "hcc": 35.272,
+    "lcc": 43.578342,
+    "mcc": 33.738823,
+    "prate": 2.8190969e-05,
+    "r": 18.359747,
+    "sde": 0.36937004,
+    "t": 278.5223,
+    "tcc": 66.841606,
+    "u10": -0.0022310058,
+    "u100": 0.0823025,
+    "v10": 0.06219831,
+    "v100": 0.0797807,
+    "vis": 19628.32,
+    "u": 11.645444,
+    "v": 0.12330122,
+}
+
+GFS_STD = _to_data_array(GFS_STD)
+GFS_MEAN = _to_data_array(GFS_MEAN)
+
+
 NWP_STDS = NWPStatDict(
     ukv=UKV_STD,
     ecmwf=ECMWF_STD,
+    gfs=GFS_STD
 )
 NWP_MEANS = NWPStatDict(
     ukv=UKV_MEAN,
     ecmwf=ECMWF_MEAN,
+    gfs=GFS_MEAN
 )
 
 # ------ Satellite

ocf_data_sampler/load/utils.py

@@ -40,4 +40,4 @@ def get_xr_data_array_from_xr_dataset(ds: xr.Dataset) -> xr.DataArray:
 
     datavars = list(ds.var())
     assert len(datavars) == 1, "Cannot open as xr.DataArray: dataset contains multiple variables"
-    return ds[datavars[0]]
+    return ds[datavars[0]]

ocf_data_sampler/torch_datasets/datasets/site.py

@@ -1,6 +1,6 @@
 """Torch dataset for sites"""
 import logging
-
+import numpy as np
 import pandas as pd
 import xarray as xr
 from typing import Tuple
@@ -421,3 +421,27 @@ def convert_to_numpy_and_combine(
     combined_sample = fill_nans_in_arrays(combined_sample)
 
     return combined_sample
+
+
+def coarsen_data(xr_data: xr.Dataset, coarsen_to_deg: float=0.1):
+    """
+    Coarsen the data to a specified resolution in degrees.
+    
+    Args:
+        xr_data: xarray dataset to coarsen
+        coarsen_to_deg: resolution to coarsen to in degrees
+    """
+
+    if "latitude" in xr_data.coords and "longitude" in xr_data.coords:
+        step = np.abs(xr_data.latitude.values[1]-xr_data.latitude.values[0])
+        step = np.round(step,4)
+        coarsen_factor = int(coarsen_to_deg/step)
+        if coarsen_factor > 1:
+            xr_data = xr_data.coarsen(
+                latitude=coarsen_factor,
+                longitude=coarsen_factor,
+                boundary="pad",
+                coord_func="min"
+            ).mean()
+        
+    return xr_data

{ocf_data_sampler-0.0.51.dist-info → ocf_data_sampler-0.0.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.0.51
+Version: 0.0.53
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org
@@ -56,7 +56,7 @@ Requires-Dist: mkdocs-material>=8.0; extra == "docs"
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -136,6 +136,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://timothyajaniportfolio-b6v3zq29k-timthegreat.vercel.app/"><img src="https://avatars.githubusercontent.com/u/60073728?v=4?s=100" width="100px;" alt="Ajani Timothy"/><br /><sub><b>Ajani Timothy</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Tim1119" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://rupeshmangalam.vercel.app/"><img src="https://avatars.githubusercontent.com/u/91172425?v=4?s=100" width="100px;" alt="Rupesh Mangalam"/><br /><sub><b>Rupesh Mangalam</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=RupeshMangalam21" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://siddharth7113.github.io"><img src="https://avatars.githubusercontent.com/u/114160268?v=4?s=100" width="100px;" alt="Siddharth"/><br /><sub><b>Siddharth</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=siddharth7113" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sachin-G13"><img src="https://avatars.githubusercontent.com/u/190184500?v=4?s=100" width="100px;" alt="Sachin-G13"/><br /><sub><b>Sachin-G13</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sachin-G13" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.0.51.dist-info → ocf_data_sampler-0.0.53.dist-info}/RECORD

@@ -1,5 +1,5 @@
 ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/constants.py,sha256=G2VfkE_-veq_0hNBQQOQCtCsfC37O5-QG9mJWEmln5s,4153
+ocf_data_sampler/constants.py,sha256=S4-pkGdKj9HxQcMpI0iL90Ob_37Dloejn3c-niqdar0,5063
 ocf_data_sampler/utils.py,sha256=rKA0BHAyAG4f90zEcgxp25EEYrXS-aOVNzttZ6Mzv2k,250
 ocf_data_sampler/config/__init__.py,sha256=YXnAkgHViHB26hSsjiv32b6EbpG-A1kKTkARJf0_RkY,212
 ocf_data_sampler/config/load.py,sha256=4f7vPHAIAmd-55tPxoIzn7F_TI_ue4NxkDcLPoVWl0g,943
@@ -11,7 +11,7 @@ ocf_data_sampler/load/gsp.py,sha256=Gcr1JVUOPKhFRDCSHtfPDjxx0BtyyEhXrZvGEKLPJ5I,
 ocf_data_sampler/load/load_dataset.py,sha256=Ua3RaUg4PIYJkD9BKqTfN8IWUbezbhThJGgEkd9PcaE,1587
 ocf_data_sampler/load/satellite.py,sha256=3KlA1fx4SwxdzM-jC1WRaONXO0D6m0WxORnEnwUnZrA,2967
 ocf_data_sampler/load/site.py,sha256=P83uz01WBDzoZajdOH0m8FQt4-buKDlUk19N548KqhA,1086
-ocf_data_sampler/load/utils.py,sha256=EQGvVWlGMoSOdbDYuMfVAa0v6wmAOPmHIAemdrTB5v4,1406
+ocf_data_sampler/load/utils.py,sha256=sAEkPMS9LXVCrc5pANQo97zaoEItVg9hoNj2ZWfx_Ug,1405
 ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
 ocf_data_sampler/load/nwp/nwp.py,sha256=O4QnajEZem8BvBgTcYYDBhRhgqPYuJkolHmpMRmrXEA,610
 ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -38,7 +38,7 @@ ocf_data_sampler/select/spatial_slice_for_dataset.py,sha256=3tRrMBXr7s4CnClbVSIq
 ocf_data_sampler/select/time_slice_for_dataset.py,sha256=P7cAARfDzjttGDvpKt2zuA4WkLoTmSXy_lBpI8RiA6k,4249
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=nJUa2KzVa84ZoM0PT2AbDz26ennmAYc7M7WJVfypPMs,85
 ocf_data_sampler/torch_datasets/datasets/pvnet_uk_regional.py,sha256=xxeX4Js9LQpydehi3BS7k9psqkYGzgJuM17uTYux40M,8742
-ocf_data_sampler/torch_datasets/datasets/site.py,sha256=v7plMF_WJPkfwnJAUFf_8gXAy8SXE5Og_fgZMEm4c20,15257
+ocf_data_sampler/torch_datasets/datasets/site.py,sha256=5T8nkTMUHHFidZRuFOunYeKAqNuyZ8V7sikBoBOBwwA,16033
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=hIbekql64eXsNDFIoEc--GWxwdVWrh2qKegdOi70Bow,874
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=Qo65qUHtle_bW5tLTYr7empHTRv-lpjvfx_6GNJj3Xg,4371
 scripts/refactor_site.py,sha256=asZ27hQ4IyXgCCUaFJqcz1ObBNcV2W3ywqHBpSXA_fc,1728
@@ -65,9 +65,9 @@ tests/select/test_select_time_slice.py,sha256=nYrdlmZlGEygJKiE26bADiluNPN1qt5kD4
 tests/torch_datasets/conftest.py,sha256=eRCzHE7cxS4AoskExkCGFDBeqItktAYNAdkfpMoFCeE,629
 tests/torch_datasets/test_merge_and_fill_utils.py,sha256=ueA0A7gZaWEgNdsU8p3CnKuvSnlleTUjEhSw2HUUROM,1229
 tests/torch_datasets/test_pvnet_uk_regional.py,sha256=FCiFueeFqrsXe7gWguSjBz5ZeUrvyhGbGw81gaVvkHM,5087
-tests/torch_datasets/test_site.py,sha256=0gT_7k086BBnxqbvOayiUeI-vzJsYXlx3KvACC0c6lk,6114
-ocf_data_sampler-0.0.51.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.0.51.dist-info/METADATA,sha256=fBrPrERCKjQRN6HWgInZA5aibFPQLTTC_c2Xs4u921w,11788
-ocf_data_sampler-0.0.51.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ocf_data_sampler-0.0.51.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
-ocf_data_sampler-0.0.51.dist-info/RECORD,,
+tests/torch_datasets/test_site.py,sha256=J1ZDE5V5MRlq7EuZ1zUu-aFRGTDJIiO-ZZzkOXvDdWA,6757
+ocf_data_sampler-0.0.53.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
+ocf_data_sampler-0.0.53.dist-info/METADATA,sha256=KlR1y8ot0K5afAbA_0FzdY6SLVm378QYzQpiAhShI8Y,12143
+ocf_data_sampler-0.0.53.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ocf_data_sampler-0.0.53.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
+ocf_data_sampler-0.0.53.dist-info/RECORD,,

tests/torch_datasets/test_site.py

@@ -1,7 +1,8 @@
 import pandas as pd
 import numpy as np
-from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset, convert_from_dataset_to_dict_datasets
+from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset, convert_from_dataset_to_dict_datasets, coarsen_data
 from xarray import Dataset, DataArray
+import xarray as xr
 
 from torch.utils.data import DataLoader
 
@@ -43,7 +44,6 @@ def test_site(site_config_filename):
 
     expected_data_vars = {"nwp-ukv", "satellite", "site"}
 
-    import xarray as xr
 
     sample.to_netcdf("sample.nc")
     sample = xr.open_dataset("sample.nc")
@@ -198,3 +198,18 @@ def test_process_and_combine_site_sample_dict(site_config_filename):
     assert nwp_result.shape == (4, 1, 2, 2), f"Unexpected shape for nwp-ukv : {nwp_result.shape}"
     site_result = result["site"]
     assert site_result.shape == (197,), f"Unexpected shape for site: {site_result.shape}"
+
+
+def test_potentially_coarsen(ds_nwp_ecmwf):
+    """Test potentially_coarsen function with ECMWF_UK data."""
+    nwp_data = ds_nwp_ecmwf
+    assert nwp_data.ECMWF_UK.shape[3:] == (15, 12)  # Check initial shape (lon, lat)
+
+    data = coarsen_data(xr_data=nwp_data, coarsen_to_deg=2)
+    assert data.ECMWF_UK.shape[3:] == (8, 6)  # Coarsen to every 2 degrees
+
+    data = coarsen_data(xr_data=nwp_data, coarsen_to_deg=3)
+    assert data.ECMWF_UK.shape[3:] == (5, 4)  # Coarsen to every 3 degrees
+
+    data = coarsen_data(xr_data=nwp_data, coarsen_to_deg=1)
+    assert data.ECMWF_UK.shape[3:] == (15, 12)  # No coarsening (same shape)