ocf-data-sampler 0.0.43__py3-none-any.whl → 0.0.45__py3-none-any.whl

ocf_data_sampler/numpy_sample/__init__.py

@@ -1,5 +1,6 @@
 """Conversion from Xarray to NumpySample"""
 
+from .datetime_features import make_datetime_numpy_dict
 from .gsp import convert_gsp_to_numpy_sample, GSPSampleKey
 from .nwp import convert_nwp_to_numpy_sample, NWPSampleKey
 from .satellite import convert_satellite_to_numpy_sample, SatelliteSampleKey

ocf_data_sampler/numpy_sample/datetime_features.py

@@ -0,0 +1,46 @@
+"""Functions to create trigonometric date and time inputs"""
+
+import numpy as np
+import pandas as pd
+from numpy.typing import NDArray
+
+
+def _get_date_time_in_pi(
+    dt: pd.DatetimeIndex,
+) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
+    """
+    Change the datetimes, into time and date scaled in radians
+    """
+
+    day_of_year = dt.dayofyear
+    minute_of_day = dt.minute + dt.hour * 60
+
+    # converting into positions on sin-cos circle
+    time_in_pi = (2 * np.pi) * (minute_of_day / (24 * 60))
+    date_in_pi = (2 * np.pi) * (day_of_year / 365)
+
+    return date_in_pi, time_in_pi
+
+
+def make_datetime_numpy_dict(datetimes: pd.DatetimeIndex, key_prefix: str = "wind") -> dict:
+    """ Make dictionary of datetime features"""
+
+    if datetimes.empty:
+        raise ValueError("Input datetimes is empty for 'make_datetime_numpy_dict' function")
+
+    time_numpy_sample = {}
+
+    date_in_pi, time_in_pi = _get_date_time_in_pi(datetimes)
+
+    # Store
+    date_sin_batch_key = key_prefix + "_date_sin"
+    date_cos_batch_key = key_prefix + "_date_cos"
+    time_sin_batch_key = key_prefix + "_time_sin"
+    time_cos_batch_key = key_prefix + "_time_cos"
+
+    time_numpy_sample[date_sin_batch_key] = np.sin(date_in_pi)
+    time_numpy_sample[date_cos_batch_key] = np.cos(date_in_pi)
+    time_numpy_sample[time_sin_batch_key] = np.sin(time_in_pi)
+    time_numpy_sample[time_cos_batch_key] = np.cos(time_in_pi)
+
+    return time_numpy_sample

ocf_data_sampler/numpy_sample/gsp.py

@@ -21,7 +21,7 @@ def convert_gsp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) ->
     """Convert from Xarray to NumpySample"""
 
    # Extract values from the DataArray
-    example = {
+    sample = {
         GSPSampleKey.gsp: da.values,
         GSPSampleKey.nominal_capacity_mwp: da.isel(time_utc=0)["nominal_capacity_mwp"].values,
         GSPSampleKey.effective_capacity_mwp: da.isel(time_utc=0)["effective_capacity_mwp"].values,
@@ -29,6 +29,6 @@ def convert_gsp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) ->
     }
 
     if t0_idx is not None:
-        example[GSPSampleKey.t0_idx] = t0_idx
+        sample[GSPSampleKey.t0_idx] = t0_idx
 
-    return example
+    return sample

ocf_data_sampler/numpy_sample/nwp.py

@@ -21,7 +21,7 @@ def convert_nwp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) ->
     """Convert from Xarray to NWP NumpySample"""
     
     # Create example and add t if available
-    example = {
+    sample = {
         NWPSampleKey.nwp: da.values,
         NWPSampleKey.channel_names: da.channel.values,
         NWPSampleKey.init_time_utc: da.init_time_utc.values.astype(float),
@@ -29,14 +29,14 @@ def convert_nwp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) ->
     }
 
     if "target_time_utc" in da.coords:
-        example[NWPSampleKey.target_time_utc] = da.target_time_utc.values.astype(float)
+        sample[NWPSampleKey.target_time_utc] = da.target_time_utc.values.astype(float)
 
     # TODO: Do we need this at all? Especially since it is only present in UKV data
     for sample_key, dataset_key in ((NWPSampleKey.y_osgb, "y_osgb"),(NWPSampleKey.x_osgb, "x_osgb"),):
         if dataset_key in da.coords:
-            example[sample_key] = da[dataset_key].values
+            sample[sample_key] = da[dataset_key].values
 
     if t0_idx is not None:
-        example[NWPSampleKey.t0_idx] = t0_idx
+        sample[NWPSampleKey.t0_idx] = t0_idx
         
-    return example
+    return sample

ocf_data_sampler/numpy_sample/satellite.py

@@ -13,7 +13,7 @@ class SatelliteSampleKey:
 
 def convert_satellite_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
     """Convert from Xarray to NumpySample"""
-    example = {
+    sample = {
         SatelliteSampleKey.satellite_actual: da.values,
         SatelliteSampleKey.time_utc: da.time_utc.values.astype(float),
     }
@@ -22,9 +22,9 @@ def convert_satellite_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = Non
          (SatelliteSampleKey.x_geostationary, "x_geostationary"),
         (SatelliteSampleKey.y_geostationary, "y_geostationary"),
     ):
-        example[sample_key] = da[dataset_key].values
+        sample[sample_key] = da[dataset_key].values
 
     if t0_idx is not None:
-        example[SatelliteSampleKey.t0_idx] = t0_idx
+        sample[SatelliteSampleKey.t0_idx] = t0_idx
 
-    return example
+    return sample

ocf_data_sampler/numpy_sample/site.py

@@ -18,13 +18,13 @@ def convert_site_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) ->
     """Convert from Xarray to NumpySample"""
 
     # Extract values from the DataArray
-    example = {
+    sample = {
         SiteSampleKey.generation: da.values,
         SiteSampleKey.capacity_kwp: da.isel(time_utc=0)["capacity_kwp"].values,
         SiteSampleKey.time_utc: da["time_utc"].values.astype(float),
     }
 
     if t0_idx is not None:
-        example[SiteSampleKey.t0_idx] = t0_idx
+        sample[SiteSampleKey.t0_idx] = t0_idx
 
-    return example
+    return sample

ocf_data_sampler/torch_datasets/process_and_combine.py

@@ -10,6 +10,7 @@ from ocf_data_sampler.numpy_sample import (
     convert_satellite_to_numpy_sample,
     convert_gsp_to_numpy_sample,
     make_sun_position_numpy_sample,
+
 )
 from ocf_data_sampler.numpy_sample.gsp import GSPSampleKey
 from ocf_data_sampler.numpy_sample.nwp import NWPSampleKey

ocf_data_sampler/torch_datasets/site.py

@@ -22,7 +22,9 @@ from ocf_data_sampler.torch_datasets.process_and_combine import merge_dicts, fil
 from ocf_data_sampler.numpy_sample import (
     convert_site_to_numpy_sample, 
     convert_satellite_to_numpy_sample, 
-    convert_nwp_to_numpy_sample
+    convert_nwp_to_numpy_sample,
+    make_datetime_numpy_dict,
+    make_sun_position_numpy_sample,
 )
 from ocf_data_sampler.numpy_sample import NWPSampleKey
 from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS
@@ -234,10 +236,26 @@ class SitesDataset(Dataset):
             da_sites = dataset_dict["site"]
             da_sites = da_sites / da_sites.capacity_kwp
             data_arrays.append(("site", da_sites))
-        
+
         combined_sample_dataset = self.merge_data_arrays(data_arrays)
 
-        # TODO add solar + time features for sites
+        # add datetime features
+        datetimes = pd.DatetimeIndex(combined_sample_dataset.site__time_utc.values)
+        datetime_features = make_datetime_numpy_dict(datetimes=datetimes, key_prefix="site")
+        datetime_features_xr = xr.Dataset(datetime_features, coords={"site__time_utc": datetimes})
+        combined_sample_dataset = xr.merge([combined_sample_dataset, datetime_features_xr])
+
+        # add sun features
+        sun_position_features = make_sun_position_numpy_sample(
+            datetimes=datetimes,
+            lon=combined_sample_dataset.site__longitude.values,
+            lat=combined_sample_dataset.site__latitude.values,
+            key_prefix="site",
+        )
+        sun_position_features_xr = xr.Dataset(
+            sun_position_features, coords={"site__time_utc": datetimes}
+        )
+        combined_sample_dataset = xr.merge([combined_sample_dataset, sun_position_features_xr])
 
         # Fill any nan values
         return combined_sample_dataset.fillna(0.0)

{ocf_data_sampler-0.0.43.dist-info → ocf_data_sampler-0.0.45.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.0.43
+Version: 0.0.45
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org
@@ -56,7 +56,7 @@ Requires-Dist: mkdocs-material>=8.0; extra == "docs"
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -135,6 +135,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/felix-e-h-p"><img src="https://avatars.githubusercontent.com/u/137530077?v=4?s=100" width="100px;" alt="Felix"/><br /><sub><b>Felix</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=felix-e-h-p" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://timothyajaniportfolio-b6v3zq29k-timthegreat.vercel.app/"><img src="https://avatars.githubusercontent.com/u/60073728?v=4?s=100" width="100px;" alt="Ajani Timothy"/><br /><sub><b>Ajani Timothy</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Tim1119" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://rupeshmangalam.vercel.app/"><img src="https://avatars.githubusercontent.com/u/91172425?v=4?s=100" width="100px;" alt="Rupesh Mangalam"/><br /><sub><b>Rupesh Mangalam</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=RupeshMangalam21" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://siddharth7113.github.io"><img src="https://avatars.githubusercontent.com/u/114160268?v=4?s=100" width="100px;" alt="Siddharth"/><br /><sub><b>Siddharth</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=siddharth7113" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.0.43.dist-info → ocf_data_sampler-0.0.45.dist-info}/RECORD

@@ -18,12 +18,13 @@ ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=2iR1Iy542lo51rC6XFLV-3pbUE68dWjlHa6TVJzx3ac,1280
 ocf_data_sampler/load/nwp/providers/ukv.py,sha256=79Bm7q-K_GJPYMy62SUIZbRWRF4-tIaB1dYPEgLD9vo,1207
 ocf_data_sampler/load/nwp/providers/utils.py,sha256=Sy2exG1wpXLLhMXYdsfR-DZMR3txG1_bBmBdchlc-yA,848
-ocf_data_sampler/numpy_sample/__init__.py,sha256=jWvN4BDYAgoFkdGBxjNrYioZdZNVbIcP4RsNGJCEGm0,345
+ocf_data_sampler/numpy_sample/__init__.py,sha256=nY5C6CcuxiWZ_jrXRzWtN7WyKXhJImSiVTIG6Rz4B_4,401
 ocf_data_sampler/numpy_sample/collate.py,sha256=y8QFUhskaAfOMP22aVkexwyGAwLHbNE-q1pOZ6txWKA,2227
-ocf_data_sampler/numpy_sample/gsp.py,sha256=4ZckEN7S82UY36e9z6rp5DTp10SAn_cNISBNilpOADw,1008
-ocf_data_sampler/numpy_sample/nwp.py,sha256=eEIRXP09wY99Vq-ItnfQ6U9AY1vJPLYKfUOc83V6ICo,1302
-ocf_data_sampler/numpy_sample/satellite.py,sha256=WYG9_ryOk3RWHWjgcMjIaU-YahEhTXz2xN0rQm-_TTQ,914
-ocf_data_sampler/numpy_sample/site.py,sha256=E2MQKkgUXikbIlKYU4ueNCPoBYJC-vnkbrQ_IwP2hqw,792
+ocf_data_sampler/numpy_sample/datetime_features.py,sha256=U-9uRplfZ7VYFA4qBduI8OkG2x_65RYIP8wrLG4i-Nw,1441
+ocf_data_sampler/numpy_sample/gsp.py,sha256=5UaWO_aGRRVQo82wnDaT4zBKHihOnIsXiwgPjM8vGFM,1005
+ocf_data_sampler/numpy_sample/nwp.py,sha256=_seQNWsut3IzPsrpipqImjnaM3XNHZCy5_5be6syivk,1297
+ocf_data_sampler/numpy_sample/satellite.py,sha256=8OaTvkPjzSjotcdKsa6BKmmlBKDBunbhDN4Pjo0Grxs,910
+ocf_data_sampler/numpy_sample/site.py,sha256=PIfmCtPA37dqpC8GArkryVqFrNAwqacj0iW2ikBOdSk,789
 ocf_data_sampler/numpy_sample/sun_position.py,sha256=UklhucCxCT6GMlAhCWL6c4cfWrdc1cWgegrYaqUoHOY,1611
 ocf_data_sampler/select/__init__.py,sha256=E4AJulEbO2K-o0UlG1fgaEteuf_1ZFjHTvrotXSb4YU,332
 ocf_data_sampler/select/dropout.py,sha256=HCx5Wzk8Oh2Z9vV94Jy-ALJsHtGduwvMaQOleQXp5z0,1142
@@ -36,9 +37,9 @@ ocf_data_sampler/select/select_time_slice.py,sha256=D5P_cSvnv8Qs49K5au7lPxDr9U_V
 ocf_data_sampler/select/spatial_slice_for_dataset.py,sha256=3tRrMBXr7s4CnClbVSIq7hpls3H4Y3qYTDwswcxCCCE,1763
 ocf_data_sampler/select/time_slice_for_dataset.py,sha256=LMw8KnOCKnPjD0m4UubAWERpaiQtzRKkI2cSh5a0A-M,4335
 ocf_data_sampler/torch_datasets/__init__.py,sha256=nJUa2KzVa84ZoM0PT2AbDz26ennmAYc7M7WJVfypPMs,85
-ocf_data_sampler/torch_datasets/process_and_combine.py,sha256=ygRBf70Jd6turmwLrv0Fsxo_Ge-54N5S2NUntyRxtJk,4305
+ocf_data_sampler/torch_datasets/process_and_combine.py,sha256=BaZZwGUKQf--X-3cNHcxZRn6iTNXg-sZyaYeRzQ7V68,4306
 ocf_data_sampler/torch_datasets/pvnet_uk_regional.py,sha256=QRFqbdfNchVWj4y70n-rJdFvFGvQj-WpZLdFqWjnOTw,5543
-ocf_data_sampler/torch_datasets/site.py,sha256=2fN2tw5tCXoOkh8vzRWWm_5FNGglGwJInTanJgV4Xuo,14455
+ocf_data_sampler/torch_datasets/site.py,sha256=bBqKQt0lhk90-d1ZkBCeHHEIAFCnoQbRs2AQYyulW40,15381
 ocf_data_sampler/torch_datasets/valid_time_periods.py,sha256=Qo65qUHtle_bW5tLTYr7empHTRv-lpjvfx_6GNJj3Xg,4371
 scripts/refactor_site.py,sha256=asZ27hQ4IyXgCCUaFJqcz1ObBNcV2W3ywqHBpSXA_fc,1728
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -50,6 +51,7 @@ tests/load/test_load_nwp.py,sha256=3qyyDkB1q9t3tyAwogfotNrxqUOpXXimco1CImoEWGg,7
 tests/load/test_load_satellite.py,sha256=STX5AqqmOAgUgE9R1xyq_sM3P1b8NKdGjO-hDhayfxM,524
 tests/load/test_load_sites.py,sha256=T9lSEnGPI8FQISudVYHHNTHeplNS62Vrx48jaZ6J_Jo,364
 tests/numpy_sample/test_collate.py,sha256=oWVNRR0LyF0iqaxjsq6hPMmkSGIyvGVlAM6q2rqvXqQ,840
+tests/numpy_sample/test_datetime_features.py,sha256=o4t3KeKFdGrOBQ77rNFcDuDMQSD23ileCS5T5AP3wG4,1769
 tests/numpy_sample/test_gsp.py,sha256=FLlq4SlJ-9cSRAepf4_ksA6PsUVKegnKEAc5pUojCJ0,1458
 tests/numpy_sample/test_nwp.py,sha256=yf4u7mAU0E3FQ4xAH6YjuHuHBzzFoXjHSFNkOVJUdSM,1455
 tests/numpy_sample/test_satellite.py,sha256=cCqtn5See-uSNfh89COGTUQNuFm6sIZ8QmBVHsuUeRI,1189
@@ -63,9 +65,9 @@ tests/select/test_select_time_slice.py,sha256=K1EJR5TwZa9dJf_YTEHxGtvs398iy1xS2l
 tests/torch_datasets/conftest.py,sha256=eRCzHE7cxS4AoskExkCGFDBeqItktAYNAdkfpMoFCeE,629
 tests/torch_datasets/test_process_and_combine.py,sha256=mbjQdqzLhox-U2sc1Ec68xLz95b3XOyPa7WchgxUM88,4256
 tests/torch_datasets/test_pvnet_uk_regional.py,sha256=dLY861PMyQ_buTksP8d0UXzfKsZ_CFNgceSYVGXRIRs,2134
-tests/torch_datasets/test_site.py,sha256=TSmdpD0OFid6e8eRgeB5FGxglUN1L2t4cIuMTfjA7jE,4539
-ocf_data_sampler-0.0.43.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.0.43.dist-info/METADATA,sha256=pqYvFJO9iGZnsMneHiF0WobuYrSpfxG_Cw701uBi480,11431
-ocf_data_sampler-0.0.43.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ocf_data_sampler-0.0.43.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
-ocf_data_sampler-0.0.43.dist-info/RECORD,,
+tests/torch_datasets/test_site.py,sha256=PGsGgg4jCDasxBg09hW9DA6rOGf60ECIh5YnBWyyzDE,4675
+ocf_data_sampler-0.0.45.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
+ocf_data_sampler-0.0.45.dist-info/METADATA,sha256=5QNcC3hPXLfehwBcxUVlidlPwSlMEoQWp5ln9JE_mzQ,11788
+ocf_data_sampler-0.0.45.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ocf_data_sampler-0.0.45.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
+ocf_data_sampler-0.0.45.dist-info/RECORD,,

tests/numpy_sample/test_datetime_features.py

@@ -0,0 +1,47 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from ocf_data_sampler.numpy_sample.datetime_features import make_datetime_numpy_dict
+
+
+def test_calculate_azimuth_and_elevation():
+
+    # Pick the day of the summer solstice
+    datetimes = pd.to_datetime(["2024-06-20 12:00", "2024-06-20 12:30", "2024-06-20 13:00"])
+
+    # Calculate sun angles
+    datetime_features = make_datetime_numpy_dict(datetimes)
+
+    assert len(datetime_features) == 4
+
+    assert len(datetime_features["wind_date_sin"]) == len(datetimes)
+    assert (datetime_features["wind_date_cos"] != datetime_features["wind_date_sin"]).all()
+
+    # assert all values are between -1 and 1
+    assert all(np.abs(datetime_features["wind_date_sin"]) <= 1)
+    assert all(np.abs(datetime_features["wind_date_cos"]) <= 1)
+    assert all(np.abs(datetime_features["wind_time_sin"]) <= 1)
+    assert all(np.abs(datetime_features["wind_time_cos"]) <= 1)
+
+
+def test_make_datetime_numpy_batch_custom_key_prefix():
+    # Test function correctly applies custom prefix to dict keys
+    datetimes = pd.to_datetime(["2024-06-20 12:00", "2024-06-20 12:30", "2024-06-20 13:00"])
+    key_prefix = "solar"
+
+    datetime_features = make_datetime_numpy_dict(datetimes, key_prefix=key_prefix)
+
+    # Assert dict contains expected quantity of keys and verify starting with custom prefix
+    assert len(datetime_features) == 4
+    assert all(key.startswith(key_prefix) for key in datetime_features.keys())
+
+
+def test_make_datetime_numpy_batch_empty_input():
+    # Verification that function raises error for empty input
+    datetimes = pd.DatetimeIndex([])
+
+    with pytest.raises(
+        ValueError, match="Input datetimes is empty for 'make_datetime_numpy_dict' function"
+    ):
+        make_datetime_numpy_dict(datetimes)

tests/torch_datasets/test_site.py

@@ -22,7 +22,9 @@ def test_site(site_config_filename):
     # Expected dimensions and data variables
     expected_dims = {'satellite__x_geostationary', 'site__time_utc', 'nwp-ukv__target_time_utc',
                      'nwp-ukv__x_osgb', 'satellite__channel', 'satellite__y_geostationary',
-                     'satellite__time_utc', 'nwp-ukv__channel', 'nwp-ukv__y_osgb'}
+                     'satellite__time_utc', 'nwp-ukv__channel', 'nwp-ukv__y_osgb', 'site_solar_azimuth',
+                     'site_solar_elevation', 'site_date_cos', 'site_time_cos', 'site_time_sin', 'site_date_sin'}
+
     expected_data_vars = {"nwp-ukv", "satellite", "site"}
 
     # Check dimensions