ocf-data-sampler 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

ocf_data_sampler/numpy_sample/__init__.py

@@ -1,6 +1,6 @@
 """Conversion from Xarray to NumpySample"""
 
-from .datetime_features import make_datetime_numpy_dict
+from .datetime_features import encode_datetimes
 from .gsp import convert_gsp_to_numpy_sample, GSPSampleKey
 from .nwp import convert_nwp_to_numpy_sample, NWPSampleKey
 from .satellite import convert_satellite_to_numpy_sample, SatelliteSampleKey

ocf_data_sampler/numpy_sample/datetime_features.py

@@ -6,33 +6,24 @@ import pandas as pd
 from ocf_data_sampler.numpy_sample.common_types import NumpySample
 
 
-def _get_date_time_in_pi(dt: pd.DatetimeIndex) -> tuple[np.ndarray, np.ndarray]:
-    """Create positional embeddings for the datetimes in radians.
+def encode_datetimes(datetimes: pd.DatetimeIndex) -> NumpySample:
+    """Creates dictionary of sin and cos datetime embeddings.
 
     Args:
-        dt: DatetimeIndex to create radian embeddings for
+        datetimes: DatetimeIndex to create radian embeddings for
 
     Returns:
-        Tuple of numpy arrays containing radian coordinates for date and time
+        Dictionary of datetime encodings
     """
-    day_of_year = dt.dayofyear
-    minute_of_day = dt.minute + dt.hour * 60
-
-    time_in_pi = (2 * np.pi) * (minute_of_day / (24 * 60))
-    date_in_pi = (2 * np.pi) * (day_of_year / 365)
-
-    return date_in_pi, time_in_pi
-
-
-def make_datetime_numpy_dict(datetimes: pd.DatetimeIndex, key_prefix: str = "wind") -> NumpySample:
-    """Creates dictionary of cyclical datetime features - encoded."""
-    date_in_pi, time_in_pi = _get_date_time_in_pi(datetimes)
-
-    time_numpy_sample = {}
-
-    time_numpy_sample[key_prefix + "_date_sin"] = np.sin(date_in_pi)
-    time_numpy_sample[key_prefix + "_date_cos"] = np.cos(date_in_pi)
-    time_numpy_sample[key_prefix + "_time_sin"] = np.sin(time_in_pi)
-    time_numpy_sample[key_prefix + "_time_cos"] = np.cos(time_in_pi)
-
-    return time_numpy_sample
+    day_of_year = datetimes.dayofyear
+    minute_of_day = datetimes.minute + datetimes.hour * 60
+
+    time_in_radians = (2 * np.pi) * (minute_of_day / (24 * 60))
+    date_in_radians = (2 * np.pi) * (day_of_year / 365)
+
+    return {
+        "date_sin": np.sin(date_in_radians),
+        "date_cos": np.cos(date_in_radians),
+        "time_sin": np.sin(time_in_radians),
+        "time_cos": np.cos(time_in_radians),
+    }

ocf_data_sampler/numpy_sample/site.py

@@ -13,10 +13,7 @@ class SiteSampleKey:
     time_utc = "site_time_utc"
     t0_idx = "site_t0_idx"
     id = "site_id"
-    date_sin = "site_date_sin"
-    date_cos = "site_date_cos"
-    time_sin = "site_time_sin"
-    time_cos = "site_time_cos"
+
 
 
 def convert_site_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> NumpySample:
@@ -31,10 +28,6 @@ def convert_site_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) ->
         SiteSampleKey.capacity_kwp: da.isel(time_utc=0)["capacity_kwp"].values,
         SiteSampleKey.time_utc: da["time_utc"].values.astype(float),
         SiteSampleKey.id: da["site_id"].values,
-        SiteSampleKey.date_sin: da["date_sin"].values,
-        SiteSampleKey.date_cos: da["date_cos"].values,
-        SiteSampleKey.time_sin: da["time_sin"].values,
-        SiteSampleKey.time_cos: da["time_cos"].values,
     }
 
     if t0_idx is not None:

ocf_data_sampler/torch_datasets/datasets/site.py

@@ -13,7 +13,7 @@ from ocf_data_sampler.numpy_sample import (
     convert_nwp_to_numpy_sample,
     convert_satellite_to_numpy_sample,
     convert_site_to_numpy_sample,
-    make_datetime_numpy_dict,
+    encode_datetimes,
     make_sun_position_numpy_sample,
 )
 from ocf_data_sampler.numpy_sample.collate import stack_np_samples_into_batch
@@ -249,7 +249,7 @@ class SitesDataset(Dataset):
 
         # add datetime features
         datetimes = pd.DatetimeIndex(combined_sample_dataset.site__time_utc.values)
-        datetime_features = make_datetime_numpy_dict(datetimes=datetimes, key_prefix="site_")
+        datetime_features = encode_datetimes(datetimes=datetimes)
         combined_sample_dataset = combined_sample_dataset.assign_coords(
             {k: ("site__time_utc", v) for k, v in datetime_features.items()},
         )
@@ -445,11 +445,7 @@ class SitesDatasetConcurrent(Dataset):
             da_sites = da_sites / da_sites.capacity_kwp
 
             # Convert to NumpyBatch
-            numpy_modalities.append(
-                convert_site_to_numpy_sample(
-                    da_sites,
-                ),
-            )
+            numpy_modalities.append(convert_site_to_numpy_sample(da_sites))
 
         # Only add solar position if explicitly configured
         has_solar_config = (
@@ -578,13 +574,11 @@ def convert_netcdf_to_numpy_sample(ds: xr.Dataset) -> dict:
         sample_dict["sat"] = sample_dict.pop("satellite")
 
     # process and combine the datasets
-    sample = convert_to_numpy_and_combine(
-        dataset_dict=sample_dict,
-    )
+    sample = convert_to_numpy_and_combine(dataset_dict=sample_dict)
 
-    # Extraction of solar position coords
-    solar_keys = ["solar_azimuth", "solar_elevation"]
-    for key in solar_keys:
+    # Add solar coord and datetime features
+    keys = ["solar_azimuth", "solar_elevation", "date_sin", "date_cos", "time_sin", "time_cos"]
+    for key in keys:
         if key in ds.coords:
             sample[key] = ds.coords[key].values
 
@@ -672,11 +666,7 @@ def convert_to_numpy_and_combine(dataset_dict: dict[xr.Dataset]) -> NumpySample:
     if "site" in dataset_dict:
         da_sites = dataset_dict["site"]
 
-        numpy_modalities.append(
-            convert_site_to_numpy_sample(
-                da_sites,
-            ),
-        )
+        numpy_modalities.append(convert_site_to_numpy_sample(da_sites))
 
     # Combine all the modalities and fill NaNs
     combined_sample = merge_dicts(numpy_modalities)

{ocf_data_sampler-0.3.1.dist-info → ocf_data_sampler-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.3.1
+Version: 0.4.0
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License
@@ -49,7 +49,7 @@ Requires-Dist: xarray-tensorstore==0.1.5
 # ocf-data-sampler
  
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-14-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 [![tags badge](https://img.shields.io/github/v/tag/openclimatefix/ocf-data-sampler?include_prereleases&sort=semver&color=FFAC5F)](https://github.com/openclimatefix/ocf-data-sampler/tags)
@@ -128,6 +128,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="http://siddharth7113.github.io"><img src="https://avatars.githubusercontent.com/u/114160268?v=4?s=100" width="100px;" alt="Siddharth"/><br /><sub><b>Siddharth</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=siddharth7113" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sachin-G13"><img src="https://avatars.githubusercontent.com/u/190184500?v=4?s=100" width="100px;" alt="Sachin-G13"/><br /><sub><b>Sachin-G13</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sachin-G13" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://drona-gyawali.github.io/"><img src="https://avatars.githubusercontent.com/u/170401554?v=4?s=100" width="100px;" alt="Dorna Raj Gyawali"/><br /><sub><b>Dorna Raj Gyawali</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=drona-gyawali" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/adnanhashmi25"><img src="https://avatars.githubusercontent.com/u/55550094?v=4?s=100" width="100px;" alt="Adnan Hashmi"/><br /><sub><b>Adnan Hashmi</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=adnanhashmi25" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

{ocf_data_sampler-0.3.1.dist-info → ocf_data_sampler-0.4.0.dist-info}/RECORD

@@ -22,14 +22,14 @@ ocf_data_sampler/load/nwp/providers/gfs.py,sha256=h6vm-Rfz1JGOE4P_fP1_XQJ3bugNbe
 ocf_data_sampler/load/nwp/providers/icon.py,sha256=iVZwLKRr_D74_kAu5MHir6pRKEfbTmIxFRZAxzmiYdI,1257
 ocf_data_sampler/load/nwp/providers/ukv.py,sha256=2i32VM9gnmWUpbL0qBSp_AKzuyKucXZPS8yklbcGlbc,1039
 ocf_data_sampler/load/nwp/providers/utils.py,sha256=cVwCiC8FqNpkZFSUGv1CRqIQlKdjx1sIsb2SIUlvWV8,2333
-ocf_data_sampler/numpy_sample/__init__.py,sha256=nY5C6CcuxiWZ_jrXRzWtN7WyKXhJImSiVTIG6Rz4B_4,401
+ocf_data_sampler/numpy_sample/__init__.py,sha256=eeIVC3NcLl6vVN2rgI-oY-LxwMAZYDLMUU9kzrVy1Io,393
 ocf_data_sampler/numpy_sample/collate.py,sha256=hoxIc5SoHoIs3Nx37aRZzWChpswjy9lHUgaKgHIoo80,2039
 ocf_data_sampler/numpy_sample/common_types.py,sha256=9CjYHkUTx0ObduWh43fhsybZCTXvexql7qC2ptMDoek,377
-ocf_data_sampler/numpy_sample/datetime_features.py,sha256=qoUOQzHZebnc5JiXCwm258kPLwNRNJgE5JcVHKI9b70,1278
+ocf_data_sampler/numpy_sample/datetime_features.py,sha256=ObHM42VnZB7_daQ5a42GeftoDWYtVMT-wDP8kRtY_84,857
 ocf_data_sampler/numpy_sample/gsp.py,sha256=aUHDIUSu2LMsVmR7TsTriZxVfv495QNL-scaxyJFHgQ,1149
 ocf_data_sampler/numpy_sample/nwp.py,sha256=lXqE2Il0xX5hzz76HHkiYmfDsXWWhmaA_6bSnmwbAXU,1078
 ocf_data_sampler/numpy_sample/satellite.py,sha256=RaYzYIcB1AmDrKeiqSpn4QVfBH-QMe26F1P5t1az2Jg,1111
-ocf_data_sampler/numpy_sample/site.py,sha256=zfYBjK3CJrIaKH1QdKXU7gwOxTqONt527y3nJ9TRnwc,1325
+ocf_data_sampler/numpy_sample/site.py,sha256=4S19bzCN5lswVUrmWRfwpVsBPUE7bi0OIdxsD9wgvhU,982
 ocf_data_sampler/numpy_sample/sun_position.py,sha256=5tt-zNm6aRuZMsxZPaAxyg7HeikswfZCeHWXTHuO2K0,1555
 ocf_data_sampler/select/__init__.py,sha256=mK7Wu_-j9IXGTYrOuDf5yDDuU5a306b0iGKTAooNg_s,210
 ocf_data_sampler/select/dropout.py,sha256=BYpv8L771faPOyN7SdIJ5cwkpDve-ohClj95jjsHmjg,1973
@@ -41,7 +41,7 @@ ocf_data_sampler/select/select_spatial_slice.py,sha256=Hd4jGRUfIZRoWCirOQZeoLpaU
 ocf_data_sampler/select/select_time_slice.py,sha256=HeHbwZ0CP03x0-LaJtpbSdtpLufwVTR73p6wH6O_PS8,5513
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=jfJSFcR0eO1AqeH7S3KnGjsBqVZT5w3oyi784PUR6Q0,146
 ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=v63goKEMI6UgBPnQCnIbxhFFdwuP_sxgcPYY6iNfGkc,12257
-ocf_data_sampler/torch_datasets/datasets/site.py,sha256=R9sYZz3e1zr8NAtlYQp8_DgI3wIfC7Zvaeo_73TyiA8,24936
+ocf_data_sampler/torch_datasets/datasets/site.py,sha256=RGk1tNXxC4RXHVQtChlgtOSfi4DXOVeSyGSX3wiAjiM,24799
 ocf_data_sampler/torch_datasets/sample/__init__.py,sha256=GL84vdZl_SjHDGVyh9Uekx2XhPYuZ0dnO3l6f6KXnHI,100
 ocf_data_sampler/torch_datasets/sample/base.py,sha256=cQ1oIyhdmlotejZK8B3Cw6MNvpdnBPD8G_o2h7Ye4Vc,2206
 ocf_data_sampler/torch_datasets/sample/site.py,sha256=Pq-QjAg88kWXzFR4ci5ATaTWA864eifU7wrJSpmqK4s,1292
@@ -56,7 +56,7 @@ ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
 utils/compute_icon_mean_stddev.py,sha256=a1oWMRMnny39rV-dvu8rcx85sb4bXzPFrR1gkUr4Jpg,2296
-ocf_data_sampler-0.3.1.dist-info/METADATA,sha256=pQpPqmpTlUiZnPY1Q_xZr1Z-GrKSATG_P77YYHpWm6Y,12224
-ocf_data_sampler-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ocf_data_sampler-0.3.1.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
-ocf_data_sampler-0.3.1.dist-info/RECORD,,
+ocf_data_sampler-0.4.0.dist-info/METADATA,sha256=QbbQ2msPobYgD4v0ghC-xkU3NRGtzi8fhn8A7Gb9TAk,12588
+ocf_data_sampler-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ocf_data_sampler-0.4.0.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
+ocf_data_sampler-0.4.0.dist-info/RECORD,,