ocf-data-sampler 0.0.31__py3-none-any.whl → 0.0.33__py3-none-any.whl

ocf_data_sampler/load/site.py

@@ -8,23 +8,23 @@ from ocf_data_sampler.config.model import Site
 def open_site(sites_config: Site) -> xr.DataArray:
 
     # Load site generation xr.Dataset
-    data_ds = xr.open_dataset(sites_config.file_path)
+    site_generation_ds = xr.open_dataset(sites_config.file_path)
 
     # Load site generation data
     metadata_df = pd.read_csv(sites_config.metadata_file_path, index_col="site_id")
 
-    # Add coordinates
-    ds = data_ds.assign_coords(
-        latitude=(metadata_df.latitude.to_xarray()),
-        longitude=(metadata_df.longitude.to_xarray()),
-        capacity_kwp=data_ds.capacity_kwp,
+    # Ensure metadata aligns with the site_id dimension in data_ds
+    metadata_df = metadata_df.reindex(site_generation_ds.site_id.values)
+
+    # Assign coordinates to the Dataset using the aligned metadata
+    site_generation_ds = site_generation_ds.assign_coords(
+        latitude=("site_id", metadata_df["latitude"].values),
+        longitude=("site_id", metadata_df["longitude"].values),
+        capacity_kwp=("site_id", metadata_df["capacity_kwp"].values),
     )
 
     # Sanity checks
-    assert np.isfinite(data_ds.capacity_kwp.values).all()
-    assert (data_ds.capacity_kwp.values > 0).all()
+    assert np.isfinite(site_generation_ds.capacity_kwp.values).all()
+    assert (site_generation_ds.capacity_kwp.values > 0).all()
     assert metadata_df.index.is_unique
-
-    return ds.generation_kw
-
-
+    return site_generation_ds.generation_kw

ocf_data_sampler/torch_datasets/__init__.py

@@ -1 +1,2 @@
-
+from .pvnet_uk_regional import PVNetUKRegionalDataset
+from .site import SitesDataset

ocf_data_sampler/torch_datasets/process_and_combine.py

@@ -1,6 +1,7 @@
 import numpy as np
 import pandas as pd
 import xarray as xr
+from typing import Tuple
 
 from ocf_data_sampler.config import Configuration
 from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS
@@ -9,7 +10,6 @@ from ocf_data_sampler.numpy_batch import (
     convert_satellite_to_numpy_batch,
     convert_gsp_to_numpy_batch,
     make_sun_position_numpy_batch,
-    convert_site_to_numpy_batch,
 )
 from ocf_data_sampler.numpy_batch.gsp import GSPBatchKey
 from ocf_data_sampler.numpy_batch.nwp import NWPBatchKey
@@ -73,18 +73,6 @@ def process_and_combine_datasets(
             }
         )
 
-
-    if "site" in dataset_dict:
-        site_config = config.input_data.site
-        da_sites = dataset_dict["site"]
-        da_sites = da_sites / da_sites.capacity_kwp
-
-        numpy_modalities.append(
-            convert_site_to_numpy_batch(
-                da_sites, t0_idx=-site_config.interval_start_minutes / site_config.time_resolution_minutes
-            )
-        )
-
     if target_key == 'gsp':
         # Make sun coords NumpyBatch
         datetimes = pd.date_range(
@@ -95,16 +83,6 @@ def process_and_combine_datasets(
 
         lon, lat = osgb_to_lon_lat(location.x, location.y)
 
-    elif target_key == 'site':
-        # Make sun coords NumpyBatch
-        datetimes = pd.date_range(
-            t0+minutes(site_config.interval_start_minutes),
-            t0+minutes(site_config.interval_end_minutes),
-            freq=minutes(site_config.time_resolution_minutes),
-        )
-
-        lon, lat = location.x, location.y
-
     numpy_modalities.append(
         make_sun_position_numpy_batch(datetimes, lon, lat, key_prefix=target_key)
     )
@@ -115,6 +93,47 @@ def process_and_combine_datasets(
 
     return combined_sample
 
+def process_and_combine_site_sample_dict(
+    dataset_dict: dict,
+    config: Configuration,
+) -> xr.Dataset:
+    """
+    Normalize and combine data into a single xr Dataset
+
+    Args:
+        dataset_dict: dict containing sliced xr DataArrays
+        config: Configuration for the model
+
+    Returns:
+        xr.Dataset: A merged Dataset with nans filled in.
+    
+    """
+
+    data_arrays = []
+
+    if "nwp" in dataset_dict:
+        for nwp_key, da_nwp in dataset_dict["nwp"].items():
+            # Standardise
+            provider = config.input_data.nwp[nwp_key].provider
+            da_nwp = (da_nwp - NWP_MEANS[provider]) / NWP_STDS[provider]
+            data_arrays.append((f"nwp-{provider}", da_nwp))
+          
+    if "sat" in dataset_dict:
+        # TODO add some satellite normalisation
+        da_sat = dataset_dict["sat"]
+        data_arrays.append(("satellite", da_sat))
+
+    if "site" in dataset_dict:
+        # site_config = config.input_data.site
+        da_sites = dataset_dict["site"]
+        da_sites = da_sites / da_sites.capacity_kwp
+        data_arrays.append(("sites", da_sites))
+    
+    combined_sample_dataset = merge_arrays(data_arrays)
+
+    # Fill any nan values
+    return combined_sample_dataset.fillna(0.0)
+
 
 def merge_dicts(list_of_dicts: list[dict]) -> dict:
     """Merge a list of dictionaries into a single dictionary"""
@@ -124,6 +143,59 @@ def merge_dicts(list_of_dicts: list[dict]) -> dict:
         combined_dict.update(d)
     return combined_dict
 
+def merge_arrays(normalised_data_arrays: list[Tuple[str, xr.DataArray]]) -> xr.Dataset:
+    """
+    Combine a list of DataArrays into a single Dataset with unique naming conventions.
+
+    Args:
+        list_of_arrays: List of tuples where each tuple contains:
+            - A string (key name).
+            - An xarray.DataArray.
+
+    Returns:
+        xr.Dataset: A merged Dataset with uniquely named variables, coordinates, and dimensions.
+    """
+    datasets = []
+
+    for key, data_array in normalised_data_arrays:
+        # Ensure all attributes are strings for consistency
+        data_array = data_array.assign_attrs(
+            {attr_key: str(attr_value) for attr_key, attr_value in data_array.attrs.items()}
+        )
+
+        # Convert DataArray to Dataset with the variable name as the key
+        dataset = data_array.to_dataset(name=key)
+
+        # Prepend key name to all dimension and coordinate names for uniqueness
+        dataset = dataset.rename(
+            {dim: f"{key}__{dim}" for dim in dataset.dims if dim not in dataset.coords}
+        )
+        dataset = dataset.rename(
+            {coord: f"{key}__{coord}" for coord in dataset.coords}
+        )
+
+        # Handle concatenation dimension if applicable
+        concat_dim = (
+            f"{key}__target_time_utc" if f"{key}__target_time_utc" in dataset.coords
+            else f"{key}__time_utc"
+        )
+
+        if f"{key}__init_time_utc" in dataset.coords:
+            init_coord = f"{key}__init_time_utc"
+            if dataset[init_coord].ndim == 0:  # Check if scalar
+                expanded_init_times = [dataset[init_coord].values] * len(dataset[concat_dim])
+                dataset = dataset.assign_coords({init_coord: (concat_dim, expanded_init_times)})
+
+        datasets.append(dataset)
+
+    # Ensure all datasets are valid xarray.Dataset objects
+    for ds in datasets:
+        assert isinstance(ds, xr.Dataset), f"Object is not an xr.Dataset: {type(ds)}"
+
+    # Merge all prepared datasets
+    combined_dataset = xr.merge(datasets)
+
+    return combined_dataset
 
 def fill_nans_in_arrays(batch: dict) -> dict:
     """Fills all NaN values in each np.ndarray in the batch dictionary with zeros.

ocf_data_sampler/torch_datasets/site.py

@@ -15,7 +15,7 @@ from ocf_data_sampler.select import (
     slice_datasets_by_time, slice_datasets_by_space
 )
 from ocf_data_sampler.utils import minutes
-from ocf_data_sampler.torch_datasets.process_and_combine import process_and_combine_datasets, compute
+from ocf_data_sampler.torch_datasets.process_and_combine import process_and_combine_site_sample_dict
 from ocf_data_sampler.torch_datasets.valid_time_periods import find_valid_time_periods
 
 xr.set_options(keep_attrs=True)
@@ -152,10 +152,9 @@ class SitesDataset(Dataset):
         """
         sample_dict = slice_datasets_by_space(self.datasets_dict, location, self.config)
         sample_dict = slice_datasets_by_time(sample_dict, t0, self.config)
-        sample_dict = compute(sample_dict)
-
-        sample = process_and_combine_datasets(sample_dict, self.config, t0, location, target_key='site')
 
+        sample = process_and_combine_site_sample_dict(sample_dict, self.config)
+        sample = sample.compute()
         return sample
 
     def get_location_from_site_id(self, site_id):

{ocf_data_sampler-0.0.31.dist-info → ocf_data_sampler-0.0.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.31
+Version: 0.0.33
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.0.31.dist-info → ocf_data_sampler-0.0.33.dist-info}/RECORD

@@ -10,7 +10,7 @@ ocf_data_sampler/load/__init__.py,sha256=MjgfxilTzyz1RYFoBEeAXmE9hyjknLvdmlHPmlA
 ocf_data_sampler/load/gsp.py,sha256=Gcr1JVUOPKhFRDCSHtfPDjxx0BtyyEhXrZvGEKLPJ5I,759
 ocf_data_sampler/load/load_dataset.py,sha256=Ua3RaUg4PIYJkD9BKqTfN8IWUbezbhThJGgEkd9PcaE,1587
 ocf_data_sampler/load/satellite.py,sha256=3KlA1fx4SwxdzM-jC1WRaONXO0D6m0WxORnEnwUnZrA,2967
-ocf_data_sampler/load/site.py,sha256=ROif2XXIIgBz-JOOiHymTq1CMXswJ3AzENU9DJmYpcU,782
+ocf_data_sampler/load/site.py,sha256=P83uz01WBDzoZajdOH0m8FQt4-buKDlUk19N548KqhA,1086
 ocf_data_sampler/load/utils.py,sha256=EQGvVWlGMoSOdbDYuMfVAa0v6wmAOPmHIAemdrTB5v4,1406
 ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
 ocf_data_sampler/load/nwp/nwp.py,sha256=O4QnajEZem8BvBgTcYYDBhRhgqPYuJkolHmpMRmrXEA,610
@@ -34,10 +34,10 @@ ocf_data_sampler/select/select_spatial_slice.py,sha256=WNxwur9Q5oetvogATw8-hNejD
 ocf_data_sampler/select/select_time_slice.py,sha256=D5P_cSvnv8Qs49K5au7lPxDr9U_VmDn42s5leMzHt0k,6122
 ocf_data_sampler/select/spatial_slice_for_dataset.py,sha256=3tRrMBXr7s4CnClbVSIq7hpls3H4Y3qYTDwswcxCCCE,1763
 ocf_data_sampler/select/time_slice_for_dataset.py,sha256=LMw8KnOCKnPjD0m4UubAWERpaiQtzRKkI2cSh5a0A-M,4335
-ocf_data_sampler/torch_datasets/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/torch_datasets/process_and_combine.py,sha256=4k6f6PlMqrg3luMwGw3764iOyfuUNUePKyoikYGaRMI,4953
+ocf_data_sampler/torch_datasets/__init__.py,sha256=nJUa2KzVa84ZoM0PT2AbDz26ennmAYc7M7WJVfypPMs,85
+ocf_data_sampler/torch_datasets/process_and_combine.py,sha256=WwwuzxXoq8S70R-tWABXUMO854TG8GWYnNhb1IU8MRY,7526
 ocf_data_sampler/torch_datasets/pvnet_uk_regional.py,sha256=QRFqbdfNchVWj4y70n-rJdFvFGvQj-WpZLdFqWjnOTw,5543
-ocf_data_sampler/torch_datasets/site.py,sha256=lo2ULurfWNu9vzBC6H4pdKMMpUMIT8_FWC1l_1mgIOM,6596
+ocf_data_sampler/torch_datasets/site.py,sha256=NYuhgm9ti9SRt1dcb_WrFYYo14NgVdOsaoPbc5FsnaA,6560
 ocf_data_sampler/torch_datasets/valid_time_periods.py,sha256=Qo65qUHtle_bW5tLTYr7empHTRv-lpjvfx_6GNJj3Xg,4371
 scripts/refactor_site.py,sha256=asZ27hQ4IyXgCCUaFJqcz1ObBNcV2W3ywqHBpSXA_fc,1728
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -56,11 +56,11 @@ tests/select/test_fill_time_periods.py,sha256=o59f2YRe5b0vJrG3B0aYZkYeHnpNk4s6EJ
 tests/select/test_find_contiguous_time_periods.py,sha256=kOga_V7er5We7ewMARXaKdM3agOhsvZYx8inXtUn1PM,5976
 tests/select/test_location.py,sha256=_WZk2FPYeJ-nIfCJS6Sp_yaVEEo7m31DmMFoZzgyCts,2712
 tests/select/test_select_spatial_slice.py,sha256=7EX9b6g-pMdACQx3yefjs5do2s-Rho2UmKevV4oglsU,5147
-tests/select/test_select_time_slice.py,sha256=QOhoR3qsr7RBGze4yohcViZ-ad1zYQzIKzxlnf0ymnU,9603
-tests/torch_datasets/test_pvnet_uk_regional.py,sha256=8gxjJO8FhY-ImX6eGnihDFsa8fhU2Zb4bVJaToJwuwo,2653
-tests/torch_datasets/test_site.py,sha256=yTv6tAT6lha5yLYJiC8DNms1dct8o_ObPV97dHZyT7I,2719
-ocf_data_sampler-0.0.31.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.0.31.dist-info/METADATA,sha256=rFtc-y0PkztBWSGazWfr7WsPRo7SdnccosWrlaLJmk8,9559
-ocf_data_sampler-0.0.31.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ocf_data_sampler-0.0.31.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
-ocf_data_sampler-0.0.31.dist-info/RECORD,,
+tests/select/test_select_time_slice.py,sha256=K1EJR5TwZa9dJf_YTEHxGtvs398iy1xS2lr1BgJZkoo,9603
+tests/torch_datasets/test_pvnet_uk_regional.py,sha256=ZNyrisyhM1vw4q2qcHDuvX2uRi-v3U8Y8lOYx7cd8yM,2635
+tests/torch_datasets/test_site.py,sha256=YuVjWTI14_kmEOx23XE5J_RZ8UalCKD2xRv6mqYizB8,2872
+ocf_data_sampler-0.0.33.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
+ocf_data_sampler-0.0.33.dist-info/METADATA,sha256=HWhJVLbJkMM9t5i_GTDbgfit1essTQh_02sOOOgolAY,9559
+ocf_data_sampler-0.0.33.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ocf_data_sampler-0.0.33.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
+ocf_data_sampler-0.0.33.dist-info/RECORD,,

tests/select/test_select_time_slice.py

@@ -197,7 +197,7 @@ def test_select_time_slice_nwp_with_dropout_and_accum(da_nwp_like, t0_str):
     t0 = pd.Timestamp(f"2024-01-02 {t0_str}")
     interval_start = pd.Timedelta(-6, "h")
     interval_end = pd.Timedelta(3, "h")
-    freq = pd.Timedelta("1H")
+    freq = pd.Timedelta("1h")
     dropout_timedelta = pd.Timedelta("-2h")
 
     t0_delayed = (t0 + dropout_timedelta).floor(NWP_FREQ)

tests/torch_datasets/test_pvnet_uk_regional.py

@@ -1,7 +1,7 @@
 import pytest
 import tempfile
 
-from ocf_data_sampler.torch_datasets.pvnet_uk_regional import PVNetUKRegionalDataset
+from ocf_data_sampler.torch_datasets import PVNetUKRegionalDataset
 from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configuration
 from ocf_data_sampler.numpy_batch import NWPBatchKey, GSPBatchKey, SatelliteBatchKey
 

tests/torch_datasets/test_site.py

@@ -1,11 +1,12 @@
 import pandas as pd
 import pytest
 
-from ocf_data_sampler.torch_datasets.site import SitesDataset
+from ocf_data_sampler.torch_datasets import SitesDataset
 from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configuration
 from ocf_data_sampler.numpy_batch.nwp import NWPBatchKey
 from ocf_data_sampler.numpy_batch.site import SiteBatchKey
 from ocf_data_sampler.numpy_batch.satellite import SatelliteBatchKey
+from xarray import Dataset
 
 
 @pytest.fixture()
@@ -34,31 +35,26 @@ def test_site(site_config_filename):
     # Generate a sample
     sample = dataset[0]
 
-    assert isinstance(sample, dict)
+    assert isinstance(sample, Dataset)
 
-    for key in [
-        NWPBatchKey.nwp,
-        SatelliteBatchKey.satellite_actual,
-        SiteBatchKey.generation,
-        SiteBatchKey.site_solar_azimuth,
-        SiteBatchKey.site_solar_elevation,
-    ]:
-        assert key in sample
+    # Expected dimensions and data variables
+    expected_dims = {'satellite__x_geostationary', 'sites__time_utc', 'nwp-ukv__target_time_utc',
+                     'nwp-ukv__x_osgb', 'satellite__channel', 'satellite__y_geostationary',
+                     'satellite__time_utc', 'nwp-ukv__channel', 'nwp-ukv__y_osgb'}
+    expected_data_vars = {"nwp-ukv", "satellite", "sites"}
 
-    for nwp_source in ["ukv"]:
-        assert nwp_source in sample[NWPBatchKey.nwp]
+    # Check dimensions
+    assert set(sample.dims) == expected_dims, f"Missing or extra dimensions: {set(sample.dims) ^ expected_dims}"
+    # Check data variables
+    assert set(sample.data_vars) == expected_data_vars, f"Missing or extra data variables: {set(sample.data_vars) ^ expected_data_vars}"
 
     # check the shape of the data is correct
     # 30 minutes of 5 minute data (inclusive), one channel, 2x2 pixels
-    assert sample[SatelliteBatchKey.satellite_actual].shape == (7, 1, 2, 2)
+    assert sample["satellite"].values.shape == (7, 1, 2, 2)
     # 3 hours of 60 minute data (inclusive), one channel, 2x2 pixels
-    assert sample[NWPBatchKey.nwp]["ukv"][NWPBatchKey.nwp].shape == (4, 1, 2, 2)
-    # 3 hours of 30 minute data (inclusive)
-    assert sample[SiteBatchKey.generation].shape == (4,)
-    # Solar angles have same shape as GSP data
-    assert sample[SiteBatchKey.site_solar_azimuth].shape == (4,)
-    assert sample[SiteBatchKey.site_solar_elevation].shape == (4,)
-
+    assert sample["nwp-ukv"].values.shape == (4, 1, 2, 2)
+    # 1.5 hours of 30 minute data (inclusive)
+    assert sample["sites"].values.shape == (4,)
 
 def test_site_time_filter_start(site_config_filename):