ocf-data-sampler 0.0.19__py3-none-any.whl → 0.0.42__py3-none-any.whl

tests/torch_datasets/test_site.py

@@ -0,0 +1,129 @@
+import pandas as pd
+
+from ocf_data_sampler.torch_datasets import SitesDataset
+from ocf_data_sampler.torch_datasets.site import convert_from_dataset_to_dict_datasets
+import numpy as np
+from xarray import Dataset, DataArray
+
+
+def test_site(site_config_filename):
+
+    # Create dataset object
+    dataset = SitesDataset(site_config_filename)
+
+    assert len(dataset) == 10 * 41
+    # TODO check 41
+
+    # Generate a sample
+    sample = dataset[0]
+
+    assert isinstance(sample, Dataset)
+
+    # Expected dimensions and data variables
+    expected_dims = {'satellite__x_geostationary', 'site__time_utc', 'nwp-ukv__target_time_utc',
+                     'nwp-ukv__x_osgb', 'satellite__channel', 'satellite__y_geostationary',
+                     'satellite__time_utc', 'nwp-ukv__channel', 'nwp-ukv__y_osgb'}
+    expected_data_vars = {"nwp-ukv", "satellite", "site"}
+
+    # Check dimensions
+    assert set(sample.dims) == expected_dims, f"Missing or extra dimensions: {set(sample.dims) ^ expected_dims}"
+    # Check data variables
+    assert set(sample.data_vars) == expected_data_vars, f"Missing or extra data variables: {set(sample.data_vars) ^ expected_data_vars}"
+
+    # check the shape of the data is correct
+    # 30 minutes of 5 minute data (inclusive), one channel, 2x2 pixels
+    assert sample["satellite"].values.shape == (7, 1, 2, 2)
+    # 3 hours of 60 minute data (inclusive), one channel, 2x2 pixels
+    assert sample["nwp-ukv"].values.shape == (4, 1, 2, 2)
+    # 1.5 hours of 30 minute data (inclusive)
+    assert sample["site"].values.shape == (4,)
+
+def test_site_time_filter_start(site_config_filename):
+
+    # Create dataset object
+    dataset = SitesDataset(site_config_filename, start_time="2024-01-01")
+
+    assert len(dataset) == 0
+
+
+def test_site_time_filter_end(site_config_filename):
+
+    # Create dataset object
+    dataset = SitesDataset(site_config_filename, end_time="2000-01-01")
+
+    assert len(dataset) == 0
+
+
+def test_site_get_sample(site_config_filename):
+
+    # Create dataset object
+    dataset = SitesDataset(site_config_filename)
+
+    assert len(dataset) == 410
+    sample = dataset.get_sample(t0=pd.Timestamp("2023-01-01 12:00"), site_id=1)
+
+
+def test_convert_from_dataset_to_dict_datasets(site_config_filename):
+    # Create dataset object
+    dataset = SitesDataset(site_config_filename)
+
+    # Generate two samples
+    sample_xr = dataset[0]
+
+    sample = convert_from_dataset_to_dict_datasets(sample_xr)
+
+    assert isinstance(sample, dict)
+
+    print(sample.keys())
+
+    for key in ["nwp", "satellite", "site"]:
+        assert key in sample
+
+def test_process_and_combine_site_sample_dict(site_config_filename):
+    # Load config
+    # config = load_yaml_configuration(pvnet_config_filename)
+    site_ds = SitesDataset(site_config_filename)
+    # Specify minimal structure for testing
+    raw_nwp_values = np.random.rand(4, 1, 2, 2)  # Single channel
+    fake_site_values = np.random.rand(197)
+    site_dict = {
+        "nwp": {
+            "ukv": DataArray(
+                raw_nwp_values,
+                dims=["time_utc", "channel", "y", "x"],
+                coords={
+                    "time_utc": pd.date_range("2024-01-01 00:00", periods=4, freq="h"),
+                    "channel": ["dswrf"],  # Single channel
+                },
+            )
+        },
+        "site": DataArray(
+            fake_site_values,
+            dims=["time_utc"],
+            coords={
+                    "time_utc": pd.date_range("2024-01-01 00:00", periods=197, freq="15min"),
+                    "capacity_kwp": 1000,
+                    "site_id": 1,
+                    "longitude": -3.5,
+                    "latitude": 51.5
+                }
+        )
+    }
+    print(f"Input site_dict: {site_dict}")
+
+    # Call function
+    result = site_ds.process_and_combine_site_sample_dict(site_dict)
+
+    # Assert to validate output structure
+    assert isinstance(result, Dataset), "Result should be an xarray.Dataset"
+    assert len(result.data_vars) > 0, "Dataset should contain data variables"
+
+    # Validate variable via assertion and shape of such
+    expected_variables = ["nwp-ukv", "site"]
+    for expected_variable in expected_variables:
+        assert expected_variable in result.data_vars, f"Expected variable '{expected_variable}' not found"
+    
+    nwp_result = result["nwp-ukv"]
+    assert nwp_result.shape == (4, 1, 2, 2), f"Unexpected shape for nwp-ukv : {nwp_result.shape}"
+    site_result = result["site"]
+    assert site_result.shape == (197,), f"Unexpected shape for site: {site_result.shape}"

ocf_data_sampler/numpy_batch/__init__.py

@@ -1,7 +0,0 @@
-"""Conversion from Xarray to NumpyBatch"""
-
-from .gsp import convert_gsp_to_numpy_batch
-from .nwp import convert_nwp_to_numpy_batch
-from .satellite import convert_satellite_to_numpy_batch
-from .sun_position import make_sun_position_numpy_batch
-

ocf_data_sampler/numpy_batch/gsp.py

@@ -1,20 +0,0 @@
-"""Convert GSP to Numpy Batch"""
-
-import xarray as xr
-from ocf_datapipes.batch import BatchKey, NumpyBatch
-
-
-def convert_gsp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> NumpyBatch:
-    """Convert from Xarray to NumpyBatch"""
-
-    example: NumpyBatch = {
-        BatchKey.gsp: da.values,
-        BatchKey.gsp_nominal_capacity_mwp: da.isel(time_utc=0)["nominal_capacity_mwp"].values,
-        BatchKey.gsp_effective_capacity_mwp: da.isel(time_utc=0)["effective_capacity_mwp"].values,
-        BatchKey.gsp_time_utc: da["time_utc"].values.astype(float),
-    }
-
-    if t0_idx is not None:
-        example[BatchKey.gsp_t0_idx] = t0_idx
-
-    return example

ocf_data_sampler/numpy_batch/nwp.py

@@ -1,33 +0,0 @@
-"""Convert NWP to NumpyBatch"""
-
-import pandas as pd
-import xarray as xr
-
-from ocf_datapipes.batch import NWPBatchKey, NWPNumpyBatch
-
-
-def convert_nwp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> NWPNumpyBatch:
-    """Convert from Xarray to NWP NumpyBatch"""
-
-    example: NWPNumpyBatch = {
-        NWPBatchKey.nwp: da.values,
-        NWPBatchKey.nwp_channel_names: da.channel.values,
-        NWPBatchKey.nwp_init_time_utc: da.init_time_utc.values.astype(float),
-        NWPBatchKey.nwp_step: (da.step.values / pd.Timedelta("1h")).astype(int),
-    }
-
-    if "target_time_utc" in da.coords:
-        example[NWPBatchKey.nwp_target_time_utc] = da.target_time_utc.values.astype(float)
-
-    # TODO: Do we need this at all? Especially since it is only present in UKV data
-    for batch_key, dataset_key in (
-        (NWPBatchKey.nwp_y_osgb, "y_osgb"),
-        (NWPBatchKey.nwp_x_osgb, "x_osgb"),
-    ):
-        if dataset_key in da.coords:
-            example[batch_key] = da[dataset_key].values
-
-    if t0_idx is not None:
-        example[NWPBatchKey.nwp_t0_idx] = t0_idx
-
-    return example

ocf_data_sampler/numpy_batch/satellite.py

@@ -1,23 +0,0 @@
-"""Convert Satellite to NumpyBatch"""
-import xarray as xr
-
-from ocf_datapipes.batch import BatchKey, NumpyBatch
-
-
-def convert_satellite_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> NumpyBatch:
-    """Convert from Xarray to NumpyBatch"""
-    example: NumpyBatch = {
-        BatchKey.satellite_actual: da.values,
-        BatchKey.satellite_time_utc: da.time_utc.values.astype(float),
-    }
-
-    for batch_key, dataset_key in (
-         (BatchKey.satellite_x_geostationary, "x_geostationary"),
-        (BatchKey.satellite_y_geostationary, "y_geostationary"),
-    ):
-        example[batch_key] = da[dataset_key].values
-
-    if t0_idx is not None:
-        example[BatchKey.satellite_t0_idx] = t0_idx
-
-    return example

ocf_data_sampler-0.0.19.dist-info/METADATA

@@ -1,22 +0,0 @@
-Metadata-Version: 2.1
-Name: ocf_data_sampler
-Version: 0.0.19
-Summary: Sample from weather data for renewable energy prediction
-Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
-Author-email: info@openclimatefix.org
-License: MIT
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: numpy
-Requires-Dist: pandas
-Requires-Dist: xarray
-Requires-Dist: zarr
-Requires-Dist: dask
-Requires-Dist: ocf-blosc2
-Requires-Dist: ocf-datapipes ==3.3.39
-Requires-Dist: pvlib
-
-# OCF Data Sampler
-[![ease of contribution: easy](https://img.shields.io/badge/ease%20of%20contribution:%20easy-32bd50)](https://github.com/openclimatefix/ocf-meta-repo?tab=readme-ov-file#overview-of-ocfs-nowcasting-repositories) 
-
-A repo for sampling from weather data for renewable energy prediction

ocf_data_sampler-0.0.19.dist-info/RECORD

@@ -1,32 +0,0 @@
-ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
-ocf_data_sampler/load/__init__.py,sha256=MjgfxilTzyz1RYFoBEeAXmE9hyjknLvdmlHPmlAoiQY,44
-ocf_data_sampler/load/gsp.py,sha256=Gcr1JVUOPKhFRDCSHtfPDjxx0BtyyEhXrZvGEKLPJ5I,759
-ocf_data_sampler/load/satellite.py,sha256=3KlA1fx4SwxdzM-jC1WRaONXO0D6m0WxORnEnwUnZrA,2967
-ocf_data_sampler/load/utils.py,sha256=EQGvVWlGMoSOdbDYuMfVAa0v6wmAOPmHIAemdrTB5v4,1406
-ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
-ocf_data_sampler/load/nwp/nwp.py,sha256=O4QnajEZem8BvBgTcYYDBhRhgqPYuJkolHmpMRmrXEA,610
-ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=vW-p3vCyQ-CofKo555-gE7VDi5hlpjtjTLfHqWF0HEE,1175
-ocf_data_sampler/load/nwp/providers/ukv.py,sha256=79Bm7q-K_GJPYMy62SUIZbRWRF4-tIaB1dYPEgLD9vo,1207
-ocf_data_sampler/load/nwp/providers/utils.py,sha256=Sy2exG1wpXLLhMXYdsfR-DZMR3txG1_bBmBdchlc-yA,848
-ocf_data_sampler/numpy_batch/__init__.py,sha256=mrtqwbGik5Zc9MYP5byfCTBm08wMtS2XnTsypC4fPMo,245
-ocf_data_sampler/numpy_batch/gsp.py,sha256=3gwSj0k29JyA8_09zovB8f8Pr-dVhCuMSO1-k4QKAOg,668
-ocf_data_sampler/numpy_batch/nwp.py,sha256=Rv0yfDj902Z2oCwdlRjOs3Kh-F5Fgxjjylh99-lQ9ws,1105
-ocf_data_sampler/numpy_batch/satellite.py,sha256=e6eoNmiiHtzZbDVtBolFzDuE3qwhHN6bL9H86emAUsk,732
-ocf_data_sampler/numpy_batch/sun_position.py,sha256=UW6-WtjrKdCkcguolHUDSLhYFfarknQzzjlCX8YdEOM,1700
-ocf_data_sampler/select/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/select/dropout.py,sha256=3aDqlL3U9kzzIkIHV44h5_ZbfOUSg1iChHKingXk7-s,1064
-ocf_data_sampler/select/fill_time_periods.py,sha256=iTtMjIPFYG5xtUYYedAFBLjTWWUa7t7WQ0-yksWf0-E,440
-ocf_data_sampler/select/find_contiguous_time_periods.py,sha256=6ioB8LeFpFNBMgKDxrgG3zqzNjkBF_jlV9yye2ZYT2E,11925
-ocf_data_sampler/select/select_spatial_slice.py,sha256=7BSzOFPMSBWpBWXSajWTfI8luUVsSgh4zN-rkr-AuUs,11470
-ocf_data_sampler/select/select_time_slice.py,sha256=41cch1fQr59fZgv7UHsNGc3OvoynrixT3bmr3_1d7cU,6628
-ocf_data_sampler/torch_datasets/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/torch_datasets/pvnet_uk_regional.py,sha256=NGrq5e6NOX8npz_45io7gVlx6eYfI-AW0rxcQjSOBQE,19167
-tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/conftest.py,sha256=OcArgF60paroZQqoP7xExRBF34nEyMuXd7dS7hD6p3w,5393
-ocf_data_sampler-0.0.19.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.0.19.dist-info/METADATA,sha256=rmnF4WJbPaLoeBFmJ-ZWgwY4FpoxcND8ZvQHMrnv8b8,801
-ocf_data_sampler-0.0.19.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-ocf_data_sampler-0.0.19.dist-info/top_level.txt,sha256=KaQn5qzkJGJP6hKWqsVAc9t0cMLjVvSTk8-kTrW79SA,23
-ocf_data_sampler-0.0.19.dist-info/RECORD,,