ocf-data-sampler 0.0.18__py3-none-any.whl → 0.0.21__py3-none-any.whl

tests/select/test_select_time_slice.py

@@ -0,0 +1,284 @@
+from ocf_data_sampler.select.select_time_slice import select_time_slice, select_time_slice_nwp
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+import pytest
+
+
+NWP_FREQ = pd.Timedelta("3H")
+
+@pytest.fixture(scope="module")
+def da_sat_like():
+    """Create dummy data which looks like satellite data"""
+    x = np.arange(-100, 100)
+    y = np.arange(-100, 100)
+    datetimes = pd.date_range("2024-01-02 00:00", "2024-01-03 00:00", freq="5min")
+
+    da_sat = xr.DataArray(
+        np.random.normal(size=(len(datetimes), len(x), len(y))),
+        coords=dict(
+            time_utc=(["time_utc"], datetimes),
+            x_geostationary=(["x_geostationary"], x),
+            y_geostationary=(["y_geostationary"], y),
+        )
+    )
+    return da_sat
+
+
+@pytest.fixture(scope="module")
+def da_nwp_like():
+    """Create dummy data which looks like NWP data"""
+
+    x = np.arange(-100, 100)
+    y = np.arange(-100, 100)
+    datetimes = pd.date_range("2024-01-02 00:00", "2024-01-03 00:00", freq=NWP_FREQ)
+    steps = pd.timedelta_range("0H", "16H", freq="1H")
+    channels = ["t", "dswrf"]
+
+    da_nwp = xr.DataArray(
+        np.random.normal(size=(len(datetimes), len(steps), len(channels), len(x), len(y))),
+        coords=dict(
+            init_time_utc=(["init_time_utc"], datetimes),
+            step=(["step"], steps),
+            channel=(["channel"], channels),
+            x_osgb=(["x_osgb"], x),
+            y_osgb=(["y_osgb"], y),
+        )
+    )
+    return da_nwp
+
+
+@pytest.mark.parametrize("t0_str", ["12:30", "12:40", "12:00"])
+def test_select_time_slice(da_sat_like, t0_str):
+    """Test the basic functionality of select_time_slice"""
+
+    # Slice parameters
+    t0 = pd.Timestamp(f"2024-01-02 {t0_str}")
+    forecast_duration = pd.Timedelta("0min")
+    history_duration = pd.Timedelta("60min")
+    freq = pd.Timedelta("5min")
+
+    # Expect to return these timestamps from the selection
+    expected_datetimes = pd.date_range(t0 - history_duration, t0 + forecast_duration, freq=freq)
+
+    # Make the selection using the `[x]_duration` parameters
+    sat_sample = select_time_slice(
+        ds=da_sat_like,
+        t0=t0,
+        history_duration=history_duration,
+        forecast_duration=forecast_duration,
+        sample_period_duration=freq,
+    )
+
+    # Check the returned times are as expected
+    assert (sat_sample.time_utc == expected_datetimes).all()
+
+    # Make the selection using the `interval_[x]` parameters
+    sat_sample = select_time_slice(
+        ds=da_sat_like,
+        t0=t0,
+        interval_start=-history_duration,
+        interval_end=forecast_duration,
+        sample_period_duration=freq,
+    )
+
+    # Check the returned times are as expected
+    assert (sat_sample.time_utc == expected_datetimes).all()
+
+
+@pytest.mark.parametrize("t0_str", ["00:00", "00:25", "11:00", "11:55"])
+def test_select_time_slice_out_of_bounds(da_sat_like, t0_str):
+    """Test the behaviour of select_time_slice when the selection is out of bounds"""
+
+    # Slice parameters
+    t0 = pd.Timestamp(f"2024-01-02 {t0_str}")
+    forecast_duration = pd.Timedelta("30min")
+    history_duration = pd.Timedelta("60min")
+    freq = pd.Timedelta("5min")
+
+    # The data is available between these times
+    min_time = da_sat_like.time_utc.min()
+    max_time = da_sat_like.time_utc.max()
+
+    # Expect to return these timestamps from the selection
+    expected_datetimes = pd.date_range(t0 - history_duration, t0 + forecast_duration, freq=freq)
+
+    # Make the partially out of bounds selection
+    sat_sample = select_time_slice(
+        ds=da_sat_like,
+        t0=t0,
+        history_duration=history_duration,
+        forecast_duration=forecast_duration,
+        sample_period_duration=freq,
+        fill_selection=True
+    )
+
+    # Check the returned times are as expected
+    assert (sat_sample.time_utc == expected_datetimes).all()
+
+    
+    # Check all the values before the first timestamp available in the data are NaN
+    all_nan_space = sat_sample.isnull().all(dim=("x_geostationary", "y_geostationary"))
+    if expected_datetimes[0] < min_time:    
+        assert all_nan_space.sel(time_utc=slice(None, min_time-freq)).all(dim="time_utc") 
+
+    # Check all the values before the first timestamp available in the data are NaN
+    if expected_datetimes[-1] > max_time:    
+        assert all_nan_space.sel(time_utc=slice(max_time+freq, None)).all(dim="time_utc")   
+
+    # Check that none of the values between the first and last available timestamp are NaN
+    any_nan_space = sat_sample.isnull().any(dim=("x_geostationary", "y_geostationary"))
+    assert not any_nan_space.sel(time_utc=slice(min_time, max_time)).any(dim="time_utc")
+
+
+@pytest.mark.parametrize("t0_str", ["10:00", "11:00", "12:00"])
+def test_select_time_slice_nwp_basic(da_nwp_like, t0_str):
+    """Test the basic functionality of select_time_slice_nwp"""
+
+    # Slice parameters
+    t0 = pd.Timestamp(f"2024-01-02 {t0_str}")
+    forecast_duration = pd.Timedelta("6H")
+    history_duration = pd.Timedelta("3H")
+    freq = pd.Timedelta("1H")
+
+    # Make the selection
+    da_slice = select_time_slice_nwp(
+        da_nwp_like,
+        t0,
+        sample_period_duration=freq,
+        history_duration=history_duration,
+        forecast_duration=forecast_duration,
+        dropout_timedeltas = None,
+        dropout_frac = 0,
+        accum_channels = [],
+        channel_dim_name = "channel",
+    )
+
+    # Check the target-times are as expected
+    expected_target_times = pd.date_range(t0 - history_duration, t0 + forecast_duration, freq=freq)
+    assert (da_slice.target_time_utc==expected_target_times).all()
+
+    # Check the init-times are as expected
+    # - Forecast frequency is `NWP_FREQ`, and we can't have selected future init-times
+    expected_init_times = pd.to_datetime(
+        [t if t<t0 else t0 for t in expected_target_times]
+    ).floor(NWP_FREQ)
+    assert (da_slice.init_time_utc==expected_init_times).all()
+
+
+@pytest.mark.parametrize("dropout_hours", [1, 2, 5])
+def test_select_time_slice_nwp_with_dropout(da_nwp_like, dropout_hours):
+    """Test the functionality of select_time_slice_nwp with dropout"""
+
+    t0 = pd.Timestamp("2024-01-02 12:00")
+    forecast_duration = pd.Timedelta("6H")
+    history_duration = pd.Timedelta("3H")
+    freq = pd.Timedelta("1H")
+    dropout_timedelta = pd.Timedelta(f"-{dropout_hours}H")
+
+    da_slice = select_time_slice_nwp(
+        da_nwp_like,
+        t0,
+        sample_period_duration=freq,
+        history_duration=history_duration,
+        forecast_duration=forecast_duration,
+        dropout_timedeltas = [dropout_timedelta],
+        dropout_frac = 1,
+        accum_channels = [],
+        channel_dim_name = "channel",
+    )
+
+    # Check the target-times are as expected
+    expected_target_times = pd.date_range(t0 - history_duration, t0 + forecast_duration, freq=freq)
+    assert (da_slice.target_time_utc==expected_target_times).all()
+
+    # Check the init-times are as expected considering the delay
+    t0_delayed = t0 + dropout_timedelta
+    expected_init_times = pd.to_datetime(
+        [t if t<t0_delayed else t0_delayed for t in expected_target_times]
+    ).floor(NWP_FREQ)
+    assert (da_slice.init_time_utc==expected_init_times).all()
+
+
+@pytest.mark.parametrize("t0_str", ["10:00", "11:00", "12:00"])
+def test_select_time_slice_nwp_with_dropout_and_accum(da_nwp_like, t0_str):
+    """Test the functionality of select_time_slice_nwp with dropout and accumulated variables"""
+
+    # Slice parameters
+    t0 = pd.Timestamp(f"2024-01-02 {t0_str}")
+    forecast_duration = pd.Timedelta("6H")
+    history_duration = pd.Timedelta("3H")
+    freq = pd.Timedelta("1H")
+    dropout_timedelta = pd.Timedelta("-2H")
+
+    t0_delayed = (t0 + dropout_timedelta).floor(NWP_FREQ)
+
+    da_slice = select_time_slice_nwp(
+        da_nwp_like,
+        t0,
+        sample_period_duration=freq,
+        history_duration=history_duration,
+        forecast_duration=forecast_duration,
+        dropout_timedeltas=[dropout_timedelta],
+        dropout_frac=1,
+        accum_channels=["dswrf"],
+        channel_dim_name="channel",
+    )
+
+    # Check the target-times are as expected
+    expected_target_times = pd.date_range(t0 - history_duration, t0 + forecast_duration, freq=freq)
+    assert (da_slice.target_time_utc==expected_target_times).all()
+
+    # Check the init-times are as expected considering the delay
+    expected_init_times = pd.to_datetime(
+        [t if t<t0_delayed else t0_delayed for t in expected_target_times]
+    ).floor(NWP_FREQ)
+    assert (da_slice.init_time_utc==expected_init_times).all()
+
+    # Check channels are as expected
+    assert (da_slice.channel.values == ["t", "diff_dswrf"]).all()
+
+    # Check the accummulated channel has been differenced correctly
+
+    # This part of the data is pulled from the init-time: t0_delayed
+    da_slice_accum = da_slice.sel(
+        target_time_utc=slice(t0_delayed, None), 
+        channel="diff_dswrf"
+    )
+
+    # Get the original data for the t0_delayed init-time, and diff it along steps
+    # then select the steps which are expected to be used in the above slice
+    da_orig_diffed = (
+        da_nwp_like.sel(
+            init_time_utc=t0_delayed, 
+            channel="dswrf", 
+        ).diff(dim="step", label="lower")
+        .sel(step=slice(t0-t0_delayed - history_duration, t0-t0_delayed + forecast_duration))
+    )
+
+    # Check the values are the same
+    assert (da_slice_accum.values == da_orig_diffed.values).all()
+
+    # Check the non-accummulated channel has not been differenced
+
+    # This part of the data is pulled from the init-time: t0_delayed
+    da_slice_nonaccum = da_slice.sel(
+        target_time_utc=slice(t0_delayed, None), 
+        channel="t"
+    )
+
+    # Get the original data for the t0_delayed init-time, and select the steps which are expected 
+    # to be used in the above slice
+    da_orig = (
+        da_nwp_like.sel(
+            init_time_utc=t0_delayed, 
+            channel="t", 
+        )
+        .sel(step=slice(t0-t0_delayed - history_duration, t0-t0_delayed + forecast_duration))
+    )
+
+    # Check the values are the same
+    assert (da_slice_nonaccum.values == da_orig.values).all()
+
+

tests/torch_datasets/test_pvnet_uk_regional.py

@@ -0,0 +1,72 @@
+import pytest
+import tempfile
+
+from ocf_data_sampler.torch_datasets.pvnet_uk_regional import PVNetUKRegionalDataset
+from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configuration
+from ocf_datapipes.batch import BatchKey, NWPBatchKey
+
+
+@pytest.fixture()
+def pvnet_config_filename(tmp_path, config_filename, nwp_ukv_zarr_path, uk_gsp_zarr_path, sat_zarr_path):
+
+    # adjust config to point to the zarr file
+    config = load_yaml_configuration(config_filename)
+    config.input_data.nwp['ukv'].nwp_zarr_path = nwp_ukv_zarr_path
+    config.input_data.satellite.satellite_zarr_path = sat_zarr_path
+    config.input_data.gsp.gsp_zarr_path = uk_gsp_zarr_path
+
+    filename = f"{tmp_path}/configuration.yaml"
+    save_yaml_configuration(config, filename)
+    return filename
+
+
+def test_pvnet(pvnet_config_filename):
+
+    # Create dataset object
+    dataset = PVNetUKRegionalDataset(pvnet_config_filename)
+
+    assert len(dataset.locations) == 317 # no of GSPs not including the National level
+    # NB. I have not checked this value is in fact correct, but it does seem to stay constant
+    assert len(dataset.valid_t0_times) == 39
+    assert len(dataset) == 317*39
+
+    # Generate a sample
+    sample = dataset[0]
+
+    assert isinstance(sample, dict)
+
+    for key in [
+        BatchKey.nwp, BatchKey.satellite_actual, BatchKey.gsp,
+        BatchKey.gsp_solar_azimuth, BatchKey.gsp_solar_elevation,
+    ]:
+        assert key in sample
+    
+    for nwp_source in ["ukv"]:
+        assert nwp_source in sample[BatchKey.nwp]
+
+    # check the shape of the data is correct
+    # 30 minutes of 5 minute data (inclusive), one channel, 2x2 pixels
+    assert sample[BatchKey.satellite_actual].shape == (7, 1, 2, 2)
+    # 3 hours of 60 minute data (inclusive), one channel, 2x2 pixels
+    assert sample[BatchKey.nwp]["ukv"][NWPBatchKey.nwp].shape == (4, 1, 2, 2)
+    # 3 hours of 30 minute data (inclusive)
+    assert sample[BatchKey.gsp].shape == (7,)
+    # Solar angles have same shape as GSP data
+    assert sample[BatchKey.gsp_solar_azimuth].shape == (7,)
+    assert sample[BatchKey.gsp_solar_elevation].shape == (7,)
+
+def test_pvnet_no_gsp(pvnet_config_filename):
+
+    # load config
+    config = load_yaml_configuration(pvnet_config_filename)
+    # remove gsp
+    config.input_data.gsp.gsp_zarr_path = ''
+
+    # save temp config file
+    with tempfile.NamedTemporaryFile() as temp_config_file:
+        save_yaml_configuration(config, temp_config_file.name)
+        # Create dataset object
+        dataset = PVNetUKRegionalDataset(temp_config_file.name)
+
+        # Generate a sample
+        _ = dataset[0]

ocf_data_sampler-0.0.18.dist-info/METADATA

@@ -1,22 +0,0 @@
-Metadata-Version: 2.1
-Name: ocf_data_sampler
-Version: 0.0.18
-Summary: Sample from weather data for renewable energy prediction
-Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
-Author-email: info@openclimatefix.org
-License: MIT
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: numpy
-Requires-Dist: pandas
-Requires-Dist: xarray
-Requires-Dist: zarr
-Requires-Dist: dask
-Requires-Dist: ocf-blosc2
-Requires-Dist: ocf-datapipes ==3.3.39
-Requires-Dist: pvlib
-
-# OCF Data Sampler
-[![ease of contribution: easy](https://img.shields.io/badge/ease%20of%20contribution:%20easy-32bd50)](https://github.com/openclimatefix/ocf-meta-repo?tab=readme-ov-file#overview-of-ocfs-nowcasting-repositories) 
-
-A repo for sampling from weather data for renewable energy prediction

ocf_data_sampler-0.0.18.dist-info/RECORD

@@ -1,32 +0,0 @@
-ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
-ocf_data_sampler/load/__init__.py,sha256=MjgfxilTzyz1RYFoBEeAXmE9hyjknLvdmlHPmlAoiQY,44
-ocf_data_sampler/load/gsp.py,sha256=Gcr1JVUOPKhFRDCSHtfPDjxx0BtyyEhXrZvGEKLPJ5I,759
-ocf_data_sampler/load/satellite.py,sha256=3KlA1fx4SwxdzM-jC1WRaONXO0D6m0WxORnEnwUnZrA,2967
-ocf_data_sampler/load/utils.py,sha256=EQGvVWlGMoSOdbDYuMfVAa0v6wmAOPmHIAemdrTB5v4,1406
-ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
-ocf_data_sampler/load/nwp/nwp.py,sha256=O4QnajEZem8BvBgTcYYDBhRhgqPYuJkolHmpMRmrXEA,610
-ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=vW-p3vCyQ-CofKo555-gE7VDi5hlpjtjTLfHqWF0HEE,1175
-ocf_data_sampler/load/nwp/providers/ukv.py,sha256=79Bm7q-K_GJPYMy62SUIZbRWRF4-tIaB1dYPEgLD9vo,1207
-ocf_data_sampler/load/nwp/providers/utils.py,sha256=Sy2exG1wpXLLhMXYdsfR-DZMR3txG1_bBmBdchlc-yA,848
-ocf_data_sampler/numpy_batch/__init__.py,sha256=mrtqwbGik5Zc9MYP5byfCTBm08wMtS2XnTsypC4fPMo,245
-ocf_data_sampler/numpy_batch/gsp.py,sha256=3gwSj0k29JyA8_09zovB8f8Pr-dVhCuMSO1-k4QKAOg,668
-ocf_data_sampler/numpy_batch/nwp.py,sha256=Rv0yfDj902Z2oCwdlRjOs3Kh-F5Fgxjjylh99-lQ9ws,1105
-ocf_data_sampler/numpy_batch/satellite.py,sha256=e6eoNmiiHtzZbDVtBolFzDuE3qwhHN6bL9H86emAUsk,732
-ocf_data_sampler/numpy_batch/sun_position.py,sha256=UW6-WtjrKdCkcguolHUDSLhYFfarknQzzjlCX8YdEOM,1700
-ocf_data_sampler/select/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/select/dropout.py,sha256=3aDqlL3U9kzzIkIHV44h5_ZbfOUSg1iChHKingXk7-s,1064
-ocf_data_sampler/select/fill_time_periods.py,sha256=iTtMjIPFYG5xtUYYedAFBLjTWWUa7t7WQ0-yksWf0-E,440
-ocf_data_sampler/select/find_contiguous_time_periods.py,sha256=6ioB8LeFpFNBMgKDxrgG3zqzNjkBF_jlV9yye2ZYT2E,11925
-ocf_data_sampler/select/select_spatial_slice.py,sha256=7BSzOFPMSBWpBWXSajWTfI8luUVsSgh4zN-rkr-AuUs,11470
-ocf_data_sampler/select/select_time_slice.py,sha256=41cch1fQr59fZgv7UHsNGc3OvoynrixT3bmr3_1d7cU,6628
-ocf_data_sampler/torch_datasets/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/torch_datasets/pvnet_uk_regional.py,sha256=W8AfmfkLywRjMJlPSJ6u3ZhB7ShwtiQuM4BZs5de_LA,18941
-tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/conftest.py,sha256=OcArgF60paroZQqoP7xExRBF34nEyMuXd7dS7hD6p3w,5393
-ocf_data_sampler-0.0.18.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.0.18.dist-info/METADATA,sha256=EJKObEAl-_R7a_1xgtXZQ-eXvg1Ljsgbl5QNG1mb1m4,801
-ocf_data_sampler-0.0.18.dist-info/WHEEL,sha256=5Mi1sN9lKoFv_gxcPtisEVrJZihrm_beibeg5R6xb4I,91
-ocf_data_sampler-0.0.18.dist-info/top_level.txt,sha256=KaQn5qzkJGJP6hKWqsVAc9t0cMLjVvSTk8-kTrW79SA,23
-ocf_data_sampler-0.0.18.dist-info/RECORD,,