ocf-data-sampler 0.1.11__py3-none-any.whl → 0.1.16__py3-none-any.whl

tests/config/test_load.py

@@ -1,7 +0,0 @@
-from ocf_data_sampler.config import Configuration, load_yaml_configuration
-
-
-def test_load_yaml_configuration(test_config_filename):
-    loaded_config = load_yaml_configuration(test_config_filename)
-    assert isinstance(loaded_config, Configuration)
-        

tests/config/test_save.py

@@ -1,28 +0,0 @@
-"""Tests for configuration saving functionality."""
-import os
-from ocf_data_sampler.config import Configuration, save_yaml_configuration, load_yaml_configuration
-
-
-def test_save_yaml_configuration_basic(tmp_path):
-    """Save an empty configuration object"""
-    config = Configuration()
-
-    filepath = f"{tmp_path}/config.yaml"
-    save_yaml_configuration(config, filepath)
-    
-    assert os.path.exists(filepath)
-
-
-def test_save_load_yaml_configuration(tmp_path, test_config_filename):
-    """Make sure a saved configuration is the same after loading"""
-
-    # Start with this config
-    initial_config = load_yaml_configuration(test_config_filename)
-    
-    # Save it
-    filepath = f"{tmp_path}/config.yaml"
-    save_yaml_configuration(initial_config, filepath)
-    
-    # Load it and check it is still the same
-    loaded_config = load_yaml_configuration(filepath)
-    assert loaded_config == initial_config

tests/conftest.py

@@ -1,319 +0,0 @@
-import pytest
-
-import os
-import numpy as np
-import pandas as pd
-import xarray as xr
-import dask.array
-
-from ocf_data_sampler.config.model import Site
-from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configuration
-
-
-_top_test_directory = os.path.dirname(os.path.realpath(__file__))
-
-@pytest.fixture()
-def test_config_filename():
-    return f"{_top_test_directory}/test_data/configs/test_config.yaml"
-
-
-@pytest.fixture(scope="session")
-def config_filename():
-    return f"{_top_test_directory}/test_data/configs/pvnet_test_config.yaml"
-
-
-@pytest.fixture(scope="session")
-def session_tmp_path(tmp_path_factory):
-    return tmp_path_factory.mktemp("data")
-
-
-@pytest.fixture(scope="session")
-def sat_zarr_path(session_tmp_path):
-
-    # Define coords for satellite-like dataset
-    variables = [
-        'IR_016', 'IR_039', 'IR_087', 'IR_097', 'IR_108', 'IR_120', 
-        'IR_134', 'VIS006', 'VIS008', 'WV_062', 'WV_073',
-    ]
-    x = np.linspace(start=15002, stop=-1824245, num=100)
-    y = np.linspace(start=4191563, stop=5304712, num=100)
-    times = pd.date_range("2023-01-01 00:00", "2023-01-01 23:55", freq="5min")
-    
-    area_string = (
-        """msg_seviri_rss_3km:
-        description: MSG SEVIRI Rapid Scanning Service area definition with 3 km resolution
-        projection:
-            proj: geos
-            lon_0: 9.5
-            h: 35785831
-            x_0: 0
-            y_0: 0
-            a: 6378169
-            rf: 295.488065897014
-            no_defs: null
-            type: crs
-        shape:
-            height: 298
-            width: 615
-        area_extent:
-            lower_left_xy: [28503.830075263977, 5090183.970808983]
-            upper_right_xy: [-1816744.1169023514, 4196063.827395439]
-            units: m
-        """
-    )
-    
-    # Create satellite-like data with some NaNs
-    data = dask.array.zeros(
-        shape=(len(variables), len(times), len(y), len(x)), 
-        chunks=(-1, 10, -1, -1),
-        dtype=np.float32
-    )
-    data [:, 10, :, :] = np.nan
-    
-    ds = xr.DataArray(
-        data=data,
-        coords=dict(
-            variable=variables,
-            time=times,
-            y_geostationary=y,
-            x_geostationary=x,
-        ),
-        attrs=dict(area=area_string),
-    ).to_dataset(name="data")
-
-    # Save temporarily as a zarr
-    zarr_path = session_tmp_path / "test_sat.zarr"
-    ds.to_zarr(zarr_path)
-
-    yield zarr_path
-
-
-@pytest.fixture(scope="session")
-def ds_nwp_ukv():
-    init_times = pd.date_range(start="2023-01-01 00:00", freq="180min", periods=24 * 7)
-    steps = pd.timedelta_range("0h", "10h", freq="1h")
-
-    x = np.linspace(-239_000, 857_000, 50)
-    y = np.linspace(-183_000, 1225_000, 100)
-    variables = ["si10", "dswrf", "t", "prate"]
-
-    coords = (
-        ("init_time", init_times),
-        ("variable", variables),
-        ("step", steps),
-        ("x", x),
-        ("y", y),
-    )
-
-    nwp_array_shape = tuple(len(coord_values) for _, coord_values in coords)
-
-    nwp_data = xr.DataArray(
-        np.random.uniform(0, 200, size=nwp_array_shape).astype(np.float32),
-        coords=coords,
-    )
-    return nwp_data.to_dataset(name="UKV")
-
-
-@pytest.fixture(scope="session")
-def nwp_ukv_zarr_path(session_tmp_path, ds_nwp_ukv):
-    ds = ds_nwp_ukv.chunk(
-        {
-            "init_time": 1, 
-            "step": -1, 
-            "variable": -1,
-            "x": 50, 
-            "y": 50,
-        }
-    )
-    zarr_path = session_tmp_path / "ukv_nwp.zarr"
-    ds.to_zarr(zarr_path)
-    yield zarr_path
-
-
-@pytest.fixture()
-def ds_nwp_ukv_time_sliced():
-
-    t0 = pd.to_datetime("2024-01-02 00:00")
-
-    x = np.arange(-100, 100, 10)
-    y = np.arange(-100, 100, 10)
-    steps = pd.timedelta_range("0h", "8h", freq="1h")
-    target_times = t0 + steps
-
-    channels = ["t", "dswrf"]
-    init_times = pd.to_datetime([t0]*len(steps))
-    
-    # Create dummy time-sliced NWP data
-    da_nwp = xr.DataArray(
-        np.random.normal(size=(len(target_times), len(channels), len(x), len(y))),
-        coords=dict(
-            target_time_utc=(["target_time_utc"], target_times),
-            channel=(["channel"], channels),
-            x_osgb=(["x_osgb"], x),
-            y_osgb=(["y_osgb"], y),
-        )
-    )
-
-    # Add extra non-coordinate dimensions
-    da_nwp = da_nwp.assign_coords(
-        init_time_utc=("target_time_utc", init_times),
-        step=("target_time_utc", steps),
-    )
-
-    return da_nwp
-
-
-@pytest.fixture(scope="session")
-def ds_nwp_ecmwf():
-    init_times = pd.date_range(start="2023-01-01 00:00", freq="6h", periods=24 * 7)
-    steps = pd.timedelta_range("0h", "14h", freq="1h")
-
-    lons = np.arange(-12, 3)
-    lats = np.arange(48, 60)
-    variables = ["t2m","dswrf", "mcc"]
-
-    coords = (
-        ("init_time", init_times),
-        ("variable", variables),
-        ("step", steps),
-        ("longitude", lons),
-        ("latitude", lats),
-    )
-
-    nwp_array_shape = tuple(len(coord_values) for _, coord_values in coords)
-
-    nwp_data = xr.DataArray(
-        np.random.uniform(0, 200, size=nwp_array_shape).astype(np.float32),
-        coords=coords,
-    )
-    return nwp_data.to_dataset(name="ECMWF_UK")
-
-
-@pytest.fixture(scope="session")
-def nwp_ecmwf_zarr_path(session_tmp_path, ds_nwp_ecmwf):
-    ds = ds_nwp_ecmwf.chunk(
-        {
-            "init_time": 1, 
-            "step": -1, 
-            "variable": -1,
-            "longitude": 50, 
-            "latitude": 50,
-        }
-    )
-
-    zarr_path = session_tmp_path / "ukv_ecmwf.zarr"
-    ds.to_zarr(zarr_path)
-    yield zarr_path
-
-
-@pytest.fixture(scope="session")
-def ds_uk_gsp():
-    times = pd.date_range("2023-01-01 00:00", "2023-01-02 00:00", freq="30min")
-    gsp_ids = np.arange(0, 318)
-    capacity = np.ones((len(times), len(gsp_ids)))
-    generation = np.random.uniform(0, 200, size=(len(times), len(gsp_ids))).astype(np.float32)
-
-    coords = (
-        ("datetime_gmt", times),
-        ("gsp_id", gsp_ids),
-    )
-
-    da_cap = xr.DataArray(
-        capacity,
-        coords=coords,
-    )
-
-    da_gen = xr.DataArray(
-        generation,
-        coords=coords,
-    )
-
-    return xr.Dataset({
-        "capacity_mwp": da_cap, 
-        "installedcapacity_mwp": da_cap, 
-        "generation_mw":da_gen
-    })
-
-
-@pytest.fixture(scope="session")
-def data_sites(session_tmp_path) -> Site:
-    """
-    Make fake data for sites
-    Returns: filename for netcdf file, and csv metadata
-    """
-    times = pd.date_range("2023-01-01 00:00", "2023-01-02 00:00", freq="30min")
-    site_ids = list(range(0,10))
-    capacity_kwp_1d = np.array([0.1,1.1,4,6,8,9,15,2,3,4])
-    # these are quite specific for the fake satellite data
-    longitude = np.arange(-4, -3, 0.1)
-    latitude = np.arange(51, 52, 0.1)
-
-    generation = np.random.uniform(0, 200, size=(len(times), len(site_ids))).astype(np.float32)
-
-    # repeat capacity in new dims len(times) times
-    capacity_kwp = (np.tile(capacity_kwp_1d, len(times))).reshape(len(times),10)
-
-    coords = (
-        ("time_utc", times),
-        ("site_id", site_ids),
-    )
-
-    da_cap = xr.DataArray(
-        capacity_kwp,
-        coords=coords,
-    )
-
-    da_gen = xr.DataArray(
-        generation,
-        coords=coords,
-    )
-
-    # metadata
-    meta_df = pd.DataFrame(columns=[], data = [])
-    meta_df['site_id'] = site_ids
-    meta_df['capacity_kwp'] = capacity_kwp_1d
-    meta_df['longitude'] = longitude
-    meta_df['latitude'] = latitude
-
-    generation = xr.Dataset({
-        "capacity_kwp": da_cap,
-        "generation_kw": da_gen,
-    })
-
-    filename = f"{session_tmp_path}/sites.netcdf"
-    filename_csv = f"{session_tmp_path}/sites_metadata.csv"
-    generation.to_netcdf(filename)
-    meta_df.to_csv(filename_csv)
-
-    site = Site(
-        file_path=filename,
-        metadata_file_path=filename_csv,
-        interval_start_minutes=-30,
-        interval_end_minutes=60,
-        time_resolution_minutes=30,
-    )
-
-    yield site
-
-
-@pytest.fixture(scope="session")
-def uk_gsp_zarr_path(session_tmp_path, ds_uk_gsp):
-    zarr_path = session_tmp_path / "uk_gsp.zarr"
-    ds_uk_gsp.to_zarr(zarr_path)
-    yield zarr_path
-
-
-@pytest.fixture()
-def pvnet_config_filename(
-    tmp_path, config_filename, nwp_ukv_zarr_path, uk_gsp_zarr_path, sat_zarr_path
-):
-
-    # adjust config to point to the zarr file
-    config = load_yaml_configuration(config_filename)
-    config.input_data.nwp["ukv"].zarr_path = nwp_ukv_zarr_path
-    config.input_data.satellite.zarr_path = sat_zarr_path
-    config.input_data.gsp.zarr_path = uk_gsp_zarr_path
-
-    filename = f"{tmp_path}/configuration.yaml"
-    save_yaml_configuration(config, filename)
-    return filename

tests/load/test_load_gsp.py

@@ -1,15 +0,0 @@
-from ocf_data_sampler.load.gsp import open_gsp
-import xarray as xr
-
-
-def test_open_gsp(uk_gsp_zarr_path):
-    da = open_gsp(uk_gsp_zarr_path)
-
-    assert isinstance(da, xr.DataArray)
-    assert da.dims == ("time_utc", "gsp_id")
-
-    assert "nominal_capacity_mwp" in da.coords
-    assert "effective_capacity_mwp" in da.coords
-    assert "x_osgb" in da.coords
-    assert "y_osgb" in da.coords
-    assert da.shape == (49, 318)

tests/load/test_load_nwp.py

@@ -1,21 +0,0 @@
-import pandas as pd
-from xarray import DataArray
-import numpy as np
-
-from ocf_data_sampler.load.nwp import open_nwp
-
-
-def test_load_ukv(nwp_ukv_zarr_path):
-    da = open_nwp(zarr_path=nwp_ukv_zarr_path, provider="ukv")
-    assert isinstance(da, DataArray)
-    assert da.dims == ("init_time_utc", "step", "channel", "x_osgb", "y_osgb")
-    assert da.shape == (24 * 7, 11, 4, 50, 100)
-    assert np.issubdtype(da.dtype, np.number)
-
-
-def test_load_ecmwf(nwp_ecmwf_zarr_path):
-    da = open_nwp(zarr_path=nwp_ecmwf_zarr_path, provider="ecmwf")
-    assert isinstance(da, DataArray)
-    assert da.dims == ("init_time_utc", "step", "channel", "longitude", "latitude")
-    assert da.shape == (24 * 7, 15, 3, 15, 12)
-    assert np.issubdtype(da.dtype, np.number)

tests/load/test_load_satellite.py

@@ -1,17 +0,0 @@
-from ocf_data_sampler.load.satellite import open_sat_data
-import xarray as xr
-import numpy as np
-
-
-def test_open_satellite(sat_zarr_path):
-    da = open_sat_data(zarr_path=sat_zarr_path)
-
-    assert isinstance(da, xr.DataArray)
-    assert da.dims == ("time_utc", "channel", "x_geostationary", "y_geostationary")
-    # 288 is 1 days of data at 5 minutes intervals, 12 * 24
-    # There are 11 channels
-    # There are 100 x 100 pixels
-    assert da.shape == (288, 11, 100, 100)
-    assert np.issubdtype(da.dtype, np.number)
-
-

tests/load/test_load_sites.py

@@ -1,14 +0,0 @@
-from ocf_data_sampler.load.site import open_site
-import xarray as xr
-
-
-def test_open_site(data_sites):
-    da = open_site(data_sites.file_path, data_sites.metadata_file_path)
-
-    assert isinstance(da, xr.DataArray)
-    assert da.dims == ("time_utc", "site_id")
-
-    assert "capacity_kwp" in da.coords
-    assert "latitude" in da.coords
-    assert "longitude" in da.coords
-    assert da.shape == (49, 10)

tests/numpy_sample/test_collate.py

@@ -1,21 +0,0 @@
-from ocf_data_sampler.numpy_sample.collate import stack_np_samples_into_batch
-from ocf_data_sampler.torch_datasets.datasets.pvnet_uk import PVNetUKRegionalDataset
-
-
-def test_stack_np_samples_into_batch(pvnet_config_filename):
-
-    # Create dataset object
-    dataset = PVNetUKRegionalDataset(pvnet_config_filename)
-
-    # Generate 2 samples
-    sample1 = dataset[0]
-    sample2 = dataset[1]
-
-    batch = stack_np_samples_into_batch([sample1, sample2])
-
-    assert isinstance(batch, dict)
-    assert "nwp" in batch
-    assert isinstance(batch["nwp"], dict)
-    assert "ukv" in batch["nwp"]
-    assert "gsp" in batch
-    assert "satellite_actual" in batch

tests/numpy_sample/test_datetime_features.py

@@ -1,37 +0,0 @@
-import numpy as np
-import pandas as pd
-import pytest
-
-from ocf_data_sampler.numpy_sample.datetime_features import make_datetime_numpy_dict
-
-
-def test_calculate_azimuth_and_elevation():
-
-    # Pick the day of the summer solstice
-    datetimes = pd.to_datetime(["2024-06-20 12:00", "2024-06-20 12:30", "2024-06-20 13:00"])
-
-    # Calculate sun angles
-    datetime_features = make_datetime_numpy_dict(datetimes)
-
-    assert len(datetime_features) == 4
-
-    assert len(datetime_features["wind_date_sin"]) == len(datetimes)
-    assert (datetime_features["wind_date_cos"] != datetime_features["wind_date_sin"]).all()
-
-    # assert all values are between -1 and 1
-    assert all(np.abs(datetime_features["wind_date_sin"]) <= 1)
-    assert all(np.abs(datetime_features["wind_date_cos"]) <= 1)
-    assert all(np.abs(datetime_features["wind_time_sin"]) <= 1)
-    assert all(np.abs(datetime_features["wind_time_cos"]) <= 1)
-
-
-def test_make_datetime_numpy_batch_custom_key_prefix():
-    # Test function correctly applies custom prefix to dict keys
-    datetimes = pd.to_datetime(["2024-06-20 12:00", "2024-06-20 12:30", "2024-06-20 13:00"])
-    key_prefix = "solar"
-
-    datetime_features = make_datetime_numpy_dict(datetimes, key_prefix=key_prefix)
-
-    # Assert dict contains expected quantity of keys and verify starting with custom prefix
-    assert len(datetime_features) == 4
-    assert all(key.startswith(key_prefix) for key in datetime_features.keys())

tests/numpy_sample/test_gsp.py

@@ -1,38 +0,0 @@
-from ocf_data_sampler.load.gsp import open_gsp
-import numpy as np
-
-from ocf_data_sampler.numpy_sample import convert_gsp_to_numpy_sample, GSPSampleKey
-
-def test_convert_gsp_to_numpy_sample(uk_gsp_zarr_path):
-    
-    da = (
-        open_gsp(uk_gsp_zarr_path)
-        .isel(time_utc=slice(0, 10))
-        .sel(gsp_id=1)
-    )
-
-    numpy_sample = convert_gsp_to_numpy_sample(da)
-
-    # Test data structure
-    assert isinstance(numpy_sample, dict), "Should be dict"
-    assert set(numpy_sample.keys()).issubset({
-        GSPSampleKey.gsp,
-        GSPSampleKey.nominal_capacity_mwp,
-        GSPSampleKey.effective_capacity_mwp,
-        GSPSampleKey.time_utc,
-    }), "Unexpected keys"
-
-    # Assert data content and capacity values
-    assert np.array_equal(numpy_sample[GSPSampleKey.gsp], da.values), "GSP values mismatch"
-    assert isinstance(numpy_sample[GSPSampleKey.time_utc], np.ndarray), "Time UTC should be numpy array"
-    assert numpy_sample[GSPSampleKey.time_utc].dtype == float, "Time UTC should be float type"
-    assert numpy_sample[GSPSampleKey.nominal_capacity_mwp] == da.isel(time_utc=0)["nominal_capacity_mwp"].values
-    assert numpy_sample[GSPSampleKey.effective_capacity_mwp] == da.isel(time_utc=0)["effective_capacity_mwp"].values
-
-    # Test with t0_idx
-    t0_idx = 5
-    numpy_sample_with_t0 = convert_gsp_to_numpy_sample(da, t0_idx=t0_idx)
-    assert numpy_sample_with_t0[GSPSampleKey.t0_idx] == t0_idx, "t0_idx not correctly set"
-
-
-

tests/numpy_sample/test_nwp.py

@@ -1,13 +0,0 @@
-from ocf_data_sampler.numpy_sample import convert_nwp_to_numpy_sample, NWPSampleKey
-
-
-def test_convert_nwp_to_numpy_sample(ds_nwp_ukv_time_sliced):
-
-    # Call the function
-    numpy_sample = convert_nwp_to_numpy_sample(ds_nwp_ukv_time_sliced)
-
-    # Assert the output type
-    assert isinstance(numpy_sample, dict)
-
-    # Assert the shape of the numpy sample
-    assert (numpy_sample[NWPSampleKey.nwp] == ds_nwp_ukv_time_sliced.values).all()

tests/numpy_sample/test_satellite.py

@@ -1,40 +0,0 @@
-
-import numpy as np
-import pandas as pd
-import xarray as xr
-
-import pytest
-
-from ocf_data_sampler.numpy_sample import convert_satellite_to_numpy_sample, SatelliteSampleKey
-
-
-@pytest.fixture(scope="module")
-def da_sat_like():
-    """Create dummy data which looks like satellite data"""
-    x = np.arange(-100, 100, 10)
-    y = np.arange(-100, 100, 10)
-    datetimes = pd.date_range("2024-01-01 12:00", "2024-01-01 12:30", freq="5min")
-    channels = ["VIS008", "IR016"]
-
-    da_sat = xr.DataArray(
-        np.random.normal(size=(len(datetimes), len(channels), len(x), len(y))),
-        coords=dict(
-            time_utc=(["time_utc"], datetimes),
-            channel=(["channel"], channels),
-            x_geostationary=(["x_geostationary"], x),
-            y_geostationary=(["y_geostationary"], y),
-        )
-    )
-    return da_sat
-
-
-def test_convert_satellite_to_numpy_sample(da_sat_like):
-
-    # Call the function
-    numpy_sample = convert_satellite_to_numpy_sample(da_sat_like)
-
-    # Assert the output type
-    assert isinstance(numpy_sample, dict)
-
-    # Assert the shape of the numpy sample
-    assert (numpy_sample[SatelliteSampleKey.satellite_actual] == da_sat_like.values).all()

tests/numpy_sample/test_sun_position.py

@@ -1,81 +0,0 @@
-import numpy as np
-import pandas as pd
-import pytest
-
-from ocf_data_sampler.numpy_sample.sun_position import (
-    calculate_azimuth_and_elevation, make_sun_position_numpy_sample
-)
-
-from ocf_data_sampler.numpy_sample import GSPSampleKey
-
-
-@pytest.mark.parametrize("lat", [0, 5, 10, 23.5])
-def test_calculate_azimuth_and_elevation(lat):
-
-    # Pick the day of the summer solstice
-    datetimes = pd.to_datetime(["2024-06-20 12:00"])
-
-    # Calculate sun angles
-    azimuth, elevation = calculate_azimuth_and_elevation(datetimes, lon=0, lat=lat)
-
-    assert len(azimuth)==len(datetimes)
-    assert len(elevation)==len(datetimes)
-
-    # elevation should be close to (90 - (23.5-lat) degrees
-    assert np.abs(elevation - (90-23.5+lat)) < 1
-
-
-def test_calculate_azimuth_and_elevation_random():
-    """Test that the function produces the expected range of azimuths and elevations"""
-
-    # Set seed so we know the test should pass
-    np.random.seed(0)
-
-    # Pick the day of the summer solstice
-    datetimes = pd.to_datetime(["2024-06-20 12:00"])
-
-    # Pick 100 random locations and measure their azimuth and elevations
-    azimuths = []
-    elevations = []
-
-    for _ in range(100):
-
-        lon = np.random.uniform(low=0, high=360)
-        lat = np.random.uniform(low=-90, high=90)
-
-        # Calculate sun angles
-        azimuth, elevation = calculate_azimuth_and_elevation(datetimes, lon=lon, lat=lat)
-
-        azimuths.append(azimuth.item())
-        elevations.append(elevation.item())
-
-    azimuths = np.array(azimuths)
-    elevations = np.array(elevations)
-
-    assert (0<=azimuths).all() and (azimuths<=360).all()
-    assert (-90<=elevations).all() and (elevations<=90).all()
-
-    # Azimuth range is [0, 360]
-    assert azimuths.min() < 30
-    assert azimuths.max() > 330
-
-    # Elevation range is [-90, 90]
-    assert elevations.min() < -70
-    assert elevations.max() > 70
-
-
-def test_make_sun_position_numpy_sample():
-
-    datetimes = pd.date_range("2024-06-20 12:00", "2024-06-20 16:00", freq="30min")
-    lon, lat = 0, 51.5
-
-    sample = make_sun_position_numpy_sample(datetimes, lon, lat, key_prefix="gsp")
-
-    assert GSPSampleKey.solar_elevation in sample
-    assert GSPSampleKey.solar_azimuth in sample
-
-    # The solar coords are normalised in the function
-    assert (sample[GSPSampleKey.solar_elevation]>=0).all()
-    assert (sample[GSPSampleKey.solar_elevation]<=1).all()
-    assert (sample[GSPSampleKey.solar_azimuth]>=0).all()
-    assert (sample[GSPSampleKey.solar_azimuth]<=1).all()