ocf-data-sampler 0.1.11__py3-none-any.whl → 0.1.17__py3-none-any.whl

tests/test_sample/test_site_sample.py

@@ -1,165 +0,0 @@
-"""
-Site class testing - SiteSample
-"""
-
-import pytest
-import numpy as np
-import xarray as xr
-import pandas as pd
-
-from pathlib import Path
-from xarray import Dataset
-
-from ocf_data_sampler.sample.site import SiteSample
-
-
-@pytest.fixture
-def sample_data():
-    """ Fixture creation with sample data """
-
-    #  Time periods specified
-    init_time = pd.Timestamp("2023-01-01 00:00")
-    target_times = pd.date_range("2023-01-01 00:00", periods=4, freq="1h")
-    sat_times = pd.date_range("2023-01-01 00:00", periods=7, freq="5min")
-    site_times = pd.date_range("2023-01-01 00:00", periods=4, freq="30min")
-
-    # Defined steps for NWP data
-    steps = [(t - init_time) for t in target_times]
-    
-    # Create the sample dataset
-    return Dataset(
-        data_vars={
-            'nwp-ukv': (
-                ["nwp-ukv__target_time_utc", "nwp-ukv__channel", 
-                 "nwp-ukv__y_osgb", "nwp-ukv__x_osgb"],
-                np.random.rand(4, 1, 2, 2)
-            ),
-            'satellite': (
-                ["satellite__time_utc", "satellite__channel",
-                 "satellite__y_geostationary", "satellite__x_geostationary"],
-                np.random.rand(7, 1, 2, 2)
-            ),
-            'site': (["site__time_utc"], np.random.rand(4))
-        },
-        coords={
-            # NWP coords
-            'nwp-ukv__target_time_utc': target_times,
-            'nwp-ukv__channel': ['dswrf'],
-            'nwp-ukv__y_osgb': [0, 1],
-            'nwp-ukv__x_osgb': [0, 1],
-            'nwp-ukv__init_time_utc': init_time,
-            'nwp-ukv__step': ('nwp-ukv__target_time_utc', steps),
-            
-            # Sat coords
-            'satellite__time_utc': sat_times,
-            'satellite__channel': ['vis'],
-            'satellite__y_geostationary': [0, 1],
-            'satellite__x_geostationary': [0, 1],
-            
-            # Site coords
-            'site__time_utc': site_times,
-            'site__capacity_kwp': 1000.0,
-            'site__site_id': 1,
-            'site__longitude': -3.5,
-            'site__latitude': 51.5,
-            
-            # Site features as coords
-            'site__solar_azimuth': ('site__time_utc', np.random.rand(4)),
-            'site__solar_elevation': ('site__time_utc', np.random.rand(4)),
-            'site__date_cos': ('site__time_utc', np.random.rand(4)),
-            'site__date_sin': ('site__time_utc', np.random.rand(4)),
-            'site__time_cos': ('site__time_utc', np.random.rand(4)),
-            'site__time_sin': ('site__time_utc', np.random.rand(4))
-        }
-    )
-
-
-def test_site_sample_with_data(sample_data):
-    """ Testing of defined sample with actual data """
-    sample = SiteSample(sample_data)
-    
-    # Assert data structure
-    assert isinstance(sample._data, Dataset)
-    
-    # Assert dimensions / shapes
-    expected_dims = {
-        "satellite__x_geostationary",
-        "site__time_utc",
-        "nwp-ukv__target_time_utc",
-        "nwp-ukv__x_osgb",
-        "satellite__channel",
-        "satellite__y_geostationary",
-        "satellite__time_utc",
-        "nwp-ukv__channel",
-        "nwp-ukv__y_osgb",
-    }
-    assert set(sample._data.dims) == expected_dims    
-    assert sample._data["satellite"].values.shape == (7, 1, 2, 2)
-    assert sample._data["nwp-ukv"].values.shape == (4, 1, 2, 2)
-    assert sample._data["site"].values.shape == (4,)
-
-
-def test_save_load(tmp_path, sample_data):
-    """ Save and load functionality """
-    sample = SiteSample(sample_data)
-    filepath = tmp_path / "test_sample.nc"
-    sample.save(filepath)
-    
-    # Assert file exists and has content
-    assert filepath.exists()
-    assert filepath.stat().st_size > 0
-    
-    # Load and verify
-    loaded = SiteSample.load(filepath)
-    assert isinstance(loaded, SiteSample)
-    assert isinstance(loaded._data, Dataset)
-    
-    # Compare original / loaded data
-    xr.testing.assert_identical(sample._data, loaded._data)
-
-
-def test_invalid_data_type():
-    """ Handling of invalid data types """
-    
-    with pytest.raises(TypeError, match="Data must be xarray Dataset"):
-        _ = SiteSample({"invalid": "data"})
-
-
-def test_to_numpy(sample_data):
-    """ To numpy conversion """
-    sample = SiteSample(sample_data)
-    numpy_data = sample.to_numpy()
-    
-    # Assert structure
-    assert isinstance(numpy_data, dict)
-    assert 'site' in numpy_data
-    assert 'nwp' in numpy_data
-    
-    # Check site - numpy array instead of dict
-    site_data = numpy_data['site']
-    assert isinstance(site_data, np.ndarray)
-    assert site_data.ndim == 1
-    assert len(site_data) == 4
-    assert np.all(site_data >= 0) and np.all(site_data <= 1)
-    
-    # Check NWP
-    assert 'ukv' in numpy_data['nwp']
-    nwp_data = numpy_data['nwp']['ukv']
-    assert 'nwp' in nwp_data
-    assert nwp_data['nwp'].shape == (4, 1, 2, 2)
-
-
-def test_data_consistency(sample_data):
-    """ Consistency of data across operations """
-    sample = SiteSample(sample_data)
-    numpy_data = sample.to_numpy()
-    
-    # Assert components remain consistent after conversion above
-    assert numpy_data['nwp']['ukv']['nwp'].shape == (4, 1, 2, 2)
-    assert 'site' in numpy_data
-    
-    # Update site data checks to expect numpy array
-    assert isinstance(numpy_data['site'], np.ndarray)
-    assert numpy_data['site'].shape == (4,)
-    assert np.all(numpy_data['site'] >= 0)
-    assert np.all(numpy_data['site'] <= 1)

tests/test_sample/test_uk_regional_sample.py

@@ -1,136 +0,0 @@
-"""
-UK Regional class testing - UKRegionalSample
-"""
-
-import pytest
-import numpy as np
-import torch
-import tempfile
-
-from ocf_data_sampler.numpy_sample import (
-    GSPSampleKey,
-    SatelliteSampleKey,
-    NWPSampleKey
-)
-
-from ocf_data_sampler.sample.uk_regional import UKRegionalSample
-
-
-@pytest.fixture
-def pvnet_config_filename(tmp_path):
-    """ Minimal config file - testing """
-    config_content = """
-    input_data:
-        gsp:
-            zarr_path: ""
-            time_resolution_minutes: 30
-            interval_start_minutes: -180
-            interval_end_minutes: 0
-        nwp:
-            ukv:
-                zarr_path: ""
-                image_size_pixels_height: 64
-                image_size_pixels_width: 64
-                time_resolution_minutes: 60
-                interval_start_minutes: -180
-                interval_end_minutes: 0
-                channels: ["t", "dswrf"]
-                provider: "ukv"
-        satellite:
-            zarr_path: ""
-            image_size_pixels_height: 64
-            image_size_pixels_width: 64
-            time_resolution_minutes: 30
-            interval_start_minutes: -180
-            interval_end_minutes: 0
-            channels: ["HRV"]
-    """
-    config_file = tmp_path / "test_config.yaml"
-    config_file.write_text(config_content)
-    return str(config_file)
-
-@pytest.fixture
-def numpy_sample():
-    """ Synthetic data generation """
-
-    # Field / spatial coordinates
-    nwp_data = {
-        'nwp': np.random.rand(4, 1, 2, 2),
-        'x': np.array([1, 2]),
-        'y': np.array([1, 2]),
-        NWPSampleKey.channel_names: ['test_channel']
-    }
-    
-    return {
-        'nwp': {
-            'ukv': nwp_data
-        },
-        GSPSampleKey.gsp: np.random.rand(7),
-        SatelliteSampleKey.satellite_actual: np.random.rand(7, 1, 2, 2),
-        GSPSampleKey.solar_azimuth: np.random.rand(7),
-        GSPSampleKey.solar_elevation: np.random.rand(7)
-    }
-
-
-def test_sample_save_load(numpy_sample):
-   sample = UKRegionalSample(numpy_sample)
-   
-   with tempfile.NamedTemporaryFile(suffix='.pt') as tf:
-       sample.save(tf.name)
-       loaded = UKRegionalSample.load(tf.name)
-       
-       assert set(loaded._data.keys()) == set(sample._data.keys())
-       assert isinstance(loaded._data['nwp'], dict)
-       assert 'ukv' in loaded._data['nwp']
-
-       assert loaded._data[GSPSampleKey.gsp].shape == (7,)
-       assert loaded._data[SatelliteSampleKey.satellite_actual].shape == (7, 1, 2, 2) 
-       assert loaded._data[GSPSampleKey.solar_azimuth].shape == (7,)
-       assert loaded._data[GSPSampleKey.solar_elevation].shape == (7,)
-
-       np.testing.assert_array_almost_equal(
-           loaded._data[GSPSampleKey.gsp], 
-           sample._data[GSPSampleKey.gsp]
-       )
-
-
-def test_load_corrupted_file():
-    """ Test loading - corrupted / empty file """
-
-    with tempfile.NamedTemporaryFile(suffix='.pt') as tf:
-        with open(tf.name, 'wb') as f:
-            f.write(b'corrupted data')
-        
-        with pytest.raises(Exception):
-            UKRegionalSample.load(tf.name)
-
-
-def test_to_numpy():
-    """ To numpy conversion check """
-
-    data = {
-        'nwp': {
-            'ukv': {
-                'nwp': np.random.rand(4, 1, 2, 2),
-                'x': np.array([1, 2]),
-                'y': np.array([1, 2])
-            }
-        },
-        GSPSampleKey.gsp: np.random.rand(7),
-        SatelliteSampleKey.satellite_actual: np.random.rand(7, 1, 2, 2),
-        GSPSampleKey.solar_azimuth: np.random.rand(7),
-        GSPSampleKey.solar_elevation: np.random.rand(7)
-    }
-
-    sample = UKRegionalSample(data)
-    
-    numpy_data = sample.to_numpy()
-    
-    # Check returned data matches
-    assert numpy_data == sample._data
-    assert len(numpy_data) == len(sample._data)
-    
-    # Assert specific keys and types
-    assert 'nwp' in numpy_data
-    assert isinstance(numpy_data['nwp']['ukv']['nwp'], np.ndarray)
-    assert numpy_data[GSPSampleKey.gsp].shape == (7,)

tests/torch_datasets/test_merge_and_fill_utils.py

@@ -1,40 +0,0 @@
-import numpy as np
-
-from ocf_data_sampler.torch_datasets.utils.merge_and_fill_utils import (
-    merge_dicts,
-    fill_nans_in_arrays,
-)
-
-def test_merge_dicts():
-    """Test merge_dicts function"""
-    dict1 = {"a": 1, "b": 2}
-    dict2 = {"c": 3, "d": 4}
-    dict3 = {"e": 5}
-    
-    result = merge_dicts([dict1, dict2, dict3])
-    assert result == {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}
-    
-    # Test key overwriting
-    dict4 = {"a": 10, "f": 6}
-    result = merge_dicts([dict1, dict4])
-    assert result["a"] == 10
-
-
-def test_fill_nans_in_arrays():
-    """Test the fill_nans_in_arrays function"""
-    array_with_nans = np.array([1.0, np.nan, 3.0, np.nan])
-    nested_dict = {
-        "array1": array_with_nans,
-        "nested": {
-            "array2": np.array([np.nan, 2.0, np.nan, 4.0])
-        },
-        "string_key": "not_an_array"
-    }
-    
-    result = fill_nans_in_arrays(nested_dict)
-    
-    assert np.array_equal(result["array1"], np.array([1.0, 0.0, 3.0, 0.0]))
-    assert np.array_equal(result["nested"]["array2"], np.array([0.0, 2.0, 0.0, 4.0]))
-    assert result["string_key"] == "not_an_array"
-
-

tests/torch_datasets/test_pvnet_uk.py

@@ -1,154 +0,0 @@
-import numpy as np
-import pandas as pd
-import xarray as xr
-import dask.array
-
-from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configuration
-from ocf_data_sampler.torch_datasets.datasets.pvnet_uk import (
-    PVNetUKRegionalDataset,
-    PVNetUKConcurrentDataset,
-    process_and_combine_datasets, 
-    compute,
-)
-from ocf_data_sampler.select.location import Location
-
-
-def test_process_and_combine_datasets(pvnet_config_filename, ds_nwp_ukv_time_sliced):
-
-    config = load_yaml_configuration(pvnet_config_filename)
-    
-    t0 = pd.Timestamp("2024-01-01 00:00")
-    location = Location(coordinate_system="osgb", x=1234, y=5678, id=1)
-
-    sat_data = xr.DataArray(
-        np.random.rand(7, 1, 2, 2),
-        dims=["time_utc", "channel", "y", "x"],
-        coords={
-            "time_utc": pd.date_range("2024-01-01 00:00", periods=7, freq="5min"),
-            "channel": ["HRV"],
-            "x_geostationary": (["y", "x"], np.array([[1, 2], [1, 2]])),
-            "y_geostationary": (["y", "x"], np.array([[1, 1], [2, 2]]))
-        }
-    )
-
-    dataset_dict = {
-        "nwp": {"ukv": ds_nwp_ukv_time_sliced},
-        "sat": sat_data
-    }
-
-    sample = process_and_combine_datasets(dataset_dict, config, t0, location)
-
-    assert isinstance(sample, dict)
-    assert "nwp" in sample
-    assert sample["satellite_actual"].shape == sat_data.shape
-    assert sample["nwp"]["ukv"]["nwp"].shape == ds_nwp_ukv_time_sliced.shape
-    assert "gsp_id" in sample
-
-
-def test_compute():
-    """Test compute function with dask array"""
-    da_dask = xr.DataArray(dask.array.random.random((5, 5)))
-
-    # Create a nested dictionary with dask array
-    lazy_data_dict = {
-        "array1": da_dask,
-        "nested": {
-            "array2": da_dask
-        }
-    }
-
-    computed_data_dict = compute(lazy_data_dict)
-
-    # Assert that the result is no longer lazy
-    assert isinstance(computed_data_dict["array1"].data, np.ndarray)
-    assert isinstance(computed_data_dict["nested"]["array2"].data, np.ndarray)
-
-    
-def test_pvnet_uk_regional_dataset(pvnet_config_filename):
-
-    # Create dataset object
-    dataset = PVNetUKRegionalDataset(pvnet_config_filename)
-
-    assert len(dataset.locations) == 317 # Number of regional GSPs
-    # NB. I have not checked the value (39 below) is in fact correct
-    assert len(dataset.valid_t0_times) == 39
-    assert len(dataset) == 317*39
-
-    # Generate a sample
-    sample = dataset[0]
-
-    assert isinstance(sample, dict)
-
-    for key in [
-        "nwp", "satellite_actual", "gsp",
-        "gsp_solar_azimuth", "gsp_solar_elevation",
-    ]:
-        assert key in sample
-    
-    for nwp_source in ["ukv"]:
-        assert nwp_source in sample["nwp"]
-
-    # Check the shape of the data is correct
-    # 30 minutes of 5 minute data (inclusive), one channel, 2x2 pixels
-    assert sample["satellite_actual"].shape == (7, 1, 2, 2)
-    # 3 hours of 60 minute data (inclusive), one channel, 2x2 pixels
-    assert sample["nwp"]["ukv"]["nwp"].shape == (4, 1, 2, 2)
-    # 3 hours of 30 minute data (inclusive)
-    assert sample["gsp"].shape == (7,)
-    # Solar angles have same shape as GSP data
-    assert sample["gsp_solar_azimuth"].shape == (7,)
-    assert sample["gsp_solar_elevation"].shape == (7,)
-
-
-def test_pvnet_no_gsp(tmp_path, pvnet_config_filename):
-
-    # Create new config without GSP inputs
-    config = load_yaml_configuration(pvnet_config_filename)
-    config.input_data.gsp.zarr_path = ''
-    new_config_path = tmp_path / "pvnet_config_no_gsp.yaml"
-    save_yaml_configuration(config, new_config_path)
-    
-    # Create dataset object
-    dataset = PVNetUKRegionalDataset(new_config_path)
-
-    # Generate a sample
-    _ = dataset[0]
-
-    
-def test_pvnet_uk_concurrent_dataset(pvnet_config_filename):
-
-    # Create dataset object using a limited set of GSPs for test
-    gsp_ids = [1,2,3]
-    num_gsps = len(gsp_ids)
-    
-    dataset = PVNetUKConcurrentDataset(pvnet_config_filename, gsp_ids=gsp_ids)
-
-    assert len(dataset.locations) == num_gsps # Number of regional GSPs
-    # NB. I have not checked the value (39 below) is in fact correct
-    assert len(dataset.valid_t0_times) == 39
-    assert len(dataset) == 39
-
-    # Generate a sample
-    sample = dataset[0]
-
-    assert isinstance(sample, dict)
-
-    for key in [
-        "nwp", "satellite_actual", "gsp",
-        "gsp_solar_azimuth", "gsp_solar_elevation",
-    ]:
-        assert key in sample
-    
-    for nwp_source in ["ukv"]:
-        assert nwp_source in sample["nwp"]
-
-    # Check the shape of the data is correct
-    # 30 minutes of 5 minute data (inclusive), one channel, 2x2 pixels
-    assert sample["satellite_actual"].shape == (num_gsps, 7, 1, 2, 2)
-    # 3 hours of 60 minute data (inclusive), one channel, 2x2 pixels
-    assert sample["nwp"]["ukv"]["nwp"].shape == (num_gsps, 4, 1, 2, 2)
-    # 3 hours of 30 minute data (inclusive)
-    assert sample["gsp"].shape == (num_gsps, 7,)
-    # Solar angles have same shape as GSP data
-    assert sample["gsp_solar_azimuth"].shape == (num_gsps, 7,)
-    assert sample["gsp_solar_elevation"].shape == (num_gsps, 7,)