ocf-data-sampler 0.1.11__py3-none-any.whl → 0.1.17__py3-none-any.whl

ocf_data_sampler/torch_datasets/utils/validate_channels.py

@@ -1,82 +0,0 @@
-import xarray as xr
-
-from ocf_data_sampler.config import Configuration
-from ocf_data_sampler.constants import NWP_MEANS, NWP_STDS, RSS_MEAN, RSS_STD
-
-
-def validate_channels(
-    data_channels: list,
-    means_channels: list,
-    stds_channels: list,
-    source_name: str | None = None
-) -> None:
-    """
-    Validates that all channels in data have corresponding normalisation constants.
-
-    Args:
-        data_channels: Set of channels from the data
-        means_channels: Set of channels from means constants
-        stds_channels: Set of channels from stds constants
-        source_name: Name of data source (e.g., 'ecmwf', 'satellite') for error messages
-
-    Raises:
-        ValueError: If there's a mismatch between data channels and normalisation constants
-    """
-
-    data_set = set(data_channels)
-    means_set = set(means_channels)
-    stds_set = set(stds_channels)
-
-    # Find missing channels in means
-    missing_in_means = data_set - means_set
-    if missing_in_means:
-        raise ValueError(
-            f"The following channels for {source_name} are missing in normalisation means: "
-            f"{missing_in_means}"
-        )
-            
-    # Find missing channels in stds
-    missing_in_stds = data_set - stds_set
-    if missing_in_stds:
-        raise ValueError(
-            f"The following channels for {source_name} are missing in normalisation stds: "
-            f"{missing_in_stds}"
-        )
-
-
-def validate_nwp_channels(config: Configuration) -> None:
-    """Validate that NWP channels in config have corresponding normalisation constants.
-    
-    Args:
-        config: Configuration object containing NWP channel information
-        
-    Raises:
-        ValueError: If there's a mismatch between configured NWP channels and normalisation constants
-    """
-    if hasattr(config.input_data, "nwp"):
-        for nwp_key, nwp_config in config.input_data.nwp.items():
-            provider = nwp_config.provider
-            validate_channels(
-                data_channels=nwp_config.channels,
-                means_channels=NWP_MEANS[provider].channel.values,
-                stds_channels=NWP_STDS[provider].channel.values,
-                source_name=provider
-            )
-
-
-def validate_satellite_channels(config: Configuration) -> None:
-    """Validate that satellite channels in config have corresponding normalisation constants.
-    
-    Args:
-        config: Configuration object containing satellite channel information
-        
-    Raises:
-        ValueError: If there's a mismatch between configured satellite channels and normalisation constants
-    """
-    if hasattr(config.input_data, "satellite"):
-        validate_channels(
-            data_channels=config.input_data.satellite.channels,
-            means_channels=RSS_MEAN.channel.values,
-            stds_channels=RSS_STD.channel.values,
-            source_name="satellite"
-        )

ocf_data_sampler-0.1.11.dist-info/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Open Climate Fix
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

ocf_data_sampler-0.1.11.dist-info/RECORD

@@ -1,82 +0,0 @@
-ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-ocf_data_sampler/constants.py,sha256=0HYNmqwBaHVTAEEx9qzk6WD9YInh0gSKLeI3pyq7aNs,5077
-ocf_data_sampler/utils.py,sha256=rKA0BHAyAG4f90zEcgxp25EEYrXS-aOVNzttZ6Mzv2k,250
-ocf_data_sampler/config/__init__.py,sha256=O29mbH0XG2gIY1g3BaveGCnpBO2SFqdu-qzJ7a6evl0,223
-ocf_data_sampler/config/load.py,sha256=sKCKmhkkeFvvkNL5xmnFvdAulaCtV4-rigPsFvVDPDc,634
-ocf_data_sampler/config/model.py,sha256=8PO-23uVy_JjWOJKgaZWdNMehQsAI-Jn8t0lcmBycwg,6992
-ocf_data_sampler/config/save.py,sha256=OqCPT3e0d7vMI2g2iRzmifPD7GscDkFQztU_qE5I0JY,1066
-ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
-ocf_data_sampler/load/__init__.py,sha256=T5Zj1PGt0aiiNEN7Ra1Ac-cBsNKhphmmHy_8g7XU_w0,219
-ocf_data_sampler/load/gsp.py,sha256=uRxEORH7J99JAJ-D38nm0iJFOQh7dkm_NCXcpbYkyvo,857
-ocf_data_sampler/load/load_dataset.py,sha256=PHUGSm4hFHfS9nfIP2KjHHCp325O4br7uGBdQH_DP7g,1603
-ocf_data_sampler/load/satellite.py,sha256=SEQZ9oPe-asEeZeEMDkB1xWK5hErhWMagxohFcBl6KI,2294
-ocf_data_sampler/load/site.py,sha256=hMdoF6sn2PcSBfF2soj7nuQoK9SItaxDXco5nk2n-44,1232
-ocf_data_sampler/load/utils.py,sha256=sAEkPMS9LXVCrc5pANQo97zaoEItVg9hoNj2ZWfx_Ug,1405
-ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
-ocf_data_sampler/load/nwp/nwp.py,sha256=Jyq1dE7DN0iSe6iSEGA76uu9LoeJz9FzfEUkq6ZZExQ,565
-ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=8rYZKdV62AdczVNSOJ2G0BM4-fRFRV0_y5zkHgNYkQs,1004
-ocf_data_sampler/load/nwp/providers/ukv.py,sha256=dM_kvUI0xk9xEdslXqZGjOPP96PEw3qAci5mPUgUvxA,1014
-ocf_data_sampler/load/nwp/providers/utils.py,sha256=MFOZ5ZXLu3-SxYVJExdlo30b3y3s5ebRx3_6DO-33FQ,780
-ocf_data_sampler/numpy_sample/__init__.py,sha256=nY5C6CcuxiWZ_jrXRzWtN7WyKXhJImSiVTIG6Rz4B_4,401
-ocf_data_sampler/numpy_sample/collate.py,sha256=oX5axq30sCsSquhNbmWAVMjM54HT1v3MCMopYHcO5Q0,1950
-ocf_data_sampler/numpy_sample/datetime_features.py,sha256=D0RajbnBjg15qjYk16h2H0XO4wH3fw-x0--4VC2nq0s,1204
-ocf_data_sampler/numpy_sample/gsp.py,sha256=uBquCFCoWuhJKY8sXpgsTCUDWUuLuv1XeixtFnFw6KU,1115
-ocf_data_sampler/numpy_sample/nwp.py,sha256=Tiba-es23XeyMoEPgZUpLT6EnJCGU9A_1MdY6qkE7bM,1015
-ocf_data_sampler/numpy_sample/satellite.py,sha256=RdXMdGGXysUx-AdL9T33yFOlxprtIdPNBKKX99-mhpY,991
-ocf_data_sampler/numpy_sample/site.py,sha256=TvoEU85fmjYW8pD9UZOyUUACjimdQYxEzulQXunRO6Q,1425
-ocf_data_sampler/numpy_sample/sun_position.py,sha256=ithM--eztAhiIQ1g52tlxgj-tMKbsJzx8mk6CgV2tzk,1613
-ocf_data_sampler/sample/__init__.py,sha256=zdS73NTnxFX_j8uh9tT-IXiURB6635wbneM1koWYV1o,169
-ocf_data_sampler/sample/base.py,sha256=IH3HbfqEUwjHmq-h2eJYLd8Jk-0ZcOylnehMyCPMV38,2223
-ocf_data_sampler/sample/site.py,sha256=ONf2Yz5zi8Ombd_znA4T7NXbO01F76kQsBZv6rfnC74,1343
-ocf_data_sampler/sample/uk_regional.py,sha256=KhJ5Ik1pZRp7PgIJjGIrE4i7SQnIdVjUbBHnfn-7ghg,2649
-ocf_data_sampler/select/__init__.py,sha256=E4AJulEbO2K-o0UlG1fgaEteuf_1ZFjHTvrotXSb4YU,332
-ocf_data_sampler/select/dropout.py,sha256=Pgov9P7rQMkSdqluG_hwm8loGyYNFOg-3PJUBLN_kjU,1526
-ocf_data_sampler/select/fill_time_periods.py,sha256=EIcXG-77aQVOAYNwbDBEv6SGf6DO2p1WMEf96iW4MEM,596
-ocf_data_sampler/select/find_contiguous_time_periods.py,sha256=IwPQwvgu4cOiAZ5Gbjflv3fnQCcs0EVK0g4V6yqqSgw,11129
-ocf_data_sampler/select/geospatial.py,sha256=4xL-9y674jjoaXeqE52NHCHVfknciE4OEGsZtn9DvP4,4911
-ocf_data_sampler/select/location.py,sha256=26Y5ZjfFngShBwXieuWSoOA-RLaRzci4TTmcDk3Wg7U,2015
-ocf_data_sampler/select/select_spatial_slice.py,sha256=WNxwur9Q5oetvogATw8-hNejDuEwrXHzuZIovFDjNJA,11488
-ocf_data_sampler/select/select_time_slice.py,sha256=9M-yvDv9K77XfEys_OIR31_aVB56sNWk3BnCnkCgcPI,4725
-ocf_data_sampler/select/spatial_slice_for_dataset.py,sha256=3tRrMBXr7s4CnClbVSIq7hpls3H4Y3qYTDwswcxCCCE,1763
-ocf_data_sampler/select/time_slice_for_dataset.py,sha256=Z7pOiilSHScxmBKZNG18K5J-S4ifdXXAYGZoHRHD3AY,4324
-ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=jfJSFcR0eO1AqeH7S3KnGjsBqVZT5w3oyi784PUR6Q0,146
-ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=ZgfvVCcEU3dj3RoY0zdBdKGppC7Wm81qecqB17gYTmE,12286
-ocf_data_sampler/torch_datasets/datasets/site.py,sha256=_uHmqg-VJu-MHgXc5JFDX1noPfH6E8nY4XhQmsrOav4,16325
-ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=hIbekql64eXsNDFIoEc--GWxwdVWrh2qKegdOi70Bow,874
-ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=Qo65qUHtle_bW5tLTYr7empHTRv-lpjvfx_6GNJj3Xg,4371
-ocf_data_sampler/torch_datasets/utils/validate_channels.py,sha256=u2EpiFAKAOHpmvINhOUJCT8Vbc-cle6qJ3YNVse4yLs,2884
-scripts/refactor_site.py,sha256=xaJGxt2_WObIPrPAnRiOMMB68r-5Q51jWRx409AcscM,1747
-tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/conftest.py,sha256=k7nM3u2YJmkMupN4SIbJP3BRoxNR1dpIoo2fPFf0abg,8588
-tests/config/test_config.py,sha256=CzYVhAUpgT4lvQdIddtVxtJeMqYL_TJolfeIwaaohq4,3969
-tests/config/test_load.py,sha256=8nui2UsgK_eufWGD74yXvf-6eY_SxBFKhDmGYUtRQxw,260
-tests/config/test_save.py,sha256=BxSd2S50-bRPIXP_4iX0B6Wt7pRFJnUbLYtzfLaqlAs,915
-tests/load/test_load_gsp.py,sha256=aT_nqaSXmUTcdHzuTT7AmXJr3R31k4OEN-Fv3eLxlQE,424
-tests/load/test_load_nwp.py,sha256=3qyyDkB1q9t3tyAwogfotNrxqUOpXXimco1CImoEWGg,753
-tests/load/test_load_satellite.py,sha256=IQ8ISRZKCEoi8IsJoPpXZJTolD0mwjnl2E7762RM_PM,524
-tests/load/test_load_sites.py,sha256=6V-U3_EtBklkV7w-hOoR4nba3dSaZ_cnjuRWFs8kYVU,405
-tests/numpy_sample/test_collate.py,sha256=RqHCD5_LTRpe4r6kqC_2TKhmhM_IHYM0ZtFUvSjDqcM,654
-tests/numpy_sample/test_datetime_features.py,sha256=iR9WdBLj1nIBNqoaTFE9rkUaH1eKFJSNb96nwiEaQH0,1449
-tests/numpy_sample/test_gsp.py,sha256=FLlq4SlJ-9cSRAepf4_ksA6PsUVKegnKEAc5pUojCJ0,1458
-tests/numpy_sample/test_nwp.py,sha256=Lnd-PMa6gI-fSIJkSZ554QiHFfnwxeXZxLg-rpuBv1U,442
-tests/numpy_sample/test_satellite.py,sha256=cCqtn5See-uSNfh89COGTUQNuFm6sIZ8QmBVHsuUeRI,1189
-tests/numpy_sample/test_sun_position.py,sha256=_ENYzsNBVPdNXf--FI-UUFqw2u5w7_zqw6LcENU2uZM,2504
-tests/select/test_dropout.py,sha256=aQuSSqZF9RxBjN9-ogkQ8O-_zktAM30CrT1Lz7j1hMg,2222
-tests/select/test_fill_time_periods.py,sha256=o59f2YRe5b0vJrG3B0aYZkYeHnpNk4s6EJxdXZluNQg,907
-tests/select/test_find_contiguous_time_periods.py,sha256=kOga_V7er5We7ewMARXaKdM3agOhsvZYx8inXtUn1PM,5976
-tests/select/test_location.py,sha256=_WZk2FPYeJ-nIfCJS6Sp_yaVEEo7m31DmMFoZzgyCts,2712
-tests/select/test_select_spatial_slice.py,sha256=7EX9b6g-pMdACQx3yefjs5do2s-Rho2UmKevV4oglsU,5147
-tests/select/test_select_time_slice.py,sha256=nYrdlmZlGEygJKiE26bADiluNPN1qt5kD4FrI2vtxUw,9686
-tests/test_sample/test_base.py,sha256=sD9NZghYQWbkAcQP9YXypWZowqYkO3xeNMH-_mEoD5I,4833
-tests/test_sample/test_site_sample.py,sha256=8HNenhIWYouCQu4y389PDQGokSPI5jQ4lS4CG-eA1Y8,5382
-tests/test_sample/test_uk_regional_sample.py,sha256=MFibX9-M8mFK7vwMPu58gAG2VoY6y7w7chW5BlZclwk,3962
-tests/torch_datasets/test_merge_and_fill_utils.py,sha256=GtuQg82BM1eHQjT7Ik1x1zaVcuc7KJO4_NC9stXsd4s,1123
-tests/torch_datasets/test_pvnet_uk.py,sha256=hgD_IDa4D8cgc4cgK1UqKYkT6sFlrTMAvgVn_iwD5_4,5086
-tests/torch_datasets/test_site.py,sha256=t57vAR_RRWcbG_kEFk6VrFCYzVxwFG6qJKBnRHF02fM,7000
-tests/torch_datasets/test_validate_channels_utils.py,sha256=Rzdweu98j1of45jCOUrSiBtyPlf-dDaCceulf0H7ml8,2921
-ocf_data_sampler-0.1.11.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.1.11.dist-info/METADATA,sha256=d8wctSlRyDbP1_yYHFvIGQgEC8DmOkM8h-ITI4XFuPw,12174
-ocf_data_sampler-0.1.11.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ocf_data_sampler-0.1.11.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
-ocf_data_sampler-0.1.11.dist-info/RECORD,,

tests/config/test_config.py

@@ -1,113 +0,0 @@
-import pytest
-from pydantic import ValidationError
-from ocf_data_sampler.config import load_yaml_configuration, Configuration
-
-
-def test_default_configuration():
-    """Test default pydantic class"""
-    _ = Configuration()
-
-
-def test_extra_field_error():
-    """
-    Check an extra parameters in config causes error
-    """
-
-    configuration = Configuration()
-    configuration_dict = configuration.model_dump()
-    configuration_dict["extra_field"] = "extra_value"
-    with pytest.raises(ValidationError, match="Extra inputs are not permitted"):
-        _ = Configuration(**configuration_dict)
-
-
-def test_incorrect_interval_start_minutes(test_config_filename):
-    """
-    Check a history length not divisible by time resolution causes error
-    """
-
-    configuration = load_yaml_configuration(test_config_filename)
-
-    configuration.input_data.nwp['ukv'].interval_start_minutes = -1111
-    with pytest.raises(
-        ValueError, 
-        match="interval_start_minutes.*must be divisible.*time_resolution_minutes.*"
-    ):
-        _ = Configuration(**configuration.model_dump())
-
-
-def test_incorrect_interval_end_minutes(test_config_filename):
-    """
-    Check a forecast length not divisible by time resolution causes error
-    """
-
-    configuration = load_yaml_configuration(test_config_filename)
-
-    configuration.input_data.nwp['ukv'].interval_end_minutes = 1111
-    with pytest.raises(
-        ValueError, 
-        match="interval_end_minutes.*must be divisible.*time_resolution_minutes.*"
-    ):
-        _ = Configuration(**configuration.model_dump())
-
-
-def test_incorrect_nwp_provider(test_config_filename):
-    """
-    Check an unexpected nwp provider causes error
-    """
-
-    configuration = load_yaml_configuration(test_config_filename)
-
-    configuration.input_data.nwp['ukv'].provider = "unexpected_provider"
-    with pytest.raises(Exception, match="NWP provider"):
-        _ = Configuration(**configuration.model_dump())
-
-
-def test_incorrect_dropout(test_config_filename):
-    """
-    Check a dropout timedelta over 0 causes error and 0 doesn't
-    """
-
-    configuration = load_yaml_configuration(test_config_filename)
-
-    # check a positive number is not allowed
-    configuration.input_data.nwp['ukv'].dropout_timedeltas_minutes = [120]
-    with pytest.raises(Exception, match="Dropout timedeltas must be negative"):
-        _ = Configuration(**configuration.model_dump())
-
-    # check 0 is allowed
-    configuration.input_data.nwp['ukv'].dropout_timedeltas_minutes = [0]
-    _ = Configuration(**configuration.model_dump())
-
-
-def test_incorrect_dropout_fraction(test_config_filename):
-    """
-    Check dropout fraction outside of range causes error
-    """
-
-    configuration = load_yaml_configuration(test_config_filename)
-
-    configuration.input_data.nwp['ukv'].dropout_fraction= 1.1
-
-    with pytest.raises(ValidationError,  match="Input should be less than or equal to 1"):
-        _ = Configuration(**configuration.model_dump())
-
-    configuration.input_data.nwp['ukv'].dropout_fraction= -0.1
-    with pytest.raises(ValidationError, match="Input should be greater than or equal to 0"):
-        _ = Configuration(**configuration.model_dump())
-
-
-def test_inconsistent_dropout_use(test_config_filename):
-    """
-    Check dropout fraction outside of range causes error
-    """
-
-    configuration = load_yaml_configuration(test_config_filename)
-    configuration.input_data.satellite.dropout_fraction= 1.0
-    configuration.input_data.satellite.dropout_timedeltas_minutes = []
-
-    with pytest.raises(ValueError, match="To dropout fraction > 0 requires a list of dropout timedeltas"):
-        _ = Configuration(**configuration.model_dump())
-    configuration.input_data.satellite.dropout_fraction= 0.0
-    configuration.input_data.satellite.dropout_timedeltas_minutes = [-120, -60]
-    with pytest.raises(ValueError, match="To use dropout timedeltas dropout fraction should be > 0"):
-        _ = Configuration(**configuration.model_dump())

tests/config/test_load.py

@@ -1,7 +0,0 @@
-from ocf_data_sampler.config import Configuration, load_yaml_configuration
-
-
-def test_load_yaml_configuration(test_config_filename):
-    loaded_config = load_yaml_configuration(test_config_filename)
-    assert isinstance(loaded_config, Configuration)
-        

tests/config/test_save.py

@@ -1,28 +0,0 @@
-"""Tests for configuration saving functionality."""
-import os
-from ocf_data_sampler.config import Configuration, save_yaml_configuration, load_yaml_configuration
-
-
-def test_save_yaml_configuration_basic(tmp_path):
-    """Save an empty configuration object"""
-    config = Configuration()
-
-    filepath = f"{tmp_path}/config.yaml"
-    save_yaml_configuration(config, filepath)
-    
-    assert os.path.exists(filepath)
-
-
-def test_save_load_yaml_configuration(tmp_path, test_config_filename):
-    """Make sure a saved configuration is the same after loading"""
-
-    # Start with this config
-    initial_config = load_yaml_configuration(test_config_filename)
-    
-    # Save it
-    filepath = f"{tmp_path}/config.yaml"
-    save_yaml_configuration(initial_config, filepath)
-    
-    # Load it and check it is still the same
-    loaded_config = load_yaml_configuration(filepath)
-    assert loaded_config == initial_config

tests/conftest.py

@@ -1,319 +0,0 @@
-import pytest
-
-import os
-import numpy as np
-import pandas as pd
-import xarray as xr
-import dask.array
-
-from ocf_data_sampler.config.model import Site
-from ocf_data_sampler.config import load_yaml_configuration, save_yaml_configuration
-
-
-_top_test_directory = os.path.dirname(os.path.realpath(__file__))
-
-@pytest.fixture()
-def test_config_filename():
-    return f"{_top_test_directory}/test_data/configs/test_config.yaml"
-
-
-@pytest.fixture(scope="session")
-def config_filename():
-    return f"{_top_test_directory}/test_data/configs/pvnet_test_config.yaml"
-
-
-@pytest.fixture(scope="session")
-def session_tmp_path(tmp_path_factory):
-    return tmp_path_factory.mktemp("data")
-
-
-@pytest.fixture(scope="session")
-def sat_zarr_path(session_tmp_path):
-
-    # Define coords for satellite-like dataset
-    variables = [
-        'IR_016', 'IR_039', 'IR_087', 'IR_097', 'IR_108', 'IR_120', 
-        'IR_134', 'VIS006', 'VIS008', 'WV_062', 'WV_073',
-    ]
-    x = np.linspace(start=15002, stop=-1824245, num=100)
-    y = np.linspace(start=4191563, stop=5304712, num=100)
-    times = pd.date_range("2023-01-01 00:00", "2023-01-01 23:55", freq="5min")
-    
-    area_string = (
-        """msg_seviri_rss_3km:
-        description: MSG SEVIRI Rapid Scanning Service area definition with 3 km resolution
-        projection:
-            proj: geos
-            lon_0: 9.5
-            h: 35785831
-            x_0: 0
-            y_0: 0
-            a: 6378169
-            rf: 295.488065897014
-            no_defs: null
-            type: crs
-        shape:
-            height: 298
-            width: 615
-        area_extent:
-            lower_left_xy: [28503.830075263977, 5090183.970808983]
-            upper_right_xy: [-1816744.1169023514, 4196063.827395439]
-            units: m
-        """
-    )
-    
-    # Create satellite-like data with some NaNs
-    data = dask.array.zeros(
-        shape=(len(variables), len(times), len(y), len(x)), 
-        chunks=(-1, 10, -1, -1),
-        dtype=np.float32
-    )
-    data [:, 10, :, :] = np.nan
-    
-    ds = xr.DataArray(
-        data=data,
-        coords=dict(
-            variable=variables,
-            time=times,
-            y_geostationary=y,
-            x_geostationary=x,
-        ),
-        attrs=dict(area=area_string),
-    ).to_dataset(name="data")
-
-    # Save temporarily as a zarr
-    zarr_path = session_tmp_path / "test_sat.zarr"
-    ds.to_zarr(zarr_path)
-
-    yield zarr_path
-
-
-@pytest.fixture(scope="session")
-def ds_nwp_ukv():
-    init_times = pd.date_range(start="2023-01-01 00:00", freq="180min", periods=24 * 7)
-    steps = pd.timedelta_range("0h", "10h", freq="1h")
-
-    x = np.linspace(-239_000, 857_000, 50)
-    y = np.linspace(-183_000, 1225_000, 100)
-    variables = ["si10", "dswrf", "t", "prate"]
-
-    coords = (
-        ("init_time", init_times),
-        ("variable", variables),
-        ("step", steps),
-        ("x", x),
-        ("y", y),
-    )
-
-    nwp_array_shape = tuple(len(coord_values) for _, coord_values in coords)
-
-    nwp_data = xr.DataArray(
-        np.random.uniform(0, 200, size=nwp_array_shape).astype(np.float32),
-        coords=coords,
-    )
-    return nwp_data.to_dataset(name="UKV")
-
-
-@pytest.fixture(scope="session")
-def nwp_ukv_zarr_path(session_tmp_path, ds_nwp_ukv):
-    ds = ds_nwp_ukv.chunk(
-        {
-            "init_time": 1, 
-            "step": -1, 
-            "variable": -1,
-            "x": 50, 
-            "y": 50,
-        }
-    )
-    zarr_path = session_tmp_path / "ukv_nwp.zarr"
-    ds.to_zarr(zarr_path)
-    yield zarr_path
-
-
-@pytest.fixture()
-def ds_nwp_ukv_time_sliced():
-
-    t0 = pd.to_datetime("2024-01-02 00:00")
-
-    x = np.arange(-100, 100, 10)
-    y = np.arange(-100, 100, 10)
-    steps = pd.timedelta_range("0h", "8h", freq="1h")
-    target_times = t0 + steps
-
-    channels = ["t", "dswrf"]
-    init_times = pd.to_datetime([t0]*len(steps))
-    
-    # Create dummy time-sliced NWP data
-    da_nwp = xr.DataArray(
-        np.random.normal(size=(len(target_times), len(channels), len(x), len(y))),
-        coords=dict(
-            target_time_utc=(["target_time_utc"], target_times),
-            channel=(["channel"], channels),
-            x_osgb=(["x_osgb"], x),
-            y_osgb=(["y_osgb"], y),
-        )
-    )
-
-    # Add extra non-coordinate dimensions
-    da_nwp = da_nwp.assign_coords(
-        init_time_utc=("target_time_utc", init_times),
-        step=("target_time_utc", steps),
-    )
-
-    return da_nwp
-
-
-@pytest.fixture(scope="session")
-def ds_nwp_ecmwf():
-    init_times = pd.date_range(start="2023-01-01 00:00", freq="6h", periods=24 * 7)
-    steps = pd.timedelta_range("0h", "14h", freq="1h")
-
-    lons = np.arange(-12, 3)
-    lats = np.arange(48, 60)
-    variables = ["t2m","dswrf", "mcc"]
-
-    coords = (
-        ("init_time", init_times),
-        ("variable", variables),
-        ("step", steps),
-        ("longitude", lons),
-        ("latitude", lats),
-    )
-
-    nwp_array_shape = tuple(len(coord_values) for _, coord_values in coords)
-
-    nwp_data = xr.DataArray(
-        np.random.uniform(0, 200, size=nwp_array_shape).astype(np.float32),
-        coords=coords,
-    )
-    return nwp_data.to_dataset(name="ECMWF_UK")
-
-
-@pytest.fixture(scope="session")
-def nwp_ecmwf_zarr_path(session_tmp_path, ds_nwp_ecmwf):
-    ds = ds_nwp_ecmwf.chunk(
-        {
-            "init_time": 1, 
-            "step": -1, 
-            "variable": -1,
-            "longitude": 50, 
-            "latitude": 50,
-        }
-    )
-
-    zarr_path = session_tmp_path / "ukv_ecmwf.zarr"
-    ds.to_zarr(zarr_path)
-    yield zarr_path
-
-
-@pytest.fixture(scope="session")
-def ds_uk_gsp():
-    times = pd.date_range("2023-01-01 00:00", "2023-01-02 00:00", freq="30min")
-    gsp_ids = np.arange(0, 318)
-    capacity = np.ones((len(times), len(gsp_ids)))
-    generation = np.random.uniform(0, 200, size=(len(times), len(gsp_ids))).astype(np.float32)
-
-    coords = (
-        ("datetime_gmt", times),
-        ("gsp_id", gsp_ids),
-    )
-
-    da_cap = xr.DataArray(
-        capacity,
-        coords=coords,
-    )
-
-    da_gen = xr.DataArray(
-        generation,
-        coords=coords,
-    )
-
-    return xr.Dataset({
-        "capacity_mwp": da_cap, 
-        "installedcapacity_mwp": da_cap, 
-        "generation_mw":da_gen
-    })
-
-
-@pytest.fixture(scope="session")
-def data_sites(session_tmp_path) -> Site:
-    """
-    Make fake data for sites
-    Returns: filename for netcdf file, and csv metadata
-    """
-    times = pd.date_range("2023-01-01 00:00", "2023-01-02 00:00", freq="30min")
-    site_ids = list(range(0,10))
-    capacity_kwp_1d = np.array([0.1,1.1,4,6,8,9,15,2,3,4])
-    # these are quite specific for the fake satellite data
-    longitude = np.arange(-4, -3, 0.1)
-    latitude = np.arange(51, 52, 0.1)
-
-    generation = np.random.uniform(0, 200, size=(len(times), len(site_ids))).astype(np.float32)
-
-    # repeat capacity in new dims len(times) times
-    capacity_kwp = (np.tile(capacity_kwp_1d, len(times))).reshape(len(times),10)
-
-    coords = (
-        ("time_utc", times),
-        ("site_id", site_ids),
-    )
-
-    da_cap = xr.DataArray(
-        capacity_kwp,
-        coords=coords,
-    )
-
-    da_gen = xr.DataArray(
-        generation,
-        coords=coords,
-    )
-
-    # metadata
-    meta_df = pd.DataFrame(columns=[], data = [])
-    meta_df['site_id'] = site_ids
-    meta_df['capacity_kwp'] = capacity_kwp_1d
-    meta_df['longitude'] = longitude
-    meta_df['latitude'] = latitude
-
-    generation = xr.Dataset({
-        "capacity_kwp": da_cap,
-        "generation_kw": da_gen,
-    })
-
-    filename = f"{session_tmp_path}/sites.netcdf"
-    filename_csv = f"{session_tmp_path}/sites_metadata.csv"
-    generation.to_netcdf(filename)
-    meta_df.to_csv(filename_csv)
-
-    site = Site(
-        file_path=filename,
-        metadata_file_path=filename_csv,
-        interval_start_minutes=-30,
-        interval_end_minutes=60,
-        time_resolution_minutes=30,
-    )
-
-    yield site
-
-
-@pytest.fixture(scope="session")
-def uk_gsp_zarr_path(session_tmp_path, ds_uk_gsp):
-    zarr_path = session_tmp_path / "uk_gsp.zarr"
-    ds_uk_gsp.to_zarr(zarr_path)
-    yield zarr_path
-
-
-@pytest.fixture()
-def pvnet_config_filename(
-    tmp_path, config_filename, nwp_ukv_zarr_path, uk_gsp_zarr_path, sat_zarr_path
-):
-
-    # adjust config to point to the zarr file
-    config = load_yaml_configuration(config_filename)
-    config.input_data.nwp["ukv"].zarr_path = nwp_ukv_zarr_path
-    config.input_data.satellite.zarr_path = sat_zarr_path
-    config.input_data.gsp.zarr_path = uk_gsp_zarr_path
-
-    filename = f"{tmp_path}/configuration.yaml"
-    save_yaml_configuration(config, filename)
-    return filename

tests/load/test_load_gsp.py

@@ -1,15 +0,0 @@
-from ocf_data_sampler.load.gsp import open_gsp
-import xarray as xr
-
-
-def test_open_gsp(uk_gsp_zarr_path):
-    da = open_gsp(uk_gsp_zarr_path)
-
-    assert isinstance(da, xr.DataArray)
-    assert da.dims == ("time_utc", "gsp_id")
-
-    assert "nominal_capacity_mwp" in da.coords
-    assert "effective_capacity_mwp" in da.coords
-    assert "x_osgb" in da.coords
-    assert "y_osgb" in da.coords
-    assert da.shape == (49, 318)