ocf-data-sampler 0.0.19__tar.gz → 0.0.22__tar.gz

ocf_data_sampler-0.0.22/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.1
+Name: ocf_data_sampler
+Version: 0.0.22
+Summary: Sample from weather data for renewable energy prediction
+Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
+Author-email: info@openclimatefix.org
+Maintainer: Open Climate Fix Ltd
+License: MIT License
+        
+        Copyright (c) 2023 Open Climate Fix
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: homepage, https://github.com/openclimatefix
+Project-URL: repository, https://github.com/openclimatefix/ocf-data-sampler
+Keywords: weather data,renewable energy prediction,sample weather data
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: xarray
+Requires-Dist: zarr
+Requires-Dist: dask
+Requires-Dist: ocf_blosc2
+Requires-Dist: pvlib
+Requires-Dist: pydantic
+Requires-Dist: pyproj
+Requires-Dist: pathy
+Requires-Dist: pyaml_env
+Requires-Dist: pyresample
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.2; extra == "docs"
+Requires-Dist: mkdocs-material>=8.0; extra == "docs"
+
+# OCF Data Sampler
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-5-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+[![ease of contribution: easy](https://img.shields.io/badge/ease%20of%20contribution:%20easy-32bd50)](https://github.com/openclimatefix/ocf-meta-repo?tab=readme-ov-file#overview-of-ocfs-nowcasting-repositories) 
+
+A repo for sampling from weather data for renewable energy prediction
+
+## Contributors ✨
+
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dfulu"><img src="https://avatars.githubusercontent.com/u/41546094?v=4?s=100" width="100px;" alt="James Fulton"/><br /><sub><b>James Fulton</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=dfulu" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/AUdaltsova"><img src="https://avatars.githubusercontent.com/u/43303448?v=4?s=100" width="100px;" alt="Alexandra Udaltsova"/><br /><sub><b>Alexandra Udaltsova</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=AUdaltsova" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sukh-P"><img src="https://avatars.githubusercontent.com/u/42407101?v=4?s=100" width="100px;" alt="Sukhil Patel"/><br /><sub><b>Sukhil Patel</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sukh-P" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/peterdudfield"><img src="https://avatars.githubusercontent.com/u/34686298?v=4?s=100" width="100px;" alt="Peter Dudfield"/><br /><sub><b>Peter Dudfield</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=peterdudfield" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/VikramsDataScience"><img src="https://avatars.githubusercontent.com/u/45002417?v=4?s=100" width="100px;" alt="Vikram Pande"/><br /><sub><b>Vikram Pande</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=VikramsDataScience" title="Code">💻</a></td>
+    </tr>
+  </tbody>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

ocf_data_sampler-0.0.22/README.md

@@ -0,0 +1,33 @@
+# OCF Data Sampler
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-5-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+[![ease of contribution: easy](https://img.shields.io/badge/ease%20of%20contribution:%20easy-32bd50)](https://github.com/openclimatefix/ocf-meta-repo?tab=readme-ov-file#overview-of-ocfs-nowcasting-repositories) 
+
+A repo for sampling from weather data for renewable energy prediction
+
+## Contributors ✨
+
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dfulu"><img src="https://avatars.githubusercontent.com/u/41546094?v=4?s=100" width="100px;" alt="James Fulton"/><br /><sub><b>James Fulton</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=dfulu" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/AUdaltsova"><img src="https://avatars.githubusercontent.com/u/43303448?v=4?s=100" width="100px;" alt="Alexandra Udaltsova"/><br /><sub><b>Alexandra Udaltsova</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=AUdaltsova" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sukh-P"><img src="https://avatars.githubusercontent.com/u/42407101?v=4?s=100" width="100px;" alt="Sukhil Patel"/><br /><sub><b>Sukhil Patel</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=Sukh-P" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/peterdudfield"><img src="https://avatars.githubusercontent.com/u/34686298?v=4?s=100" width="100px;" alt="Peter Dudfield"/><br /><sub><b>Peter Dudfield</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=peterdudfield" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/VikramsDataScience"><img src="https://avatars.githubusercontent.com/u/45002417?v=4?s=100" width="100px;" alt="Vikram Pande"/><br /><sub><b>Vikram Pande</b></sub></a><br /><a href="https://github.com/openclimatefix/ocf-data-sampler/commits?author=VikramsDataScience" title="Code">💻</a></td>
+    </tr>
+  </tbody>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

ocf_data_sampler-0.0.22/ocf_data_sampler/config/__init__.py

@@ -0,0 +1,5 @@
+"""Configuration model"""
+
+from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.config.save import save_yaml_configuration
+from ocf_data_sampler.config.load import load_yaml_configuration

ocf_data_sampler-0.0.22/ocf_data_sampler/config/load.py

@@ -0,0 +1,33 @@
+"""Loading configuration functions.
+
+Example:
+
+    from ocf_data_sampler.config import load_yaml_configuration
+    configuration = load_yaml_configuration(filename)
+"""
+
+import fsspec
+from pathy import Pathy
+from pyaml_env import parse_config
+
+from ocf_data_sampler.config import Configuration
+
+
+def load_yaml_configuration(filename: str | Pathy) -> Configuration:
+    """
+    Load a yaml file which has a configuration in it
+
+    Args:
+        filename: the file name that you want to load.  Will load from local, AWS, or GCP
+            depending on the protocol suffix (e.g. 's3://bucket/config.yaml').
+
+    Returns:pydantic class
+
+    """
+    # load the file to a dictionary
+    with fsspec.open(filename, mode="r") as stream:
+        configuration = parse_config(data=stream)
+        # this means we can load ENVs in the yaml file
+    # turn into pydantic class
+    configuration = Configuration(**configuration)
+    return configuration

ocf_data_sampler-0.0.22/ocf_data_sampler/config/model.py

@@ -0,0 +1,249 @@
+"""Configuration model for the dataset.
+
+All paths must include the protocol prefix.  For local files,
+it's sufficient to just start with a '/'.  For aws, start with 's3://',
+for gcp start with 'gs://'.
+
+Example:
+
+    from ocf_data_sampler.config import Configuration
+    config = Configuration(**config_dict)
+"""
+
+import logging
+from typing import Dict, List, Optional
+from typing_extensions import Self
+
+from pydantic import BaseModel, Field, RootModel, field_validator, ValidationInfo, model_validator
+from ocf_data_sampler.constants import NWP_PROVIDERS
+
+logger = logging.getLogger(__name__)
+
+providers = ["pvoutput.org", "solar_sheffield_passiv"]
+
+
+class Base(BaseModel):
+    """Pydantic Base model where no extras can be added"""
+
+    class Config:
+        """config class"""
+
+        extra = "forbid"  # forbid use of extra kwargs
+
+
+class General(Base):
+    """General pydantic model"""
+
+    name: str = Field("example", description="The name of this configuration file.")
+    description: str = Field(
+        "example configuration", description="Description of this configuration file"
+    )
+
+
+class DataSourceMixin(Base):
+    """Mixin class, to add forecast and history minutes"""
+
+    forecast_minutes: int = Field(
+        ...,
+        ge=0,
+        description="how many minutes to forecast in the future. ",
+    )
+    history_minutes: int = Field(
+        ...,
+        ge=0,
+        description="how many historic minutes to use. ",
+    )
+
+
+# noinspection PyMethodParameters
+class DropoutMixin(Base):
+    """Mixin class, to add dropout minutes"""
+
+    dropout_timedeltas_minutes: Optional[List[int]] = Field(
+        default=None,
+        description="List of possible minutes before t0 where data availability may start. Must be "
+        "negative or zero.",
+    )
+
+    dropout_fraction: float = Field(0, description="Chance of dropout being applied to each sample")
+
+    @field_validator("dropout_timedeltas_minutes")
+    def dropout_timedeltas_minutes_negative(cls, v: List[int]) -> List[int]:
+        """Validate 'dropout_timedeltas_minutes'"""
+        if v is not None:
+            for m in v:
+                assert m <= 0, "Dropout timedeltas must be negative"
+        return v
+
+    @field_validator("dropout_fraction")
+    def dropout_fraction_valid(cls, v: float) -> float:
+        """Validate 'dropout_fraction'"""
+        assert 0 <= v <= 1, "Dropout fraction must be between 0 and 1"
+        return v
+
+    @model_validator(mode="after")
+    def dropout_instructions_consistent(self) -> Self:
+        if self.dropout_fraction == 0:
+            if self.dropout_timedeltas_minutes is not None:
+                raise ValueError("To use dropout timedeltas dropout fraction should be > 0")
+        else:
+            if self.dropout_timedeltas_minutes is None:
+                raise ValueError("To dropout fraction > 0 requires a list of dropout timedeltas")
+        return self
+
+
+# noinspection PyMethodParameters
+class TimeResolutionMixin(Base):
+    """Time resolution mix in"""
+
+    time_resolution_minutes: int = Field(
+        ...,
+        description="The temporal resolution of the data in minutes",
+    )
+
+
+class Satellite(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """Satellite configuration model"""
+
+    # Todo: remove 'satellite' from names
+    satellite_zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path or list of paths which hold the satellite zarr",
+    )
+    satellite_channels: list[str] = Field(
+        ..., description="the satellite channels that are used"
+    )
+    satellite_image_size_pixels_height: int = Field(
+        ...,
+        description="The number of pixels of the height of the region of interest"
+        " for non-HRV satellite channels.",
+    )
+
+    satellite_image_size_pixels_width: int = Field(
+        ...,
+        description="The number of pixels of the width of the region "
+        "of interest for non-HRV satellite channels.",
+    )
+
+    live_delay_minutes: int = Field(
+        ..., description="The expected delay in minutes of the satellite data"
+    )
+
+
+# noinspection PyMethodParameters
+class NWP(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """NWP configuration model"""
+
+    nwp_zarr_path: str | tuple[str] | list[str] = Field(
+        ...,
+        description="The path which holds the NWP zarr",
+    )
+    nwp_channels: list[str] = Field(
+        ..., description="the channels used in the nwp data"
+    )
+    nwp_accum_channels: list[str] = Field([], description="the nwp channels which need to be diffed")
+    nwp_image_size_pixels_height: int = Field(..., description="The size of NWP spacial crop in pixels")
+    nwp_image_size_pixels_width: int = Field(..., description="The size of NWP spacial crop in pixels")
+
+    nwp_provider: str = Field(..., description="The provider of the NWP data")
+
+    max_staleness_minutes: Optional[int] = Field(
+        None,
+        description="Sets a limit on how stale an NWP init time is allowed to be whilst still being"
+        " used to construct an example. If set to None, then the max staleness is set according to"
+        " the maximum forecast horizon of the NWP and the requested forecast length.",
+    )
+
+
+    @field_validator("nwp_provider")
+    def validate_nwp_provider(cls, v: str) -> str:
+        """Validate 'nwp_provider'"""
+        if v.lower() not in NWP_PROVIDERS:
+            message = f"NWP provider {v} is not in {NWP_PROVIDERS}"
+            logger.warning(message)
+            raise Exception(message)
+        return v
+
+    # Todo: put into time mixin when moving intervals there
+    @field_validator("forecast_minutes")
+    def forecast_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "Forecast duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+    @field_validator("history_minutes")
+    def history_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "History duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+
+class MultiNWP(RootModel):
+    """Configuration for multiple NWPs"""
+
+    root: Dict[str, NWP]
+
+    def __getattr__(self, item):
+        return self.root[item]
+
+    def __getitem__(self, item):
+        return self.root[item]
+
+    def __len__(self):
+        return len(self.root)
+
+    def __iter__(self):
+        return iter(self.root)
+
+    def keys(self):
+        """Returns dictionary-like keys"""
+        return self.root.keys()
+
+    def items(self):
+        """Returns dictionary-like items"""
+        return self.root.items()
+
+
+# noinspection PyMethodParameters
+class GSP(DataSourceMixin, TimeResolutionMixin, DropoutMixin):
+    """GSP configuration model"""
+
+    gsp_zarr_path: str = Field(..., description="The path which holds the GSP zarr")
+
+    @field_validator("forecast_minutes")
+    def forecast_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "Forecast duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+    @field_validator("history_minutes")
+    def history_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
+        if v % info.data["time_resolution_minutes"] != 0:
+            message = "History duration must be divisible by time resolution"
+            logger.error(message)
+            raise Exception(message)
+        return v
+
+
+# noinspection PyPep8Naming
+class InputData(Base):
+    """
+    Input data model.
+    """
+
+    satellite: Optional[Satellite] = None
+    nwp: Optional[MultiNWP] = None
+    gsp: Optional[GSP] = None
+
+
+class Configuration(Base):
+    """Configuration model for the dataset"""
+
+    general: General = General()
+    input_data: InputData = InputData()

ocf_data_sampler-0.0.22/ocf_data_sampler/config/save.py

@@ -0,0 +1,36 @@
+"""Save functions for the configuration model.
+
+Example:
+
+    from ocf_data_sampler.config import save_yaml_configuration
+    configuration = save_yaml_configuration(config, filename)
+"""
+
+import json
+
+import fsspec
+import yaml
+from pathy import Pathy
+
+from ocf_data_sampler.config import Configuration
+
+
+def save_yaml_configuration(
+    configuration: Configuration, filename: str | Pathy
+):
+    """
+    Save a local yaml file which has the configuration in it.
+
+    If `filename` is None then saves to configuration.output_data.filepath / configuration.yaml.
+
+    Will save to GCP, AWS, or local, depending on the protocol suffix of filepath.
+    """
+    # make a dictionary from the configuration,
+    # Note that we make the object json'able first, so that it can be saved to a yaml file
+    d = json.loads(configuration.model_dump_json())
+    if filename is None:
+        filename = Pathy(configuration.output_data.filepath) / "configuration.yaml"
+
+    # save to a yaml file
+    with fsspec.open(filename, "w") as yaml_file:
+        yaml.safe_dump(d, yaml_file, default_flow_style=False)

ocf_data_sampler-0.0.22/ocf_data_sampler/constants.py

@@ -0,0 +1,135 @@
+import xarray as xr
+import numpy as np
+
+
+NWP_PROVIDERS = [
+    "ukv",
+    "ecmwf",
+]
+
+
+def _to_data_array(d):
+    return xr.DataArray(
+        [d[k] for k in d.keys()],
+        coords={"channel": [k for k in d.keys()]},
+    ).astype(np.float32)
+
+
+class NWPStatDict(dict):
+    """Custom dictionary class to hold NWP normalization stats"""
+
+    def __getitem__(self, key):
+        if key not in NWP_PROVIDERS:
+            raise KeyError(f"{key} is not a supported NWP provider - {NWP_PROVIDERS}")
+        elif key in self.keys():
+            return super().__getitem__(key)
+        else:
+            raise KeyError(
+                f"Values for {key} not yet available in ocf-data-sampler {list(self.keys())}"
+            )
+
+# ------ UKV
+# Means and std computed WITH version_7 and higher, MetOffice values
+UKV_STD = {
+    "cdcb": 2126.99350113,
+    "lcc": 39.33210726,
+    "mcc": 41.91144559,
+    "hcc": 38.07184418,
+    "sde": 0.1029753,
+    "hcct": 18382.63958991,
+    "dswrf": 190.47216887,
+    "dlwrf": 39.45988077,
+    "h": 1075.77812282,
+    "t": 4.38818501,
+    "r": 11.45012499,
+    "dpt": 4.57250482,
+    "vis": 21578.97975625,
+    "si10": 3.94718813,
+    "wdir10": 94.08407495,
+    "prmsl": 1252.71790539,
+    "prate": 0.00021497,
+}
+UKV_MEAN = {
+    "cdcb": 1412.26599062,
+    "lcc": 50.08362643,
+    "mcc": 40.88984494,
+    "hcc": 29.11949682,
+    "sde": 0.00289545,
+    "hcct": -18345.97478167,
+    "dswrf": 111.28265039,
+    "dlwrf": 325.03130139,
+    "h": 2096.51991356,
+    "t": 283.64913206,
+    "r": 81.79229501,
+    "dpt": 280.54379901,
+    "vis": 32262.03285118,
+    "si10": 6.88348448,
+    "wdir10": 199.41891636,
+    "prmsl": 101321.61574029,
+    "prate": 3.45793433e-05,
+}
+
+UKV_STD = _to_data_array(UKV_STD)
+UKV_MEAN = _to_data_array(UKV_MEAN)
+
+# ------ ECMWF
+# These were calculated from 100 random init times of UK data from 2020-2023
+ECMWF_STD = {
+    "dlwrf": 15855867.0,
+    "dswrf": 13025427.0,
+    "duvrs": 1445635.25,
+    "hcc": 0.42244860529899597,
+    "lcc": 0.3791404366493225,
+    "mcc": 0.38039860129356384,
+    "prate": 9.81039775069803e-05,
+    "sde": 0.000913831521756947,
+    "sr": 16294988.0,
+    "t2m": 3.692270040512085,
+    "tcc": 0.37487083673477173,
+    "u10": 5.531515598297119,
+    "u100": 7.2320556640625,
+    "u200": 8.049470901489258,
+    "v10": 5.411230564117432,
+    "v100": 6.944501876831055,
+    "v200": 7.561611652374268,
+    "diff_dlwrf": 131942.03125,
+    "diff_dswrf": 715366.3125,
+    "diff_duvrs": 81605.25,
+    "diff_sr": 818950.6875,
+}
+ECMWF_MEAN = {
+    "dlwrf": 27187026.0,
+    "dswrf": 11458988.0,
+    "duvrs": 1305651.25,
+    "hcc": 0.3961029052734375,
+    "lcc": 0.44901806116104126,
+    "mcc": 0.3288780450820923,
+    "prate": 3.108070450252853e-05,
+    "sde": 8.107526082312688e-05,
+    "sr": 12905302.0,
+    "t2m": 283.48333740234375,
+    "tcc": 0.7049227356910706,
+    "u10": 1.7677178382873535,
+    "u100": 2.393547296524048,
+    "u200": 2.7963004112243652,
+    "v10": 0.985887885093689,
+    "v100": 1.4244288206100464,
+    "v200": 1.6010299921035767,
+    "diff_dlwrf": 1136464.0,
+    "diff_dswrf": 420584.6875,
+    "diff_duvrs": 48265.4765625,
+    "diff_sr": 469169.5,
+}
+
+ECMWF_STD = _to_data_array(ECMWF_STD)
+ECMWF_MEAN = _to_data_array(ECMWF_MEAN)
+
+NWP_STDS = NWPStatDict(
+    ukv=UKV_STD,
+    ecmwf=ECMWF_STD,
+)
+NWP_MEANS = NWPStatDict(
+    ukv=UKV_MEAN,
+    ecmwf=ECMWF_MEAN,
+)
+

ocf_data_sampler-0.0.22/ocf_data_sampler/numpy_batch/gsp.py

@@ -0,0 +1,33 @@
+"""Convert GSP to Numpy Batch"""
+
+import xarray as xr
+
+
+class GSPBatchKey:
+
+    gsp = 'gsp'
+    gsp_nominal_capacity_mwp = 'gsp_nominal_capacity_mwp'
+    gsp_effective_capacity_mwp = 'gsp_effective_capacity_mwp'
+    gsp_time_utc = 'gsp_time_utc'
+    gsp_t0_idx = 'gsp_t0_idx'
+    gsp_solar_azimuth = 'gsp_solar_azimuth'
+    gsp_solar_elevation = 'gsp_solar_elevation'
+    gsp_id = 'gsp_id'
+    gsp_x_osgb = 'gsp_x_osgb'
+    gsp_y_osgb = 'gsp_y_osgb'
+
+
+def convert_gsp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
+    """Convert from Xarray to NumpyBatch"""
+
+    example = {
+        GSPBatchKey.gsp: da.values,
+        GSPBatchKey.gsp_nominal_capacity_mwp: da.isel(time_utc=0)["nominal_capacity_mwp"].values,
+        GSPBatchKey.gsp_effective_capacity_mwp: da.isel(time_utc=0)["effective_capacity_mwp"].values,
+        GSPBatchKey.gsp_time_utc: da["time_utc"].values.astype(float),
+    }
+
+    if t0_idx is not None:
+        example[GSPBatchKey.gsp_t0_idx] = t0_idx
+
+    return example

{ocf_data_sampler-0.0.19 → ocf_data_sampler-0.0.22}/ocf_data_sampler/numpy_batch/nwp.py

@@ -3,13 +3,23 @@
 import pandas as pd
 import xarray as xr
 
-from ocf_datapipes.batch import NWPBatchKey, NWPNumpyBatch
 
+class NWPBatchKey:
 
-def convert_nwp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> NWPNumpyBatch:
+    nwp = 'nwp'
+    nwp_channel_names = 'nwp_channel_names'
+    nwp_init_time_utc = 'nwp_init_time_utc'
+    nwp_step = 'nwp_step'
+    nwp_target_time_utc = 'nwp_target_time_utc'
+    nwp_t0_idx = 'nwp_t0_idx'
+    nwp_y_osgb = 'nwp_y_osgb'
+    nwp_x_osgb = 'nwp_x_osgb'
+
+
+def convert_nwp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
     """Convert from Xarray to NWP NumpyBatch"""
 
-    example: NWPNumpyBatch = {
+    example = {
         NWPBatchKey.nwp: da.values,
         NWPBatchKey.nwp_channel_names: da.channel.values,
         NWPBatchKey.nwp_init_time_utc: da.init_time_utc.values.astype(float),

ocf_data_sampler-0.0.22/ocf_data_sampler/numpy_batch/satellite.py

@@ -0,0 +1,30 @@
+"""Convert Satellite to NumpyBatch"""
+import xarray as xr
+
+
+class SatelliteBatchKey:
+
+    satellite_actual = 'satellite_actual'
+    satellite_time_utc = 'satellite_time_utc'
+    satellite_x_geostationary = 'satellite_x_geostationary'
+    satellite_y_geostationary = 'satellite_y_geostationary'
+    satellite_t0_idx = 'satellite_t0_idx'
+
+
+def convert_satellite_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> dict:
+    """Convert from Xarray to NumpyBatch"""
+    example = {
+        SatelliteBatchKey.satellite_actual: da.values,
+        SatelliteBatchKey.satellite_time_utc: da.time_utc.values.astype(float),
+    }
+
+    for batch_key, dataset_key in (
+         (SatelliteBatchKey.satellite_x_geostationary, "x_geostationary"),
+        (SatelliteBatchKey.satellite_y_geostationary, "y_geostationary"),
+    ):
+        example[batch_key] = da[dataset_key].values
+
+    if t0_idx is not None:
+        example[SatelliteBatchKey.satellite_t0_idx] = t0_idx
+
+    return example