ocf-data-sampler 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

ocf_data_sampler/load/__init__.py

@@ -0,0 +1 @@
+from ocf_blosc2 import Blosc2  # noqa: F401

ocf_data_sampler/load/gsp.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+import pkg_resources
+
+import pandas as pd
+import xarray as xr
+
+
+def open_gsp(zarr_path: str | Path) -> xr.DataArray:
+
+    # Load GSP generation xr.Dataset
+    ds = xr.open_zarr(zarr_path)
+
+    # Rename to standard time name
+    ds = ds.rename({"datetime_gmt": "time_utc"})
+
+    # Load UK GSP locations
+    df_gsp_loc = pd.read_csv(
+        pkg_resources.resource_filename(__name__, "../data/uk_gsp_locations.csv"),
+        index_col="gsp_id",
+    )
+
+    # Add coordinates
+    ds = ds.assign_coords(
+        x_osgb=(df_gsp_loc.x_osgb.to_xarray()),
+        y_osgb=(df_gsp_loc.y_osgb.to_xarray()),
+        nominal_capacity_mwp=ds.installedcapacity_mwp,
+        effective_capacity_mwp=ds.capacity_mwp,
+
+    )
+
+    return ds.generation_mw
+
+

ocf_data_sampler/load/nwp/__init__.py

@@ -0,0 +1 @@
+from .nwp import open_nwp

ocf_data_sampler/load/nwp/nwp.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+import xarray as xr
+
+from ocf_data_sampler.load.nwp.providers.ukv import open_ukv
+from ocf_data_sampler.load.nwp.providers.ecmwf import open_ifs
+
+
+def open_nwp(zarr_path: Path | str | list[Path] | list[str], provider: str) -> xr.DataArray:
+    """Opens NWP Zarr
+
+    Args:
+        zarr_path: Path to the Zarr file
+        provider: NWP provider
+    """
+
+    if provider.lower() == "ukv":
+        _open_nwp = open_ukv
+    elif provider.lower() == "ecmwf":
+        _open_nwp = open_ifs
+    else:
+        raise ValueError(f"Unknown provider: {provider}")
+    return _open_nwp(zarr_path)
+

ocf_data_sampler/load/satellite.py

@@ -0,0 +1,101 @@
+"""Satellite loader"""
+
+import subprocess
+from pathlib import Path
+
+import pandas as pd
+import xarray as xr
+from ocf_data_sampler.load.utils import check_time_unique_increasing, make_spatial_coords_increasing
+
+
+def _get_single_sat_data(zarr_path: Path | str) -> xr.DataArray:
+    """Helper function to open a zarr from either local or GCP path.
+
+    The local or GCP path may contain wildcard matching (*)
+
+    Args:
+        zarr_path: Path to zarr file
+    """
+
+    # These kwargs are used if zarr path contains "*"
+    openmf_kwargs = dict(
+        engine="zarr",
+        concat_dim="time",
+        combine="nested",
+        chunks="auto",
+        join="override",
+    )
+
+    # Need to generate list of files if using GCP bucket storage
+    if "gs://" in str(zarr_path) and "*" in str(zarr_path):
+        result_string = subprocess.run(
+            f"gsutil ls -d {zarr_path}".split(" "), stdout=subprocess.PIPE
+        ).stdout.decode("utf-8")
+        files = result_string.splitlines()
+
+        ds = xr.open_mfdataset(files, **openmf_kwargs)
+
+    elif "*" in str(zarr_path):  # Multi-file dataset
+        ds = xr.open_mfdataset(zarr_path, **openmf_kwargs)
+    else:
+        ds = xr.open_dataset(zarr_path, engine="zarr", chunks="auto")
+    ds = ds.drop_duplicates("time").sortby("time")
+
+    return ds
+
+
+def open_sat_data(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArray:
+    """Lazily opens the Zarr store.
+
+    Args:
+      zarr_path: Cloud URL or local path pattern, or list of these. If GCS URL, it must start with
+          'gs://'.
+
+    Example:
+        With wild cards and GCS path:
+        ```
+        zarr_paths = [
+            "gs://bucket/2020_nonhrv_split_*.zarr",
+            "gs://bucket/2019_nonhrv_split_*.zarr",
+        ]
+        ds = open_sat_data(zarr_paths)
+        ```
+        Without wild cards and with local path:
+        ```
+        zarr_paths = [
+            "/data/2020_nonhrv.zarr",
+            "/data/2019_nonhrv.zarr",
+        ]
+        ds = open_sat_data(zarr_paths)
+        ```
+    """
+
+    # Open the data
+    if isinstance(zarr_path, (list, tuple)):
+        ds = xr.combine_nested(
+            [_get_single_sat_data(path) for path in zarr_path],
+            concat_dim="time",
+            combine_attrs="override",
+            join="override",
+        )
+    else:
+        ds = _get_single_sat_data(zarr_path)
+
+    # Rename
+    ds = ds.rename(
+        {
+            "variable": "channel",
+            "time": "time_utc",
+        }
+    )
+
+    # Check the timestmps are unique and increasing
+    check_time_unique_increasing(ds.time_utc)
+
+    # Make sure the spatial coords are in increasing order
+    ds = make_spatial_coords_increasing(ds, x_coord="x_geostationary", y_coord="y_geostationary")
+
+    ds = ds.transpose("time_utc", "channel", "x_geostationary", "y_geostationary")
+
+    # TODO: should we control the dtype of the DataArray?
+    return ds.data

ocf_data_sampler/load/utils.py

@@ -0,0 +1,29 @@
+import xarray as xr
+import pandas as pd
+
+def check_time_unique_increasing(datetimes) -> None:
+    """Check that the time dimension is unique and increasing"""
+    time = pd.DatetimeIndex(datetimes)
+    assert time.is_unique
+    assert time.is_monotonic_increasing
+
+def make_spatial_coords_increasing(ds: xr.Dataset, x_coord: str, y_coord: str) -> xr.Dataset:
+    """Make sure the spatial coordinates are in increasing order
+    
+    Args:
+        ds: Xarray Dataset
+        x_coord: Name of the x coordinate
+        y_coord: Name of the y coordinate
+    """
+
+    # Make sure the coords are in increasing order
+    if ds[x_coord][0] > ds[x_coord][-1]:
+        ds = ds.isel({x_coord:slice(None, None, -1)})
+    if ds[y_coord][0] > ds[y_coord][-1]:
+       ds = ds.isel({y_coord:slice(None, None, -1)})
+
+    # Check the coords are all increasing now
+    assert (ds[x_coord].diff(dim=x_coord) > 0).all()
+    assert (ds[y_coord].diff(dim=y_coord) > 0).all()
+
+    return ds

ocf_data_sampler/numpy_batch/nwp.py

@@ -13,7 +13,7 @@ def convert_nwp_to_numpy_batch(da: xr.DataArray, t0_idx: int | None = None) -> N
         NWPBatchKey.nwp: da.values,
         NWPBatchKey.nwp_channel_names: da.channel.values,
         NWPBatchKey.nwp_init_time_utc: da.init_time_utc.values.astype(float),
-        NWPBatchKey.nwp_step: (da.step.values / pd.Timedelta("1H")).astype(int),
+        NWPBatchKey.nwp_step: (da.step.values / pd.Timedelta("1h")).astype(int),
     }
 
     if "target_time_utc" in da.coords:

ocf_data_sampler/select/select_spatial_slice.py

@@ -124,8 +124,10 @@ def _get_idx_of_pixel_closest_to_poi_geostationary(
     center_geostationary = Location(x=x, y=y, coordinate_system="geostationary")
 
     # Check that the requested point lies within the data
-    assert da[x_dim].min() < x < da[x_dim].max()
-    assert da[y_dim].min() < y < da[y_dim].max()
+    assert da[x_dim].min() < x < da[x_dim].max(), \
+        f"{x} is not in the interval {da[x_dim].min().values}: {da[x_dim].max().values}"
+    assert da[y_dim].min() < y < da[y_dim].max(), \
+        f"{y} is not in the interval {da[y_dim].min().values}: {da[y_dim].max().values}"
 
     # Get the index into x and y nearest to x_center_geostationary and y_center_geostationary:
     x_index_at_center = searchsorted(

{ocf_data_sampler-0.0.8.dist-info → ocf_data_sampler-0.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.8
+Version: 0.0.10
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.0.8.dist-info → ocf_data_sampler-0.0.10.dist-info}/RECORD

@@ -1,22 +1,28 @@
 ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
+ocf_data_sampler/load/__init__.py,sha256=MjgfxilTzyz1RYFoBEeAXmE9hyjknLvdmlHPmlAoiQY,44
+ocf_data_sampler/load/gsp.py,sha256=ewuypEah4dfj_gIcYwZoLHiA2ZjUj7KPeyBYKwT1qdo,761
+ocf_data_sampler/load/satellite.py,sha256=RcF0HmpV2PKedOdqcTc6dDk4qdQZAdTLYwmMuNpSTZQ,2906
+ocf_data_sampler/load/utils.py,sha256=tkhuhL3YzJucAtaCH572OxBkYvcEDpSed83yg02O8jg,966
+ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
+ocf_data_sampler/load/nwp/nwp.py,sha256=O4QnajEZem8BvBgTcYYDBhRhgqPYuJkolHmpMRmrXEA,610
 ocf_data_sampler/numpy_batch/__init__.py,sha256=mrtqwbGik5Zc9MYP5byfCTBm08wMtS2XnTsypC4fPMo,245
 ocf_data_sampler/numpy_batch/gsp.py,sha256=EL0_cJJNyvkQQcOat9vFA61pF4lema3BP_vB4ZS788U,805
-ocf_data_sampler/numpy_batch/nwp.py,sha256=AdEBPOjvEnyWJ8UAVqrBfSXPGMSl3sC1_Rfxxr0A1xM,1105
+ocf_data_sampler/numpy_batch/nwp.py,sha256=Rv0yfDj902Z2oCwdlRjOs3Kh-F5Fgxjjylh99-lQ9ws,1105
 ocf_data_sampler/numpy_batch/satellite.py,sha256=e6eoNmiiHtzZbDVtBolFzDuE3qwhHN6bL9H86emAUsk,732
 ocf_data_sampler/numpy_batch/sun_position.py,sha256=UW6-WtjrKdCkcguolHUDSLhYFfarknQzzjlCX8YdEOM,1700
 ocf_data_sampler/select/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 ocf_data_sampler/select/dropout.py,sha256=JYbjG5e8d48te7xj4I9pTWk43d6ksjGeyKFLSTuAOlY,1062
 ocf_data_sampler/select/fill_time_periods.py,sha256=iTtMjIPFYG5xtUYYedAFBLjTWWUa7t7WQ0-yksWf0-E,440
 ocf_data_sampler/select/find_contiguous_time_periods.py,sha256=6ioB8LeFpFNBMgKDxrgG3zqzNjkBF_jlV9yye2ZYT2E,11925
-ocf_data_sampler/select/select_spatial_slice.py,sha256=ZKYrEVyUB83Amayc_cYkEw4g93Veone8d3DWrYJCZb8,11286
+ocf_data_sampler/select/select_spatial_slice.py,sha256=7BSzOFPMSBWpBWXSajWTfI8luUVsSgh4zN-rkr-AuUs,11470
 ocf_data_sampler/select/select_time_slice.py,sha256=XuksC9N03c5rV9OeWtxjGuoGyeJJGy4JMJe3w7m6oaw,6654
 ocf_data_sampler/torch_datasets/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 ocf_data_sampler/torch_datasets/pvnet_uk_regional.py,sha256=rVKFfoHqSfm4C-eOXiqi5GwBJdMewRMIikvpjEJXi1s,17477
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/conftest.py,sha256=OcArgF60paroZQqoP7xExRBF34nEyMuXd7dS7hD6p3w,5393
-ocf_data_sampler-0.0.8.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.0.8.dist-info/METADATA,sha256=kMfeb2SacYux-bEozcQPMpFRYNH9YEGu2luYR-Hx9JE,587
-ocf_data_sampler-0.0.8.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-ocf_data_sampler-0.0.8.dist-info/top_level.txt,sha256=KaQn5qzkJGJP6hKWqsVAc9t0cMLjVvSTk8-kTrW79SA,23
-ocf_data_sampler-0.0.8.dist-info/RECORD,,
+ocf_data_sampler-0.0.10.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
+ocf_data_sampler-0.0.10.dist-info/METADATA,sha256=eXXMZ7ogKXx0j3Krj508liDesXTjUzM2c0NlzOz1P6Q,588
+ocf_data_sampler-0.0.10.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+ocf_data_sampler-0.0.10.dist-info/top_level.txt,sha256=KaQn5qzkJGJP6hKWqsVAc9t0cMLjVvSTk8-kTrW79SA,23
+ocf_data_sampler-0.0.10.dist-info/RECORD,,