ocf-data-sampler 0.2.33__tar.gz → 0.2.35__tar.gz

{ocf_data_sampler-0.2.33 → ocf_data_sampler-0.2.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.33
+Version: 0.2.35
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.33 → ocf_data_sampler-0.2.35}/ocf_data_sampler/load/gsp.py

@@ -2,6 +2,7 @@
 
 from importlib.resources import files
 
+import numpy as np
 import pandas as pd
 import xarray as xr
 
@@ -26,11 +27,12 @@ def get_gsp_boundaries(version: str) -> pd.DataFrame:
     )
 
 
-def open_gsp(zarr_path: str,
-             boundaries_version: str = "20220314",
-             public: bool = False,
-    ) -> xr.DataArray:
-    """Open the GSP data.
+def open_gsp(
+    zarr_path: str,
+    boundaries_version: str = "20220314",
+    public: bool = False,
+) -> xr.DataArray:
+    """Open the GSP data and validates its data types.
 
     Args:
         zarr_path: Path to the GSP zarr data
@@ -44,18 +46,16 @@ def open_gsp(zarr_path: str,
     # Load UK GSP locations
     df_gsp_loc = get_gsp_boundaries(boundaries_version)
 
-    backend_kwargs ={}
+    backend_kwargs = {}
     # Open the GSP generation data
     if public:
-        backend_kwargs ={"storage_options":{"anon": True}}
+        backend_kwargs = {"storage_options": {"anon": True}}
         # Currently only compatible with S3 bucket.
 
-    ds = (
-        xr.open_dataset(zarr_path,engine="zarr",backend_kwargs=backend_kwargs)
-        .rename({"datetime_gmt": "time_utc"})
+    ds = xr.open_dataset(zarr_path, engine="zarr", backend_kwargs=backend_kwargs).rename(
+        {"datetime_gmt": "time_utc"},
     )
 
-
     if not (ds.gsp_id.isin(df_gsp_loc.index)).all():
         raise ValueError(
             "Some GSP IDs in the GSP generation data are not available in the locations file.",
@@ -72,4 +72,24 @@ def open_gsp(zarr_path: str,
         effective_capacity_mwp=ds.capacity_mwp,
     )
 
-    return ds.generation_mw
+    gsp_da = ds.generation_mw
+
+    # Validate data types directly in loading function
+    if not np.issubdtype(gsp_da.dtype, np.floating):
+        raise TypeError(f"generation_mw should be floating, not {gsp_da.dtype}")
+
+    coord_dtypes = {
+        "time_utc": np.datetime64,
+        "gsp_id": np.integer,
+        "nominal_capacity_mwp": np.floating,
+        "effective_capacity_mwp": np.floating,
+        "x_osgb": np.floating,
+        "y_osgb": np.floating,
+    }
+
+    for coord, expected_dtype in coord_dtypes.items():
+        if not np.issubdtype(gsp_da.coords[coord].dtype, expected_dtype):
+            dtype = gsp_da.coords[coord].dtype
+            raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
+
+    return gsp_da

ocf_data_sampler-0.2.35/ocf_data_sampler/load/nwp/nwp.py

@@ -0,0 +1,115 @@
+"""Module for opening NWP data."""
+
+import numpy as np
+import xarray as xr
+
+from ocf_data_sampler.load.nwp.providers.cloudcasting import open_cloudcasting
+from ocf_data_sampler.load.nwp.providers.ecmwf import open_ifs
+from ocf_data_sampler.load.nwp.providers.gfs import open_gfs
+from ocf_data_sampler.load.nwp.providers.icon import open_icon_eu
+from ocf_data_sampler.load.nwp.providers.ukv import open_ukv
+
+
+def _validate_nwp_data(data_array: xr.DataArray, provider: str) -> None:
+    """Validates the structure and data types of a loaded NWP DataArray.
+
+    This helper function is extracted to keep the main `open_nwp` function clean.
+
+    Args:
+        data_array: The xarray.DataArray to validate.
+        provider: The NWP provider name.
+
+    Raises:
+        TypeError: If the data or any coordinate has an unexpected dtype.
+        ValueError: If a required coordinate is missing.
+    """
+    if not np.issubdtype(data_array.dtype, np.number):
+        raise TypeError(f"NWP data for {provider} should be numeric, not {data_array.dtype}")
+
+    common_expected_dtypes = {
+        "init_time_utc": np.datetime64,
+        "step": np.timedelta64,
+        "channel": (np.str_, np.object_),
+    }
+
+    geographic_spatial_dtypes = {
+        "latitude": np.floating,
+        "longitude": np.floating,
+    }
+
+    provider_specific_spatial_dtypes = {
+        "ecmwf": geographic_spatial_dtypes,
+        "icon-eu": geographic_spatial_dtypes,
+        "gfs": geographic_spatial_dtypes,
+        "mo_global": geographic_spatial_dtypes,
+        "ukv": {
+            "x_osgb": np.floating,
+            "y_osgb": np.floating,
+        },
+        "cloudcasting": {
+            "x_geostationary": np.floating,
+            "y_geostationary": np.floating,
+        },
+    }
+
+    expected_dtypes = {
+        **common_expected_dtypes,
+        **provider_specific_spatial_dtypes.get(provider, {}),
+    }
+
+    for coord, expected_dtype in expected_dtypes.items():
+        if coord not in data_array.coords:
+            raise ValueError(f"Coordinate '{coord}' missing for provider '{provider}'")
+
+        actual_dtype = data_array.coords[coord].dtype
+
+        if not np.issubdtype(actual_dtype, expected_dtype):
+            if isinstance(expected_dtype, tuple):
+                expected_name_str = " or ".join([t.__name__ for t in expected_dtype])
+            else:
+                expected_name_str = expected_dtype.__name__
+
+            err_msg = (
+                f"'{coord}' for {provider} should be {expected_name_str}, "
+                f"not {actual_dtype.name}"
+            )
+            raise TypeError(err_msg)
+
+
+def open_nwp(
+    zarr_path: str | list[str],
+    provider: str,
+    public: bool = False,
+) -> xr.DataArray:
+    """Opens NWP zarr and validates its structure and data types.
+
+    Args:
+        zarr_path: path to the zarr file
+        provider: NWP provider
+        public: Whether the data is public or private (only for GFS)
+    """
+    provider = provider.lower()
+
+    kwargs = {
+        "zarr_path": zarr_path,
+    }
+    if provider == "ukv":
+        _open_nwp = open_ukv
+    elif provider in ["ecmwf", "mo_global"]:
+        _open_nwp = open_ifs
+    elif provider == "icon-eu":
+        _open_nwp = open_icon_eu
+    elif provider == "gfs":
+        _open_nwp = open_gfs
+        # GFS has a public/private flag
+        if public:
+            kwargs["public"] = True
+    elif provider == "cloudcasting":
+        _open_nwp = open_cloudcasting
+    else:
+        raise ValueError(f"Unknown provider: {provider}")
+
+    data_array = _open_nwp(**kwargs)
+    _validate_nwp_data(data_array, provider)
+
+    return data_array

ocf_data_sampler-0.2.35/ocf_data_sampler/load/nwp/providers/icon.py

@@ -0,0 +1,37 @@
+"""DWD ICON Loading."""
+
+import xarray as xr
+
+from ocf_data_sampler.load.nwp.providers.utils import open_zarr_paths
+from ocf_data_sampler.load.utils import check_time_unique_increasing, make_spatial_coords_increasing
+
+
+def open_icon_eu(zarr_path: str | list[str]) -> xr.DataArray:
+    """Opens the ICON data.
+
+    ICON EU Data is now expected to be on a regular lat/lon grid,
+    with a 'channel' dimension directly available (as per the updated fixture).
+    The 'isobaricInhPa' dimension is expected to be already handled.
+
+    Args:
+        zarr_path: Path to the zarr(s) to open
+
+    Returns:
+        Xarray DataArray of the NWP data
+    """
+    # Open and check initially
+    ds = open_zarr_paths(zarr_path, time_dim="init_time_utc")
+
+    if "icon_eu_data" in ds.data_vars:
+        nwp = ds["icon_eu_data"]
+    else:
+        raise ValueError("Could not find 'icon_eu_data' DataArray in the ICON-EU Zarr file.")
+
+    check_time_unique_increasing(nwp.init_time_utc)
+
+    # 0-78 one hour steps, rest 3 hour steps
+    nwp = nwp.isel(step=slice(0, 78))
+    nwp = nwp.transpose("init_time_utc", "step", "channel", "longitude", "latitude")
+    nwp = make_spatial_coords_increasing(nwp, x_coord="longitude", y_coord="latitude")
+
+    return nwp

{ocf_data_sampler-0.2.33 → ocf_data_sampler-0.2.35}/ocf_data_sampler/load/satellite.py

@@ -1,5 +1,5 @@
 """Satellite loader."""
-
+import numpy as np
 import xarray as xr
 
 from ocf_data_sampler.load.utils import (
@@ -44,7 +44,7 @@ def get_single_sat_data(zarr_path: str) -> xr.Dataset:
 
 
 def open_sat_data(zarr_path: str | list[str]) -> xr.DataArray:
-    """Lazily opens the zarr store.
+    """Lazily opens the zarr store and validates data types.
 
     Args:
       zarr_path: Cloud URL or local path pattern, or list of these. If GCS URL,
@@ -72,5 +72,22 @@ def open_sat_data(zarr_path: str | list[str]) -> xr.DataArray:
     ds = make_spatial_coords_increasing(ds, x_coord="x_geostationary", y_coord="y_geostationary")
     ds = ds.transpose("time_utc", "channel", "x_geostationary", "y_geostationary")
 
-    # TODO: should we control the dtype of the DataArray?
-    return get_xr_data_array_from_xr_dataset(ds)
+    data_array = get_xr_data_array_from_xr_dataset(ds)
+
+    # Validate data types directly loading function
+    if not np.issubdtype(data_array.dtype, np.number):
+        raise TypeError(f"Satellite data should be numeric, not {data_array.dtype}")
+
+    coord_dtypes = {
+        "time_utc": np.datetime64,
+        "channel": np.str_,
+        "x_geostationary": np.floating,
+        "y_geostationary": np.floating,
+    }
+
+    for coord, expected_dtype in coord_dtypes.items():
+        if not np.issubdtype(data_array.coords[coord].dtype, expected_dtype):
+            dtype = data_array.coords[coord].dtype
+            raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
+
+    return data_array

{ocf_data_sampler-0.2.33 → ocf_data_sampler-0.2.35}/ocf_data_sampler/load/site.py

@@ -16,7 +16,6 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
         xr.DataArray: The opened site generation data
     """
     generation_ds = xr.open_dataset(generation_file_path)
-
     metadata_df = pd.read_csv(metadata_file_path, index_col="site_id")
 
     if not metadata_df.index.is_unique:
@@ -38,4 +37,23 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
     if not (generation_ds.capacity_kwp.values > 0).all():
         raise ValueError("capacity_kwp contains non-positive values")
 
-    return generation_ds.generation_kw
+    site_da = generation_ds.generation_kw
+
+    # Validate data types directly in loading function
+    if not np.issubdtype(site_da.dtype, np.floating):
+        raise TypeError(f"Generation data should be float, not {site_da.dtype}")
+
+    coord_dtypes = {
+        "time_utc": np.datetime64,
+        "site_id": np.integer,
+        "capacity_kwp": np.floating,
+        "latitude": np.floating,
+        "longitude": np.floating,
+    }
+
+    for coord, expected_dtype in coord_dtypes.items():
+        if not np.issubdtype(site_da.coords[coord].dtype, expected_dtype):
+            dtype = site_da.coords[coord].dtype
+            raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
+
+    return site_da

{ocf_data_sampler-0.2.33 → ocf_data_sampler-0.2.35}/ocf_data_sampler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.33
+Version: 0.2.35
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

ocf_data_sampler-0.2.33/ocf_data_sampler/load/nwp/nwp.py

@@ -1,47 +0,0 @@
-"""Module for opening NWP data."""
-
-import xarray as xr
-
-from ocf_data_sampler.load.nwp.providers.cloudcasting import open_cloudcasting
-from ocf_data_sampler.load.nwp.providers.ecmwf import open_ifs
-from ocf_data_sampler.load.nwp.providers.gfs import open_gfs
-from ocf_data_sampler.load.nwp.providers.icon import open_icon_eu
-from ocf_data_sampler.load.nwp.providers.ukv import open_ukv
-
-
-def open_nwp(zarr_path: str | list[str], provider: str, public: bool = False) -> xr.DataArray:
-    """Opens NWP zarr.
-
-    Args:
-        zarr_path: path to the zarr file
-        provider: NWP provider
-        public: Whether the data is public or private (only for GFS)
-
-    Returns:
-        Xarray DataArray of the NWP data
-    """
-    provider = provider.lower()
-
-    kwargs = {
-        "zarr_path": zarr_path,
-    }
-
-    if provider == "ukv":
-        _open_nwp = open_ukv
-    elif provider in ["ecmwf", "mo_global"]:
-        _open_nwp = open_ifs
-    elif provider == "icon-eu":
-        _open_nwp = open_icon_eu
-    elif provider == "gfs":
-        _open_nwp = open_gfs
-
-        # GFS has a public/private flag
-        if public:
-            kwargs["public"] = True
-
-    elif provider == "cloudcasting":
-        _open_nwp = open_cloudcasting
-    else:
-        raise ValueError(f"Unknown provider: {provider}")
-
-    return _open_nwp(**kwargs)

ocf_data_sampler-0.2.33/ocf_data_sampler/load/nwp/providers/icon.py

@@ -1,46 +0,0 @@
-"""DWD ICON Loading."""
-
-import xarray as xr
-
-from ocf_data_sampler.load.nwp.providers.utils import open_zarr_paths
-from ocf_data_sampler.load.utils import check_time_unique_increasing, make_spatial_coords_increasing
-
-
-def remove_isobaric_lelvels_from_coords(nwp: xr.Dataset) -> xr.Dataset:
-    """Removes the isobaric levels from the coordinates of the NWP data.
-
-    Args:
-        nwp: NWP data
-
-    Returns:
-        NWP data without isobaric levels in the coordinates
-    """
-    variables_to_drop = [var for var in nwp.data_vars if "isobaricInhPa" in nwp[var].dims]
-    return nwp.drop_vars(["isobaricInhPa", *variables_to_drop])
-
-
-def open_icon_eu(zarr_path: str | list[str]) -> xr.Dataset:
-    """Opens the ICON data.
-
-    ICON EU Data is on a regular lat/lon grid
-    It has data on multiple pressure levels, as well as the surface
-    Each of the variables is its own data variable
-
-    Args:
-        zarr_path: Path to the zarr(s) to open
-
-    Returns:
-        Xarray DataArray of the NWP data
-    """
-    # Open the data
-    nwp = open_zarr_paths(zarr_path, time_dim="time")
-    nwp = nwp.rename({"time": "init_time_utc"})
-    # Sanity checks.
-    check_time_unique_increasing(nwp.init_time_utc)
-    # 0-78 one hour steps, rest 3 hour steps
-    nwp = nwp.isel(step=slice(0, 78))
-    nwp = remove_isobaric_lelvels_from_coords(nwp)
-    nwp = nwp.to_array().rename({"variable": "channel"})
-    nwp = nwp.transpose("init_time_utc", "step", "channel", "longitude", "latitude")
-    nwp = make_spatial_coords_increasing(nwp, x_coord="longitude", y_coord="latitude")
-    return nwp