ocf-data-sampler 0.2.32__py3-none-any.whl → 0.2.34__py3-none-any.whl

ocf_data_sampler/config/model.py

@@ -214,6 +214,17 @@ class NWP(TimeWindowMixin, DropoutMixin, SpatialWindowMixin, NormalisationConsta
     )
     public: bool = Field(False, description="Whether the NWP data is public or private")
 
+    @model_validator(mode="after")
+    def validate_accum_channels_subset(self) -> "NWP":
+        """Validate accum_channels is subset of channels."""
+        invalid_channels = set(self.accum_channels) - set(self.channels)
+        if invalid_channels:
+            raise ValueError(
+                f"NWP provider '{self.provider}': all values in 'accum_channels' should "
+                f"be present in 'channels'. Extra values found: {invalid_channels}",
+            )
+        return self
+
     @field_validator("provider")
     def validate_provider(cls, v: str) -> str:
         """Validator for 'provider'."""

ocf_data_sampler/load/gsp.py

@@ -2,6 +2,7 @@
 
 from importlib.resources import files
 
+import numpy as np
 import pandas as pd
 import xarray as xr
 
@@ -26,11 +27,12 @@ def get_gsp_boundaries(version: str) -> pd.DataFrame:
     )
 
 
-def open_gsp(zarr_path: str,
-             boundaries_version: str = "20220314",
-             public: bool = False,
-    ) -> xr.DataArray:
-    """Open the GSP data.
+def open_gsp(
+    zarr_path: str,
+    boundaries_version: str = "20220314",
+    public: bool = False,
+) -> xr.DataArray:
+    """Open the GSP data and validates its data types.
 
     Args:
         zarr_path: Path to the GSP zarr data
@@ -44,18 +46,16 @@ def open_gsp(zarr_path: str,
     # Load UK GSP locations
     df_gsp_loc = get_gsp_boundaries(boundaries_version)
 
-    backend_kwargs ={}
+    backend_kwargs = {}
     # Open the GSP generation data
     if public:
-        backend_kwargs ={"storage_options":{"anon": True}}
+        backend_kwargs = {"storage_options": {"anon": True}}
         # Currently only compatible with S3 bucket.
 
-    ds = (
-        xr.open_dataset(zarr_path,engine="zarr",backend_kwargs=backend_kwargs)
-        .rename({"datetime_gmt": "time_utc"})
+    ds = xr.open_dataset(zarr_path, engine="zarr", backend_kwargs=backend_kwargs).rename(
+        {"datetime_gmt": "time_utc"},
     )
 
-
     if not (ds.gsp_id.isin(df_gsp_loc.index)).all():
         raise ValueError(
             "Some GSP IDs in the GSP generation data are not available in the locations file.",
@@ -72,4 +72,24 @@ def open_gsp(zarr_path: str,
         effective_capacity_mwp=ds.capacity_mwp,
     )
 
-    return ds.generation_mw
+    gsp_da = ds.generation_mw
+
+    # Validate data types directly in loading function
+    if not np.issubdtype(gsp_da.dtype, np.floating):
+        raise TypeError(f"generation_mw should be floating, not {gsp_da.dtype}")
+
+    coord_dtypes = {
+        "time_utc": np.datetime64,
+        "gsp_id": np.integer,
+        "nominal_capacity_mwp": np.floating,
+        "effective_capacity_mwp": np.floating,
+        "x_osgb": np.floating,
+        "y_osgb": np.floating,
+    }
+
+    for coord, expected_dtype in coord_dtypes.items():
+        if not np.issubdtype(gsp_da.coords[coord].dtype, expected_dtype):
+            dtype = gsp_da.coords[coord].dtype
+            raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
+
+    return gsp_da

ocf_data_sampler/load/nwp/nwp.py

@@ -1,5 +1,6 @@
 """Module for opening NWP data."""
 
+import numpy as np
 import xarray as xr
 
 from ocf_data_sampler.load.nwp.providers.cloudcasting import open_cloudcasting
@@ -9,23 +10,85 @@ from ocf_data_sampler.load.nwp.providers.icon import open_icon_eu
 from ocf_data_sampler.load.nwp.providers.ukv import open_ukv
 
 
-def open_nwp(zarr_path: str | list[str], provider: str, public: bool = False) -> xr.DataArray:
-    """Opens NWP zarr.
+def _validate_nwp_data(data_array: xr.DataArray, provider: str) -> None:
+    """Validates the structure and data types of a loaded NWP DataArray.
+
+    This helper function is extracted to keep the main `open_nwp` function clean.
+
+    Args:
+        data_array: The xarray.DataArray to validate.
+        provider: The NWP provider name.
+
+    Raises:
+        TypeError: If the data or any coordinate has an unexpected dtype.
+        ValueError: If a required coordinate is missing.
+    """
+    if not np.issubdtype(data_array.dtype, np.number):
+        raise TypeError(f"NWP data for {provider} should be numeric, not {data_array.dtype}")
+
+    common_expected_dtypes = {
+        "init_time_utc": np.datetime64,
+        "step": np.timedelta64,
+        "channel": np.str_,
+    }
+
+    geographic_spatial_dtypes = {
+        "latitude": np.floating,
+        "longitude": np.floating,
+    }
+
+    provider_specific_spatial_dtypes = {
+        "ecmwf": geographic_spatial_dtypes,
+        "icon-eu": geographic_spatial_dtypes,
+        "gfs": geographic_spatial_dtypes,
+        "mo_global": geographic_spatial_dtypes,
+        "ukv": {
+            "x_osgb": np.floating,
+            "y_osgb": np.floating,
+        },
+        "cloudcasting": {
+            "x_geostationary": np.floating,
+            "y_geostationary": np.floating,
+        },
+    }
+
+    expected_dtypes = {
+        **common_expected_dtypes,
+        **provider_specific_spatial_dtypes.get(provider, {}),
+    }
+
+    if not expected_dtypes:
+        raise ValueError(f"Unknown provider: {provider}")
+
+    for coord, expected_dtype in expected_dtypes.items():
+        if coord not in data_array.coords:
+            raise ValueError(f"Coordinate '{coord}' missing for provider '{provider}'")
+        if not np.issubdtype(data_array.coords[coord].dtype, expected_dtype):
+            actual_dtype = data_array.coords[coord].dtype
+            err_msg = (
+                f"'{coord}' for {provider} should be {expected_dtype.__name__}, "
+                f"not {actual_dtype}"
+            )
+            raise TypeError(err_msg)
+
+
+def open_nwp(
+    zarr_path: str | list[str],
+    provider: str,
+    public: bool = False,
+) -> xr.DataArray:
+    """Opens NWP zarr and validates its structure and data types.
 
     Args:
         zarr_path: path to the zarr file
         provider: NWP provider
         public: Whether the data is public or private (only for GFS)
-
-    Returns:
-        Xarray DataArray of the NWP data
     """
     provider = provider.lower()
 
     kwargs = {
         "zarr_path": zarr_path,
     }
-
     if provider == "ukv":
         _open_nwp = open_ukv
     elif provider in ["ecmwf", "mo_global"]:
@@ -34,14 +97,15 @@ def open_nwp(zarr_path: str | list[str], provider: str, public: bool = False) ->
         _open_nwp = open_icon_eu
     elif provider == "gfs":
         _open_nwp = open_gfs
-
         # GFS has a public/private flag
         if public:
             kwargs["public"] = True
-
     elif provider == "cloudcasting":
         _open_nwp = open_cloudcasting
     else:
         raise ValueError(f"Unknown provider: {provider}")
 
-    return _open_nwp(**kwargs)
+    data_array = _open_nwp(**kwargs)
+    _validate_nwp_data(data_array, provider)
+
+    return data_array

ocf_data_sampler/load/nwp/providers/icon.py

@@ -6,25 +6,12 @@ from ocf_data_sampler.load.nwp.providers.utils import open_zarr_paths
 from ocf_data_sampler.load.utils import check_time_unique_increasing, make_spatial_coords_increasing
 
 
-def remove_isobaric_lelvels_from_coords(nwp: xr.Dataset) -> xr.Dataset:
-    """Removes the isobaric levels from the coordinates of the NWP data.
-
-    Args:
-        nwp: NWP data
-
-    Returns:
-        NWP data without isobaric levels in the coordinates
-    """
-    variables_to_drop = [var for var in nwp.data_vars if "isobaricInhPa" in nwp[var].dims]
-    return nwp.drop_vars(["isobaricInhPa", *variables_to_drop])
-
-
-def open_icon_eu(zarr_path: str | list[str]) -> xr.Dataset:
+def open_icon_eu(zarr_path: str | list[str]) -> xr.DataArray:
     """Opens the ICON data.
 
-    ICON EU Data is on a regular lat/lon grid
-    It has data on multiple pressure levels, as well as the surface
-    Each of the variables is its own data variable
+    ICON EU Data is now expected to be on a regular lat/lon grid,
+    with a 'channel' dimension directly available (as per the updated fixture).
+    The 'isobaricInhPa' dimension is expected to be already handled.
 
     Args:
         zarr_path: Path to the zarr(s) to open
@@ -32,15 +19,19 @@ def open_icon_eu(zarr_path: str | list[str]) -> xr.Dataset:
     Returns:
         Xarray DataArray of the NWP data
     """
-    # Open the data
-    nwp = open_zarr_paths(zarr_path, time_dim="time")
-    nwp = nwp.rename({"time": "init_time_utc"})
-    # Sanity checks.
+    # Open and check initially
+    ds = open_zarr_paths(zarr_path, time_dim="init_time_utc")
+
+    if "icon_eu_data" in ds.data_vars:
+        nwp = ds["icon_eu_data"]
+    else:
+        raise ValueError("Could not find 'icon_eu_data' DataArray in the ICON-EU Zarr file.")
+
     check_time_unique_increasing(nwp.init_time_utc)
+
     # 0-78 one hour steps, rest 3 hour steps
     nwp = nwp.isel(step=slice(0, 78))
-    nwp = remove_isobaric_lelvels_from_coords(nwp)
-    nwp = nwp.to_array().rename({"variable": "channel"})
     nwp = nwp.transpose("init_time_utc", "step", "channel", "longitude", "latitude")
     nwp = make_spatial_coords_increasing(nwp, x_coord="longitude", y_coord="latitude")
+
     return nwp

ocf_data_sampler/load/satellite.py

@@ -1,5 +1,5 @@
 """Satellite loader."""
-
+import numpy as np
 import xarray as xr
 
 from ocf_data_sampler.load.utils import (
@@ -44,7 +44,7 @@ def get_single_sat_data(zarr_path: str) -> xr.Dataset:
 
 
 def open_sat_data(zarr_path: str | list[str]) -> xr.DataArray:
-    """Lazily opens the zarr store.
+    """Lazily opens the zarr store and validates data types.
 
     Args:
       zarr_path: Cloud URL or local path pattern, or list of these. If GCS URL,
@@ -72,5 +72,22 @@ def open_sat_data(zarr_path: str | list[str]) -> xr.DataArray:
     ds = make_spatial_coords_increasing(ds, x_coord="x_geostationary", y_coord="y_geostationary")
     ds = ds.transpose("time_utc", "channel", "x_geostationary", "y_geostationary")
 
-    # TODO: should we control the dtype of the DataArray?
-    return get_xr_data_array_from_xr_dataset(ds)
+    data_array = get_xr_data_array_from_xr_dataset(ds)
+
+    # Validate data types directly loading function
+    if not np.issubdtype(data_array.dtype, np.number):
+        raise TypeError(f"Satellite data should be numeric, not {data_array.dtype}")
+
+    coord_dtypes = {
+        "time_utc": np.datetime64,
+        "channel": np.str_,
+        "x_geostationary": np.floating,
+        "y_geostationary": np.floating,
+    }
+
+    for coord, expected_dtype in coord_dtypes.items():
+        if not np.issubdtype(data_array.coords[coord].dtype, expected_dtype):
+            dtype = data_array.coords[coord].dtype
+            raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
+
+    return data_array

ocf_data_sampler/load/site.py

@@ -16,7 +16,6 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
         xr.DataArray: The opened site generation data
     """
     generation_ds = xr.open_dataset(generation_file_path)
-
     metadata_df = pd.read_csv(metadata_file_path, index_col="site_id")
 
     if not metadata_df.index.is_unique:
@@ -38,4 +37,23 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
     if not (generation_ds.capacity_kwp.values > 0).all():
         raise ValueError("capacity_kwp contains non-positive values")
 
-    return generation_ds.generation_kw
+    site_da = generation_ds.generation_kw
+
+    # Validate data types directly in loading function
+    if not np.issubdtype(site_da.dtype, np.floating):
+        raise TypeError(f"Generation data should be float, not {site_da.dtype}")
+
+    coord_dtypes = {
+        "time_utc": np.datetime64,
+        "site_id": np.integer,
+        "capacity_kwp": np.floating,
+        "latitude": np.floating,
+        "longitude": np.floating,
+    }
+
+    for coord, expected_dtype in coord_dtypes.items():
+        if not np.issubdtype(site_da.coords[coord].dtype, expected_dtype):
+            dtype = site_da.coords[coord].dtype
+            raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
+
+    return site_da

{ocf_data_sampler-0.2.32.dist-info → ocf_data_sampler-0.2.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.32
+Version: 0.2.34
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.32.dist-info → ocf_data_sampler-0.2.34.dist-info}/RECORD

@@ -2,23 +2,23 @@ ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,
 ocf_data_sampler/utils.py,sha256=DjuneGGisl08ENvPZV_lrcX4b2NCKJC1ZpXgIpxuQi4,290
 ocf_data_sampler/config/__init__.py,sha256=O29mbH0XG2gIY1g3BaveGCnpBO2SFqdu-qzJ7a6evl0,223
 ocf_data_sampler/config/load.py,sha256=LL-7wemI8o4KPkx35j-wQ3HjsMvDgqXr7G46IcASfnU,632
-ocf_data_sampler/config/model.py,sha256=UwVQOjRBthbwhAWR5Rcs5cSXG3imLZ5pnd8vBeFseLE,10623
+ocf_data_sampler/config/model.py,sha256=xX2PPywEFGYDsx_j9DX1GlwMRq3ovJR-mhmysMt_mO0,11116
 ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdRaI,1064
 ocf_data_sampler/data/uk_gsp_locations_20220314.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
 ocf_data_sampler/data/uk_gsp_locations_20250109.csv,sha256=XZISFatnbpO9j8LwaxNKFzQSjs6hcHFsV8a9uDDpy2E,9055334
 ocf_data_sampler/load/__init__.py,sha256=-vQP9g0UOWdVbjEGyVX_ipa7R1btmiETIKAf6aw4d78,201
-ocf_data_sampler/load/gsp.py,sha256=winSW3ibFbpsOr0ZRIjYUlqSW5C6SUb0dxkRZm3E8GI,2195
+ocf_data_sampler/load/gsp.py,sha256=IrTA6z9quN08imKGHJLf8gRktarxn1-utNMNFD0zWQs,2944
 ocf_data_sampler/load/load_dataset.py,sha256=WjB3DvHbDQQYYnPmDFOWg_TQPgARZ5pu8fiRZSGtIg0,2099
-ocf_data_sampler/load/satellite.py,sha256=E7Ln7Y60Qr1RTV-_R71YoxXQM-Ca7Y1faIo3oKB2eFk,2292
-ocf_data_sampler/load/site.py,sha256=zOzlWk6pYZBB5daqG8URGksmDXWKrkutUvN8uALAIh8,1468
+ocf_data_sampler/load/satellite.py,sha256=Gsc3oyPydEZLy6slUDtIpBCYLxWy9P3pD1VyI4W9-2w,2944
+ocf_data_sampler/load/site.py,sha256=WtOy20VMHJIY0IwEemCdcecSDUGcVaLUown-4ixJw90,2147
 ocf_data_sampler/load/utils.py,sha256=sZ0-zzconcLkVQwAkCYrqKDo98Hrh5ChdiQJv5Bh91g,2040
 ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
-ocf_data_sampler/load/nwp/nwp.py,sha256=S0wsxQHKa-OZchmvhTAOSG1HLtAVRTo_ElXAIwz1pXo,1332
+ocf_data_sampler/load/nwp/nwp.py,sha256=1GCoIX0_KmEmIB99e_C9JDo6wlrvsF13oC5r3lrqYBo,3495
 ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ocf_data_sampler/load/nwp/providers/cloudcasting.py,sha256=fozXpB3a2rNqQgnpRDC7xunxffh7Wwmc0kkCiYmDVJ4,1521
 ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=an-gXsZwkPQvRXeza1U_4MNU5yEnVm0_8tn03rxTudI,997
 ocf_data_sampler/load/nwp/providers/gfs.py,sha256=glBbo2kXtcTjQv_VNqA32lsdCCGB114Ovm-cibRWxTA,1088
-ocf_data_sampler/load/nwp/providers/icon.py,sha256=6MkOfUk5dmv0XJZLrKMy1e8xipj2fHCTkYXuff7MgUY,1584
+ocf_data_sampler/load/nwp/providers/icon.py,sha256=BnY3vAa5pHn1cyrImj0ymaRFKHanNtfD9_JO-4p2IZY,1241
 ocf_data_sampler/load/nwp/providers/ukv.py,sha256=Ka1KFZcJYPwr5vuxo-xWGVQC0pudheqGBonUnbyJCMg,1016
 ocf_data_sampler/load/nwp/providers/utils.py,sha256=NrzE3JAtoc6oEywJHxPUdi_I4UJgJ_l5dxLZ4DLKvcg,1124
 ocf_data_sampler/numpy_sample/__init__.py,sha256=nY5C6CcuxiWZ_jrXRzWtN7WyKXhJImSiVTIG6Rz4B_4,401
@@ -55,7 +55,7 @@ ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
 utils/compute_icon_mean_stddev.py,sha256=a1oWMRMnny39rV-dvu8rcx85sb4bXzPFrR1gkUr4Jpg,2296
-ocf_data_sampler-0.2.32.dist-info/METADATA,sha256=NZ29xNAjkgJgfpfnvHqjh3cn7aU8vlzOVBjjab9Ksj4,12184
-ocf_data_sampler-0.2.32.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ocf_data_sampler-0.2.32.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
-ocf_data_sampler-0.2.32.dist-info/RECORD,,
+ocf_data_sampler-0.2.34.dist-info/METADATA,sha256=C6ux4M-XRmQsnG-yLwRYVdkKW6qPxP47KLGACzJRIss,12184
+ocf_data_sampler-0.2.34.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ocf_data_sampler-0.2.34.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
+ocf_data_sampler-0.2.34.dist-info/RECORD,,