ocf-data-sampler 0.5.24__py3-none-any.whl → 0.5.26__py3-none-any.whl

ocf_data_sampler/load/nwp/providers/utils.py

@@ -75,7 +75,7 @@ def _tensostore_open_zarr_paths(zarr_path: str | list[str], time_dim: str) -> xr
         zarr_path = sorted(glob(zarr_path))
 
     if isinstance(zarr_path, list | tuple):
-        ds = open_zarrs(zarr_path, concat_dim=time_dim).sortby(time_dim)
+        ds = open_zarrs(zarr_path, concat_dim=time_dim, data_source="nwp").sortby(time_dim)
     else:
         ds = open_zarr(zarr_path)
     return ds

ocf_data_sampler/load/open_xarray_tensorstore.py

@@ -14,6 +14,7 @@ References:
     [2] https://www.apache.org/licenses/LICENSE-2.0
 """
 
+import logging
 import os.path
 import re
 
@@ -26,6 +27,7 @@ from xarray_tensorstore import (
     _TensorStoreAdapter,
 )
 
+logger = logging.getLogger(__name__)
 
 def _zarr_spec_from_path(path: str, zarr_format: int) -> ...:
     if re.match(r"\w+\://", path):  # path is a URI
@@ -127,6 +129,7 @@ def open_zarrs(
     concat_dim: str,
     context: ts.Context | None = None,
     mask_and_scale: bool = True,
+    data_source: str = "unknown",
 ) -> xr.Dataset:
     """Open multiple zarrs with TensorStore.
 
@@ -135,6 +138,7 @@ def open_zarrs(
         concat_dim: Dimension along which to concatenate the data variables.
         context: TensorStore context.
         mask_and_scale: Whether to mask and scale the data.
+        data_source: Which data source is being opened. Used for warning context.
 
     Returns:
         Concatenated Dataset with all data variables opened via TensorStore.
@@ -143,13 +147,28 @@ def open_zarrs(
         context = ts.Context()
 
     ds_list = [xr.open_zarr(p, mask_and_scale=mask_and_scale, decode_timedelta=True) for p in paths]
-    ds = xr.concat(
-        ds_list,
-        dim=concat_dim,
-        data_vars="minimal",
-        compat="equals",
-        combine_attrs="drop_conflicts",
-    )
+    try:
+        ds = xr.concat(
+            ds_list,
+            dim=concat_dim,
+            data_vars="minimal",
+            compat="equals",
+            combine_attrs="drop_conflicts",
+            join="exact",
+        )
+    except ValueError:
+        logger.warning(f"Coordinate mismatch found in {data_source} input data. "
+                       f"The coordinates will be overwritten! "
+                       f"This might be fine for satellite data. "
+                       f"Proceed with caution.")
+        ds = xr.concat(
+            ds_list,
+            dim=concat_dim,
+            data_vars="minimal",
+            compat="equals",
+            combine_attrs="drop_conflicts",
+            join="override",
+        )
 
     if mask_and_scale:
         _raise_if_mask_and_scale_used_for_data_vars(ds)

ocf_data_sampler/load/satellite.py

@@ -19,7 +19,7 @@ def open_sat_data(zarr_path: str | list[str]) -> xr.DataArray:
     """
     # Open the data
     if isinstance(zarr_path, list | tuple):
-        ds = open_zarrs(zarr_path, concat_dim="time")
+        ds = open_zarrs(zarr_path, concat_dim="time", data_source="satellite")
     else:
         ds = open_zarr(zarr_path)
 

ocf_data_sampler/load/site.py

@@ -31,9 +31,10 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
         capacity_kwp=("site_id", metadata_df["capacity_kwp"].values),
     )
 
-    # Sanity checks
-    if not np.isfinite(generation_ds.generation_kw.values).all():
-        raise ValueError("generation_kw contains non-finite values")
+    # Sanity checks, to prevent inf or negative values
+    # Note NaNs are allowed in generation_kw as can have non overlapping time periods for sites
+    if np.isinf(generation_ds.generation_kw.values).all():
+        raise ValueError("generation_kw contains infinite (+/- inf) values")
     if not (generation_ds.capacity_kwp.values > 0).all():
         raise ValueError("capacity_kwp contains non-positive values")
 
@@ -43,17 +44,21 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
     if not np.issubdtype(site_da.dtype, np.floating):
         raise TypeError(f"Generation data should be float, not {site_da.dtype}")
 
+
     coord_dtypes = {
-        "time_utc": np.datetime64,
-        "site_id": np.integer,
-        "capacity_kwp": np.floating,
-        "latitude": np.floating,
-        "longitude": np.floating,
-    }
-
-    for coord, expected_dtype in coord_dtypes.items():
-        if not np.issubdtype(site_da.coords[coord].dtype, expected_dtype):
+    "time_utc": (np.datetime64,),
+    "site_id": (np.integer,),
+    "capacity_kwp": (np.integer, np.floating),
+    "latitude": (np.floating,),
+    "longitude": (np.floating,),
+}
+    for coord, expected_dtypes in coord_dtypes.items():
+        if not any(np.issubdtype(site_da.coords[coord].dtype, dt) for dt in expected_dtypes):
             dtype = site_da.coords[coord].dtype
-            raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
+            allowed = ", ".join(dt.__name__ for dt in expected_dtypes)
+            raise TypeError(f"{coord} should be one of ({allowed}), not {dtype}")
 
-    return site_da
+    # Load the data eagerly into memory by calling compute
+    # this makes the dataset faster to sample from, but
+    # at the cost of a little extra memory usage
+    return site_da.compute()

{ocf_data_sampler-0.5.24.dist-info → ocf_data_sampler-0.5.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.5.24
+Version: 0.5.26
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.5.24.dist-info → ocf_data_sampler-0.5.26.dist-info}/RECORD

@@ -9,9 +9,9 @@ ocf_data_sampler/data/uk_gsp_locations_20250109.csv,sha256=XZISFatnbpO9j8LwaxNKF
 ocf_data_sampler/load/__init__.py,sha256=-vQP9g0UOWdVbjEGyVX_ipa7R1btmiETIKAf6aw4d78,201
 ocf_data_sampler/load/gsp.py,sha256=zsQ39dZBS45qd86lGfCZUjheLRTtMzIUozj-j8c87UQ,3058
 ocf_data_sampler/load/load_dataset.py,sha256=K8rWykjII-3g127If7WRRFivzHNx3SshCvZj4uQlf28,2089
-ocf_data_sampler/load/open_xarray_tensorstore.py,sha256=kAqlIavGe1dcCPkzAtoZo2dFS-tW36E-wRE_3w1HMfg,5620
-ocf_data_sampler/load/satellite.py,sha256=B-m0_Py_D0GwzwX5o-ixyeXntV5Z4k4MbmMBHZLUWMM,1831
-ocf_data_sampler/load/site.py,sha256=WtOy20VMHJIY0IwEemCdcecSDUGcVaLUown-4ixJw90,2147
+ocf_data_sampler/load/open_xarray_tensorstore.py,sha256=YglCBeKa4mSjUU5qlcMOLZXUtFrPFWVKDeKHLjs_YbA,6353
+ocf_data_sampler/load/satellite.py,sha256=5o5SfcplQfZFlm3JJq73j8_m_cWKpFtKk0tTKGjjCuE,1856
+ocf_data_sampler/load/site.py,sha256=bpFABjpvlstn6yJ6OPVlPZms-CjJdxNwCkQafpnj0Ik,2539
 ocf_data_sampler/load/utils.py,sha256=AGL0aOOQPrgqNBTjlBtR7Qg1PyQov3DFJo-y198u8pY,2044
 ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
 ocf_data_sampler/load/nwp/nwp.py,sha256=0E9shei3Mq1N7F-fBlEKY5Hm0_kI7ysY_rffnWIshvk,3612
@@ -21,7 +21,7 @@ ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=P7JqfssmQq8eHKKXaBexsxts325A
 ocf_data_sampler/load/nwp/providers/gfs.py,sha256=h6vm-Rfz1JGOE4P_fP1_XQJ3bugNbeNAIyt56N8B1Dc,1066
 ocf_data_sampler/load/nwp/providers/icon.py,sha256=iVZwLKRr_D74_kAu5MHir6pRKEfbTmIxFRZAxzmiYdI,1257
 ocf_data_sampler/load/nwp/providers/ukv.py,sha256=2i32VM9gnmWUpbL0qBSp_AKzuyKucXZPS8yklbcGlbc,1039
-ocf_data_sampler/load/nwp/providers/utils.py,sha256=IjJ3w7zDgXNFaVa4TMk8yVCvdzfrIRu5tn1OaaQ7Zso,2304
+ocf_data_sampler/load/nwp/providers/utils.py,sha256=Pyp1U3kUpUoqv6ArEtYBfEfzjO2JGQFlUOL-ndsEmlY,2323
 ocf_data_sampler/numpy_sample/__init__.py,sha256=5bdpzM8hMAEe0XRSZ9AZFQdqEeBsEPhaF79Y8bDx3GQ,407
 ocf_data_sampler/numpy_sample/collate.py,sha256=hoxIc5SoHoIs3Nx37aRZzWChpswjy9lHUgaKgHIoo80,2039
 ocf_data_sampler/numpy_sample/common_types.py,sha256=9CjYHkUTx0ObduWh43fhsybZCTXvexql7qC2ptMDoek,377
@@ -59,7 +59,7 @@ ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0Wrg
 ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul3l0EP73Ik002fStr_bhsZh9mQqEU,4735
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
-ocf_data_sampler-0.5.24.dist-info/METADATA,sha256=r5w9nbLOb6vB8rIALgrp02EY7HM6KJ43hE0xp3xLGEE,13541
-ocf_data_sampler-0.5.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ocf_data_sampler-0.5.24.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
-ocf_data_sampler-0.5.24.dist-info/RECORD,,
+ocf_data_sampler-0.5.26.dist-info/METADATA,sha256=NQUuT_DW3MI5F3ZFyupsHs3nfTLLU2Fo2PYzn9PxNqM,13541
+ocf_data_sampler-0.5.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ocf_data_sampler-0.5.26.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
+ocf_data_sampler-0.5.26.dist-info/RECORD,,