ocf-data-sampler 0.1.3__tar.gz → 0.1.5__tar.gz

{ocf_data_sampler-0.1.3/ocf_data_sampler.egg-info → ocf_data_sampler-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.3
+Version: 0.1.5
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.3 → ocf_data_sampler-0.1.5}/ocf_data_sampler/load/satellite.py

@@ -12,15 +12,20 @@ from ocf_data_sampler.load.utils import (
 
 
 def _get_single_sat_data(zarr_path: Path | str) -> xr.Dataset:
-    """Helper function to open a zarr from either local or GCP path.
-
-    The local or GCP path may contain wildcard matching (*)
+    """Helper function to open a Zarr from either a local or GCP path.
 
     Args:
-        zarr_path: Path to zarr file
+        zarr_path: Path to a Zarr file. Wildcards (*) are supported **only** for local paths. 
+                   GCS paths (gs://) **do not support** wildcards.
+
+    Returns:
+        An xarray Dataset containing satellite data.
+
+    Raises:
+        ValueError: If a wildcard (*) is used in a GCS (gs://) path.
     """
 
-    # These kwargs are used if zarr path contains "*"
+    # These kwargs are used if the path contains "*"
     openmf_kwargs = dict(
         engine="zarr",
         concat_dim="time",
@@ -29,19 +34,17 @@ def _get_single_sat_data(zarr_path: Path | str) -> xr.Dataset:
         join="override",
     )
 
-    # Need to generate list of files if using GCP bucket storage
+    # Raise an error if a wildcard is used in a GCP path
     if "gs://" in str(zarr_path) and "*" in str(zarr_path):
-        result_string = subprocess.run(
-            f"gsutil ls -d {zarr_path}".split(" "), stdout=subprocess.PIPE
-        ).stdout.decode("utf-8")
-        files = result_string.splitlines()
-
-        ds = xr.open_mfdataset(files, **openmf_kwargs)
+        raise ValueError("Wildcard (*) paths are not supported for GCP (gs://) URLs.")
 
-    elif "*" in str(zarr_path):  # Multi-file dataset
+    # Handle multi-file dataset for local paths
+    if "*" in str(zarr_path):
         ds = xr.open_mfdataset(zarr_path, **openmf_kwargs)
     else:
         ds = xr.open_dataset(zarr_path, engine="zarr", chunks="auto")
+
+    # Ensure time is unique and sorted
     ds = ds.drop_duplicates("time").sortby("time")
 
     return ds
@@ -53,24 +56,6 @@ def open_sat_data(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArra
     Args:
       zarr_path: Cloud URL or local path pattern, or list of these. If GCS URL, it must start with
           'gs://'.
-
-    Example:
-        With wild cards and GCS path:
-        ```
-        zarr_paths = [
-            "gs://bucket/2020_nonhrv_split_*.zarr",
-            "gs://bucket/2019_nonhrv_split_*.zarr",
-        ]
-        ds = open_sat_data(zarr_paths)
-        ```
-        Without wild cards and with local path:
-        ```
-        zarr_paths = [
-            "/data/2020_nonhrv.zarr",
-            "/data/2019_nonhrv.zarr",
-        ]
-        ds = open_sat_data(zarr_paths)
-        ```
     """
 
     # Open the data
@@ -84,7 +69,7 @@ def open_sat_data(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArra
     else:
         ds = _get_single_sat_data(zarr_path)
 
-    # Rename
+    # Rename dimensions
     ds = ds.rename(
         {
             "variable": "channel",
@@ -92,13 +77,13 @@ def open_sat_data(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArra
         }
     )
 
-    # Check the timestamps are unique and increasing
+    # Check timestamps
     check_time_unique_increasing(ds.time_utc)
 
-    # Make sure the spatial coords are in increasing order
+    # Ensure spatial coordinates are sorted
     ds = make_spatial_coords_increasing(ds, x_coord="x_geostationary", y_coord="y_geostationary")
 
     ds = ds.transpose("time_utc", "channel", "x_geostationary", "y_geostationary")
-
     # TODO: should we control the dtype of the DataArray?
+
     return get_xr_data_array_from_xr_dataset(ds)

{ocf_data_sampler-0.1.3 → ocf_data_sampler-0.1.5}/ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -84,16 +84,6 @@ def process_and_combine_datasets(
             )
         )
 
-        # Add coordinate data
-        # TODO: Do we need all of these?
-        numpy_modalities.append(
-            {
-                GSPSampleKey.gsp_id: location.id,
-                GSPSampleKey.x_osgb: location.x,
-                GSPSampleKey.y_osgb: location.y,
-            }
-        )
-
     if target_key == 'gsp':
         # Make sun coords NumpySample
         datetimes = pd.date_range(
@@ -104,6 +94,14 @@ def process_and_combine_datasets(
 
         lon, lat = osgb_to_lon_lat(location.x, location.y)
 
+        numpy_modalities.append(
+            {
+                GSPSampleKey.gsp_id: location.id,
+                GSPSampleKey.x_osgb: location.x,
+                GSPSampleKey.y_osgb: location.y,
+            }
+        )
+
     numpy_modalities.append(
         make_sun_position_numpy_sample(datetimes, lon, lat, key_prefix=target_key)
     )

{ocf_data_sampler-0.1.3 → ocf_data_sampler-0.1.5/ocf_data_sampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.3
+Version: 0.1.5
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.3 → ocf_data_sampler-0.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ocf_data_sampler"
-version = "0.1.3"
+version = "0.1.5"
 license = { file = "LICENSE" }
 readme = "README.md"
 description = "Sample from weather data for renewable energy prediction"

{ocf_data_sampler-0.1.3 → ocf_data_sampler-0.1.5}/tests/torch_datasets/test_pvnet_uk.py

@@ -55,6 +55,7 @@ def test_process_and_combine_datasets(pvnet_config_filename):
     assert "nwp" in sample
     assert sample["satellite_actual"].shape == (7, 1, 2, 2)
     assert sample["nwp"]["ukv"]["nwp"].shape == (4, 1, 2, 2)
+    assert "gsp_id" in sample
 
 
 def test_compute():