ocf-data-sampler 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

ocf_data_sampler/load/satellite.py

@@ -12,15 +12,20 @@ from ocf_data_sampler.load.utils import (
 
 
 def _get_single_sat_data(zarr_path: Path | str) -> xr.Dataset:
-    """Helper function to open a zarr from either local or GCP path.
-
-    The local or GCP path may contain wildcard matching (*)
+    """Helper function to open a Zarr from either a local or GCP path.
 
     Args:
-        zarr_path: Path to zarr file
+        zarr_path: Path to a Zarr file. Wildcards (*) are supported **only** for local paths. 
+                   GCS paths (gs://) **do not support** wildcards.
+
+    Returns:
+        An xarray Dataset containing satellite data.
+
+    Raises:
+        ValueError: If a wildcard (*) is used in a GCS (gs://) path.
     """
 
-    # These kwargs are used if zarr path contains "*"
+    # These kwargs are used if the path contains "*"
     openmf_kwargs = dict(
         engine="zarr",
         concat_dim="time",
@@ -29,19 +34,17 @@ def _get_single_sat_data(zarr_path: Path | str) -> xr.Dataset:
         join="override",
     )
 
-    # Need to generate list of files if using GCP bucket storage
+    # Raise an error if a wildcard is used in a GCP path
     if "gs://" in str(zarr_path) and "*" in str(zarr_path):
-        result_string = subprocess.run(
-            f"gsutil ls -d {zarr_path}".split(" "), stdout=subprocess.PIPE
-        ).stdout.decode("utf-8")
-        files = result_string.splitlines()
-
-        ds = xr.open_mfdataset(files, **openmf_kwargs)
+        raise ValueError("Wildcard (*) paths are not supported for GCP (gs://) URLs.")
 
-    elif "*" in str(zarr_path):  # Multi-file dataset
+    # Handle multi-file dataset for local paths
+    if "*" in str(zarr_path):
         ds = xr.open_mfdataset(zarr_path, **openmf_kwargs)
     else:
         ds = xr.open_dataset(zarr_path, engine="zarr", chunks="auto")
+
+    # Ensure time is unique and sorted
     ds = ds.drop_duplicates("time").sortby("time")
 
     return ds
@@ -53,24 +56,6 @@ def open_sat_data(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArra
     Args:
       zarr_path: Cloud URL or local path pattern, or list of these. If GCS URL, it must start with
           'gs://'.
-
-    Example:
-        With wild cards and GCS path:
-        ```
-        zarr_paths = [
-            "gs://bucket/2020_nonhrv_split_*.zarr",
-            "gs://bucket/2019_nonhrv_split_*.zarr",
-        ]
-        ds = open_sat_data(zarr_paths)
-        ```
-        Without wild cards and with local path:
-        ```
-        zarr_paths = [
-            "/data/2020_nonhrv.zarr",
-            "/data/2019_nonhrv.zarr",
-        ]
-        ds = open_sat_data(zarr_paths)
-        ```
     """
 
     # Open the data
@@ -84,7 +69,7 @@ def open_sat_data(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArra
     else:
         ds = _get_single_sat_data(zarr_path)
 
-    # Rename
+    # Rename dimensions
     ds = ds.rename(
         {
             "variable": "channel",
@@ -92,13 +77,13 @@ def open_sat_data(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArra
         }
     )
 
-    # Check the timestamps are unique and increasing
+    # Check timestamps
     check_time_unique_increasing(ds.time_utc)
 
-    # Make sure the spatial coords are in increasing order
+    # Ensure spatial coordinates are sorted
     ds = make_spatial_coords_increasing(ds, x_coord="x_geostationary", y_coord="y_geostationary")
 
     ds = ds.transpose("time_utc", "channel", "x_geostationary", "y_geostationary")
-
     # TODO: should we control the dtype of the DataArray?
+
     return get_xr_data_array_from_xr_dataset(ds)

ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -84,16 +84,6 @@ def process_and_combine_datasets(
             )
         )
 
-        # Add coordinate data
-        # TODO: Do we need all of these?
-        numpy_modalities.append(
-            {
-                GSPSampleKey.gsp_id: location.id,
-                GSPSampleKey.x_osgb: location.x,
-                GSPSampleKey.y_osgb: location.y,
-            }
-        )
-
     if target_key == 'gsp':
         # Make sun coords NumpySample
         datetimes = pd.date_range(
@@ -104,6 +94,14 @@ def process_and_combine_datasets(
 
         lon, lat = osgb_to_lon_lat(location.x, location.y)
 
+        numpy_modalities.append(
+            {
+                GSPSampleKey.gsp_id: location.id,
+                GSPSampleKey.x_osgb: location.x,
+                GSPSampleKey.y_osgb: location.y,
+            }
+        )
+
     numpy_modalities.append(
         make_sun_position_numpy_sample(datetimes, lon, lat, key_prefix=target_key)
     )

{ocf_data_sampler-0.1.3.dist-info → ocf_data_sampler-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf_data_sampler
-Version: 0.1.3
+Version: 0.1.5
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.1.3.dist-info → ocf_data_sampler-0.1.5.dist-info}/RECORD

@@ -9,7 +9,7 @@ ocf_data_sampler/data/uk_gsp_locations.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZt
 ocf_data_sampler/load/__init__.py,sha256=MjgfxilTzyz1RYFoBEeAXmE9hyjknLvdmlHPmlAoiQY,44
 ocf_data_sampler/load/gsp.py,sha256=Gcr1JVUOPKhFRDCSHtfPDjxx0BtyyEhXrZvGEKLPJ5I,759
 ocf_data_sampler/load/load_dataset.py,sha256=Ua3RaUg4PIYJkD9BKqTfN8IWUbezbhThJGgEkd9PcaE,1587
-ocf_data_sampler/load/satellite.py,sha256=3KlA1fx4SwxdzM-jC1WRaONXO0D6m0WxORnEnwUnZrA,2967
+ocf_data_sampler/load/satellite.py,sha256=f2Q7FSyySOf7DeHxcigHd-vk-J-U4S2pXg_CnhnhuwU,2571
 ocf_data_sampler/load/site.py,sha256=P83uz01WBDzoZajdOH0m8FQt4-buKDlUk19N548KqhA,1086
 ocf_data_sampler/load/utils.py,sha256=sAEkPMS9LXVCrc5pANQo97zaoEItVg9hoNj2ZWfx_Ug,1405
 ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
@@ -41,7 +41,7 @@ ocf_data_sampler/select/select_time_slice.py,sha256=9M-yvDv9K77XfEys_OIR31_aVB56
 ocf_data_sampler/select/spatial_slice_for_dataset.py,sha256=3tRrMBXr7s4CnClbVSIq7hpls3H4Y3qYTDwswcxCCCE,1763
 ocf_data_sampler/select/time_slice_for_dataset.py,sha256=Z7pOiilSHScxmBKZNG18K5J-S4ifdXXAYGZoHRHD3AY,4324
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=jfJSFcR0eO1AqeH7S3KnGjsBqVZT5w3oyi784PUR6Q0,146
-ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=rodvVSR4sh8qZ2hLdI8qAc3lyxq5U7cVGfS4rRKCzbs,11944
+ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=4lqniFbUNt1qWSct4ISavXg9C7FM5cdVu48JHd7A9Pk,11873
 ocf_data_sampler/torch_datasets/datasets/site.py,sha256=5T8nkTMUHHFidZRuFOunYeKAqNuyZ8V7sikBoBOBwwA,16033
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=hIbekql64eXsNDFIoEc--GWxwdVWrh2qKegdOi70Bow,874
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=Qo65qUHtle_bW5tLTYr7empHTRv-lpjvfx_6GNJj3Xg,4371
@@ -70,10 +70,10 @@ tests/test_sample/test_base.py,sha256=ljtB38MmscTGN6OvUgclBceNnfx6m7AN8iHYDml9XW
 tests/test_sample/test_site_sample.py,sha256=Gln-Or060cUWvA7Q7c1vsthgCttOAM2z9yBI9zUIrDw,6238
 tests/test_sample/test_uk_regional_sample.py,sha256=gkeQWC2wC757jKJz_QBmDMFQjn3R54q_tEo948yyxCY,4840
 tests/torch_datasets/test_merge_and_fill_utils.py,sha256=GtuQg82BM1eHQjT7Ik1x1zaVcuc7KJO4_NC9stXsd4s,1123
-tests/torch_datasets/test_pvnet_uk.py,sha256=OzT9ArdnWPa3iJKggxc2-7npkDqWmZyS5pzM4M08NZU,5566
+tests/torch_datasets/test_pvnet_uk.py,sha256=loueo7PUUYJVda3-vBn3bQIC_zgrTAThfx-GTDcBOZg,5596
 tests/torch_datasets/test_site.py,sha256=5MH5zkHFJXekwpnV6nHuSxt_sRNu9_mxiUjfWqmEhr0,6966
-ocf_data_sampler-0.1.3.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
-ocf_data_sampler-0.1.3.dist-info/METADATA,sha256=c5LEfePIqtFvxzWabUufEZkAhIZqp8ep-cHLUH61zAU,12173
-ocf_data_sampler-0.1.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ocf_data_sampler-0.1.3.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
-ocf_data_sampler-0.1.3.dist-info/RECORD,,
+ocf_data_sampler-0.1.5.dist-info/LICENSE,sha256=F-Q3UFCR-BECSocV55BFDpn4YKxve9PKrm-lTt6o_Tg,1073
+ocf_data_sampler-0.1.5.dist-info/METADATA,sha256=PetECVCNM6jys05FuPsOVmntGurbxTuW3n1_j7CYCLE,12173
+ocf_data_sampler-0.1.5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ocf_data_sampler-0.1.5.dist-info/top_level.txt,sha256=Faob6N6cFdPc5eUpCTYcXgCaNhi4XLLteUL5W5ayYmg,31
+ocf_data_sampler-0.1.5.dist-info/RECORD,,

tests/torch_datasets/test_pvnet_uk.py

@@ -55,6 +55,7 @@ def test_process_and_combine_datasets(pvnet_config_filename):
     assert "nwp" in sample
     assert sample["satellite_actual"].shape == (7, 1, 2, 2)
     assert sample["nwp"]["ukv"]["nwp"].shape == (4, 1, 2, 2)
+    assert "gsp_id" in sample
 
 
 def test_compute():