ocf-data-sampler 0.0.10__tar.gz → 0.0.12__tar.gz

{ocf_data_sampler-0.0.10/ocf_data_sampler.egg-info → ocf_data_sampler-0.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.10
+Version: 0.0.12
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

ocf_data_sampler-0.0.12/ocf_data_sampler/load/nwp/providers/ecmwf.py

@@ -0,0 +1,37 @@
+"""ECMWF provider loaders"""
+from pathlib import Path
+import xarray as xr
+from ocf_data_sampler.load.nwp.providers.utils import open_zarr_paths
+from ocf_data_sampler.load.utils import check_time_unique_increasing, make_spatial_coords_increasing
+
+def open_ifs(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArray:
+    """
+    Opens the ECMWF IFS NWP data
+
+    Args:
+        zarr_path: Path to the zarr to open
+
+    Returns:
+        Xarray DataArray of the NWP data
+    """
+    # Open the data
+    ds = open_zarr_paths(zarr_path)
+
+    # Rename
+    ds = ds.rename(
+        {
+            "init_time": "init_time_utc",
+            "variable": "channel",
+        }
+    )
+
+    # Check the timestmps are unique and increasing
+    check_time_unique_increasing(ds.init_time_utc)
+
+    # Make sure the spatial coords are in increasing order
+    ds = make_spatial_coords_increasing(ds, x_coord="longitude", y_coord="latitude")
+
+    ds = ds.transpose("init_time_utc", "step", "channel", "longitude", "latitude")
+    
+    # TODO: should we control the dtype of the DataArray?
+    return ds.ECMWF_UK

ocf_data_sampler-0.0.12/ocf_data_sampler/load/nwp/providers/ukv.py

@@ -0,0 +1,45 @@
+"""UKV provider loaders"""
+
+import xarray as xr
+
+from pathlib import Path
+
+from ocf_data_sampler.load.nwp.providers.utils import open_zarr_paths
+from ocf_data_sampler.load.utils import check_time_unique_increasing, make_spatial_coords_increasing
+
+
+def open_ukv(zarr_path: Path | str | list[Path] | list[str]) -> xr.DataArray:
+    """
+    Opens the NWP data
+
+    Args:
+        zarr_path: Path to the zarr to open
+
+    Returns:
+        Xarray DataArray of the NWP data
+    """
+    # Open the data
+    ds = open_zarr_paths(zarr_path)
+
+    # Rename
+    ds = ds.rename(
+        {
+            "init_time": "init_time_utc",
+            "variable": "channel",
+            "x": "x_osgb",
+            "y": "y_osgb",
+        }
+    )
+
+    # Check the timestmps are unique and increasing
+    check_time_unique_increasing(ds.init_time_utc)
+
+    # Make sure the spatial coords are in increasing order
+    ds = make_spatial_coords_increasing(ds, x_coord="x_osgb", y_coord="y_osgb")
+
+    ds = ds.transpose("init_time_utc", "step", "channel", "x_osgb", "y_osgb")
+
+    # TODO: should we control the dtype of the DataArray?
+    return ds.UKV
+
+

ocf_data_sampler-0.0.12/ocf_data_sampler/load/nwp/providers/utils.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+import xarray as xr
+
+
+def open_zarr_paths(
+        zarr_path: Path | str | list[Path] | list[str], 
+        time_dim: str = "init_time"
+    ) -> xr.Dataset:
+    """Opens the NWP data
+
+    Args:
+        zarr_path: Path to the zarr(s) to open
+        time_dim: Name of the time dimension
+
+    Returns:
+        The opened Xarray Dataset
+    """
+    if type(zarr_path) in [list, tuple] or "*" in str(zarr_path):  # Multi-file dataset
+        ds = xr.open_mfdataset(
+            zarr_path,
+            engine="zarr",
+            concat_dim=time_dim,
+            combine="nested",
+            chunks="auto",
+        ).sortby(time_dim)
+    else:
+        ds = xr.open_dataset(
+            zarr_path,
+            engine="zarr",
+            consolidated=True,
+            mode="r",
+            chunks="auto",
+        )
+    return ds

{ocf_data_sampler-0.0.10 → ocf_data_sampler-0.0.12}/ocf_data_sampler/torch_datasets/pvnet_uk_regional.py

@@ -114,7 +114,7 @@ def find_valid_t0_times(
 
     assert set(datasets_dict.keys()).issubset({"nwp", "sat", "gsp"})
 
-    contiguous_time_periods = []  # Used to store contiguous time periods from each data source
+    contiguous_time_periods: dict[str: pd.DataFrame] = {}  # Used to store contiguous time periods from each data source
 
     if "nwp" in datasets_dict:
         for nwp_key, nwp_config in config.input_data.nwp.items():
@@ -158,7 +158,7 @@ def find_valid_t0_times(
                 max_dropout=max_dropout,
             )
 
-            contiguous_time_periods.append(time_periods)
+            contiguous_time_periods[f'nwp_{nwp_key}'] = time_periods
 
     if "sat" in datasets_dict:
         sat_config = config.input_data.satellite
@@ -170,7 +170,7 @@ def find_valid_t0_times(
             forecast_duration=minutes(sat_config.forecast_minutes),
         )
 
-        contiguous_time_periods.append(time_periods)
+        contiguous_time_periods['sat'] = time_periods
 
     # GSP always assumed to be in data
     gsp_config = config.input_data.gsp
@@ -182,15 +182,22 @@ def find_valid_t0_times(
         forecast_duration=minutes(gsp_config.forecast_minutes),
     )
 
-    contiguous_time_periods.append(time_periods)
+    contiguous_time_periods['gsp'] = time_periods
+
+    # just get the values (no the keys)
+    contiguous_time_periods_values = list(contiguous_time_periods.values())
 
     # Find joint overlapping contiguous time periods
-    if len(contiguous_time_periods) > 1:
+    if len(contiguous_time_periods_values) > 1:
         valid_time_periods = intersection_of_multiple_dataframes_of_periods(
-            contiguous_time_periods
+            contiguous_time_periods_values
         )
     else:
-        valid_time_periods = contiguous_time_periods[0]
+        valid_time_periods = contiguous_time_periods_values[0]
+
+    # check there are some valid time periods
+    if len(valid_time_periods.keys()) == 0:
+        raise ValueError(f"No valid time periods found, {contiguous_time_periods=}")
 
     # Fill out the contiguous time periods to get the t0 times
     valid_t0_times = fill_time_periods(

{ocf_data_sampler-0.0.10 → ocf_data_sampler-0.0.12/ocf_data_sampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.10
+Version: 0.0.12
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.0.10 → ocf_data_sampler-0.0.12}/ocf_data_sampler.egg-info/SOURCES.txt

@@ -16,6 +16,10 @@ ocf_data_sampler/load/satellite.py
 ocf_data_sampler/load/utils.py
 ocf_data_sampler/load/nwp/__init__.py
 ocf_data_sampler/load/nwp/nwp.py
+ocf_data_sampler/load/nwp/providers/__init__.py
+ocf_data_sampler/load/nwp/providers/ecmwf.py
+ocf_data_sampler/load/nwp/providers/ukv.py
+ocf_data_sampler/load/nwp/providers/utils.py
 ocf_data_sampler/numpy_batch/__init__.py
 ocf_data_sampler/numpy_batch/gsp.py
 ocf_data_sampler/numpy_batch/nwp.py

{ocf_data_sampler-0.0.10 → ocf_data_sampler-0.0.12}/setup.py

@@ -10,7 +10,7 @@ install_requires = (this_directory / "requirements.txt").read_text().splitlines(
 
 setup(
     name="ocf_data_sampler",
-    version="0.0.10",
+    version="0.0.12",
     license="MIT",
     description="Sample from weather data for renewable energy prediction",
     author="James Fulton, Peter Dudfield, and the Open Climate Fix team",