ocf-data-sampler 0.0.12__tar.gz → 0.0.14__tar.gz

{ocf_data_sampler-0.0.12/ocf_data_sampler.egg-info → ocf_data_sampler-0.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.12
+Version: 0.0.14
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.0.12 → ocf_data_sampler-0.0.14}/ocf_data_sampler/torch_datasets/pvnet_uk_regional.py

@@ -4,7 +4,7 @@ import numpy as np
 import pandas as pd
 import xarray as xr
 from torch.utils.data import Dataset
-
+import pkg_resources
 
 from ocf_data_sampler.load.gsp import open_gsp
 from ocf_data_sampler.load.nwp import open_nwp
@@ -37,8 +37,6 @@ from ocf_datapipes.utils.geospatial import osgb_to_lon_lat
 from ocf_datapipes.utils.consts import (
     NWP_MEANS,
     NWP_STDS,
-    RSS_MEAN,
-    RSS_STD,
 )
 
 from ocf_datapipes.training.common import concat_xr_time_utc, normalize_gsp
@@ -69,11 +67,12 @@ def get_dataset_dict(config: Configuration) -> dict[xr.DataArray, dict[xr.DataAr
 
     datasets_dict = {}
 
-    # We always assume GSP will be included
-    da_gsp = open_gsp(zarr_path=in_config.gsp.gsp_zarr_path)
+    # Load GSP data unless the path is None
+    if in_config.gsp.gsp_zarr_path:
+        da_gsp = open_gsp(zarr_path=in_config.gsp.gsp_zarr_path)
 
-    # Remove national GSP
-    datasets_dict["gsp"] = da_gsp.sel(gsp_id=slice(1, None))
+        # Remove national GSP
+        datasets_dict["gsp"] = da_gsp.sel(gsp_id=slice(1, None))
 
     # Load NWP data if in config
     if in_config.nwp:
@@ -172,19 +171,19 @@ def find_valid_t0_times(
 
         contiguous_time_periods['sat'] = time_periods
 
-    # GSP always assumed to be in data
-    gsp_config = config.input_data.gsp
+    if "gsp" in datasets_dict:
+        gsp_config = config.input_data.gsp
 
-    time_periods = find_contiguous_t0_periods(
-        pd.DatetimeIndex(datasets_dict["gsp"]["time_utc"]),
-        sample_period_duration=minutes(gsp_config.time_resolution_minutes),
-        history_duration=minutes(gsp_config.history_minutes),
-        forecast_duration=minutes(gsp_config.forecast_minutes),
-    )
+        time_periods = find_contiguous_t0_periods(
+            pd.DatetimeIndex(datasets_dict["gsp"]["time_utc"]),
+            sample_period_duration=minutes(gsp_config.time_resolution_minutes),
+            history_duration=minutes(gsp_config.history_minutes),
+            forecast_duration=minutes(gsp_config.forecast_minutes),
+        )
 
-    contiguous_time_periods['gsp'] = time_periods
+        contiguous_time_periods['gsp'] = time_periods
 
-    # just get the values (no the keys)
+    # just get the values (not the keys)
     contiguous_time_periods_values = list(contiguous_time_periods.values())
 
     # Find joint overlapping contiguous time periods
@@ -196,7 +195,7 @@ def find_valid_t0_times(
         valid_time_periods = contiguous_time_periods_values[0]
 
     # check there are some valid time periods
-    if len(valid_time_periods.keys()) == 0:
+    if len(valid_time_periods) == 0:
         raise ValueError(f"No valid time periods found, {contiguous_time_periods=}")
 
     # Fill out the contiguous time periods to get the t0 times
@@ -248,8 +247,8 @@ def slice_datasets_by_space(
             width_pixels=sat_config.satellite_image_size_pixels_width,
         )
 
-    # GSP always assumed to be in data
-    sliced_datasets_dict["gsp"] = datasets_dict["gsp"].sel(gsp_id=location.id)
+    if "gsp" in datasets_dict:
+        sliced_datasets_dict["gsp"] = datasets_dict["gsp"].sel(gsp_id=location.id)
 
     return sliced_datasets_dict
     
@@ -314,33 +313,33 @@ def slice_datasets_by_time(
             sat_dropout_time,
         )
 
-    # GSP always assumed to be included
-    gsp_config = config.input_data.gsp
+    if "gsp" in datasets_dict:
+        gsp_config = config.input_data.gsp
 
-    sliced_datasets_dict["gsp_future"] = select_time_slice(
-        datasets_dict["gsp"],
-        t0,
-        sample_period_duration=minutes(gsp_config.time_resolution_minutes),
-        interval_start=minutes(30),
-        interval_end=minutes(gsp_config.forecast_minutes),
-    )
-        
-    sliced_datasets_dict["gsp"] = select_time_slice(
-        datasets_dict["gsp"],
-        t0,
-        sample_period_duration=minutes(gsp_config.time_resolution_minutes),
-        interval_start=-minutes(gsp_config.history_minutes),
-        interval_end=minutes(0),
-    )
+        sliced_datasets_dict["gsp_future"] = select_time_slice(
+            datasets_dict["gsp"],
+            t0,
+            sample_period_duration=minutes(gsp_config.time_resolution_minutes),
+            interval_start=minutes(30),
+            interval_end=minutes(gsp_config.forecast_minutes),
+        )
+            
+        sliced_datasets_dict["gsp"] = select_time_slice(
+            datasets_dict["gsp"],
+            t0,
+            sample_period_duration=minutes(gsp_config.time_resolution_minutes),
+            interval_start=-minutes(gsp_config.history_minutes),
+            interval_end=minutes(0),
+        )
 
-    # Dropout on the GSP, but not the future GSP
-    gsp_dropout_time = draw_dropout_time(
-        t0,
-        dropout_timedeltas=minutes(gsp_config.dropout_timedeltas_minutes),
-        dropout_frac=gsp_config.dropout_fraction,
-    )
+        # Dropout on the GSP, but not the future GSP
+        gsp_dropout_time = draw_dropout_time(
+            t0,
+            dropout_timedeltas=minutes(gsp_config.dropout_timedeltas_minutes),
+            dropout_frac=gsp_config.dropout_fraction,
+        )
 
-    sliced_datasets_dict["gsp"] = apply_dropout_time(sliced_datasets_dict["gsp"], gsp_dropout_time)
+        sliced_datasets_dict["gsp"] = apply_dropout_time(sliced_datasets_dict["gsp"], gsp_dropout_time)
 
     return sliced_datasets_dict
 
@@ -379,23 +378,23 @@ def process_and_combine_datasets(
         numpy_modalities.append({BatchKey.nwp: nwp_numpy_modalities})
 
     if "sat" in dataset_dict:
-        # Standardise
-        # TODO: Since satellite is in range 0-1 already, so we don't need to standardize
-        da_sat = (dataset_dict["sat"] - RSS_MEAN) / RSS_STD
+        # Satellite is already in the range [0-1] so no need to standardise
+        da_sat = dataset_dict["sat"]
+
         # Convert to NumpyBatch
         numpy_modalities.append(convert_satellite_to_numpy_batch(da_sat))
 
-    # GSP always assumed to be in data
     gsp_config = config.input_data.gsp
-    da_gsp = concat_xr_time_utc([dataset_dict["gsp"], dataset_dict["gsp_future"]])
-    da_gsp = normalize_gsp(da_gsp)
-
-    numpy_modalities.append(
-        convert_gsp_to_numpy_batch(
-            da_gsp, 
-            t0_idx=gsp_config.history_minutes / gsp_config.time_resolution_minutes
+    if "gsp" in dataset_dict:
+        da_gsp = concat_xr_time_utc([dataset_dict["gsp"], dataset_dict["gsp_future"]])
+        da_gsp = normalize_gsp(da_gsp)
+
+        numpy_modalities.append(
+            convert_gsp_to_numpy_batch(
+                da_gsp, 
+                t0_idx=gsp_config.history_minutes / gsp_config.time_resolution_minutes
+            )
         )
-    )
 
     # Make sun coords NumpyBatch
     datetimes = pd.date_range(
@@ -440,6 +439,29 @@ def get_locations(ga_gsp: xr.DataArray) -> list[Location]:
     return locations
 
 
+def get_gsp_locations() -> list[Location]:
+    """Get list of locations of all GSPs"""
+    locations = []
+
+    # Load UK GSP locations
+    df_gsp_loc = pd.read_csv(
+        pkg_resources.resource_filename(__name__, "../data/uk_gsp_locations.csv"),
+        index_col="gsp_id",
+    )
+
+    for gsp_id in np.arange(1, 318):
+        locations.append(
+            Location(
+                coordinate_system = "osgb",
+                x=df_gsp_loc.loc[gsp_id].x_osgb,
+                y=df_gsp_loc.loc[gsp_id].y_osgb,
+                id=gsp_id,
+            )
+        )
+    return locations
+
+
+
 class PVNetUKRegionalDataset(Dataset):
     def __init__(
         self, 
@@ -470,7 +492,7 @@ class PVNetUKRegionalDataset(Dataset):
             valid_t0_times = valid_t0_times[valid_t0_times<=pd.Timestamp(end_time)]
 
         # Construct list of locations to sample from
-        locations = get_locations(datasets_dict["gsp"])
+        locations = get_gsp_locations()
 
         # Construct a lookup for locations - useful for users to construct sample by GSP ID
         location_lookup = {loc.id: loc for loc in locations}
@@ -540,6 +562,5 @@ class PVNetUKRegionalDataset(Dataset):
         assert gsp_id in self.location_lookup
 
         location = self.location_lookup[gsp_id]
-
         
-        return self._get_sample(t0, location)
+        return self._get_sample(t0, location)

{ocf_data_sampler-0.0.12 → ocf_data_sampler-0.0.14/ocf_data_sampler.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocf_data_sampler
-Version: 0.0.12
+Version: 0.0.14
 Summary: Sample from weather data for renewable energy prediction
 Author: James Fulton, Peter Dudfield, and the Open Climate Fix team
 Author-email: info@openclimatefix.org

{ocf_data_sampler-0.0.12 → ocf_data_sampler-0.0.14}/setup.py

@@ -10,7 +10,7 @@ install_requires = (this_directory / "requirements.txt").read_text().splitlines(
 
 setup(
     name="ocf_data_sampler",
-    version="0.0.12",
+    version="0.0.14",
     license="MIT",
     description="Sample from weather data for renewable energy prediction",
     author="James Fulton, Peter Dudfield, and the Open Climate Fix team",