ocf-data-sampler 0.2.20__tar.gz → 0.2.22__tar.gz

{ocf_data_sampler-0.2.20 → ocf_data_sampler-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.20
+Version: 0.2.22
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.20 → ocf_data_sampler-0.2.22}/ocf_data_sampler/load/gsp.py

@@ -48,7 +48,7 @@ def open_gsp(zarr_path: str, boundaries_version: str = "20220314") -> xr.DataArr
 
     if not (ds.gsp_id.isin(df_gsp_loc.index)).all():
         raise ValueError(
-            "Some GSP IDs in the GSP generation data are available in the locations file.",
+            "Some GSP IDs in the GSP generation data are not available in the locations file.",
         )
 
     # Select the locations by the GSP IDs in the generation data

{ocf_data_sampler-0.2.20 → ocf_data_sampler-0.2.22}/ocf_data_sampler/select/dropout.py

@@ -9,19 +9,22 @@ import pandas as pd
 import xarray as xr
 
 
-def draw_dropout_time(
+def apply_sampled_dropout_time(
     t0: pd.Timestamp,
     dropout_timedeltas: list[pd.Timedelta],
     dropout_frac: float,
-) -> pd.Timestamp:
-    """Randomly pick a dropout time from a list of timedeltas.
+    da: xr.DataArray,
+) -> xr.DataArray:
+    """Randomly pick a dropout time from a list of timedeltas and apply dropout time to the data.
 
     Args:
         t0: The forecast init-time
         dropout_timedeltas: List of timedeltas relative to t0 to pick from
         dropout_frac: Probability that dropout will be applied.
             This should be between 0 and 1 inclusive
+        da: Xarray DataArray with 'time_utc' coordinate
     """
+    # sample dropout time
     if dropout_frac > 0 and len(dropout_timedeltas) == 0:
         raise ValueError("To apply dropout, dropout_timedeltas must be provided")
 
@@ -37,21 +40,8 @@ def draw_dropout_time(
     else:
         dropout_time = t0 + np.random.choice(dropout_timedeltas)
 
-    return dropout_time
-
-
-def apply_dropout_time(
-    ds: xr.DataArray,
-    dropout_time: pd.Timestamp | None,
-) -> xr.DataArray:
-    """Apply dropout time to the data.
-
-    Args:
-        ds: Xarray DataArray with 'time_utc' coordinate
-        dropout_time: Time after which data is set to NaN
-    """
+    # apply dropout time
     if dropout_time is None:
-        return ds
-    else:
-        # This replaces the times after the dropout with NaNs
-        return ds.where(ds.time_utc <= dropout_time)
+        return da
+    # This replaces the times after the dropout with NaNs
+    return da.where(da.time_utc <= dropout_time)

{ocf_data_sampler-0.2.20 → ocf_data_sampler-0.2.22}/ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -1,6 +1,5 @@
 """Torch dataset for UK PVNet."""
 
-import numpy as np
 import pandas as pd
 import xarray as xr
 from torch.utils.data import Dataset
@@ -257,22 +256,12 @@ class PVNetUKRegionalDataset(AbstractPVNetUKDataset):
         # Construct a lookup for locations - useful for users to construct sample by GSP ID
         location_lookup = {loc.id: loc for loc in self.locations}
 
-        # Construct indices for sampling
-        t_index, loc_index = np.meshgrid(
-            np.arange(len(self.valid_t0_times)),
-            np.arange(len(self.locations)),
-        )
-
-        # Make array of all possible (t0, location) coordinates. Each row is a single coordinate
-        index_pairs = np.stack((t_index.ravel(), loc_index.ravel())).T
-
         # Assign coords and indices to self
         self.location_lookup = location_lookup
-        self.index_pairs = index_pairs
 
     @override
     def __len__(self) -> int:
-        return len(self.index_pairs)
+        return len(self.locations)*len(self.valid_t0_times)
 
     def _get_sample(self, t0: pd.Timestamp, location: Location) -> NumpySample:
         """Generate the PVNet sample for given coordinates.
@@ -290,7 +279,16 @@ class PVNetUKRegionalDataset(AbstractPVNetUKDataset):
     @override
     def __getitem__(self, idx: int) -> NumpySample:
         # Get the coordinates of the sample
-        t_index, loc_index = self.index_pairs[idx]
+
+        if idx >= len(self):
+            raise ValueError(f"Index {idx} out of range for dataset of length {len(self)}")
+
+        # t_index will be between 0 and len(self.valid_t0_times)-1
+        t_index = idx % len(self.valid_t0_times)
+
+        # For each location, there are len(self.valid_t0_times) possible samples
+        loc_index = idx // len(self.valid_t0_times)
+
         location = self.locations[loc_index]
         t0 = self.valid_t0_times[t_index]
 

{ocf_data_sampler-0.2.20 → ocf_data_sampler-0.2.22}/ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py

@@ -4,7 +4,7 @@ import pandas as pd
 import xarray as xr
 
 from ocf_data_sampler.config import Configuration
-from ocf_data_sampler.select.dropout import apply_dropout_time, draw_dropout_time
+from ocf_data_sampler.select.dropout import apply_sampled_dropout_time
 from ocf_data_sampler.select.select_time_slice import select_time_slice, select_time_slice_nwp
 from ocf_data_sampler.utils import minutes
 
@@ -51,17 +51,12 @@ def slice_datasets_by_time(
             interval_end=minutes(sat_config.interval_end_minutes),
         )
 
-        # Randomly sample dropout
-        sat_dropout_time = draw_dropout_time(
+        # Apply the randomly sampled dropout
+        sliced_datasets_dict["sat"] = apply_sampled_dropout_time(
             t0,
             dropout_timedeltas=minutes(sat_config.dropout_timedeltas_minutes),
             dropout_frac=sat_config.dropout_fraction,
-        )
-
-        # Apply the dropout
-        sliced_datasets_dict["sat"] = apply_dropout_time(
-            sliced_datasets_dict["sat"],
-            sat_dropout_time,
+            da=sliced_datasets_dict["sat"],
         )
 
     if "gsp" in datasets_dict:
@@ -76,15 +71,11 @@ def slice_datasets_by_time(
         )
 
         # Dropout on the past GSP, but not the future GSP
-        gsp_dropout_time = draw_dropout_time(
+        da_gsp_past = apply_sampled_dropout_time(
             t0,
             dropout_timedeltas=minutes(gsp_config.dropout_timedeltas_minutes),
             dropout_frac=gsp_config.dropout_fraction,
-        )
-
-        da_gsp_past = apply_dropout_time(
-            da_gsp_past,
-            gsp_dropout_time,
+            da=da_gsp_past,
         )
 
         da_gsp_future = select_time_slice(
@@ -100,25 +91,30 @@ def slice_datasets_by_time(
     if "site" in datasets_dict:
         site_config = config.input_data.site
 
-        sliced_datasets_dict["site"] = select_time_slice(
+        da_site_past = select_time_slice(
             datasets_dict["site"],
             t0,
             time_resolution=minutes(site_config.time_resolution_minutes),
             interval_start=minutes(site_config.interval_start_minutes),
-            interval_end=minutes(site_config.interval_end_minutes),
+            interval_end=minutes(0),
         )
 
-        # Randomly sample dropout
-        site_dropout_time = draw_dropout_time(
+        # Apply the randomly sampled dropout on the past site not the future
+        da_site_past = apply_sampled_dropout_time(
             t0,
             dropout_timedeltas=minutes(site_config.dropout_timedeltas_minutes),
             dropout_frac=site_config.dropout_fraction,
+            da=da_site_past,
         )
 
-        # Apply the dropout
-        sliced_datasets_dict["site"] = apply_dropout_time(
-            sliced_datasets_dict["site"],
-            site_dropout_time,
+        da_site_future = select_time_slice(
+            datasets_dict["site"],
+            t0,
+            time_resolution=minutes(site_config.time_resolution_minutes),
+            interval_start=minutes(site_config.time_resolution_minutes),
+            interval_end=minutes(site_config.interval_end_minutes),
         )
 
+        sliced_datasets_dict["site"] = xr.concat([da_site_past, da_site_future], dim="time_utc")
+
     return sliced_datasets_dict

{ocf_data_sampler-0.2.20 → ocf_data_sampler-0.2.22}/ocf_data_sampler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.20
+Version: 0.2.22
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License