ocf-data-sampler 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

ocf_data_sampler/load/gsp.py

@@ -52,9 +52,12 @@ def open_gsp(
         backend_kwargs = {"storage_options": {"anon": True}}
         # Currently only compatible with S3 bucket.
 
-    ds = xr.open_dataset(zarr_path, engine="zarr", backend_kwargs=backend_kwargs).rename(
-        {"datetime_gmt": "time_utc"},
-    )
+    ds = xr.open_dataset(
+        zarr_path,
+        engine="zarr",
+        chunks=None,
+        backend_kwargs=backend_kwargs,
+    ).rename({"datetime_gmt": "time_utc"})
 
     if not (ds.gsp_id.isin(df_gsp_loc.index)).all():
         raise ValueError(

ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -21,7 +21,7 @@ from ocf_data_sampler.numpy_sample.nwp import NWPSampleKey
 from ocf_data_sampler.select import Location, fill_time_periods
 from ocf_data_sampler.select.geospatial import osgb_to_lon_lat
 from ocf_data_sampler.torch_datasets.utils import (
-    channel_dict_to_dataarray,
+    config_normalization_values_to_dicts,
     find_valid_time_periods,
     slice_datasets_by_space,
     slice_datasets_by_time,
@@ -110,11 +110,14 @@ class AbstractPVNetUKDataset(Dataset):
         self.config = config
         self.datasets_dict = datasets_dict
 
+        # Extract the normalisation values from the config for faster access
+        means_dict, stds_dict = config_normalization_values_to_dicts(config)
+        self.means_dict = means_dict
+        self.stds_dict = stds_dict
 
-    @staticmethod
     def process_and_combine_datasets(
+        self,
         dataset_dict: dict,
-        config: Configuration,
         t0: pd.Timestamp,
         location: Location,
     ) -> NumpySample:
@@ -122,7 +125,6 @@ class AbstractPVNetUKDataset(Dataset):
 
         Args:
             dataset_dict: Dictionary of xarray datasets
-            config: Configuration object
             t0: init-time for sample
             location: location of the sample
         """
@@ -134,13 +136,8 @@ class AbstractPVNetUKDataset(Dataset):
             for nwp_key, da_nwp in dataset_dict["nwp"].items():
 
                 # Standardise and convert to NumpyBatch
-
-                da_channel_means = channel_dict_to_dataarray(
-                    config.input_data.nwp[nwp_key].channel_means,
-                )
-                da_channel_stds = channel_dict_to_dataarray(
-                    config.input_data.nwp[nwp_key].channel_stds,
-                )
+                da_channel_means = self.means_dict["nwp"][nwp_key]
+                da_channel_stds = self.stds_dict["nwp"][nwp_key]
 
                 da_nwp = (da_nwp - da_channel_means) / da_channel_stds
 
@@ -153,15 +150,15 @@ class AbstractPVNetUKDataset(Dataset):
             da_sat = dataset_dict["sat"]
 
             # Standardise and convert to NumpyBatch
-            da_channel_means = channel_dict_to_dataarray(config.input_data.satellite.channel_means)
-            da_channel_stds = channel_dict_to_dataarray(config.input_data.satellite.channel_stds)
+            da_channel_means = self.means_dict["sat"]
+            da_channel_stds = self.stds_dict["sat"]
 
             da_sat = (da_sat - da_channel_means) / da_channel_stds
 
             numpy_modalities.append(convert_satellite_to_numpy_sample(da_sat))
 
         if "gsp" in dataset_dict:
-            gsp_config = config.input_data.gsp
+            gsp_config = self.config.input_data.gsp
             da_gsp = dataset_dict["gsp"]
             da_gsp = da_gsp / da_gsp.effective_capacity_mwp
 
@@ -183,13 +180,8 @@ class AbstractPVNetUKDataset(Dataset):
         )
 
         # Only add solar position if explicitly configured
-        has_solar_config = (
-            hasattr(config.input_data, "solar_position") and
-            config.input_data.solar_position is not None
-        )
-
-        if has_solar_config:
-            solar_config = config.input_data.solar_position
+        if self.config.input_data.solar_position is not None:
+            solar_config = self.config.input_data.solar_position
 
             # Create datetime range for solar position calculation
             datetimes = pd.date_range(
@@ -264,7 +256,7 @@ class PVNetUKRegionalDataset(AbstractPVNetUKDataset):
         sample_dict = slice_datasets_by_time(sample_dict, t0, self.config)
         sample_dict = compute(sample_dict)
 
-        return self.process_and_combine_datasets(sample_dict, self.config, t0, location)
+        return self.process_and_combine_datasets(sample_dict, t0, location)
 
     @override
     def __getitem__(self, idx: int) -> NumpySample:
@@ -330,7 +322,6 @@ class PVNetUKConcurrentDataset(AbstractPVNetUKDataset):
             gsp_sample_dict = slice_datasets_by_space(sample_dict, location, self.config)
             gsp_numpy_sample = self.process_and_combine_datasets(
                 gsp_sample_dict,
-                self.config,
                 t0,
                 location,
             )

ocf_data_sampler/torch_datasets/datasets/site.py

@@ -25,7 +25,7 @@ from ocf_data_sampler.select import (
     intersection_of_multiple_dataframes_of_periods,
 )
 from ocf_data_sampler.torch_datasets.utils import (
-    channel_dict_to_dataarray,
+    config_normalization_values_to_dicts,
     find_valid_time_periods,
     slice_datasets_by_space,
     slice_datasets_by_time,
@@ -62,6 +62,8 @@ def process_and_combine_datasets(
     dataset_dict: dict,
     config: Configuration,
     t0: pd.Timestamp,
+    means_dict: dict[str, xr.DataArray | dict[str, xr.DataArray]],
+    stds_dict: dict[str, xr.DataArray | dict[str, xr.DataArray]],
 ) -> NumpySample:
     """Normalise and convert data to numpy arrays.
 
@@ -69,6 +71,8 @@ def process_and_combine_datasets(
         dataset_dict: Dictionary of xarray datasets
         config: Configuration object
         t0: init-time for sample
+        means_dict: Nested dictionary of mean values for the input data sources
+        stds_dict: Nested dictionary of std values for the input data sources
     """
     numpy_modalities = []
 
@@ -79,12 +83,8 @@ def process_and_combine_datasets(
 
             # Standardise and convert to NumpyBatch
 
-            da_channel_means = channel_dict_to_dataarray(
-                config.input_data.nwp[nwp_key].channel_means,
-            )
-            da_channel_stds = channel_dict_to_dataarray(
-                config.input_data.nwp[nwp_key].channel_stds,
-            )
+            da_channel_means = means_dict["nwp"][nwp_key]
+            da_channel_stds = stds_dict["nwp"][nwp_key]
 
             da_nwp = (da_nwp - da_channel_means) / da_channel_stds
 
@@ -97,8 +97,8 @@ def process_and_combine_datasets(
         da_sat = dataset_dict["sat"]
 
         # Standardise and convert to NumpyBatch
-        da_channel_means = channel_dict_to_dataarray(config.input_data.satellite.channel_means)
-        da_channel_stds = channel_dict_to_dataarray(config.input_data.satellite.channel_stds)
+        da_channel_means = means_dict["sat"]
+        da_channel_stds = stds_dict["sat"]
 
         da_sat = (da_sat - da_channel_means) / da_channel_stds
 
@@ -109,11 +109,7 @@ def process_and_combine_datasets(
         da_sites = da_sites / da_sites.capacity_kwp
 
         # Convert to NumpyBatch
-        numpy_modalities.append(
-            convert_site_to_numpy_sample(
-                da_sites,
-            ),
-        )
+        numpy_modalities.append(convert_site_to_numpy_sample(da_sites))
 
         # add datetime features
         datetimes = pd.DatetimeIndex(da_sites.time_utc.values)
@@ -193,6 +189,11 @@ class SitesDataset(Dataset):
         # Assign coords and indices to self
         self.valid_t0_and_site_ids = valid_t0_and_site_ids
 
+        # Extract the normalisation values from the config for faster access
+        means_dict, stds_dict = config_normalization_values_to_dicts(config)
+        self.means_dict = means_dict
+        self.stds_dict = stds_dict
+
     def find_valid_t0_and_site_ids(
         self,
         datasets_dict: dict,
@@ -273,7 +274,13 @@ class SitesDataset(Dataset):
 
         sample_dict = compute(sample_dict)
 
-        return process_and_combine_datasets(sample_dict, self.config, t0)
+        return process_and_combine_datasets(
+            sample_dict,
+            self.config,
+            t0,
+            self.means_dict,
+            self.stds_dict,
+        )
 
     def get_sample(self, t0: pd.Timestamp, site_id: int) -> dict:
         """Generate a sample for a given site id and t0.
@@ -332,6 +339,11 @@ class SitesDatasetConcurrent(Dataset):
         # Assign coords and indices to self
         self.valid_t0s = valid_t0s
 
+        # Extract the normalisation values from the config for faster access
+        means_dict, stds_dict = config_normalization_values_to_dicts(config)
+        self.means_dict = means_dict
+        self.stds_dict = stds_dict
+
     def find_valid_t0s(
         self,
         datasets_dict: dict,
@@ -406,6 +418,8 @@ class SitesDatasetConcurrent(Dataset):
                 site_sample_dict,
                 self.config,
                 t0,
+                self.means_dict,
+                self.stds_dict,
             )
             site_samples.append(site_numpy_sample)
 

ocf_data_sampler/torch_datasets/utils/__init__.py

@@ -1,4 +1,4 @@
-from .channel_dict_to_dataarray import channel_dict_to_dataarray
+from .config_normalization_values_to_dicts import config_normalization_values_to_dicts
 from .merge_and_fill_utils import fill_nans_in_arrays, merge_dicts
 from .valid_time_periods import find_valid_time_periods
 from .spatial_slice_for_dataset import slice_datasets_by_space

ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py

@@ -0,0 +1,57 @@
+"""Utility function for converting channel dictionaries to xarray DataArrays."""
+
+import xarray as xr
+
+from ocf_data_sampler.config import Configuration
+
+
+def channel_dict_to_dataarray(channel_dict: dict[str, float]) -> xr.DataArray:
+    """Converts a dictionary of channel values to a DataArray.
+
+    Args:
+        channel_dict: Dictionary mapping channel names (str) to their values (float).
+
+    Returns:
+        xr.DataArray: A 1D DataArray with channels as coordinates.
+    """
+    return xr.DataArray(
+        list(channel_dict.values()),
+        coords={"channel": list(channel_dict.keys())},
+    )
+
+def config_normalization_values_to_dicts(
+    config: Configuration,
+) -> tuple[dict[str, xr.DataArray | dict[str, xr.DataArray]]]:
+    """Construct DataArrays of mean and std values from the config normalisation constants.
+
+    Args:
+        config: Data configuration.
+
+    Returns:
+        Means dict
+        Stds dict
+    """
+    means_dict = {}
+    stds_dict = {}
+
+    if config.input_data.nwp is not None:
+
+        means_dict["nwp"] = {}
+        stds_dict["nwp"] = {}
+
+        for nwp_key in config.input_data.nwp:
+            # Standardise and convert to NumpyBatch
+
+            means_dict["nwp"][nwp_key] = channel_dict_to_dataarray(
+                config.input_data.nwp[nwp_key].channel_means,
+            )
+            stds_dict["nwp"][nwp_key] = channel_dict_to_dataarray(
+                config.input_data.nwp[nwp_key].channel_stds,
+            )
+
+    if config.input_data.satellite is not None:
+
+        means_dict["sat"] = channel_dict_to_dataarray(config.input_data.satellite.channel_means)
+        stds_dict["sat"] = channel_dict_to_dataarray(config.input_data.satellite.channel_stds)
+
+    return means_dict, stds_dict

{ocf_data_sampler-0.5.1.dist-info → ocf_data_sampler-0.5.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.5.1
+Version: 0.5.3
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.5.1.dist-info → ocf_data_sampler-0.5.3.dist-info}/RECORD

@@ -7,7 +7,7 @@ ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdR
 ocf_data_sampler/data/uk_gsp_locations_20220314.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
 ocf_data_sampler/data/uk_gsp_locations_20250109.csv,sha256=XZISFatnbpO9j8LwaxNKFzQSjs6hcHFsV8a9uDDpy2E,9055334
 ocf_data_sampler/load/__init__.py,sha256=-vQP9g0UOWdVbjEGyVX_ipa7R1btmiETIKAf6aw4d78,201
-ocf_data_sampler/load/gsp.py,sha256=IrTA6z9quN08imKGHJLf8gRktarxn1-utNMNFD0zWQs,2944
+ocf_data_sampler/load/gsp.py,sha256=d30jQWnwFaLj6rKNMHdz1qD8fzF8q--RNnEXT7bGiX0,2981
 ocf_data_sampler/load/load_dataset.py,sha256=K8rWykjII-3g127If7WRRFivzHNx3SshCvZj4uQlf28,2089
 ocf_data_sampler/load/open_tensorstore_zarrs.py,sha256=_RHWe0GmrBSA9s1TH5I9VCMPpeZEsuRuhDt5Vyyx5Fo,2725
 ocf_data_sampler/load/satellite.py,sha256=RylkJz8avxdM5pK_liaTlD1DTboyPMgykXJ4_Ek9WBA,1840
@@ -40,14 +40,14 @@ ocf_data_sampler/select/location.py,sha256=AZvGR8y62opiW7zACGXjoOtBEWRfSLOZIA73O
 ocf_data_sampler/select/select_spatial_slice.py,sha256=Hd4jGRUfIZRoWCirOQZeoLpaUnStB6KyFSTPX69wZLw,8790
 ocf_data_sampler/select/select_time_slice.py,sha256=HeHbwZ0CP03x0-LaJtpbSdtpLufwVTR73p6wH6O_PS8,5513
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=o0SsEXXZ6k9iL__5_RN1Sf60lw_eqK91P3UFEHAD2k0,102
-ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=v63goKEMI6UgBPnQCnIbxhFFdwuP_sxgcPYY6iNfGkc,12257
-ocf_data_sampler/torch_datasets/datasets/site.py,sha256=_0A2kRq8B5WL5zWjKxNY9snAl_GwptohUt7c6DDa2AA,14812
+ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=876oLukvb1nLtZQ8HBN3PWfN7urKH2xa45tVar7XrbM,12010
+ocf_data_sampler/torch_datasets/datasets/site.py,sha256=nn6N8daGxllYwCCiFKbCJANTl84NrDRl-nbNGcfXc3U,15429
 ocf_data_sampler/torch_datasets/sample/__init__.py,sha256=GL84vdZl_SjHDGVyh9Uekx2XhPYuZ0dnO3l6f6KXnHI,100
 ocf_data_sampler/torch_datasets/sample/base.py,sha256=cQ1oIyhdmlotejZK8B3Cw6MNvpdnBPD8G_o2h7Ye4Vc,2206
 ocf_data_sampler/torch_datasets/sample/site.py,sha256=40NwNTqjL1WVhPdwe02zDHHfDLG2u_bvCfRCtGAtFc0,1466
 ocf_data_sampler/torch_datasets/sample/uk_regional.py,sha256=Xx5cBYUyaM6PGUWQ76MHT9hwj6IJ7WAOxbpmYFbJGhc,10483
-ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=N7i_hHtWUDiJqsiJoDx4T_QuiYOuvIyulPrn6xEA4TY,309
-ocf_data_sampler/torch_datasets/utils/channel_dict_to_dataarray.py,sha256=un2IiyoAmTDIymdeMiPU899_86iCDMD-oIifjHlNyqw,555
+ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=_UHLL_yRzhLJVHi6ROSaSe8TGw80CAhU325uCZj7XkY,331
+ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py,sha256=jS3DkAwOF1W3AQnvsdkBJ1C8Unm93kQbS8hgTCtFv2A,1743
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=we7BTxRH7B7jKayDT7YfNyfI3zZClz2Bk-HXKQIokgU,956
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py,sha256=Hvz0wHSWMYYamf2oHNiGlzJcM4cAH6pL_7ZEvIBL2dE,1882
 ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=8E4a5v9dqr-sZOyBruuO-tjLPBbjtpYtdFY5z23aqnU,4365
@@ -56,7 +56,7 @@ ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
 utils/compute_icon_mean_stddev.py,sha256=a1oWMRMnny39rV-dvu8rcx85sb4bXzPFrR1gkUr4Jpg,2296
-ocf_data_sampler-0.5.1.dist-info/METADATA,sha256=sd5ucgDgrjrwa8vImToOUdU3BCWM-fMSsDHTS51p4Zc,12580
-ocf_data_sampler-0.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ocf_data_sampler-0.5.1.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
-ocf_data_sampler-0.5.1.dist-info/RECORD,,
+ocf_data_sampler-0.5.3.dist-info/METADATA,sha256=9gg1K9SNIX6pJ-PXQptutiLU9fo7FsnrKM6vdHbpQYg,12580
+ocf_data_sampler-0.5.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ocf_data_sampler-0.5.3.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
+ocf_data_sampler-0.5.3.dist-info/RECORD,,

ocf_data_sampler/torch_datasets/utils/channel_dict_to_dataarray.py

@@ -1,18 +0,0 @@
-"""Utility function for converting channel dictionaries to xarray DataArrays."""
-
-import xarray as xr
-
-
-def channel_dict_to_dataarray(channel_dict: dict[str, float]) -> xr.DataArray:
-    """Converts a dictionary of channel values to a DataArray.
-
-    Args:
-        channel_dict: Dictionary mapping channel names (str) to their values (float).
-
-    Returns:
-        xr.DataArray: A 1D DataArray with channels as coordinates.
-    """
-    return xr.DataArray(
-        list(channel_dict.values()),
-        coords={"channel": list(channel_dict.keys())},
-    )