ocf-data-sampler 0.5.12__py3-none-any.whl → 0.5.14__py3-none-any.whl

ocf_data_sampler/config/model.py

@@ -172,23 +172,6 @@ class NormalisationConstantsMixin(Base):
     """Normalisation constants for multiple channels."""
     normalisation_constants: dict[str, NormalisationValues]
 
-    @property
-    def channel_means(self) -> dict[str, float]:
-        """Return the channel means."""
-        return {
-            channel: norm_values.mean
-            for channel, norm_values in self.normalisation_constants.items()
-        }
-
-
-    @property
-    def channel_stds(self) -> dict[str, float]:
-        """Return the channel standard deviations."""
-        return {
-            channel: norm_values.std
-            for channel, norm_values in self.normalisation_constants.items()
-        }
-
 
 class Satellite(TimeWindowMixin, DropoutMixin, SpatialWindowMixin, NormalisationConstantsMixin):
     """Satellite configuration model."""

ocf_data_sampler/select/select_spatial_slice.py

@@ -25,13 +25,13 @@ def _get_pixel_index_location(da: xr.DataArray, location: Location) -> tuple[int
     x, y = location.in_coord_system(target_coords)
 
     # Check that requested point lies within the data
-    if not (da[x_dim].min() < x < da[x_dim].max()):
+    if not (da[x_dim].values[0] < x < da[x_dim].values[-1]):
         raise ValueError(
-            f"{x} is not in the interval {da[x_dim].min().values}: {da[x_dim].max().values}",
+            f"{x} is not in the interval {da[x_dim].values[0]}: {da[x_dim].values[-1]}",
         )
-    if not (da[y_dim].min() < y < da[y_dim].max()):
+    if not (da[y_dim].values[0] < y < da[y_dim].values[-1]):
         raise ValueError(
-            f"{y} is not in the interval {da[y_dim].min().values}: {da[y_dim].max().values}",
+            f"{y} is not in the interval {da[y_dim].values[0]}: {da[y_dim].values[-1]}",
         )
 
     x_index = da.get_index(x_dim)

ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -137,10 +137,10 @@ class AbstractPVNetUKDataset(Dataset):
             for nwp_key, da_nwp in dataset_dict["nwp"].items():
 
                 # Standardise and convert to NumpyBatch
-                da_channel_means = self.means_dict["nwp"][nwp_key]
-                da_channel_stds = self.stds_dict["nwp"][nwp_key]
+                channel_means = self.means_dict["nwp"][nwp_key]
+                channel_stds = self.stds_dict["nwp"][nwp_key]
 
-                da_nwp = (da_nwp - da_channel_means) / da_channel_stds
+                da_nwp = (da_nwp - channel_means) / channel_stds
 
                 nwp_numpy_modalities[nwp_key] = convert_nwp_to_numpy_sample(da_nwp)
 
@@ -151,17 +151,17 @@ class AbstractPVNetUKDataset(Dataset):
             da_sat = dataset_dict["sat"]
 
             # Standardise and convert to NumpyBatch
-            da_channel_means = self.means_dict["sat"]
-            da_channel_stds = self.stds_dict["sat"]
+            channel_means = self.means_dict["sat"]
+            channel_stds = self.stds_dict["sat"]
 
-            da_sat = (da_sat - da_channel_means) / da_channel_stds
+            da_sat = (da_sat - channel_means) / channel_stds
 
             numpy_modalities.append(convert_satellite_to_numpy_sample(da_sat))
 
         if "gsp" in dataset_dict:
             gsp_config = self.config.input_data.gsp
             da_gsp = dataset_dict["gsp"]
-            da_gsp = da_gsp / da_gsp.effective_capacity_mwp
+            da_gsp = da_gsp / da_gsp.effective_capacity_mwp.values
 
             # Convert to NumpyBatch
             numpy_modalities.append(

ocf_data_sampler/torch_datasets/datasets/site.py

@@ -82,10 +82,10 @@ def process_and_combine_datasets(
 
             # Standardise and convert to NumpyBatch
 
-            da_channel_means = means_dict["nwp"][nwp_key]
-            da_channel_stds = stds_dict["nwp"][nwp_key]
+            channel_means = means_dict["nwp"][nwp_key]
+            channel_stds = stds_dict["nwp"][nwp_key]
 
-            da_nwp = (da_nwp - da_channel_means) / da_channel_stds
+            da_nwp = (da_nwp - channel_means) / channel_stds
 
             nwp_numpy_modalities[nwp_key] = convert_nwp_to_numpy_sample(da_nwp)
 
@@ -96,16 +96,16 @@ def process_and_combine_datasets(
         da_sat = dataset_dict["sat"]
 
         # Standardise and convert to NumpyBatch
-        da_channel_means = means_dict["sat"]
-        da_channel_stds = stds_dict["sat"]
+        channel_means = means_dict["sat"]
+        channel_stds = stds_dict["sat"]
 
-        da_sat = (da_sat - da_channel_means) / da_channel_stds
+        da_sat = (da_sat - channel_means) / channel_stds
 
         numpy_modalities.append(convert_satellite_to_numpy_sample(da_sat))
 
     if "site" in dataset_dict:
         da_sites = dataset_dict["site"]
-        da_sites = da_sites / da_sites.capacity_kwp
+        da_sites = da_sites / da_sites.capacity_kwp.values
 
         # Convert to NumpyBatch
         numpy_modalities.append(convert_site_to_numpy_sample(da_sites))

ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py

@@ -1,28 +1,14 @@
-"""Utility function for converting channel dictionaries to xarray DataArrays."""
+"""Utility function for converting normalisation constants in the config to arrays."""
 
-import xarray as xr
+import numpy as np
 
 from ocf_data_sampler.config import Configuration
 
 
-def channel_dict_to_dataarray(channel_dict: dict[str, float]) -> xr.DataArray:
-    """Converts a dictionary of channel values to a DataArray.
-
-    Args:
-        channel_dict: Dictionary mapping channel names (str) to their values (float).
-
-    Returns:
-        xr.DataArray: A 1D DataArray with channels as coordinates.
-    """
-    return xr.DataArray(
-        list(channel_dict.values()),
-        coords={"channel": list(channel_dict.keys())},
-    )
-
 def config_normalization_values_to_dicts(
     config: Configuration,
-) -> tuple[dict[str, xr.DataArray | dict[str, xr.DataArray]]]:
-    """Construct DataArrays of mean and std values from the config normalisation constants.
+) -> tuple[dict[str, np.ndarray | dict[str, np.ndarray]]]:
+    """Construct numpy arrays of mean and std values from the config normalisation constants.
 
     Args:
         config: Data configuration.
@@ -40,18 +26,34 @@ def config_normalization_values_to_dicts(
         stds_dict["nwp"] = {}
 
         for nwp_key in config.input_data.nwp:
-            # Standardise and convert to NumpyBatch
+            nwp_config = config.input_data.nwp[nwp_key]
 
-            means_dict["nwp"][nwp_key] = channel_dict_to_dataarray(
-                config.input_data.nwp[nwp_key].channel_means,
-            )
-            stds_dict["nwp"][nwp_key] = channel_dict_to_dataarray(
-                config.input_data.nwp[nwp_key].channel_stds,
-            )
+            means_list = []
+            stds_list = []
+
+            for channel in list(nwp_config.channels):
+                # These accumulated channels are diffed and renamed
+                if channel in nwp_config.accum_channels:
+                    channel =f"diff_{channel}"
+
+                means_list.append(nwp_config.normalisation_constants[channel].mean)
+                stds_list.append(nwp_config.normalisation_constants[channel].std)
+
+            means_dict["nwp"][nwp_key] = np.array(means_list)[None, :, None, None]
+            stds_dict["nwp"][nwp_key] = np.array(stds_list)[None, :, None, None]
 
     if config.input_data.satellite is not None:
+        sat_config = config.input_data.satellite
+
+        means_list = []
+        stds_list = []
+
+        for channel in list(config.input_data.satellite.channels):
+            means_list.append(sat_config.normalisation_constants[channel].mean)
+            stds_list.append(sat_config.normalisation_constants[channel].std)
 
-        means_dict["sat"] = channel_dict_to_dataarray(config.input_data.satellite.channel_means)
-        stds_dict["sat"] = channel_dict_to_dataarray(config.input_data.satellite.channel_stds)
+        # Convert to array and expand dimensions so we can normalise the 4D sat and NWP sources
+        means_dict["sat"] = np.array(means_list)[None, :, None, None]
+        stds_dict["sat"] = np.array(stds_list)[None, :, None, None]
 
     return means_dict, stds_dict

{ocf_data_sampler-0.5.12.dist-info → ocf_data_sampler-0.5.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.5.12
+Version: 0.5.14
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.5.12.dist-info → ocf_data_sampler-0.5.14.dist-info}/RECORD

@@ -2,7 +2,7 @@ ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,
 ocf_data_sampler/utils.py,sha256=CTJf9bjHjO8vOJebUtXiMpvgwUpF7gEOjjaoE77fhTk,1177
 ocf_data_sampler/config/__init__.py,sha256=O29mbH0XG2gIY1g3BaveGCnpBO2SFqdu-qzJ7a6evl0,223
 ocf_data_sampler/config/load.py,sha256=LL-7wemI8o4KPkx35j-wQ3HjsMvDgqXr7G46IcASfnU,632
-ocf_data_sampler/config/model.py,sha256=Jss8UDJAaQIBDr9megX2pERoT0ocFmwLNFC8pCWN6VA,12386
+ocf_data_sampler/config/model.py,sha256=ucddp09yM4HGZKVuW-0N8vLGqLVAo_S4mPT89N_iG-0,11881
 ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdRaI,1064
 ocf_data_sampler/data/uk_gsp_locations_20220314.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
 ocf_data_sampler/data/uk_gsp_locations_20250109.csv,sha256=XZISFatnbpO9j8LwaxNKFzQSjs6hcHFsV8a9uDDpy2E,9055334
@@ -37,18 +37,18 @@ ocf_data_sampler/select/fill_time_periods.py,sha256=TlGxp1xiAqnhdWfLy0pv3FuZc00d
 ocf_data_sampler/select/find_contiguous_time_periods.py,sha256=etkr6LuB7zxkfzWJ6SgHiULdRuFzFlq5bOUNd257Qx4,11545
 ocf_data_sampler/select/geospatial.py,sha256=rvMy_e--3tm-KAy9pU6b9-UMBQqH2sXykr3N_4SHYy4,6528
 ocf_data_sampler/select/location.py,sha256=Qp0di-Pgq8WLjN9IBcTVTaRM3lckhr4ZVzaDRcgVXHw,2352
-ocf_data_sampler/select/select_spatial_slice.py,sha256=0nwIRa1Wbmasxgz_PiDvXkNPUaYCdZNaUaOmkV4YIE0,7192
+ocf_data_sampler/select/select_spatial_slice.py,sha256=Ym_YJjZqeMPC5Bw_xMi7Re2-uCbUagm2KXhnAnstTHo,7200
 ocf_data_sampler/select/select_time_slice.py,sha256=HeHbwZ0CP03x0-LaJtpbSdtpLufwVTR73p6wH6O_PS8,5513
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=o0SsEXXZ6k9iL__5_RN1Sf60lw_eqK91P3UFEHAD2k0,102
-ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=1uwcbEKakXsqr_ZcSjpzf3esP5tA_YVlYgtj9vBjtdM,12115
-ocf_data_sampler/torch_datasets/datasets/site.py,sha256=Z7pyuR3VHEJf8egxnZvpsdgBv7fMg2pSc61ORAuqOwQ,15607
+ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=wVx4QKHqak2FbxtryAxsVe6wpYM2n_YKgIKpiVs6gpE,12098
+ocf_data_sampler/torch_datasets/datasets/site.py,sha256=ivdSB_YpAqL8-Q1m_uKTGU2YlNQ1bZXdwialT_UpGuo,15590
 ocf_data_sampler/torch_datasets/sample/__init__.py,sha256=GL84vdZl_SjHDGVyh9Uekx2XhPYuZ0dnO3l6f6KXnHI,100
 ocf_data_sampler/torch_datasets/sample/base.py,sha256=cQ1oIyhdmlotejZK8B3Cw6MNvpdnBPD8G_o2h7Ye4Vc,2206
 ocf_data_sampler/torch_datasets/sample/site.py,sha256=40NwNTqjL1WVhPdwe02zDHHfDLG2u_bvCfRCtGAtFc0,1466
 ocf_data_sampler/torch_datasets/sample/uk_regional.py,sha256=Xx5cBYUyaM6PGUWQ76MHT9hwj6IJ7WAOxbpmYFbJGhc,10483
 ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=TNSYuSSmFgjsvvJxtoDrH645Z64CHsNUUQ0iayTccP4,416
 ocf_data_sampler/torch_datasets/utils/add_alterate_coordinate_projections.py,sha256=w6Q4TyxNyl7PKAbhqiXvqOpnqIjwmOUcGREIvPNGYlQ,2666
-ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py,sha256=jS3DkAwOF1W3AQnvsdkBJ1C8Unm93kQbS8hgTCtFv2A,1743
+ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py,sha256=SGt1H2nXcaj44ND14-gHzvA7dkLfgjTacCq7rOkRGwg,1991
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=we7BTxRH7B7jKayDT7YfNyfI3zZClz2Bk-HXKQIokgU,956
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py,sha256=Hvz0wHSWMYYamf2oHNiGlzJcM4cAH6pL_7ZEvIBL2dE,1882
 ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=8E4a5v9dqr-sZOyBruuO-tjLPBbjtpYtdFY5z23aqnU,4365
@@ -56,7 +56,7 @@ ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0Wrg
 ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul3l0EP73Ik002fStr_bhsZh9mQqEU,4735
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
-ocf_data_sampler-0.5.12.dist-info/METADATA,sha256=2s4Pf9ZEgBo72guZFO6JI3mwJzOGRxLmkmeihiQ8Ogw,12817
-ocf_data_sampler-0.5.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ocf_data_sampler-0.5.12.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
-ocf_data_sampler-0.5.12.dist-info/RECORD,,
+ocf_data_sampler-0.5.14.dist-info/METADATA,sha256=OgS9xvqBfhmlWym0DYBBbT-IwZ3tRz_EKo2wEdErmCA,12817
+ocf_data_sampler-0.5.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ocf_data_sampler-0.5.14.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
+ocf_data_sampler-0.5.14.dist-info/RECORD,,