ocf-data-sampler 0.5.34__tar.gz → 0.5.36__tar.gz

{ocf_data_sampler-0.5.34 → ocf_data_sampler-0.5.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.5.34
+Version: 0.5.36
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.5.34 → ocf_data_sampler-0.5.36}/ocf_data_sampler/config/model.py

@@ -93,6 +93,11 @@ class DropoutMixin(Base):
         "floats (probability that dropout of the corresponding timedelta is applied)",
     )
 
+    dropout_value: float = Field(
+        default=0.0,
+        description="The value to use for dropped out values. "
+        "Idea is to use -1, but to be backwards comptaible we've put the default as 0")
+
     @field_validator("dropout_timedeltas_minutes")
     def dropout_timedeltas_minutes_negative(cls, v: list[int]) -> list[int]:
         """Validate 'dropout_timedeltas_minutes'."""

{ocf_data_sampler-0.5.34 → ocf_data_sampler-0.5.36}/ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -206,7 +206,7 @@ class AbstractPVNetUKDataset(PickleCacheMixin, Dataset):
 
         # Combine all the modalities and fill NaNs
         combined_sample = merge_dicts(numpy_modalities)
-        combined_sample = fill_nans_in_arrays(combined_sample)
+        combined_sample = fill_nans_in_arrays(combined_sample, config=self.config)
 
         return combined_sample
 

{ocf_data_sampler-0.5.34 → ocf_data_sampler-0.5.36}/ocf_data_sampler/torch_datasets/datasets/site.py

@@ -139,7 +139,7 @@ def process_and_combine_datasets(
 
     # Combine all the modalities and fill NaNs
     combined_sample = merge_dicts(numpy_modalities)
-    combined_sample = fill_nans_in_arrays(combined_sample)
+    combined_sample = fill_nans_in_arrays(combined_sample, config=config)
 
     return combined_sample
 
@@ -330,7 +330,12 @@ class SitesDatasetConcurrent(PickleCacheMixin, Dataset):
         self.config = config
 
         # get all locations
-        self.locations = get_locations(datasets_dict["site"])
+        locations = get_locations(datasets_dict["site"])
+        self.locations = add_alterate_coordinate_projections(
+            locations,
+            datasets_dict,
+            primary_coords="lon_lat",
+        )
 
         # Get t0 times where all input data is available
         valid_t0s = self.find_valid_t0s(datasets_dict)

ocf_data_sampler-0.5.36/ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py

@@ -0,0 +1,50 @@
+"""Utility functions for merging dictionaries and filling NaNs in arrays."""
+
+import numpy as np
+
+from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.numpy_sample.gsp import GSPSampleKey
+from ocf_data_sampler.numpy_sample.nwp import NWPSampleKey
+from ocf_data_sampler.numpy_sample.satellite import SatelliteSampleKey
+from ocf_data_sampler.numpy_sample.site import SiteSampleKey
+
+
+def merge_dicts(list_of_dicts: list[dict]) -> dict:
+    """Merge a list of dictionaries into a single dictionary."""
+    # TODO: This doesn't account for duplicate keys, which will be overwritten
+    combined_dict = {}
+    for d in list_of_dicts:
+        combined_dict.update(d)
+    return combined_dict
+
+
+def fill_nans_in_arrays(
+    sample: dict, config: Configuration | None = None, nwp_provider: str | None = None,
+) -> dict:
+    """Fills all NaN values in each np.ndarray in the sample dictionary.
+
+    Operation is performed in-place on the sample.
+    By default a fill value of 0.0 are used, but if a config is provided,
+    it can use the configured dropout values.
+    """
+    for k, v in sample.items():
+        if isinstance(v, np.ndarray) and np.issubdtype(v.dtype, np.number):
+            if np.isnan(v).any():
+                fill_value = 0.0
+                if config is not None:
+                    if k == GSPSampleKey.gsp:
+                        fill_value = config.input_data.gsp.dropout_value
+                    elif k == SiteSampleKey.generation:
+                        fill_value = config.input_data.site.dropout_value
+                    elif k == SatelliteSampleKey.satellite_actual:
+                        fill_value = config.input_data.satellite.dropout_value
+                    elif k == NWPSampleKey.nwp and nwp_provider in config.input_data.nwp:
+                        fill_value = config.input_data.nwp[nwp_provider].dropout_value
+
+                sample[k] = np.nan_to_num(v, copy=False, nan=fill_value)
+
+        # Recursion is included to reach NWP arrays in subdict
+        elif isinstance(v, dict):
+            fill_nans_in_arrays(v, config=config, nwp_provider=k)
+
+    return sample

{ocf_data_sampler-0.5.34 → ocf_data_sampler-0.5.36}/ocf_data_sampler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.5.34
+Version: 0.5.36
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

ocf_data_sampler-0.5.34/ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py

@@ -1,29 +0,0 @@
-"""Utility functions for merging dictionaries and filling NaNs in arrays."""
-
-import numpy as np
-
-
-def merge_dicts(list_of_dicts: list[dict]) -> dict:
-    """Merge a list of dictionaries into a single dictionary."""
-    # TODO: This doesn't account for duplicate keys, which will be overwritten
-    combined_dict = {}
-    for d in list_of_dicts:
-        combined_dict.update(d)
-    return combined_dict
-
-
-def fill_nans_in_arrays(sample: dict) -> dict:
-    """Fills all NaN values in each np.ndarray in the sample dictionary with zeros.
-
-    Operation is performed in-place on the sample.
-    """
-    for k, v in sample.items():
-        if isinstance(v, np.ndarray) and np.issubdtype(v.dtype, np.number):
-            if np.isnan(v).any():
-                sample[k] = np.nan_to_num(v, copy=False, nan=0.0)
-
-        # Recursion is included to reach NWP arrays in subdict
-        elif isinstance(v, dict):
-            fill_nans_in_arrays(v)
-
-    return sample