ocf-data-sampler 0.5.34__py3-none-any.whl → 0.5.36__py3-none-any.whl

ocf_data_sampler/config/model.py

@@ -93,6 +93,11 @@ class DropoutMixin(Base):
         "floats (probability that dropout of the corresponding timedelta is applied)",
     )
 
+    dropout_value: float = Field(
+        default=0.0,
+        description="The value to use for dropped out values. "
+        "Idea is to use -1, but to be backwards comptaible we've put the default as 0")
+
     @field_validator("dropout_timedeltas_minutes")
     def dropout_timedeltas_minutes_negative(cls, v: list[int]) -> list[int]:
         """Validate 'dropout_timedeltas_minutes'."""

ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py

@@ -206,7 +206,7 @@ class AbstractPVNetUKDataset(PickleCacheMixin, Dataset):
 
         # Combine all the modalities and fill NaNs
         combined_sample = merge_dicts(numpy_modalities)
-        combined_sample = fill_nans_in_arrays(combined_sample)
+        combined_sample = fill_nans_in_arrays(combined_sample, config=self.config)
 
         return combined_sample
 

ocf_data_sampler/torch_datasets/datasets/site.py

@@ -139,7 +139,7 @@ def process_and_combine_datasets(
 
     # Combine all the modalities and fill NaNs
     combined_sample = merge_dicts(numpy_modalities)
-    combined_sample = fill_nans_in_arrays(combined_sample)
+    combined_sample = fill_nans_in_arrays(combined_sample, config=config)
 
     return combined_sample
 
@@ -330,7 +330,12 @@ class SitesDatasetConcurrent(PickleCacheMixin, Dataset):
         self.config = config
 
         # get all locations
-        self.locations = get_locations(datasets_dict["site"])
+        locations = get_locations(datasets_dict["site"])
+        self.locations = add_alterate_coordinate_projections(
+            locations,
+            datasets_dict,
+            primary_coords="lon_lat",
+        )
 
         # Get t0 times where all input data is available
         valid_t0s = self.find_valid_t0s(datasets_dict)

ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py

@@ -2,6 +2,12 @@
 
 import numpy as np
 
+from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.numpy_sample.gsp import GSPSampleKey
+from ocf_data_sampler.numpy_sample.nwp import NWPSampleKey
+from ocf_data_sampler.numpy_sample.satellite import SatelliteSampleKey
+from ocf_data_sampler.numpy_sample.site import SiteSampleKey
+
 
 def merge_dicts(list_of_dicts: list[dict]) -> dict:
     """Merge a list of dictionaries into a single dictionary."""
@@ -12,18 +18,33 @@ def merge_dicts(list_of_dicts: list[dict]) -> dict:
     return combined_dict
 
 
-def fill_nans_in_arrays(sample: dict) -> dict:
-    """Fills all NaN values in each np.ndarray in the sample dictionary with zeros.
+def fill_nans_in_arrays(
+    sample: dict, config: Configuration | None = None, nwp_provider: str | None = None,
+) -> dict:
+    """Fills all NaN values in each np.ndarray in the sample dictionary.
 
     Operation is performed in-place on the sample.
+    By default a fill value of 0.0 are used, but if a config is provided,
+    it can use the configured dropout values.
     """
     for k, v in sample.items():
         if isinstance(v, np.ndarray) and np.issubdtype(v.dtype, np.number):
             if np.isnan(v).any():
-                sample[k] = np.nan_to_num(v, copy=False, nan=0.0)
+                fill_value = 0.0
+                if config is not None:
+                    if k == GSPSampleKey.gsp:
+                        fill_value = config.input_data.gsp.dropout_value
+                    elif k == SiteSampleKey.generation:
+                        fill_value = config.input_data.site.dropout_value
+                    elif k == SatelliteSampleKey.satellite_actual:
+                        fill_value = config.input_data.satellite.dropout_value
+                    elif k == NWPSampleKey.nwp and nwp_provider in config.input_data.nwp:
+                        fill_value = config.input_data.nwp[nwp_provider].dropout_value
+
+                sample[k] = np.nan_to_num(v, copy=False, nan=fill_value)
 
         # Recursion is included to reach NWP arrays in subdict
         elif isinstance(v, dict):
-            fill_nans_in_arrays(v)
+            fill_nans_in_arrays(v, config=config, nwp_provider=k)
 
     return sample

{ocf_data_sampler-0.5.34.dist-info → ocf_data_sampler-0.5.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.5.34
+Version: 0.5.36
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.5.34.dist-info → ocf_data_sampler-0.5.36.dist-info}/RECORD

@@ -2,7 +2,7 @@ ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,
 ocf_data_sampler/utils.py,sha256=WfmyBacjFGsv_IlUHRezNGc4ixi4wBvom_JF76iJYbY,1487
 ocf_data_sampler/config/__init__.py,sha256=O29mbH0XG2gIY1g3BaveGCnpBO2SFqdu-qzJ7a6evl0,223
 ocf_data_sampler/config/load.py,sha256=LL-7wemI8o4KPkx35j-wQ3HjsMvDgqXr7G46IcASfnU,632
-ocf_data_sampler/config/model.py,sha256=3aaMZIITDOQ6IGeod9xOYGOzft5zNVfEVBWbmWM50nA,11843
+ocf_data_sampler/config/model.py,sha256=U1AdoK85DFM_H_5ejiiAA_94aYU6ZZdj2uz7P1Uv6MU,12051
 ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdRaI,1064
 ocf_data_sampler/data/uk_gsp_locations_20220314.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
 ocf_data_sampler/data/uk_gsp_locations_20250109.csv,sha256=XZISFatnbpO9j8LwaxNKFzQSjs6hcHFsV8a9uDDpy2E,9055334
@@ -42,8 +42,8 @@ ocf_data_sampler/select/select_spatial_slice.py,sha256=Nov6foM5xPkAREsEAHHAak8jH
 ocf_data_sampler/select/select_time_slice.py,sha256=cpkdovJMvcjxSGfq9G0OJK5aDAeCXg7exWYrJnR4N2w,4116
 ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=o0SsEXXZ6k9iL__5_RN1Sf60lw_eqK91P3UFEHAD2k0,102
 ocf_data_sampler/torch_datasets/datasets/picklecache.py,sha256=b8T5lgKfiPXLwuVQuFpCQBlU-HNBrA-Z-eSwYICKvsQ,1350
-ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=Tpf1zVPtBy-815n__-zgsTeNRxlxjLwuu6UgCCyIEL0,12365
-ocf_data_sampler/torch_datasets/datasets/site.py,sha256=F3emPejGQNtm3qzWtIGSFOVVH9l3DxS9YkiSATXROHQ,15095
+ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=TDSraKUYE2wA5YF9SYO2RmAIbDJtcvfTtxr7WmYDszg,12385
+ocf_data_sampler/torch_datasets/datasets/site.py,sha256=IFMVy8c887mfT2NmcKX8ocm1OvlBhvzJNye_scfBMFQ,15265
 ocf_data_sampler/torch_datasets/sample/__init__.py,sha256=GL84vdZl_SjHDGVyh9Uekx2XhPYuZ0dnO3l6f6KXnHI,100
 ocf_data_sampler/torch_datasets/sample/base.py,sha256=cQ1oIyhdmlotejZK8B3Cw6MNvpdnBPD8G_o2h7Ye4Vc,2206
 ocf_data_sampler/torch_datasets/sample/site.py,sha256=40NwNTqjL1WVhPdwe02zDHHfDLG2u_bvCfRCtGAtFc0,1466
@@ -52,14 +52,14 @@ ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=4l1VcEmxHInU9G66zrimNMa
 ocf_data_sampler/torch_datasets/utils/add_alterate_coordinate_projections.py,sha256=w6Q4TyxNyl7PKAbhqiXvqOpnqIjwmOUcGREIvPNGYlQ,2666
 ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py,sha256=SGt1H2nXcaj44ND14-gHzvA7dkLfgjTacCq7rOkRGwg,1991
 ocf_data_sampler/torch_datasets/utils/diff_nwp_data.py,sha256=o7NpKWxKHhwMbol3xBAF087-tDgDUZeP0j8vG08E7Nc,816
-ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=we7BTxRH7B7jKayDT7YfNyfI3zZClz2Bk-HXKQIokgU,956
+ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=VkQv4wJVihObh_OiSuwKqV_w2lEOweaYgJPkm075CZc,2132
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py,sha256=Hvz0wHSWMYYamf2oHNiGlzJcM4cAH6pL_7ZEvIBL2dE,1882
 ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=1r1J2KNSo1_imN9gpVf5AupJaZ7VSnSevS1o_wck440,3925
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0WrglnX5YUAFjXXlO2GwEBHWyqo8TDuiOA,4714
 ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul3l0EP73Ik002fStr_bhsZh9mQqEU,4735
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
-ocf_data_sampler-0.5.34.dist-info/METADATA,sha256=VVnKoyWCmUnQmBBaSsUvT-OhFD1Ogd3-HGSh-jyRRps,13541
-ocf_data_sampler-0.5.34.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ocf_data_sampler-0.5.34.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
-ocf_data_sampler-0.5.34.dist-info/RECORD,,
+ocf_data_sampler-0.5.36.dist-info/METADATA,sha256=OgFiRyisHUXi5zZoScJUxJkoAF9OivTYIaEBotOlwXs,13541
+ocf_data_sampler-0.5.36.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ocf_data_sampler-0.5.36.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
+ocf_data_sampler-0.5.36.dist-info/RECORD,,