ocf-data-sampler 0.2.21__py3-none-any.whl → 0.2.23__py3-none-any.whl

ocf_data_sampler/config/model.py

@@ -13,6 +13,7 @@ from typing_extensions import override
 NWP_PROVIDERS = [
     "ukv",
     "ecmwf",
+    "mo_global",
     "gfs",
     "icon_eu",
     "cloudcasting",

ocf_data_sampler/load/gsp.py

@@ -48,7 +48,7 @@ def open_gsp(zarr_path: str, boundaries_version: str = "20220314") -> xr.DataArr
 
     if not (ds.gsp_id.isin(df_gsp_loc.index)).all():
         raise ValueError(
-            "Some GSP IDs in the GSP generation data are available in the locations file.",
+            "Some GSP IDs in the GSP generation data are not available in the locations file.",
         )
 
     # Select the locations by the GSP IDs in the generation data

ocf_data_sampler/load/nwp/nwp.py

@@ -28,7 +28,7 @@ def open_nwp(zarr_path: str | list[str], provider: str, public: bool = False) ->
 
     if provider == "ukv":
         _open_nwp = open_ukv
-    elif provider == "ecmwf":
+    elif provider in ["ecmwf", "mo_global"]:
         _open_nwp = open_ifs
     elif provider == "icon-eu":
         _open_nwp = open_icon_eu

ocf_data_sampler/select/dropout.py

@@ -9,19 +9,22 @@ import pandas as pd
 import xarray as xr
 
 
-def draw_dropout_time(
+def apply_sampled_dropout_time(
     t0: pd.Timestamp,
     dropout_timedeltas: list[pd.Timedelta],
     dropout_frac: float,
-) -> pd.Timestamp:
-    """Randomly pick a dropout time from a list of timedeltas.
+    da: xr.DataArray,
+) -> xr.DataArray:
+    """Randomly pick a dropout time from a list of timedeltas and apply dropout time to the data.
 
     Args:
         t0: The forecast init-time
         dropout_timedeltas: List of timedeltas relative to t0 to pick from
         dropout_frac: Probability that dropout will be applied.
             This should be between 0 and 1 inclusive
+        da: Xarray DataArray with 'time_utc' coordinate
     """
+    # sample dropout time
     if dropout_frac > 0 and len(dropout_timedeltas) == 0:
         raise ValueError("To apply dropout, dropout_timedeltas must be provided")
 
@@ -37,21 +40,8 @@ def draw_dropout_time(
     else:
         dropout_time = t0 + np.random.choice(dropout_timedeltas)
 
-    return dropout_time
-
-
-def apply_dropout_time(
-    ds: xr.DataArray,
-    dropout_time: pd.Timestamp | None,
-) -> xr.DataArray:
-    """Apply dropout time to the data.
-
-    Args:
-        ds: Xarray DataArray with 'time_utc' coordinate
-        dropout_time: Time after which data is set to NaN
-    """
+    # apply dropout time
     if dropout_time is None:
-        return ds
-    else:
-        # This replaces the times after the dropout with NaNs
-        return ds.where(ds.time_utc <= dropout_time)
+        return da
+    # This replaces the times after the dropout with NaNs
+    return da.where(da.time_utc <= dropout_time)

ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py

@@ -4,7 +4,7 @@ import pandas as pd
 import xarray as xr
 
 from ocf_data_sampler.config import Configuration
-from ocf_data_sampler.select.dropout import apply_dropout_time, draw_dropout_time
+from ocf_data_sampler.select.dropout import apply_sampled_dropout_time
 from ocf_data_sampler.select.select_time_slice import select_time_slice, select_time_slice_nwp
 from ocf_data_sampler.utils import minutes
 
@@ -51,17 +51,12 @@ def slice_datasets_by_time(
             interval_end=minutes(sat_config.interval_end_minutes),
         )
 
-        # Randomly sample dropout
-        sat_dropout_time = draw_dropout_time(
+        # Apply the randomly sampled dropout
+        sliced_datasets_dict["sat"] = apply_sampled_dropout_time(
             t0,
             dropout_timedeltas=minutes(sat_config.dropout_timedeltas_minutes),
             dropout_frac=sat_config.dropout_fraction,
-        )
-
-        # Apply the dropout
-        sliced_datasets_dict["sat"] = apply_dropout_time(
-            sliced_datasets_dict["sat"],
-            sat_dropout_time,
+            da=sliced_datasets_dict["sat"],
         )
 
     if "gsp" in datasets_dict:
@@ -76,15 +71,11 @@ def slice_datasets_by_time(
         )
 
         # Dropout on the past GSP, but not the future GSP
-        gsp_dropout_time = draw_dropout_time(
+        da_gsp_past = apply_sampled_dropout_time(
             t0,
             dropout_timedeltas=minutes(gsp_config.dropout_timedeltas_minutes),
             dropout_frac=gsp_config.dropout_fraction,
-        )
-
-        da_gsp_past = apply_dropout_time(
-            da_gsp_past,
-            gsp_dropout_time,
+            da=da_gsp_past,
         )
 
         da_gsp_future = select_time_slice(
@@ -100,25 +91,30 @@ def slice_datasets_by_time(
     if "site" in datasets_dict:
         site_config = config.input_data.site
 
-        sliced_datasets_dict["site"] = select_time_slice(
+        da_site_past = select_time_slice(
             datasets_dict["site"],
             t0,
             time_resolution=minutes(site_config.time_resolution_minutes),
             interval_start=minutes(site_config.interval_start_minutes),
-            interval_end=minutes(site_config.interval_end_minutes),
+            interval_end=minutes(0),
         )
 
-        # Randomly sample dropout
-        site_dropout_time = draw_dropout_time(
+        # Apply the randomly sampled dropout on the past site not the future
+        da_site_past = apply_sampled_dropout_time(
             t0,
             dropout_timedeltas=minutes(site_config.dropout_timedeltas_minutes),
             dropout_frac=site_config.dropout_fraction,
+            da=da_site_past,
         )
 
-        # Apply the dropout
-        sliced_datasets_dict["site"] = apply_dropout_time(
-            sliced_datasets_dict["site"],
-            site_dropout_time,
+        da_site_future = select_time_slice(
+            datasets_dict["site"],
+            t0,
+            time_resolution=minutes(site_config.time_resolution_minutes),
+            interval_start=minutes(site_config.time_resolution_minutes),
+            interval_end=minutes(site_config.interval_end_minutes),
         )
 
+        sliced_datasets_dict["site"] = xr.concat([da_site_past, da_site_future], dim="time_utc")
+
     return sliced_datasets_dict

{ocf_data_sampler-0.2.21.dist-info → ocf_data_sampler-0.2.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.2.21
+Version: 0.2.23
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.21.dist-info → ocf_data_sampler-0.2.23.dist-info}/RECORD

@@ -2,18 +2,18 @@ ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,
 ocf_data_sampler/utils.py,sha256=DjuneGGisl08ENvPZV_lrcX4b2NCKJC1ZpXgIpxuQi4,290
 ocf_data_sampler/config/__init__.py,sha256=O29mbH0XG2gIY1g3BaveGCnpBO2SFqdu-qzJ7a6evl0,223
 ocf_data_sampler/config/load.py,sha256=LL-7wemI8o4KPkx35j-wQ3HjsMvDgqXr7G46IcASfnU,632
-ocf_data_sampler/config/model.py,sha256=iqffLs_VDqw9jOTLWchVFK4c6FWxHAzCngSfkjLUyCY,10516
+ocf_data_sampler/config/model.py,sha256=L0-ZLeO9JHsorgbJdX4NvoFOU4lVW58DgXEgupBBuko,10533
 ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdRaI,1064
 ocf_data_sampler/data/uk_gsp_locations_20220314.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
 ocf_data_sampler/data/uk_gsp_locations_20250109.csv,sha256=XZISFatnbpO9j8LwaxNKFzQSjs6hcHFsV8a9uDDpy2E,9055334
 ocf_data_sampler/load/__init__.py,sha256=-vQP9g0UOWdVbjEGyVX_ipa7R1btmiETIKAf6aw4d78,201
-ocf_data_sampler/load/gsp.py,sha256=UfPxwHw2Dw2xYSO5Al28oTamgnEM_n_4bYXsqGwY5Tc,1884
+ocf_data_sampler/load/gsp.py,sha256=YsIlj-LBUbREHNi78JMppOM1NbSkOe4kvtIrTwDx_JQ,1888
 ocf_data_sampler/load/load_dataset.py,sha256=wSXPUQKgGRM6HC-yBXQ2IcDBQDckOSllmbGnhqikFMQ,2055
 ocf_data_sampler/load/satellite.py,sha256=E7Ln7Y60Qr1RTV-_R71YoxXQM-Ca7Y1faIo3oKB2eFk,2292
 ocf_data_sampler/load/site.py,sha256=zOzlWk6pYZBB5daqG8URGksmDXWKrkutUvN8uALAIh8,1468
 ocf_data_sampler/load/utils.py,sha256=sZ0-zzconcLkVQwAkCYrqKDo98Hrh5ChdiQJv5Bh91g,2040
 ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
-ocf_data_sampler/load/nwp/nwp.py,sha256=PNNYYREEGQT4sxGilNzuthKKOmVJdQL8R2r8bvzyEr0,1317
+ocf_data_sampler/load/nwp/nwp.py,sha256=S0wsxQHKa-OZchmvhTAOSG1HLtAVRTo_ElXAIwz1pXo,1332
 ocf_data_sampler/load/nwp/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ocf_data_sampler/load/nwp/providers/cloudcasting.py,sha256=fozXpB3a2rNqQgnpRDC7xunxffh7Wwmc0kkCiYmDVJ4,1521
 ocf_data_sampler/load/nwp/providers/ecmwf.py,sha256=an-gXsZwkPQvRXeza1U_4MNU5yEnVm0_8tn03rxTudI,997
@@ -31,7 +31,7 @@ ocf_data_sampler/numpy_sample/satellite.py,sha256=RaYzYIcB1AmDrKeiqSpn4QVfBH-QMe
 ocf_data_sampler/numpy_sample/site.py,sha256=zfYBjK3CJrIaKH1QdKXU7gwOxTqONt527y3nJ9TRnwc,1325
 ocf_data_sampler/numpy_sample/sun_position.py,sha256=5tt-zNm6aRuZMsxZPaAxyg7HeikswfZCeHWXTHuO2K0,1555
 ocf_data_sampler/select/__init__.py,sha256=mK7Wu_-j9IXGTYrOuDf5yDDuU5a306b0iGKTAooNg_s,210
-ocf_data_sampler/select/dropout.py,sha256=WVOCweTGfIjufAlnfmYiPofz6X38TxQgzkLwtiB3TrU,1712
+ocf_data_sampler/select/dropout.py,sha256=9gPyDF7bGmvSoMjMPu1j0gTZFHNFqsT3ToIo9mFNA00,1565
 ocf_data_sampler/select/fill_time_periods.py,sha256=TlGxp1xiAqnhdWfLy0pv3FuZc00dtimjWdLzr4JoTGA,865
 ocf_data_sampler/select/find_contiguous_time_periods.py,sha256=8lkWsV5i7iLCVGqQ-PGZbvWxsz3wBvLO70GSf6WeR0k,11363
 ocf_data_sampler/select/geospatial.py,sha256=CDExkl36eZOKmdJPzUr_K0Wn3axHqv5nYo-EkSiINcc,5032
@@ -49,13 +49,13 @@ ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=N7i_hHtWUDiJqsiJoDx4T_Q
 ocf_data_sampler/torch_datasets/utils/channel_dict_to_dataarray.py,sha256=un2IiyoAmTDIymdeMiPU899_86iCDMD-oIifjHlNyqw,555
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=we7BTxRH7B7jKayDT7YfNyfI3zZClz2Bk-HXKQIokgU,956
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py,sha256=Hvz0wHSWMYYamf2oHNiGlzJcM4cAH6pL_7ZEvIBL2dE,1882
-ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=1DN6VsWWdLvkpJxodZtBRDUgC4vJE2td_RP5J3ZqPNw,4268
+ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=8E4a5v9dqr-sZOyBruuO-tjLPBbjtpYtdFY5z23aqnU,4365
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0WrglnX5YUAFjXXlO2GwEBHWyqo8TDuiOA,4714
 ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul3l0EP73Ik002fStr_bhsZh9mQqEU,4735
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
 utils/compute_icon_mean_stddev.py,sha256=a1oWMRMnny39rV-dvu8rcx85sb4bXzPFrR1gkUr4Jpg,2296
-ocf_data_sampler-0.2.21.dist-info/METADATA,sha256=PKoq-iYwiK9VGZLHgAQCQJm-U97WMkb7HToAsrPNzlw,11581
-ocf_data_sampler-0.2.21.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-ocf_data_sampler-0.2.21.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
-ocf_data_sampler-0.2.21.dist-info/RECORD,,
+ocf_data_sampler-0.2.23.dist-info/METADATA,sha256=eQH416SngTjkM7gtOazOOZ0UA-Yluh7hHnsrZPben18,11581
+ocf_data_sampler-0.2.23.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+ocf_data_sampler-0.2.23.dist-info/top_level.txt,sha256=LEFU4Uk-PEo72QGLAfnVZIUEm37Q8mKuMeg_Xk-p33g,31
+ocf_data_sampler-0.2.23.dist-info/RECORD,,

{ocf_data_sampler-0.2.21.dist-info → ocf_data_sampler-0.2.23.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any