ocf-data-sampler 0.5.17__py3-none-any.whl → 0.5.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of ocf-data-sampler might be problematic. Click here for more details.

Files changed (6) hide show
  1. ocf_data_sampler/load/gsp.py +4 -2
  2. ocf_data_sampler/utils.py +10 -1
  3. {ocf_data_sampler-0.5.17.dist-info → ocf_data_sampler-0.5.19.dist-info}/METADATA +1 -1
  4. {ocf_data_sampler-0.5.17.dist-info → ocf_data_sampler-0.5.19.dist-info}/RECORD +6 -6
  5. {ocf_data_sampler-0.5.17.dist-info → ocf_data_sampler-0.5.19.dist-info}/WHEEL +0 -0
  6. {ocf_data_sampler-0.5.17.dist-info → ocf_data_sampler-0.5.19.dist-info}/top_level.txt +0 -0
ocf_data_sampler/load/gsp.py CHANGED
@@ -32,7 +32,7 @@ def open_gsp(
32
32
  boundaries_version: str = "20220314",
33
33
  public: bool = False,
34
34
  ) -> xr.DataArray:
35
- """Open the GSP data and validates its data types.
35
+ """Open and eagerly load the GSP data and validates its data types.
36
36
 
37
37
  Args:
38
38
  zarr_path: Path to the GSP zarr data
@@ -93,4 +93,6 @@ def open_gsp(
93
93
  dtype = gsp_da.coords[coord].dtype
94
94
  raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
95
95
 
96
- return gsp_da
96
+ # Below we load the data eagerly into memory - this makes the dataset faster to sample from, but
97
+ # at the cost of a little extra memory usage
98
+ return gsp_da.compute()
ocf_data_sampler/utils.py CHANGED
@@ -15,7 +15,16 @@ def minutes(minutes: int | list[float]) -> pd.Timedelta | pd.TimedeltaIndex:
15
15
 
16
16
  def compute(xarray_dict: dict) -> dict:
17
17
  """Eagerly load a nested dictionary of xarray DataArrays."""
18
- for k, v in xarray_dict.items():
18
+ # Load these keys first because they don't use tensorstore
19
+ priority_keys = ["gsp", "site"]
20
+ for key in priority_keys:
21
+ if key in xarray_dict:
22
+ xarray_dict[key] = xarray_dict[key].compute()
23
+
24
+ # Load the rest
25
+ keys = [k for k in xarray_dict if k not in priority_keys]
26
+ for k in keys:
27
+ v = xarray_dict[k]
19
28
  if isinstance(v, dict):
20
29
  xarray_dict[k] = compute(v)
21
30
  else:
{ocf_data_sampler-0.5.17.dist-info → ocf_data_sampler-0.5.19.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ocf-data-sampler
3
- Version: 0.5.17
3
+ Version: 0.5.19
4
4
  Author: James Fulton, Peter Dudfield
5
5
  Author-email: Open Climate Fix team <info@openclimatefix.org>
6
6
  License: MIT License
{ocf_data_sampler-0.5.17.dist-info → ocf_data_sampler-0.5.19.dist-info}/RECORD RENAMED
@@ -1,5 +1,5 @@
1
1
  ocf_data_sampler/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
2
- ocf_data_sampler/utils.py,sha256=CTJf9bjHjO8vOJebUtXiMpvgwUpF7gEOjjaoE77fhTk,1177
2
+ ocf_data_sampler/utils.py,sha256=WfmyBacjFGsv_IlUHRezNGc4ixi4wBvom_JF76iJYbY,1487
3
3
  ocf_data_sampler/config/__init__.py,sha256=O29mbH0XG2gIY1g3BaveGCnpBO2SFqdu-qzJ7a6evl0,223
4
4
  ocf_data_sampler/config/load.py,sha256=LL-7wemI8o4KPkx35j-wQ3HjsMvDgqXr7G46IcASfnU,632
5
5
  ocf_data_sampler/config/model.py,sha256=5ou8BZgQ9h-xyJEqHdspPKZgZO9Vr6opjSphUys7yE8,11505
@@ -7,7 +7,7 @@ ocf_data_sampler/config/save.py,sha256=m8SPw5rXjkMm1rByjh3pK5StdBi4e8ysnn3jQopdR
7
7
  ocf_data_sampler/data/uk_gsp_locations_20220314.csv,sha256=RSh7DRh55E3n8lVAaWXGTaXXHevZZtI58td4d4DhGos,10415772
8
8
  ocf_data_sampler/data/uk_gsp_locations_20250109.csv,sha256=XZISFatnbpO9j8LwaxNKFzQSjs6hcHFsV8a9uDDpy2E,9055334
9
9
  ocf_data_sampler/load/__init__.py,sha256=-vQP9g0UOWdVbjEGyVX_ipa7R1btmiETIKAf6aw4d78,201
10
- ocf_data_sampler/load/gsp.py,sha256=C-r5IidXFq1-zw_KkMjuXczjJ-5t3k4UyjXk6BjQNoU,2881
10
+ ocf_data_sampler/load/gsp.py,sha256=zsQ39dZBS45qd86lGfCZUjheLRTtMzIUozj-j8c87UQ,3058
11
11
  ocf_data_sampler/load/load_dataset.py,sha256=K8rWykjII-3g127If7WRRFivzHNx3SshCvZj4uQlf28,2089
12
12
  ocf_data_sampler/load/open_xarray_tensorstore.py,sha256=kAqlIavGe1dcCPkzAtoZo2dFS-tW36E-wRE_3w1HMfg,5620
13
13
  ocf_data_sampler/load/satellite.py,sha256=B-m0_Py_D0GwzwX5o-ixyeXntV5Z4k4MbmMBHZLUWMM,1831
@@ -58,7 +58,7 @@ ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0Wrg
58
58
  ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul3l0EP73Ik002fStr_bhsZh9mQqEU,4735
59
59
  scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
60
60
  scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
61
- ocf_data_sampler-0.5.17.dist-info/METADATA,sha256=H4nnVbAIoMjhfz0wME3sB78hUE0dAdcXSDtR5CCIIvQ,12817
62
- ocf_data_sampler-0.5.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
63
- ocf_data_sampler-0.5.17.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
64
- ocf_data_sampler-0.5.17.dist-info/RECORD,,
61
+ ocf_data_sampler-0.5.19.dist-info/METADATA,sha256=CyMcURh5QvAzzxVvu_DzbK4krMLtGtSlVjr4KURpFiw,12817
62
+ ocf_data_sampler-0.5.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
63
+ ocf_data_sampler-0.5.19.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
64
+ ocf_data_sampler-0.5.19.dist-info/RECORD,,