ocf-data-sampler 0.5.17__tar.gz → 0.5.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of ocf-data-sampler might be problematic. Click here for more details.

Files changed (71) hide show
  1. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/PKG-INFO +1 -1
  2. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/gsp.py +4 -2
  3. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/utils.py +10 -1
  4. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler.egg-info/PKG-INFO +1 -1
  5. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/LICENSE +0 -0
  6. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/README.md +0 -0
  7. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/__init__.py +0 -0
  8. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/config/__init__.py +0 -0
  9. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/config/load.py +0 -0
  10. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/config/model.py +0 -0
  11. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/config/save.py +0 -0
  12. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/data/uk_gsp_locations_20220314.csv +0 -0
  13. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/data/uk_gsp_locations_20250109.csv +0 -0
  14. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/__init__.py +0 -0
  15. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/load_dataset.py +0 -0
  16. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/__init__.py +0 -0
  17. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/nwp.py +0 -0
  18. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/providers/__init__.py +0 -0
  19. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/providers/cloudcasting.py +0 -0
  20. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/providers/ecmwf.py +0 -0
  21. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/providers/gfs.py +0 -0
  22. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/providers/icon.py +0 -0
  23. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/providers/ukv.py +0 -0
  24. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/nwp/providers/utils.py +0 -0
  25. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/open_xarray_tensorstore.py +0 -0
  26. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/satellite.py +0 -0
  27. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/site.py +0 -0
  28. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/utils.py +0 -0
  29. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/__init__.py +0 -0
  30. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/collate.py +0 -0
  31. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/common_types.py +0 -0
  32. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/datetime_features.py +0 -0
  33. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/gsp.py +0 -0
  34. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/nwp.py +0 -0
  35. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/satellite.py +0 -0
  36. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/site.py +0 -0
  37. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/numpy_sample/sun_position.py +0 -0
  38. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/__init__.py +0 -0
  39. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/diff_channels.py +0 -0
  40. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/dropout.py +0 -0
  41. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/fill_time_periods.py +0 -0
  42. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/find_contiguous_time_periods.py +0 -0
  43. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/geospatial.py +0 -0
  44. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/location.py +0 -0
  45. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/select_spatial_slice.py +0 -0
  46. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/select/select_time_slice.py +0 -0
  47. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/datasets/__init__.py +0 -0
  48. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py +0 -0
  49. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/datasets/site.py +0 -0
  50. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/sample/__init__.py +0 -0
  51. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/sample/base.py +0 -0
  52. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/sample/site.py +0 -0
  53. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/sample/uk_regional.py +0 -0
  54. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/__init__.py +0 -0
  55. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/add_alterate_coordinate_projections.py +0 -0
  56. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py +0 -0
  57. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/diff_nwp_data.py +0 -0
  58. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py +0 -0
  59. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py +0 -0
  60. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py +0 -0
  61. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/valid_time_periods.py +0 -0
  62. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/torch_datasets/utils/validation_utils.py +0 -0
  63. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler.egg-info/SOURCES.txt +0 -0
  64. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler.egg-info/dependency_links.txt +0 -0
  65. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler.egg-info/requires.txt +0 -0
  66. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler.egg-info/top_level.txt +0 -0
  67. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/pyproject.toml +0 -0
  68. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/scripts/download_gsp_location_data.py +0 -0
  69. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/scripts/refactor_site.py +0 -0
  70. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/setup.cfg +0 -0
  71. {ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/tests/test_utils.py +0 -0
{ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ocf-data-sampler
3
- Version: 0.5.17
3
+ Version: 0.5.19
4
4
  Author: James Fulton, Peter Dudfield
5
5
  Author-email: Open Climate Fix team <info@openclimatefix.org>
6
6
  License: MIT License
{ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/load/gsp.py RENAMED
@@ -32,7 +32,7 @@ def open_gsp(
32
32
  boundaries_version: str = "20220314",
33
33
  public: bool = False,
34
34
  ) -> xr.DataArray:
35
- """Open the GSP data and validates its data types.
35
+ """Open and eagerly load the GSP data and validates its data types.
36
36
 
37
37
  Args:
38
38
  zarr_path: Path to the GSP zarr data
@@ -93,4 +93,6 @@ def open_gsp(
93
93
  dtype = gsp_da.coords[coord].dtype
94
94
  raise TypeError(f"{coord} should be {expected_dtype.__name__}, not {dtype}")
95
95
 
96
- return gsp_da
96
+ # Below we load the data eagerly into memory - this makes the dataset faster to sample from, but
97
+ # at the cost of a little extra memory usage
98
+ return gsp_da.compute()
{ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler/utils.py RENAMED
@@ -15,7 +15,16 @@ def minutes(minutes: int | list[float]) -> pd.Timedelta | pd.TimedeltaIndex:
15
15
 
16
16
  def compute(xarray_dict: dict) -> dict:
17
17
  """Eagerly load a nested dictionary of xarray DataArrays."""
18
- for k, v in xarray_dict.items():
18
+ # Load these keys first because they don't use tensorstore
19
+ priority_keys = ["gsp", "site"]
20
+ for key in priority_keys:
21
+ if key in xarray_dict:
22
+ xarray_dict[key] = xarray_dict[key].compute()
23
+
24
+ # Load the rest
25
+ keys = [k for k in xarray_dict if k not in priority_keys]
26
+ for k in keys:
27
+ v = xarray_dict[k]
19
28
  if isinstance(v, dict):
20
29
  xarray_dict[k] = compute(v)
21
30
  else:
{ocf_data_sampler-0.5.17 → ocf_data_sampler-0.5.19}/ocf_data_sampler.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ocf-data-sampler
3
- Version: 0.5.17
3
+ Version: 0.5.19
4
4
  Author: James Fulton, Peter Dudfield
5
5
  Author-email: Open Climate Fix team <info@openclimatefix.org>
6
6
  License: MIT License