anemoi-datasets 0.5.16__py3-none-any.whl → 0.5.17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. anemoi/datasets/__init__.py +4 -1
  2. anemoi/datasets/__main__.py +12 -2
  3. anemoi/datasets/_version.py +9 -4
  4. anemoi/datasets/commands/cleanup.py +17 -2
  5. anemoi/datasets/commands/compare.py +18 -2
  6. anemoi/datasets/commands/copy.py +196 -14
  7. anemoi/datasets/commands/create.py +50 -7
  8. anemoi/datasets/commands/finalise-additions.py +17 -2
  9. anemoi/datasets/commands/finalise.py +17 -2
  10. anemoi/datasets/commands/init-additions.py +17 -2
  11. anemoi/datasets/commands/init.py +16 -2
  12. anemoi/datasets/commands/inspect.py +283 -62
  13. anemoi/datasets/commands/load-additions.py +16 -2
  14. anemoi/datasets/commands/load.py +16 -2
  15. anemoi/datasets/commands/patch.py +17 -2
  16. anemoi/datasets/commands/publish.py +17 -2
  17. anemoi/datasets/commands/scan.py +31 -3
  18. anemoi/datasets/compute/recentre.py +47 -11
  19. anemoi/datasets/create/__init__.py +612 -85
  20. anemoi/datasets/create/check.py +142 -20
  21. anemoi/datasets/create/chunks.py +64 -4
  22. anemoi/datasets/create/config.py +185 -21
  23. anemoi/datasets/create/filter.py +50 -0
  24. anemoi/datasets/create/filters/__init__.py +33 -0
  25. anemoi/datasets/create/filters/empty.py +37 -0
  26. anemoi/datasets/create/filters/legacy.py +93 -0
  27. anemoi/datasets/create/filters/noop.py +37 -0
  28. anemoi/datasets/create/filters/orog_to_z.py +58 -0
  29. anemoi/datasets/create/{functions/filters → filters}/pressure_level_relative_humidity_to_specific_humidity.py +33 -10
  30. anemoi/datasets/create/{functions/filters → filters}/pressure_level_specific_humidity_to_relative_humidity.py +32 -8
  31. anemoi/datasets/create/filters/rename.py +205 -0
  32. anemoi/datasets/create/{functions/filters → filters}/rotate_winds.py +43 -28
  33. anemoi/datasets/create/{functions/filters → filters}/single_level_dewpoint_to_relative_humidity.py +32 -9
  34. anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_dewpoint.py +33 -9
  35. anemoi/datasets/create/{functions/filters → filters}/single_level_relative_humidity_to_specific_humidity.py +55 -7
  36. anemoi/datasets/create/{functions/filters → filters}/single_level_specific_humidity_to_relative_humidity.py +98 -37
  37. anemoi/datasets/create/filters/speeddir_to_uv.py +95 -0
  38. anemoi/datasets/create/{functions/filters → filters}/sum.py +24 -27
  39. anemoi/datasets/create/filters/transform.py +53 -0
  40. anemoi/datasets/create/{functions/filters → filters}/unrotate_winds.py +27 -18
  41. anemoi/datasets/create/filters/uv_to_speeddir.py +94 -0
  42. anemoi/datasets/create/{functions/filters → filters}/wz_to_w.py +51 -33
  43. anemoi/datasets/create/input/__init__.py +76 -5
  44. anemoi/datasets/create/input/action.py +149 -13
  45. anemoi/datasets/create/input/concat.py +81 -10
  46. anemoi/datasets/create/input/context.py +39 -4
  47. anemoi/datasets/create/input/data_sources.py +72 -6
  48. anemoi/datasets/create/input/empty.py +21 -3
  49. anemoi/datasets/create/input/filter.py +60 -12
  50. anemoi/datasets/create/input/function.py +154 -37
  51. anemoi/datasets/create/input/join.py +86 -14
  52. anemoi/datasets/create/input/misc.py +67 -17
  53. anemoi/datasets/create/input/pipe.py +33 -6
  54. anemoi/datasets/create/input/repeated_dates.py +189 -41
  55. anemoi/datasets/create/input/result.py +202 -87
  56. anemoi/datasets/create/input/step.py +119 -22
  57. anemoi/datasets/create/input/template.py +100 -13
  58. anemoi/datasets/create/input/trace.py +62 -7
  59. anemoi/datasets/create/patch.py +52 -4
  60. anemoi/datasets/create/persistent.py +134 -17
  61. anemoi/datasets/create/size.py +15 -1
  62. anemoi/datasets/create/source.py +51 -0
  63. anemoi/datasets/create/sources/__init__.py +36 -0
  64. anemoi/datasets/create/{functions/sources → sources}/accumulations.py +296 -30
  65. anemoi/datasets/create/{functions/sources → sources}/constants.py +27 -2
  66. anemoi/datasets/create/{functions/sources → sources}/eccc_fstd.py +7 -3
  67. anemoi/datasets/create/sources/empty.py +37 -0
  68. anemoi/datasets/create/{functions/sources → sources}/forcings.py +25 -1
  69. anemoi/datasets/create/sources/grib.py +297 -0
  70. anemoi/datasets/create/{functions/sources → sources}/hindcasts.py +38 -4
  71. anemoi/datasets/create/sources/legacy.py +93 -0
  72. anemoi/datasets/create/{functions/sources → sources}/mars.py +168 -20
  73. anemoi/datasets/create/sources/netcdf.py +42 -0
  74. anemoi/datasets/create/sources/opendap.py +43 -0
  75. anemoi/datasets/create/{functions/sources/__init__.py → sources/patterns.py} +35 -4
  76. anemoi/datasets/create/sources/recentre.py +150 -0
  77. anemoi/datasets/create/{functions/sources → sources}/source.py +27 -5
  78. anemoi/datasets/create/{functions/sources → sources}/tendencies.py +64 -7
  79. anemoi/datasets/create/sources/xarray.py +92 -0
  80. anemoi/datasets/create/sources/xarray_kerchunk.py +36 -0
  81. anemoi/datasets/create/sources/xarray_support/README.md +1 -0
  82. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/__init__.py +109 -8
  83. anemoi/datasets/create/sources/xarray_support/coordinates.py +442 -0
  84. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/field.py +94 -16
  85. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/fieldlist.py +90 -25
  86. anemoi/datasets/create/sources/xarray_support/flavour.py +1036 -0
  87. anemoi/datasets/create/{functions/sources/xarray → sources/xarray_support}/grid.py +92 -31
  88. anemoi/datasets/create/sources/xarray_support/metadata.py +395 -0
  89. anemoi/datasets/create/sources/xarray_support/patch.py +91 -0
  90. anemoi/datasets/create/sources/xarray_support/time.py +391 -0
  91. anemoi/datasets/create/sources/xarray_support/variable.py +331 -0
  92. anemoi/datasets/create/sources/xarray_zarr.py +41 -0
  93. anemoi/datasets/create/{functions/sources → sources}/zenodo.py +34 -5
  94. anemoi/datasets/create/statistics/__init__.py +233 -44
  95. anemoi/datasets/create/statistics/summary.py +52 -6
  96. anemoi/datasets/create/testing.py +76 -0
  97. anemoi/datasets/create/{functions/filters/noop.py → typing.py} +6 -3
  98. anemoi/datasets/create/utils.py +97 -6
  99. anemoi/datasets/create/writer.py +26 -4
  100. anemoi/datasets/create/zarr.py +170 -23
  101. anemoi/datasets/data/__init__.py +51 -4
  102. anemoi/datasets/data/complement.py +191 -40
  103. anemoi/datasets/data/concat.py +141 -16
  104. anemoi/datasets/data/dataset.py +552 -61
  105. anemoi/datasets/data/debug.py +197 -26
  106. anemoi/datasets/data/ensemble.py +93 -8
  107. anemoi/datasets/data/fill_missing.py +165 -18
  108. anemoi/datasets/data/forwards.py +428 -56
  109. anemoi/datasets/data/grids.py +323 -97
  110. anemoi/datasets/data/indexing.py +112 -19
  111. anemoi/datasets/data/interpolate.py +92 -12
  112. anemoi/datasets/data/join.py +158 -19
  113. anemoi/datasets/data/masked.py +129 -15
  114. anemoi/datasets/data/merge.py +137 -23
  115. anemoi/datasets/data/misc.py +172 -16
  116. anemoi/datasets/data/missing.py +233 -29
  117. anemoi/datasets/data/rescale.py +111 -10
  118. anemoi/datasets/data/select.py +168 -26
  119. anemoi/datasets/data/statistics.py +67 -6
  120. anemoi/datasets/data/stores.py +149 -64
  121. anemoi/datasets/data/subset.py +159 -25
  122. anemoi/datasets/data/unchecked.py +168 -57
  123. anemoi/datasets/data/xy.py +168 -25
  124. anemoi/datasets/dates/__init__.py +191 -16
  125. anemoi/datasets/dates/groups.py +189 -47
  126. anemoi/datasets/grids.py +270 -31
  127. anemoi/datasets/testing.py +28 -1
  128. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/METADATA +9 -6
  129. anemoi_datasets-0.5.17.dist-info/RECORD +137 -0
  130. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/WHEEL +1 -1
  131. anemoi/datasets/create/functions/__init__.py +0 -66
  132. anemoi/datasets/create/functions/filters/__init__.py +0 -9
  133. anemoi/datasets/create/functions/filters/empty.py +0 -17
  134. anemoi/datasets/create/functions/filters/orog_to_z.py +0 -58
  135. anemoi/datasets/create/functions/filters/rename.py +0 -79
  136. anemoi/datasets/create/functions/filters/speeddir_to_uv.py +0 -78
  137. anemoi/datasets/create/functions/filters/uv_to_speeddir.py +0 -56
  138. anemoi/datasets/create/functions/sources/empty.py +0 -15
  139. anemoi/datasets/create/functions/sources/grib.py +0 -150
  140. anemoi/datasets/create/functions/sources/netcdf.py +0 -15
  141. anemoi/datasets/create/functions/sources/opendap.py +0 -15
  142. anemoi/datasets/create/functions/sources/recentre.py +0 -60
  143. anemoi/datasets/create/functions/sources/xarray/coordinates.py +0 -255
  144. anemoi/datasets/create/functions/sources/xarray/flavour.py +0 -472
  145. anemoi/datasets/create/functions/sources/xarray/metadata.py +0 -148
  146. anemoi/datasets/create/functions/sources/xarray/patch.py +0 -44
  147. anemoi/datasets/create/functions/sources/xarray/time.py +0 -177
  148. anemoi/datasets/create/functions/sources/xarray/variable.py +0 -188
  149. anemoi/datasets/create/functions/sources/xarray_kerchunk.py +0 -42
  150. anemoi/datasets/create/functions/sources/xarray_zarr.py +0 -15
  151. anemoi/datasets/utils/fields.py +0 -47
  152. anemoi_datasets-0.5.16.dist-info/RECORD +0 -129
  153. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/entry_points.txt +0 -0
  154. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info/licenses}/LICENSE +0 -0
  155. {anemoi_datasets-0.5.16.dist-info → anemoi_datasets-0.5.17.dist-info}/top_level.txt +0 -0
@@ -9,6 +9,7 @@
9
9
 
10
10
  import logging
11
11
  import time
12
+ from typing import Any
12
13
 
13
14
  from anemoi.utils.humanize import seconds_to_human
14
15
 
@@ -25,8 +26,14 @@ class LoadAdditions(Command):
25
26
  internal = True
26
27
  timestamp = True
27
28
 
28
- def add_arguments(self, subparser):
29
+ def add_arguments(self, subparser: Any) -> None:
30
+ """Add command line arguments to the parser.
29
31
 
32
+ Parameters
33
+ ----------
34
+ subparser : Any
35
+ The argument parser.
36
+ """
30
37
  subparser.add_argument("--parts", nargs="+", help="Only load the specified parts of the dataset.")
31
38
  subparser.add_argument(
32
39
  "--delta",
@@ -37,7 +44,14 @@ class LoadAdditions(Command):
37
44
  subparser.add_argument("path", help="Path to store the created data.")
38
45
  subparser.add_argument("--trace", action="store_true")
39
46
 
40
- def run(self, args):
47
+ def run(self, args: Any) -> None:
48
+ """Execute the command with the given arguments.
49
+
50
+ Parameters
51
+ ----------
52
+ args : Any
53
+ The command line arguments.
54
+ """
41
55
  options = vars(args)
42
56
  options.pop("command")
43
57
  now = time.time()
@@ -9,6 +9,7 @@
9
9
 
10
10
  import logging
11
11
  import time
12
+ from typing import Any
12
13
 
13
14
  from anemoi.utils.humanize import seconds_to_human
14
15
 
@@ -25,8 +26,14 @@ class Load(Command):
25
26
  internal = True
26
27
  timestamp = True
27
28
 
28
- def add_arguments(self, subparser):
29
+ def add_arguments(self, subparser: Any) -> None:
30
+ """Add arguments to the command parser.
29
31
 
32
+ Parameters
33
+ ----------
34
+ subparser : Any
35
+ The command parser.
36
+ """
30
37
  subparser.add_argument("--parts", nargs="+", help="Only load the specified parts of the dataset.")
31
38
  # subparser.add_argument(
32
39
  # "--delta",
@@ -37,7 +44,14 @@ class Load(Command):
37
44
  subparser.add_argument("--cache", help="Location to store the downloaded data.", metavar="DIR")
38
45
  subparser.add_argument("--trace", action="store_true")
39
46
 
40
- def run(self, args):
47
+ def run(self, args: Any) -> None:
48
+ """Run the command.
49
+
50
+ Parameters
51
+ ----------
52
+ args : Any
53
+ The command arguments.
54
+ """
41
55
  options = vars(args)
42
56
  options.pop("command")
43
57
  now = time.time()
@@ -9,6 +9,7 @@
9
9
 
10
10
  import logging
11
11
  import time
12
+ from typing import Any
12
13
 
13
14
  from anemoi.utils.humanize import seconds_to_human
14
15
 
@@ -25,10 +26,24 @@ class Patch(Command):
25
26
  internal = True
26
27
  timestamp = True
27
28
 
28
- def add_arguments(self, parser):
29
+ def add_arguments(self, parser: Any) -> None:
30
+ """Add command-line arguments to the parser.
31
+
32
+ Parameters
33
+ ----------
34
+ parser : Any
35
+ The argument parser instance.
36
+ """
29
37
  parser.add_argument("path", help="Path to store the created data.")
30
38
 
31
- def run(self, args):
39
+ def run(self, args: Any) -> None:
40
+ """Execute the patch command.
41
+
42
+ Parameters
43
+ ----------
44
+ args : Any
45
+ The command-line arguments.
46
+ """
32
47
  options = vars(args)
33
48
  options.pop("command")
34
49
  now = time.time()
@@ -8,6 +8,7 @@
8
8
  # nor does it submit to any jurisdiction.
9
9
 
10
10
  import logging
11
+ from typing import Any
11
12
 
12
13
  from . import Command
13
14
 
@@ -23,10 +24,24 @@ class Publish(Command):
23
24
  internal = True
24
25
  timestamp = True
25
26
 
26
- def add_arguments(self, parser):
27
+ def add_arguments(self, parser: Any) -> None:
28
+ """Add arguments to the command parser.
29
+
30
+ Parameters
31
+ ----------
32
+ parser : Any
33
+ The command parser to which arguments are added.
34
+ """
27
35
  parser.add_argument("path", help="Path of the dataset to publish.")
28
36
 
29
- def run(self, args):
37
+ def run(self, args: Any) -> None:
38
+ """Execute the publish command.
39
+
40
+ Parameters
41
+ ----------
42
+ args : Any
43
+ The arguments passed to the command.
44
+ """
30
45
  try:
31
46
  from anemoi.registry import publish_dataset
32
47
  except ImportError:
@@ -11,6 +11,7 @@ import fnmatch
11
11
  import os
12
12
  import sys
13
13
  from collections import defaultdict
14
+ from typing import Any
14
15
 
15
16
  import earthkit.data as ekd
16
17
  import tqdm
@@ -25,7 +26,14 @@ class Scan(Command):
25
26
  internal = True
26
27
  timestamp = True
27
28
 
28
- def add_arguments(self, command_parser):
29
+ def add_arguments(self, command_parser: Any) -> None:
30
+ """Add arguments to the command parser.
31
+
32
+ Parameters
33
+ ----------
34
+ command_parser : Any
35
+ The command parser to which arguments are added.
36
+ """
29
37
  command_parser.add_argument(
30
38
  "--match",
31
39
  help="Give a glob pattern to match files (default: *.grib)",
@@ -37,8 +45,28 @@ class Scan(Command):
37
45
  )
38
46
  command_parser.add_argument("paths", nargs="+", help="Paths to scan")
39
47
 
40
- def run(self, args):
41
- def match(path):
48
+ def run(self, args: Any) -> None:
49
+ """Execute the scan command.
50
+
51
+ Parameters
52
+ ----------
53
+ args : Any
54
+ The arguments passed to the command.
55
+ """
56
+
57
+ def match(path: str) -> bool:
58
+ """Check if a path matches the given pattern.
59
+
60
+ Parameters
61
+ ----------
62
+ path : str
63
+ The path to check.
64
+
65
+ Returns
66
+ -------
67
+ bool
68
+ True if the path matches, False otherwise.
69
+ """
42
70
  return fnmatch.fnmatch(path, args.match)
43
71
 
44
72
  paths = []
@@ -9,13 +9,15 @@
9
9
 
10
10
 
11
11
  import logging
12
+ from typing import Any
13
+ from typing import Dict
14
+ from typing import Optional
15
+ from typing import Tuple
12
16
 
13
17
  import numpy as np
14
18
  from earthkit.data.core.temporary import temp_file
15
19
  from earthkit.data.readers.grib.output import new_grib_output
16
20
 
17
- from anemoi.datasets.create.functions import assert_is_fieldlist
18
-
19
21
  LOG = logging.getLogger(__name__)
20
22
 
21
23
  CLIP_VARIABLES = (
@@ -33,7 +35,22 @@ CLIP_VARIABLES = (
33
35
  SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset", "time", "date", "step")
34
36
 
35
37
 
36
- def check_compatible(f1, f2, centre_field_as_mars, ensemble_field_as_mars):
38
+ def check_compatible(
39
+ f1: Any, f2: Any, centre_field_as_mars: Dict[str, Any], ensemble_field_as_mars: Dict[str, Any]
40
+ ) -> None:
41
+ """Check if two fields are compatible.
42
+
43
+ Parameters
44
+ ----------
45
+ f1 : Any
46
+ The first field.
47
+ f2 : Any
48
+ The second field.
49
+ centre_field_as_mars : Dict[str, Any]
50
+ Metadata of the centre field.
51
+ ensemble_field_as_mars : Dict[str, Any]
52
+ Metadata of the ensemble field.
53
+ """
37
54
  assert f1.mars_grid == f2.mars_grid, (f1.mars_grid, f2.mars_grid)
38
55
  assert f1.mars_area == f2.mars_area, (f1.mars_area, f2.mars_area)
39
56
  assert f1.shape == f2.shape, (f1.shape, f2.shape)
@@ -56,13 +73,32 @@ def check_compatible(f1, f2, centre_field_as_mars, ensemble_field_as_mars):
56
73
 
57
74
  def recentre(
58
75
  *,
59
- members,
60
- centre,
61
- clip_variables=CLIP_VARIABLES,
62
- alpha=1.0,
63
- output=None,
64
- ):
65
-
76
+ members: Any,
77
+ centre: Any,
78
+ clip_variables: Tuple[str, ...] = CLIP_VARIABLES,
79
+ alpha: float = 1.0,
80
+ output: Optional[str] = None,
81
+ ) -> Any:
82
+ """Recentre ensemble members around the centre field.
83
+
84
+ Parameters
85
+ ----------
86
+ members : Any
87
+ The ensemble members.
88
+ centre : Any
89
+ The centre field.
90
+ clip_variables : Tuple[str, ...], optional
91
+ Variables to clip. Defaults to CLIP_VARIABLES.
92
+ alpha : float, optional
93
+ Scaling factor. Defaults to 1.0.
94
+ output : Optional[str], optional
95
+ Output path. Defaults to None.
96
+
97
+ Returns
98
+ -------
99
+ Any
100
+ The recentred dataset or output path.
101
+ """
66
102
  keys = ["param", "level", "valid_datetime", "date", "time", "step", "number"]
67
103
 
68
104
  number_list = members.unique_values("number", progress_bar=False)["number"]
@@ -158,7 +194,7 @@ def recentre(
158
194
  from earthkit.data import from_source
159
195
 
160
196
  ds = from_source("file", path)
161
- assert_is_fieldlist(ds)
197
+
162
198
  # save a reference to the tmp file so it is deleted
163
199
  # only when the dataset is not used anymore
164
200
  ds._tmp = tmp