anemoi-datasets 0.3.3__tar.gz → 0.3.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/.github/workflows/python-publish.yml +1 -2
  2. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/.pre-commit-config.yaml +14 -26
  3. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/PKG-INFO +4 -1
  4. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/forcings.yaml +1 -1
  5. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/pyproject.toml +5 -0
  6. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/_version.py +2 -2
  7. anemoi_datasets-0.3.3/src/anemoi/datasets/commands/inspect/zarr.py → anemoi_datasets-0.3.5/src/anemoi/datasets/commands/inspect.py +27 -56
  8. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/grib.py +9 -1
  9. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/netcdf.py +20 -1
  10. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/concat.py +1 -1
  11. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/ensemble.py +1 -1
  12. anemoi_datasets-0.3.3/src/anemoi/datasets/data/forewards.py → anemoi_datasets-0.3.5/src/anemoi/datasets/data/forwards.py +6 -0
  13. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/grids.py +6 -6
  14. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/join.py +1 -1
  15. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/masked.py +7 -1
  16. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/select.py +8 -4
  17. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/statistics.py +3 -6
  18. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/stores.py +5 -2
  19. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/subset.py +7 -1
  20. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/unchecked.py +1 -1
  21. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/dates/__init__.py +2 -2
  22. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi_datasets.egg-info/PKG-INFO +4 -1
  23. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi_datasets.egg-info/SOURCES.txt +2 -4
  24. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi_datasets.egg-info/requires.txt +4 -0
  25. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/test_data.py +2 -2
  26. anemoi_datasets-0.3.3/src/anemoi/datasets/commands/inspect/__init__.py +0 -37
  27. anemoi_datasets-0.3.3/tests/requirements.txt +0 -1
  28. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/.gitignore +0 -0
  29. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/.readthedocs.yaml +0 -0
  30. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/.vscode/spellright.dict +0 -0
  31. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/LICENSE +0 -0
  32. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/README.md +0 -0
  33. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/Makefile +0 -0
  34. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/_static/logo.png +0 -0
  35. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/_static/style.css +0 -0
  36. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/_templates/.gitkeep +0 -0
  37. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/apply-fmt.sh +0 -0
  38. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/filters/empty.rst +0 -0
  39. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/filters/noop.rst +0 -0
  40. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/filters/rename.rst +0 -0
  41. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/filters/rotate_winds.rst +0 -0
  42. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/filters/select.rst +0 -0
  43. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/filters/unrotate_winds.rst +0 -0
  44. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/filters.rst +0 -0
  45. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/handling-missing-dates.rst +0 -0
  46. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/handling-missing-values.rst +0 -0
  47. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/introduction.rst +0 -0
  48. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/naming-variables.rst +0 -0
  49. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/operations.rst +0 -0
  50. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/accumulations.rst +0 -0
  51. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/accumulations1.yaml +0 -0
  52. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/accumulations2.yaml +0 -0
  53. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/forcings.rst +0 -0
  54. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/grib.rst +0 -0
  55. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/hindcasts.rst +0 -0
  56. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/mars.rst +0 -0
  57. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/mars1.yaml +0 -0
  58. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/mars2.yaml +0 -0
  59. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/netcdf.rst +0 -0
  60. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/netcdf.yaml +0 -0
  61. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/opendap.rst +0 -0
  62. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/opendap.yaml +0 -0
  63. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/recentre.rst +0 -0
  64. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/yaml/grib1.yaml +0 -0
  65. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/yaml/grib2.yaml +0 -0
  66. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/yaml/grib3.yaml +0 -0
  67. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/yaml/grib4.yaml +0 -0
  68. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/yaml/hindcasts.yaml +0 -0
  69. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources/yaml/recentre.yaml +0 -0
  70. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/sources.rst +0 -0
  71. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/statistics.rst +0 -0
  72. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/syntax.rst +0 -0
  73. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/syntax.yaml +0 -0
  74. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/Makefile +0 -0
  75. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/building1.txt +0 -0
  76. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/building1.yaml +0 -0
  77. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/building2.txt +0 -0
  78. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/building2.yaml +0 -0
  79. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/building3.txt +0 -0
  80. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/building3.yaml +0 -0
  81. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/concat.yaml +0 -0
  82. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/hindcasts.yaml +0 -0
  83. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/input.yaml +0 -0
  84. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/missing_dates.yaml +0 -0
  85. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/nan.yaml +0 -0
  86. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/building/yaml/pipe.yaml +0 -0
  87. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/check-index.sh +0 -0
  88. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/cli/compare.rst +0 -0
  89. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/cli/copy.rst +0 -0
  90. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/cli/create.rst +0 -0
  91. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/cli/inspect.rst +0 -0
  92. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/cli/introduction.rst +0 -0
  93. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/cli/scan.rst +0 -0
  94. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/conf.py +0 -0
  95. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/images.pptx +0 -0
  96. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/index.rst +0 -0
  97. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/installing.rst +0 -0
  98. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/overview.rst +0 -0
  99. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/overview_.py +0 -0
  100. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/requirements.txt +0 -0
  101. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/schemas/matrix.excalidraw +0 -0
  102. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/schemas/matrix.png +0 -0
  103. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/schemas/overview.excalidraw +0 -0
  104. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/schemas/overview.png +0 -0
  105. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/schemas/recipe.excalidraw +0 -0
  106. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/schemas/recipe.png +0 -0
  107. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/test.ipynb +0 -0
  108. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/area1_.py +0 -0
  109. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/area2_.py +0 -0
  110. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/chain_.py +0 -0
  111. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/combine_example.py +0 -0
  112. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/concat1.py +0 -0
  113. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/cutout_.py +0 -0
  114. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/drop_.py +0 -0
  115. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/end_.py +0 -0
  116. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/ensembles1_.py +0 -0
  117. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/frequency_.py +0 -0
  118. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/grids1_.py +0 -0
  119. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/join1.py +0 -0
  120. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/matching0_.py +0 -0
  121. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/matching1_.py +0 -0
  122. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/matching2_.py +0 -0
  123. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/matching3_.py +0 -0
  124. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/matching4_.py +0 -0
  125. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/misc1.py +0 -0
  126. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/misc2.py +0 -0
  127. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/missing_.py +0 -0
  128. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_cloud.py +0 -0
  129. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_combine1_.py +0 -0
  130. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_combine2_.py +0 -0
  131. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_complex.py +0 -0
  132. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_dict_.py +0 -0
  133. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_first_.py +0 -0
  134. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_list_.py +0 -0
  135. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_name.py +0 -0
  136. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_other.py +0 -0
  137. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_path.py +0 -0
  138. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/open_yaml_.py +0 -0
  139. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/rename_.py +0 -0
  140. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/reorder1_.py +0 -0
  141. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/reorder2_.py +0 -0
  142. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/select1_.py +0 -0
  143. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/select2_.py +0 -0
  144. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/shuffle_.py +0 -0
  145. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/some_attributes_.py +0 -0
  146. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/start_.py +0 -0
  147. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/statistics_.py +0 -0
  148. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/subset_example.py +0 -0
  149. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/thinning_.py +0 -0
  150. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/zip1_.py +0 -0
  151. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/code/zip2_.py +0 -0
  152. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/combining.rst +0 -0
  153. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/configuration.rst +0 -0
  154. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/configuration.toml +0 -0
  155. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/grids.rst +0 -0
  156. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/area-1.png +0 -0
  157. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/concat.png +0 -0
  158. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/cutout-1.png +0 -0
  159. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/cutout-2.png +0 -0
  160. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/cutout-3.png +0 -0
  161. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/cutout-4.png +0 -0
  162. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/join.png +0 -0
  163. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/overlay.png +0 -0
  164. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/thinning-after.png +0 -0
  165. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/images/thinning-before.png +0 -0
  166. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/introduction.rst +0 -0
  167. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/matching.rst +0 -0
  168. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/methods.rst +0 -0
  169. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/miscellaneous.rst +0 -0
  170. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/opening.rst +0 -0
  171. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/other.rst +0 -0
  172. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/selecting.rst +0 -0
  173. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/statistics.rst +0 -0
  174. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/docs/using/subsetting.rst +0 -0
  175. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/setup.cfg +0 -0
  176. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/__init__.py +0 -0
  177. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/__main__.py +0 -0
  178. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/commands/__init__.py +0 -0
  179. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/commands/compare.py +0 -0
  180. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/commands/copy.py +0 -0
  181. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/commands/create.py +0 -0
  182. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/commands/scan.py +0 -0
  183. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/compute/__init__.py +0 -0
  184. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/compute/recentre.py +0 -0
  185. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/__init__.py +0 -0
  186. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/check.py +0 -0
  187. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/chunks.py +0 -0
  188. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/config.py +0 -0
  189. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/__init__.py +0 -0
  190. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/filters/__init__.py +0 -0
  191. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/filters/empty.py +0 -0
  192. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/filters/noop.py +0 -0
  193. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/filters/rename.py +0 -0
  194. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/filters/rotate_winds.py +0 -0
  195. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/filters/unrotate_winds.py +0 -0
  196. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/__init__.py +0 -0
  197. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/accumulations.py +0 -0
  198. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/constants.py +0 -0
  199. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/empty.py +0 -0
  200. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/forcings.py +0 -0
  201. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/hindcasts.py +0 -0
  202. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/mars.py +0 -0
  203. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/opendap.py +0 -0
  204. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/recentre.py +0 -0
  205. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/source.py +0 -0
  206. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/functions/sources/tendencies.py +0 -0
  207. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/input.py +0 -0
  208. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/loaders.py +0 -0
  209. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/patch.py +0 -0
  210. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/persistent.py +0 -0
  211. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/size.py +0 -0
  212. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/statistics/__init__.py +0 -0
  213. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/statistics/summary.py +0 -0
  214. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/template.py +0 -0
  215. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/utils.py +0 -0
  216. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/writer.py +0 -0
  217. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/create/zarr.py +0 -0
  218. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/__init__.py +0 -0
  219. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/dataset.py +0 -0
  220. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/debug.css +0 -0
  221. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/debug.py +0 -0
  222. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/indexing.py +0 -0
  223. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/data/misc.py +0 -0
  224. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/dates/groups.py +0 -0
  225. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/grids.py +0 -0
  226. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi/datasets/utils/__init__.py +0 -0
  227. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi_datasets.egg-info/dependency_links.txt +0 -0
  228. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi_datasets.egg-info/entry_points.txt +0 -0
  229. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/src/anemoi_datasets.egg-info/top_level.txt +0 -0
  230. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/concat.yaml +0 -0
  231. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/data_sources.yaml +0 -0
  232. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/join.yaml +0 -0
  233. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/missing.yaml +0 -0
  234. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/nan.yaml +0 -0
  235. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/pipe.yaml +0 -0
  236. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/recentre.yaml +0 -0
  237. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create/test_create.py +0 -0
  238. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create-perturbations-full.yaml +0 -0
  239. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/create-shift.yaml +0 -0
  240. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/test_chunks.py +0 -0
  241. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/test_dates.py +0 -0
  242. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tests/test_indexing.py +0 -0
  243. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tools/.gitignore +0 -0
  244. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tools/examples/Makefile +0 -0
  245. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tools/examples/an-oper-2023-2023-2p5-6h-v1.yaml +0 -0
  246. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tools/grids/Makefile +0 -0
  247. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tools/grids/grids.ipynb +0 -0
  248. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tools/grids/grids1.yaml +0 -0
  249. {anemoi_datasets-0.3.3 → anemoi_datasets-0.3.5}/tools/grids/grids2.yaml +0 -0
@@ -42,8 +42,7 @@ jobs:
42
42
  - name: Install
43
43
  run: |
44
44
  pip install pytest
45
- pip install -e .[all]
46
- pip install -r tests/requirements.txt
45
+ pip install -e .[all,test]
47
46
  pip freeze
48
47
 
49
48
  - name: Tests
@@ -1,17 +1,14 @@
1
1
  repos:
2
-
3
2
  # Empty notebookds
4
3
  - repo: local
5
4
  hooks:
6
- - id: clear-notebooks-output
7
- name: clear-notebooks-output
8
- files: tools/.*\.ipynb$
9
- stages: [commit]
10
- language: python
11
- entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
12
- additional_dependencies: [jupyter]
13
-
14
-
5
+ - id: clear-notebooks-output
6
+ name: clear-notebooks-output
7
+ files: tools/.*\.ipynb$
8
+ stages: [commit]
9
+ language: python
10
+ entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
11
+ additional_dependencies: [jupyter]
15
12
  - repo: https://github.com/pre-commit/pre-commit-hooks
16
13
  rev: v4.6.0
17
14
  hooks:
@@ -23,13 +20,11 @@ repos:
23
20
  - id: no-commit-to-branch # Prevent committing to main / master
24
21
  - id: check-added-large-files # Check for large files added to git
25
22
  - id: check-merge-conflict # Check for files that contain merge conflict
26
-
27
23
  - repo: https://github.com/psf/black-pre-commit-mirror
28
24
  rev: 24.4.2
29
25
  hooks:
30
- - id: black
31
- args: [--line-length=120]
32
-
26
+ - id: black
27
+ args: [--line-length=120]
33
28
  - repo: https://github.com/pycqa/isort
34
29
  rev: 5.13.2
35
30
  hooks:
@@ -38,8 +33,6 @@ repos:
38
33
  - -l 120
39
34
  - --force-single-line-imports
40
35
  - --profile black
41
-
42
-
43
36
  - repo: https://github.com/astral-sh/ruff-pre-commit
44
37
  rev: v0.4.6
45
38
  hooks:
@@ -50,32 +43,27 @@ repos:
50
43
  - --fix
51
44
  - --exit-non-zero-on-fix
52
45
  - --preview
53
-
54
46
  - repo: https://github.com/sphinx-contrib/sphinx-lint
55
47
  rev: v0.9.1
56
48
  hooks:
57
- - id: sphinx-lint
58
-
49
+ - id: sphinx-lint
59
50
  # For now, we use it. But it does not support a lot of sphinx features
60
51
  - repo: https://github.com/dzhu/rstfmt
61
52
  rev: v0.0.14
62
53
  hooks:
63
- - id: rstfmt
64
- exclude: 'cli/.*' # Because we use argparse
65
-
54
+ - id: rstfmt
55
+ exclude: 'cli/.*' # Because we use argparse
66
56
  - repo: https://github.com/b8raoult/pre-commit-docconvert
67
57
  rev: "0.1.5"
68
58
  hooks:
69
59
  - id: docconvert
70
60
  args: ["numpy"]
71
-
72
61
  - repo: https://github.com/b8raoult/optional-dependencies-all
73
62
  rev: "0.0.2"
74
63
  hooks:
75
64
  - id: optional-dependencies-all
76
- args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs"]
77
-
65
+ args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs,test"]
78
66
  - repo: https://github.com/tox-dev/pyproject-fmt
79
67
  rev: "2.1.3"
80
68
  hooks:
81
- - id: pyproject-fmt
69
+ - id: pyproject-fmt
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: anemoi-datasets
3
- Version: 0.3.3
3
+ Version: 0.3.5
4
4
  Summary: A package to hold various functions to support training of ML models on ECMWF data.
5
5
  Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
6
6
  License: Apache License
@@ -238,6 +238,7 @@ Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
238
238
  Requires-Dist: entrypoints; extra == "all"
239
239
  Requires-Dist: numpy; extra == "all"
240
240
  Requires-Dist: pyproj; extra == "all"
241
+ Requires-Dist: pytest; extra == "all"
241
242
  Requires-Dist: pyyaml; extra == "all"
242
243
  Requires-Dist: requests; extra == "all"
243
244
  Requires-Dist: s3fs; extra == "all"
@@ -272,3 +273,5 @@ Provides-Extra: remote
272
273
  Requires-Dist: boto3; extra == "remote"
273
274
  Requires-Dist: requests; extra == "remote"
274
275
  Requires-Dist: s3fs; extra == "remote"
276
+ Provides-Extra: test
277
+ Requires-Dist: pytest; extra == "test"
@@ -5,7 +5,7 @@ input:
5
5
  args2: value2
6
6
  - forcings:
7
7
  template: ${input.join.0.source1}
8
- params:
8
+ param:
9
9
  - insolation
10
10
  - cos_julian_day
11
11
  - sin_julian_day
@@ -67,6 +67,7 @@ optional-dependencies.all = [
67
67
  "entrypoints",
68
68
  "numpy",
69
69
  "pyproj",
70
+ "pytest",
70
71
  "pyyaml",
71
72
  "requests",
72
73
  "s3fs",
@@ -81,6 +82,7 @@ optional-dependencies.create = [
81
82
  "entrypoints",
82
83
  "pyproj",
83
84
  ]
85
+
84
86
  optional-dependencies.dev = [
85
87
  "boto3",
86
88
 
@@ -107,6 +109,9 @@ optional-dependencies.remote = [
107
109
  "requests",
108
110
  "s3fs",
109
111
  ]
112
+ optional-dependencies.test = [
113
+ "pytest",
114
+ ]
110
115
  urls.Documentation = "https://anemoi-datasets.readthedocs.io/"
111
116
  urls.Homepage = "https://github.com/ecmwf/anemoi-datasets/"
112
117
  urls.Issues = "https://github.com/ecmwf/anemoi-datasets/issues"
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '0.3.3'
16
- __version_tuple__ = version_tuple = (0, 3, 3)
15
+ __version__ = version = '0.3.5'
16
+ __version_tuple__ = version_tuple = (0, 3, 5)
@@ -24,6 +24,9 @@ from anemoi.utils.text import table
24
24
 
25
25
  from anemoi.datasets import open_dataset
26
26
  from anemoi.datasets.data.stores import open_zarr
27
+ from anemoi.datasets.data.stores import zarr_lookup
28
+
29
+ from . import Command
27
30
 
28
31
  LOG = logging.getLogger(__name__)
29
32
 
@@ -78,46 +81,12 @@ class Version:
78
81
  self.metadata = metadata
79
82
  self.version = version
80
83
  self.dataset = None
81
- # try:
82
84
  self.dataset = open_dataset(self.path)
83
- # except Exception as e:
84
- # LOG.error("Error opening dataset '%s': %s", self.path, e)
85
85
 
86
86
  def describe(self):
87
87
  print(f"📦 Path : {self.path}")
88
88
  print(f"🔢 Format version: {self.version}")
89
89
 
90
- def probe(self):
91
- if "cos_local_time" not in self.name_to_index:
92
- print("⚠️ probe: no cos_local_time")
93
- return
94
-
95
- try:
96
- lon = self.longitudes
97
- except AttributeError:
98
- print("⚠️ probe: no longitudes")
99
- return
100
- # print(json.dumps(self.metadata, indent=4))
101
- cos_local_time = self.name_to_index["cos_local_time"]
102
- data = self.data
103
- start, end, frequency = self.first_date, self.last_date, self.frequency
104
- date = start
105
- same = 0
106
- for i in range(10):
107
- field = data[i, cos_local_time]
108
- buggy = cos_local_time_bug(lon, date).reshape(field.hape)
109
- diff = np.abs(field - buggy)
110
- if np.max(diff) < 1e-5:
111
- same += 1
112
- date += datetime.timedelta(hours=frequency)
113
- if date > end:
114
- break
115
- if same > 1:
116
- print("❌ probe: cos_local_time is buggy")
117
- return
118
-
119
- print("✅ probe: cos_local_time is fixed")
120
-
121
90
  @property
122
91
  def name_to_index(self):
123
92
  return find(self.metadata, "name_to_index")
@@ -586,29 +555,35 @@ VERSIONS = {
586
555
  }
587
556
 
588
557
 
589
- class InspectZarr:
590
- """Inspect a checkpoint or zarr file."""
558
+ class InspectZarr(Command):
559
+ """Inspect a zarr dataset."""
591
560
 
592
- def inspect_zarr(self, path, **kwargs):
593
- version = self._info(path)
561
+ def add_arguments(self, command_parser):
562
+ command_parser.add_argument("path", metavar="DATASET")
563
+ command_parser.add_argument("--detailed", action="store_true")
564
+
565
+ command_parser.add_argument("--progress", action="store_true")
566
+ command_parser.add_argument("--statistics", action="store_true")
567
+ command_parser.add_argument("--size", action="store_true", help="Print size")
594
568
 
595
- # try:
596
- # with open("/tmp/probe.json", "w") as f:
597
- # json.dump(version.metadata, f, indent=4, sort_keys=True)
598
- # except Exception:
599
- # pass
569
+ def run(self, args):
570
+ self.inspect_zarr(**vars(args))
571
+
572
+ def inspect_zarr(self, path, progress=False, statistics=False, detailed=False, size=False, **kwargs):
573
+ version = self._info(path)
600
574
 
601
575
  dotted_line()
602
576
  version.describe()
603
577
 
604
578
  try:
605
- if kwargs.get("probe"):
606
- return version.probe()
607
- if kwargs.get("progress"):
579
+ if progress:
608
580
  return version.progress()
609
- if kwargs.get("statistics"):
581
+
582
+ if statistics:
610
583
  return version.brute_force_statistics()
611
- version.info(kwargs.get("detailed"), kwargs.get("size"))
584
+
585
+ version.info(detailed, size)
586
+
612
587
  except Exception as e:
613
588
  LOG.error("Error inspecting zarr file '%s': %s", path, e)
614
589
 
@@ -616,14 +591,7 @@ class InspectZarr:
616
591
  raise
617
592
 
618
593
  def _info(self, path):
619
- if path.endswith("/"):
620
- path = path[:-1]
621
-
622
- try:
623
- z = open_zarr(path)
624
- except Exception as e:
625
- LOG.error("Error opening zarr file '%s': %s", path, e)
626
- raise
594
+ z = open_zarr(zarr_lookup(path))
627
595
 
628
596
  metadata = dict(z.attrs)
629
597
  version = metadata.get("version", "0.0.0")
@@ -640,3 +608,6 @@ class InspectZarr:
640
608
  candidate = klass
641
609
 
642
610
  return candidate(path, z, metadata, version)
611
+
612
+
613
+ command = InspectZarr
@@ -8,6 +8,8 @@
8
8
  #
9
9
 
10
10
 
11
+ import glob
12
+
11
13
  from climetlab import load_source
12
14
  from climetlab.utils.patterns import Pattern
13
15
 
@@ -22,6 +24,12 @@ def check(ds, paths, **kwargs):
22
24
  raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})")
23
25
 
24
26
 
27
+ def _expand(paths):
28
+ for path in paths:
29
+ for p in glob.glob(path):
30
+ yield p
31
+
32
+
25
33
  def execute(context, dates, path, *args, **kwargs):
26
34
  given_paths = path if isinstance(path, list) else [path]
27
35
 
@@ -35,7 +43,7 @@ def execute(context, dates, path, *args, **kwargs):
35
43
  if name in kwargs:
36
44
  raise ValueError(f"MARS interpolation parameter '{name}' not supported")
37
45
 
38
- for path in paths:
46
+ for path in _expand(paths):
39
47
  context.trace("📁", "PATH", path)
40
48
  s = load_source("file", path)
41
49
  s = s.sel(valid_datetime=dates, **kwargs)
@@ -7,10 +7,29 @@
7
7
  # nor does it submit to any jurisdiction.
8
8
  #
9
9
 
10
+ import glob
11
+
10
12
  from climetlab import load_source
11
13
  from climetlab.utils.patterns import Pattern
12
14
 
13
15
 
16
+ def _expand(paths):
17
+ for path in paths:
18
+ if path.startswith("file://"):
19
+ path = path[7:]
20
+
21
+ if path.startswith("http://"):
22
+ yield path
23
+ continue
24
+
25
+ if path.startswith("https://"):
26
+ yield path
27
+ continue
28
+
29
+ for p in glob.glob(path):
30
+ yield p
31
+
32
+
14
33
  def check(what, ds, paths, **kwargs):
15
34
  count = 1
16
35
  for k, v in kwargs.items():
@@ -32,7 +51,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
32
51
 
33
52
  levels = kwargs.get("level", kwargs.get("levelist"))
34
53
 
35
- for path in paths:
54
+ for path in _expand(paths):
36
55
  context.trace(emoji, what.upper(), path)
37
56
  s = load_source("opendap", path)
38
57
  s = s.sel(
@@ -12,7 +12,7 @@ import numpy as np
12
12
 
13
13
  from .debug import Node
14
14
  from .debug import debug_indexing
15
- from .forewards import Combined
15
+ from .forwards import Combined
16
16
  from .indexing import apply_index_to_slices_changes
17
17
  from .indexing import expand_list_indexing
18
18
  from .indexing import index_to_slices
@@ -8,7 +8,7 @@
8
8
  import logging
9
9
 
10
10
  from .debug import Node
11
- from .forewards import GivenAxis
11
+ from .forwards import GivenAxis
12
12
  from .misc import _auto_adjust
13
13
  from .misc import _open
14
14
 
@@ -91,12 +91,18 @@ class Forwards(Dataset):
91
91
  def metadata_specific(self, **kwargs):
92
92
  return super().metadata_specific(
93
93
  forward=self.forward.metadata_specific(),
94
+ **self.subclass_metadata_specific(),
94
95
  **kwargs,
95
96
  )
96
97
 
97
98
  def source(self, index):
98
99
  return self.forward.source(index)
99
100
 
101
+ def subclass_metadata_specific(self):
102
+ raise NotImplementedError(
103
+ f"subclass_metadata_specific() must be implemented in derived class {self.__class__.__name__}"
104
+ )
105
+
100
106
 
101
107
  class Combined(Forwards):
102
108
  def __init__(self, datasets):
@@ -12,8 +12,8 @@ import numpy as np
12
12
 
13
13
  from .debug import Node
14
14
  from .debug import debug_indexing
15
- from .forewards import Combined
16
- from .forewards import GivenAxis
15
+ from .forwards import Combined
16
+ from .forwards import GivenAxis
17
17
  from .indexing import apply_index_to_slices_changes
18
18
  from .indexing import expand_list_indexing
19
19
  from .indexing import index_to_slices
@@ -91,7 +91,7 @@ class Concat(Combined):
91
91
  return Node(self, [d.tree() for d in self.datasets])
92
92
 
93
93
 
94
- class Grids(GivenAxis):
94
+ class GridsBase(GivenAxis):
95
95
  def __init__(self, datasets, axis):
96
96
  super().__init__(datasets, axis)
97
97
  # Shape: (dates, variables, ensemble, 1d-values)
@@ -106,7 +106,7 @@ class Grids(GivenAxis):
106
106
  pass
107
107
 
108
108
 
109
- class ConcatGrids(Grids):
109
+ class Grids(GridsBase):
110
110
  # TODO: select the statistics of the most global grid?
111
111
  @property
112
112
  def latitudes(self):
@@ -127,7 +127,7 @@ class ConcatGrids(Grids):
127
127
  return Node(self, [d.tree() for d in self.datasets], mode="concat")
128
128
 
129
129
 
130
- class Cutout(Grids):
130
+ class Cutout(GridsBase):
131
131
  def __init__(self, datasets, axis):
132
132
  from anemoi.datasets.grids import cutout_mask
133
133
 
@@ -220,7 +220,7 @@ def grids_factory(args, kwargs):
220
220
  datasets = [_open(e) for e in grids]
221
221
  datasets, kwargs = _auto_adjust(datasets, kwargs)
222
222
 
223
- return ConcatGrids(datasets, axis=axis)._subset(**kwargs)
223
+ return Grids(datasets, axis=axis)._subset(**kwargs)
224
224
 
225
225
 
226
226
  def cutout_factory(args, kwargs):
@@ -13,7 +13,7 @@ import numpy as np
13
13
  from .debug import Node
14
14
  from .debug import Source
15
15
  from .debug import debug_indexing
16
- from .forewards import Combined
16
+ from .forwards import Combined
17
17
  from .indexing import apply_index_to_slices_changes
18
18
  from .indexing import expand_list_indexing
19
19
  from .indexing import index_to_slices
@@ -14,7 +14,7 @@ from ..grids import cropping_mask
14
14
  from .dataset import Dataset
15
15
  from .debug import Node
16
16
  from .debug import debug_indexing
17
- from .forewards import Forwards
17
+ from .forwards import Forwards
18
18
  from .indexing import apply_index_to_slices_changes
19
19
  from .indexing import expand_list_indexing
20
20
  from .indexing import index_to_slices
@@ -83,6 +83,9 @@ class Thinning(Masked):
83
83
  def tree(self):
84
84
  return Node(self, [self.forward.tree()], thinning=self.thinning, method=self.method)
85
85
 
86
+ def subclass_metadata_specific(self):
87
+ return dict(thinning=self.thinning, method=self.method)
88
+
86
89
 
87
90
  class Cropping(Masked):
88
91
  def __init__(self, forward, area):
@@ -104,3 +107,6 @@ class Cropping(Masked):
104
107
 
105
108
  def tree(self):
106
109
  return Node(self, [self.forward.tree()], area=self.area)
110
+
111
+ def metadata_specific(self, **kwargs):
112
+ return super().metadata_specific(area=self.area, **kwargs)
@@ -11,7 +11,7 @@ from functools import cached_property
11
11
  from .debug import Node
12
12
  from .debug import Source
13
13
  from .debug import debug_indexing
14
- from .forewards import Forwards
14
+ from .forwards import Forwards
15
15
  from .indexing import apply_index_to_slices_changes
16
16
  from .indexing import expand_list_indexing
17
17
  from .indexing import index_to_slices
@@ -88,6 +88,10 @@ class Select(Forwards):
88
88
  def tree(self):
89
89
  return Node(self, [self.dataset.tree()], **self.title)
90
90
 
91
+ def subclass_metadata_specific(self):
92
+ # return dict(indices=self.indices)
93
+ return {}
94
+
91
95
 
92
96
  class Rename(Forwards):
93
97
  def __init__(self, dataset, rename):
@@ -105,8 +109,8 @@ class Rename(Forwards):
105
109
  def name_to_index(self):
106
110
  return {k: i for i, k in enumerate(self.variables)}
107
111
 
108
- def metadata_specific(self, **kwargs):
109
- return super().metadata_specific(rename=self.rename, **kwargs)
110
-
111
112
  def tree(self):
112
113
  return Node(self, [self.forward.tree()], rename=self.rename)
114
+
115
+ def subclass_metadata_specific(self):
116
+ return dict(rename=self.rename)
@@ -10,7 +10,7 @@ from functools import cached_property
10
10
 
11
11
  from . import open_dataset
12
12
  from .debug import Node
13
- from .forewards import Forwards
13
+ from .forwards import Forwards
14
14
 
15
15
  LOG = logging.getLogger(__name__)
16
16
 
@@ -34,11 +34,8 @@ class Statistics(Forwards):
34
34
  delta = self.frequency
35
35
  return self._statistic.statistics_tendencies(delta)
36
36
 
37
- def metadata_specific(self, **kwargs):
38
- return super().metadata_specific(
39
- statistics=self._statistic.metadata_specific(),
40
- **kwargs,
41
- )
37
+ def subclass_metadata_specific(self):
38
+ return dict(statistics=self._statistic.metadata_specific())
42
39
 
43
40
  def tree(self):
44
41
  return Node(self, [self.forward.tree()])
@@ -347,7 +347,7 @@ class ZarrWithMissingDates(Zarr):
347
347
  return "zarr*"
348
348
 
349
349
 
350
- def zarr_lookup(name):
350
+ def zarr_lookup(name, fail=True):
351
351
 
352
352
  if name.endswith(".zarr") or name.endswith(".zip"):
353
353
  return name
@@ -372,4 +372,7 @@ def zarr_lookup(name):
372
372
  except zarr.errors.PathNotFoundError:
373
373
  pass
374
374
 
375
- raise ValueError(f"Cannot find a dataset that matched '{name}'. Tried: {tried}")
375
+ if fail:
376
+ raise ValueError(f"Cannot find a dataset that matched '{name}'. Tried: {tried}")
377
+
378
+ return None
@@ -13,7 +13,7 @@ import numpy as np
13
13
  from .debug import Node
14
14
  from .debug import Source
15
15
  from .debug import debug_indexing
16
- from .forewards import Forwards
16
+ from .forwards import Forwards
17
17
  from .indexing import apply_index_to_slices_changes
18
18
  from .indexing import expand_list_indexing
19
19
  from .indexing import index_to_slices
@@ -109,3 +109,9 @@ class Subset(Forwards):
109
109
 
110
110
  def tree(self):
111
111
  return Node(self, [self.dataset.tree()], **self.reason)
112
+
113
+ def subclass_metadata_specific(self):
114
+ return {
115
+ # "indices": self.indices,
116
+ "reason": self.reason,
117
+ }
@@ -11,7 +11,7 @@ from functools import wraps
11
11
 
12
12
  from .concat import ConcatMixin
13
13
  from .debug import Node
14
- from .forewards import Combined
14
+ from .forwards import Combined
15
15
  from .misc import _auto_adjust
16
16
  from .misc import _open
17
17
 
@@ -116,8 +116,8 @@ class StartEndDates(Dates):
116
116
  start = no_time_zone(start)
117
117
  end = no_time_zone(end)
118
118
 
119
- if end <= start:
120
- raise ValueError(f"End date {end} must be after start date {start}")
119
+ # if end <= start:
120
+ # raise ValueError(f"End date {end} must be after start date {start}")
121
121
 
122
122
  increment = datetime.timedelta(hours=frequency)
123
123
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: anemoi-datasets
3
- Version: 0.3.3
3
+ Version: 0.3.5
4
4
  Summary: A package to hold various functions to support training of ML models on ECMWF data.
5
5
  Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
6
6
  License: Apache License
@@ -238,6 +238,7 @@ Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
238
238
  Requires-Dist: entrypoints; extra == "all"
239
239
  Requires-Dist: numpy; extra == "all"
240
240
  Requires-Dist: pyproj; extra == "all"
241
+ Requires-Dist: pytest; extra == "all"
241
242
  Requires-Dist: pyyaml; extra == "all"
242
243
  Requires-Dist: requests; extra == "all"
243
244
  Requires-Dist: s3fs; extra == "all"
@@ -272,3 +273,5 @@ Provides-Extra: remote
272
273
  Requires-Dist: boto3; extra == "remote"
273
274
  Requires-Dist: requests; extra == "remote"
274
275
  Requires-Dist: s3fs; extra == "remote"
276
+ Provides-Extra: test
277
+ Requires-Dist: pytest; extra == "test"
@@ -157,9 +157,8 @@ src/anemoi/datasets/commands/__init__.py
157
157
  src/anemoi/datasets/commands/compare.py
158
158
  src/anemoi/datasets/commands/copy.py
159
159
  src/anemoi/datasets/commands/create.py
160
+ src/anemoi/datasets/commands/inspect.py
160
161
  src/anemoi/datasets/commands/scan.py
161
- src/anemoi/datasets/commands/inspect/__init__.py
162
- src/anemoi/datasets/commands/inspect/zarr.py
163
162
  src/anemoi/datasets/compute/__init__.py
164
163
  src/anemoi/datasets/compute/recentre.py
165
164
  src/anemoi/datasets/create/__init__.py
@@ -203,7 +202,7 @@ src/anemoi/datasets/data/dataset.py
203
202
  src/anemoi/datasets/data/debug.css
204
203
  src/anemoi/datasets/data/debug.py
205
204
  src/anemoi/datasets/data/ensemble.py
206
- src/anemoi/datasets/data/forewards.py
205
+ src/anemoi/datasets/data/forwards.py
207
206
  src/anemoi/datasets/data/grids.py
208
207
  src/anemoi/datasets/data/indexing.py
209
208
  src/anemoi/datasets/data/join.py
@@ -225,7 +224,6 @@ src/anemoi_datasets.egg-info/requires.txt
225
224
  src/anemoi_datasets.egg-info/top_level.txt
226
225
  tests/create-perturbations-full.yaml
227
226
  tests/create-shift.yaml
228
- tests/requirements.txt
229
227
  tests/test_chunks.py
230
228
  tests/test_data.py
231
229
  tests/test_dates.py
@@ -14,6 +14,7 @@ ecmwflibs>=0.6.3
14
14
  entrypoints
15
15
  numpy
16
16
  pyproj
17
+ pytest
17
18
  pyyaml
18
19
  requests
19
20
  s3fs
@@ -52,3 +53,6 @@ sphinx-rtd-theme
52
53
  boto3
53
54
  requests
54
55
  s3fs
56
+
57
+ [test]
58
+ pytest