anemoi-datasets 0.2.1__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.pre-commit-config.yaml +16 -5
  2. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/PKG-INFO +41 -32
  3. anemoi_datasets-0.3.1/docs/building/handling-missing-values.rst +20 -0
  4. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/accumulations.rst +5 -5
  5. anemoi_datasets-0.3.1/docs/building/sources/yaml/hindcasts.yaml +2 -0
  6. anemoi_datasets-0.2.1/docs/building/sources/yaml/perturbations.yaml → anemoi_datasets-0.3.1/docs/building/sources/yaml/recentre.yaml +2 -2
  7. anemoi_datasets-0.3.1/docs/building/sources.rst +29 -0
  8. anemoi_datasets-0.3.1/docs/building/statistics.rst +65 -0
  9. anemoi_datasets-0.3.1/docs/cli/compare.rst +18 -0
  10. anemoi_datasets-0.3.1/docs/cli/copy.rst +27 -0
  11. anemoi_datasets-0.3.1/docs/cli/create.rst +11 -0
  12. anemoi_datasets-0.3.1/docs/cli/inspect.rst +27 -0
  13. anemoi_datasets-0.3.1/docs/cli/introduction.rst +28 -0
  14. anemoi_datasets-0.3.1/docs/cli/scan.rst +10 -0
  15. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/conf.py +13 -3
  16. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/requirements.txt +1 -0
  17. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_complex.py +1 -1
  18. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_other.py +1 -1
  19. anemoi_datasets-0.3.1/pyproject.toml +121 -0
  20. anemoi_datasets-0.3.1/src/anemoi/datasets/__main__.py +28 -0
  21. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/_version.py +2 -2
  22. anemoi_datasets-0.3.1/src/anemoi/datasets/commands/__init__.py +24 -0
  23. anemoi_datasets-0.3.1/src/anemoi/datasets/commands/copy.py +305 -0
  24. anemoi_datasets-0.3.1/src/anemoi/datasets/commands/create.py +33 -0
  25. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/inspect/__init__.py +1 -5
  26. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/__init__.py +3 -0
  27. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/loaders.py +31 -0
  28. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/persistent.py +1 -3
  29. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/statistics/__init__.py +2 -2
  30. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/statistics/summary.py +1 -4
  31. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/writer.py +4 -3
  32. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/indexing.py +1 -3
  33. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/misc.py +5 -6
  34. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/stores.py +2 -6
  35. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/unchecked.py +1 -6
  36. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/PKG-INFO +41 -32
  37. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/SOURCES.txt +3 -1
  38. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/requires.txt +23 -15
  39. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_data.py +1 -3
  40. anemoi_datasets-0.2.1/docs/building/handling-missing-values.rst +0 -17
  41. anemoi_datasets-0.2.1/docs/building/sources.rst +0 -18
  42. anemoi_datasets-0.2.1/docs/building/statistics.rst +0 -30
  43. anemoi_datasets-0.2.1/docs/cli/compare.rst +0 -9
  44. anemoi_datasets-0.2.1/docs/cli/copy.rst +0 -7
  45. anemoi_datasets-0.2.1/docs/cli/create.rst +0 -9
  46. anemoi_datasets-0.2.1/docs/cli/inspect.rst +0 -7
  47. anemoi_datasets-0.2.1/docs/cli/introduction.rst +0 -23
  48. anemoi_datasets-0.2.1/pyproject.toml +0 -101
  49. anemoi_datasets-0.2.1/src/anemoi/datasets/__main__.py +0 -72
  50. anemoi_datasets-0.2.1/src/anemoi/datasets/commands/__init__.py +0 -78
  51. anemoi_datasets-0.2.1/src/anemoi/datasets/commands/copy.py +0 -247
  52. anemoi_datasets-0.2.1/src/anemoi/datasets/commands/create.py +0 -22
  53. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.github/workflows/python-publish.yml +0 -0
  54. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.gitignore +0 -0
  55. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.readthedocs.yaml +0 -0
  56. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.vscode/spellright.dict +0 -0
  57. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/LICENSE +0 -0
  58. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/README.md +0 -0
  59. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/Makefile +0 -0
  60. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/_static/logo.png +0 -0
  61. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/_static/style.css +0 -0
  62. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/_templates/.gitkeep +0 -0
  63. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/apply-fmt.sh +0 -0
  64. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/empty.rst +0 -0
  65. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/noop.rst +0 -0
  66. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/rename.rst +0 -0
  67. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/rotate_winds.rst +0 -0
  68. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/select.rst +0 -0
  69. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/unrotate_winds.rst +0 -0
  70. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters.rst +0 -0
  71. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/handling-missing-dates.rst +0 -0
  72. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/introduction.rst +0 -0
  73. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/naming-variables.rst +0 -0
  74. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/operations.rst +0 -0
  75. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/accumulations1.yaml +0 -0
  76. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/accumulations2.yaml +0 -0
  77. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/forcings.rst +0 -0
  78. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/forcings.yaml +0 -0
  79. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/grib.rst +0 -0
  80. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/hindcasts.rst +0 -0
  81. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/mars.rst +0 -0
  82. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/mars1.yaml +0 -0
  83. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/mars2.yaml +0 -0
  84. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/netcdf.rst +0 -0
  85. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/netcdf.yaml +0 -0
  86. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/opendap.rst +0 -0
  87. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/opendap.yaml +0 -0
  88. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/recentre.rst +0 -0
  89. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib1.yaml +0 -0
  90. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib2.yaml +0 -0
  91. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib3.yaml +0 -0
  92. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib4.yaml +0 -0
  93. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/syntax.rst +0 -0
  94. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/syntax.yaml +0 -0
  95. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/Makefile +0 -0
  96. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building1.txt +0 -0
  97. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building1.yaml +0 -0
  98. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building2.txt +0 -0
  99. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building2.yaml +0 -0
  100. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building3.txt +0 -0
  101. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building3.yaml +0 -0
  102. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/concat.yaml +0 -0
  103. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/hindcasts.yaml +0 -0
  104. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/input.yaml +0 -0
  105. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/missing_dates.yaml +0 -0
  106. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/nan.yaml +0 -0
  107. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/pipe.yaml +0 -0
  108. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/check-index.sh +0 -0
  109. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/images.pptx +0 -0
  110. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/index.rst +0 -0
  111. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/installing.rst +0 -0
  112. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/overview.rst +0 -0
  113. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/overview_.py +0 -0
  114. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/matrix.excalidraw +0 -0
  115. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/matrix.png +0 -0
  116. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/overview.excalidraw +0 -0
  117. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/overview.png +0 -0
  118. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/recipe.excalidraw +0 -0
  119. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/recipe.png +0 -0
  120. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/test.ipynb +0 -0
  121. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/area1_.py +0 -0
  122. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/area2_.py +0 -0
  123. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/chain_.py +0 -0
  124. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/combine_example.py +0 -0
  125. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/concat1.py +0 -0
  126. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/cutout_.py +0 -0
  127. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/drop_.py +0 -0
  128. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/end_.py +0 -0
  129. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/ensembles1_.py +0 -0
  130. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/frequency_.py +0 -0
  131. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/grids1_.py +0 -0
  132. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/join1.py +0 -0
  133. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching0_.py +0 -0
  134. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching1_.py +0 -0
  135. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching2_.py +0 -0
  136. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching3_.py +0 -0
  137. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching4_.py +0 -0
  138. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/misc1.py +0 -0
  139. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/misc2.py +0 -0
  140. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/missing_.py +0 -0
  141. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_cloud.py +0 -0
  142. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_combine1_.py +0 -0
  143. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_combine2_.py +0 -0
  144. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_dict_.py +0 -0
  145. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_first_.py +0 -0
  146. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_list_.py +0 -0
  147. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_name.py +0 -0
  148. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_path.py +0 -0
  149. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_yaml_.py +0 -0
  150. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/rename_.py +0 -0
  151. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/reorder1_.py +0 -0
  152. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/reorder2_.py +0 -0
  153. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/select1_.py +0 -0
  154. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/select2_.py +0 -0
  155. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/shuffle_.py +0 -0
  156. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/some_attributes_.py +0 -0
  157. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/start_.py +0 -0
  158. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/statistics_.py +0 -0
  159. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/subset_example.py +0 -0
  160. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/thinning_.py +0 -0
  161. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/zip1_.py +0 -0
  162. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/zip2_.py +0 -0
  163. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/combining.rst +0 -0
  164. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/configuration.rst +0 -0
  165. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/configuration.toml +0 -0
  166. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/grids.rst +0 -0
  167. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/area-1.png +0 -0
  168. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/concat.png +0 -0
  169. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-1.png +0 -0
  170. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-2.png +0 -0
  171. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-3.png +0 -0
  172. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-4.png +0 -0
  173. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/join.png +0 -0
  174. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/overlay.png +0 -0
  175. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/thinning-after.png +0 -0
  176. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/thinning-before.png +0 -0
  177. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/introduction.rst +0 -0
  178. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/matching.rst +0 -0
  179. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/methods.rst +0 -0
  180. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/miscellaneous.rst +0 -0
  181. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/opening.rst +0 -0
  182. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/other.rst +0 -0
  183. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/selecting.rst +0 -0
  184. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/statistics.rst +0 -0
  185. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/subsetting.rst +0 -0
  186. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/setup.cfg +0 -0
  187. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/__init__.py +0 -0
  188. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/compare.py +0 -0
  189. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/inspect/zarr.py +0 -0
  190. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/scan.py +0 -0
  191. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/compute/__init__.py +0 -0
  192. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/compute/recentre.py +0 -0
  193. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/check.py +0 -0
  194. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/chunks.py +0 -0
  195. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/config.py +0 -0
  196. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/__init__.py +0 -0
  197. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/__init__.py +0 -0
  198. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/empty.py +0 -0
  199. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/noop.py +0 -0
  200. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/rename.py +0 -0
  201. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/rotate_winds.py +0 -0
  202. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/unrotate_winds.py +0 -0
  203. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/__init__.py +0 -0
  204. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/accumulations.py +0 -0
  205. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/constants.py +0 -0
  206. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/empty.py +0 -0
  207. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/forcings.py +0 -0
  208. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/grib.py +0 -0
  209. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/hindcasts.py +0 -0
  210. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/mars.py +0 -0
  211. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/netcdf.py +0 -0
  212. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/opendap.py +0 -0
  213. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/recentre.py +0 -0
  214. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/source.py +0 -0
  215. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/tendencies.py +0 -0
  216. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/input.py +0 -0
  217. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/patch.py +0 -0
  218. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/size.py +0 -0
  219. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/template.py +0 -0
  220. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/utils.py +0 -0
  221. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/zarr.py +0 -0
  222. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/__init__.py +0 -0
  223. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/concat.py +0 -0
  224. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/dataset.py +0 -0
  225. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/debug.css +0 -0
  226. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/debug.py +0 -0
  227. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/ensemble.py +0 -0
  228. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/forewards.py +0 -0
  229. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/grids.py +0 -0
  230. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/join.py +0 -0
  231. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/masked.py +0 -0
  232. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/select.py +0 -0
  233. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/statistics.py +0 -0
  234. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/subset.py +0 -0
  235. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/dates/__init__.py +0 -0
  236. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/dates/groups.py +0 -0
  237. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/grids.py +0 -0
  238. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/utils/__init__.py +0 -0
  239. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/dependency_links.txt +0 -0
  240. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/entry_points.txt +0 -0
  241. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/top_level.txt +0 -0
  242. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/concat.yaml +0 -0
  243. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/data_sources.yaml +0 -0
  244. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/join.yaml +0 -0
  245. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/missing.yaml +0 -0
  246. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/nan.yaml +0 -0
  247. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/pipe.yaml +0 -0
  248. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/recentre.yaml +0 -0
  249. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/test_create.py +0 -0
  250. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create-perturbations-full.yaml +0 -0
  251. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create-shift.yaml +0 -0
  252. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/requirements.txt +0 -0
  253. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_chunks.py +0 -0
  254. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_dates.py +0 -0
  255. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_indexing.py +0 -0
  256. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/.gitignore +0 -0
  257. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/examples/Makefile +0 -0
  258. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/examples/an-oper-2023-2023-2p5-6h-v1.yaml +0 -0
  259. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/Makefile +0 -0
  260. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/grids.ipynb +0 -0
  261. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/grids1.yaml +0 -0
  262. {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/grids2.yaml +0 -0
@@ -13,19 +13,19 @@ repos:
13
13
 
14
14
 
15
15
  - repo: https://github.com/pre-commit/pre-commit-hooks
16
- rev: v4.4.0
16
+ rev: v4.6.0
17
17
  hooks:
18
18
  - id: check-yaml # Check YAML files for syntax errors only
19
19
  args: [--unsafe, --allow-multiple-documents]
20
20
  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
21
21
  - id: end-of-file-fixer # Ensure files end in a newline
22
22
  - id: trailing-whitespace # Trailing whitespace checker
23
- # - id: no-commit-to-branch # Prevent committing to main / master
23
+ - id: no-commit-to-branch # Prevent committing to main / master
24
24
  - id: check-added-large-files # Check for large files added to git
25
25
  - id: check-merge-conflict # Check for files that contain merge conflict
26
26
 
27
27
  - repo: https://github.com/psf/black-pre-commit-mirror
28
- rev: 24.1.1
28
+ rev: 24.4.2
29
29
  hooks:
30
30
  - id: black
31
31
  args: [--line-length=120]
@@ -41,7 +41,7 @@ repos:
41
41
 
42
42
 
43
43
  - repo: https://github.com/astral-sh/ruff-pre-commit
44
- rev: v0.3.0
44
+ rev: v0.4.6
45
45
  hooks:
46
46
  - id: ruff
47
47
  exclude: '(dev/.*|.*_)\.py$'
@@ -51,7 +51,6 @@ repos:
51
51
  - --exit-non-zero-on-fix
52
52
  - --preview
53
53
 
54
-
55
54
  - repo: https://github.com/sphinx-contrib/sphinx-lint
56
55
  rev: v0.9.1
57
56
  hooks:
@@ -62,9 +61,21 @@ repos:
62
61
  rev: v0.0.14
63
62
  hooks:
64
63
  - id: rstfmt
64
+ exclude: 'cli/.*' # Because we use argparse
65
65
 
66
66
  - repo: https://github.com/b8raoult/pre-commit-docconvert
67
67
  rev: "0.1.4"
68
68
  hooks:
69
69
  - id: docconvert
70
70
  args: ["numpy"]
71
+
72
+ - repo: https://github.com/b8raoult/optional-dependencies-all
73
+ rev: "0.0.2"
74
+ hooks:
75
+ - id: optional-dependencies-all
76
+ args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs"]
77
+
78
+ - repo: https://github.com/tox-dev/pyproject-fmt
79
+ rev: "2.1.3"
80
+ hooks:
81
+ - id: pyproject-fmt
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: anemoi-datasets
3
- Version: 0.2.1
3
+ Version: 0.3.1
4
4
  Summary: A package to hold various functions to support training of ML models on ECMWF data.
5
5
  Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
6
6
  License: Apache License
@@ -205,60 +205,69 @@ License: Apache License
205
205
  See the License for the specific language governing permissions and
206
206
  limitations under the License.
207
207
 
208
- Project-URL: Homepage, https://github.com/ecmwf/anemoi-datasets/
209
208
  Project-URL: Documentation, https://anemoi-datasets.readthedocs.io/
210
- Project-URL: Repository, https://github.com/ecmwf/anemoi-datasets/
209
+ Project-URL: Homepage, https://github.com/ecmwf/anemoi-datasets/
211
210
  Project-URL: Issues, https://github.com/ecmwf/anemoi-datasets/issues
212
- Keywords: tools,datasets,ai
211
+ Project-URL: Repository, https://github.com/ecmwf/anemoi-datasets/
212
+ Keywords: ai,datasets,tools
213
213
  Classifier: Development Status :: 4 - Beta
214
214
  Classifier: Intended Audience :: Developers
215
215
  Classifier: License :: OSI Approved :: Apache Software License
216
- Classifier: Programming Language :: Python :: 3
216
+ Classifier: Operating System :: OS Independent
217
+ Classifier: Programming Language :: Python :: 3 :: Only
217
218
  Classifier: Programming Language :: Python :: 3.9
218
219
  Classifier: Programming Language :: Python :: 3.10
219
220
  Classifier: Programming Language :: Python :: 3.11
221
+ Classifier: Programming Language :: Python :: 3.12
220
222
  Classifier: Programming Language :: Python :: Implementation :: CPython
221
223
  Classifier: Programming Language :: Python :: Implementation :: PyPy
222
- Classifier: Operating System :: OS Independent
223
224
  Requires-Python: >=3.9
224
225
  License-File: LICENSE
225
- Requires-Dist: anemoi-utils[provenance]>=0.1.7
226
- Requires-Dist: zarr<=2.17.0
227
- Requires-Dist: pyyaml
226
+ Requires-Dist: anemoi-utils[provenance]>=0.3
228
227
  Requires-Dist: numpy
229
- Requires-Dist: tqdm
228
+ Requires-Dist: pyyaml
230
229
  Requires-Dist: semantic-version
231
- Provides-Extra: remote
232
- Requires-Dist: boto3; extra == "remote"
233
- Requires-Dist: requests; extra == "remote"
234
- Requires-Dist: s3fs; extra == "remote"
235
- Provides-Extra: create
236
- Requires-Dist: climetlab>=0.22.1; extra == "create"
237
- Requires-Dist: earthkit-meteo; extra == "create"
238
- Requires-Dist: pyproj; extra == "create"
239
- Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
240
- Provides-Extra: docs
241
- Requires-Dist: sphinx; extra == "docs"
242
- Requires-Dist: sphinx_rtd_theme; extra == "docs"
243
- Requires-Dist: nbsphinx; extra == "docs"
244
- Requires-Dist: pandoc; extra == "docs"
230
+ Requires-Dist: tqdm
231
+ Requires-Dist: zarr<=2.17
245
232
  Provides-Extra: all
233
+ Requires-Dist: anemoi-utils[provenance]>=0.3; extra == "all"
246
234
  Requires-Dist: boto3; extra == "all"
247
- Requires-Dist: requests; extra == "all"
248
- Requires-Dist: s3fs; extra == "all"
249
235
  Requires-Dist: climetlab>=0.22.1; extra == "all"
250
236
  Requires-Dist: earthkit-meteo; extra == "all"
251
- Requires-Dist: pyproj; extra == "all"
252
237
  Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
238
+ Requires-Dist: numpy; extra == "all"
239
+ Requires-Dist: pyproj; extra == "all"
240
+ Requires-Dist: pyyaml; extra == "all"
241
+ Requires-Dist: requests; extra == "all"
242
+ Requires-Dist: s3fs; extra == "all"
243
+ Requires-Dist: semantic-version; extra == "all"
244
+ Requires-Dist: tqdm; extra == "all"
245
+ Requires-Dist: zarr<=2.17; extra == "all"
246
+ Provides-Extra: create
247
+ Requires-Dist: climetlab>=0.22.1; extra == "create"
248
+ Requires-Dist: earthkit-meteo; extra == "create"
249
+ Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
250
+ Requires-Dist: pyproj; extra == "create"
253
251
  Provides-Extra: dev
254
252
  Requires-Dist: boto3; extra == "dev"
255
- Requires-Dist: requests; extra == "dev"
256
- Requires-Dist: s3fs; extra == "dev"
257
253
  Requires-Dist: climetlab>=0.22.1; extra == "dev"
258
254
  Requires-Dist: earthkit-meteo; extra == "dev"
259
- Requires-Dist: pyproj; extra == "dev"
260
255
  Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
261
- Requires-Dist: sphinx; extra == "dev"
262
- Requires-Dist: sphinx_rtd_theme; extra == "dev"
263
256
  Requires-Dist: nbsphinx; extra == "dev"
264
257
  Requires-Dist: pandoc; extra == "dev"
258
+ Requires-Dist: pyproj; extra == "dev"
259
+ Requires-Dist: requests; extra == "dev"
260
+ Requires-Dist: s3fs; extra == "dev"
261
+ Requires-Dist: sphinx; extra == "dev"
262
+ Requires-Dist: sphinx-argparse; extra == "dev"
263
+ Requires-Dist: sphinx-rtd-theme; extra == "dev"
264
+ Provides-Extra: docs
265
+ Requires-Dist: nbsphinx; extra == "docs"
266
+ Requires-Dist: pandoc; extra == "docs"
267
+ Requires-Dist: sphinx; extra == "docs"
268
+ Requires-Dist: sphinx-argparse; extra == "docs"
269
+ Requires-Dist: sphinx-rtd-theme; extra == "docs"
270
+ Provides-Extra: remote
271
+ Requires-Dist: boto3; extra == "remote"
272
+ Requires-Dist: requests; extra == "remote"
273
+ Requires-Dist: s3fs; extra == "remote"
@@ -0,0 +1,20 @@
1
+ #########################
2
+ Handling missing values
3
+ #########################
4
+
5
+ When handling data for machine learning models, missing values (`NaNs`)
6
+ can pose a challenge, as models require complete data to operate
7
+ effectively and may crash otherwise. Ideally, we anticipate having
8
+ complete data in all fields.
9
+
10
+ However, there are scenarios where `NaNs` naturally occur, such as with
11
+ variables only relevant on land or at sea. This happens for sea surface
12
+ temperature (`sst`), for example. In such cases, the default behavior is
13
+ to reject data with `NaNs` as invalid. To accommodate `NaNs` and
14
+ accurately compute statistics based on them, you can include the
15
+ ``allow_nans`` key in the configuration.
16
+
17
+ Here's an example of how to implement it:
18
+
19
+ .. literalinclude:: yaml/nan.yaml
20
+ :language: yaml
@@ -24,9 +24,9 @@ dataset is unknown, the package assumes that the fields to use are
24
24
  accumulated since the beginning of the forecast, over a 6h period.
25
25
 
26
26
  The user can specify the desired accumulation period with the
27
- ``accumulation_period`` parameter. If its value is a single interger,
28
- the source will attempt to accumulate the variables over that period.
29
- This does not always mean that the data used is accumulated from the
27
+ ``accumulation_period`` parameter. If its value is a single integer, the
28
+ source will attempt to accumulate the variables over that period. This
29
+ does not always mean that the data used is accumulated from the
30
30
  beginning of the forecast, but the most recent data available will be
31
31
  used:
32
32
 
@@ -44,13 +44,13 @@ used:
44
44
  If the of ``accumulation_period`` value is a pair of integers `[step1,
45
45
  step2]`, the algorithm is different. The source will compute the
46
46
  accumulation between the `step1` and `step2` previous forecast that
47
- valiate at the given date at `step2`. For example, if the accumulation
47
+ validate at the given date at `step2`. For example, if the accumulation
48
48
  period is `[6, 12]`, and the valid date is 2020-10-10 18:00, the source
49
49
  will use the forecast of 2020-10-10 06:00 and the steps 6h and 12h.
50
50
 
51
51
  Please note that ``accumulation_period=6`` and ``accumulation_period=[0,
52
52
  6]`` are not equivalent. In the first case, the source can use return an
53
- accumulation bwteen step 1h and step 7h if it is the most appropriate
53
+ accumulation between step 1h and step 7h if it is the most appropriate
54
54
  data available, while in the second case, the source will always return
55
55
  the accumulation between step 0h and step 6h, if available.
56
56
 
@@ -21,6 +21,6 @@ data_sources:
21
21
  stream: oper
22
22
 
23
23
  input:
24
- perturbations:
25
- center: ${data_sources.center_source}
24
+ recentre:
25
+ centre: ${data_sources.center_source}
26
26
  members: ${data_sources.members_source}
@@ -0,0 +1,29 @@
1
+ .. _sources:
2
+
3
+ #########
4
+ Sources
5
+ #########
6
+
7
+ The source is a software component that given a list of dates and
8
+ variables will return the corresponding fields.
9
+
10
+ A `source` is responsible for reading data from the source and
11
+ converting it to a set of fields. A `source` is also responsible for
12
+ handling the metadata of the data, such as the variables names, and
13
+ more.
14
+
15
+ A example of source is ECMWF’s MARS archive, a collection of GRIB or
16
+ NetCDF files, etc.
17
+
18
+ The following `sources` are currently available:
19
+
20
+ .. toctree::
21
+ :maxdepth: 1
22
+
23
+ sources/mars
24
+ sources/grib
25
+ sources/netcdf
26
+ sources/opendap
27
+ sources/forcings
28
+ sources/accumulations
29
+ sources/recentre
@@ -0,0 +1,65 @@
1
+ .. _gathering_statistics:
2
+
3
+ ######################
4
+ Gathering statistics
5
+ ######################
6
+
7
+ *Anemoi* will collect statistics about each variables in the dataset as
8
+ it is created. These statistics are intended to be used to normalise the
9
+ data during training.
10
+
11
+ The statistics are stored in the :doc:`statistics attribute
12
+ <../using/statistics>` of the dataset. The computed statistics include
13
+ `minimum, maximum, mean, standard deviation`.
14
+
15
+ ************************
16
+ Statistics dates range
17
+ ************************
18
+
19
+ By defaults, the statistics are not computed on the whole dataset, but
20
+ on a subset of dates. This usually is done to avoid any data leakage
21
+ from the validation and test sets to the training set.
22
+
23
+ The dates subset used to compute the statistics is defined using the
24
+ following algorithm:
25
+
26
+ - If the dataset covers 20 years or more, the last 3 years are
27
+ excluded.
28
+ - If the dataset covers 10 years or more, the last year is excluded.
29
+ - Otherwise, 80% of the dataset is used.
30
+
31
+ You can override this behaviour by setting either the `start` parameter
32
+ or the `end` parameters in the `statistics` config.
33
+
34
+ Example configuration gathering statistics from 2000 to 2020 :
35
+
36
+ .. code:: yaml
37
+
38
+ statistics:
39
+ start: 2000
40
+ end: 2020
41
+
42
+ Example configuration gathering statistics from the beginning of the
43
+ dataset period to 2020 :
44
+
45
+ .. code:: yaml
46
+
47
+ statistics:
48
+ end: 2020
49
+
50
+ Example configuration gathering statistics using only 2020 data :
51
+
52
+ .. code:: yaml
53
+
54
+ statistics:
55
+ start: 2020
56
+ end: 2020
57
+
58
+ **************************
59
+ Data with missing values
60
+ **************************
61
+
62
+ If the dataset contains missing values (known as `NaNs`), an error will
63
+ be raised when trying to compute the statistics. To allow `NaNs` in the
64
+ dataset, you can set the `allow_nans` as described :doc:`here
65
+ </building/handling-missing-values>`.
@@ -0,0 +1,18 @@
1
+ compare
2
+ =======
3
+
4
+ Use this command to compare two datasets.
5
+
6
+ The command will run a quick comparison of the two datasets and output a summary of the differences.
7
+
8
+ .. warning::
9
+
10
+ This command will not compare the data in the datasets, only some of the metadata.
11
+ Subsequent versions of this command may include more detailed comparisons.
12
+
13
+
14
+ .. argparse::
15
+ :module: anemoi.datasets.__main__
16
+ :func: create_parser
17
+ :prog: anemoi-datasets
18
+ :path: compare
@@ -0,0 +1,27 @@
1
+ copy
2
+ ====
3
+
4
+
5
+ Copying a dataset from one location to another can be error-prone and time-consuming.
6
+ This command-line script allows for incremental copying.
7
+ When the copying process fails, it can be resumed.
8
+ It can be used to copy files from a local directory to a remote server, from a remote server to a local directory as long as there is a zarr backend to read and write the data.
9
+
10
+ The script uses multiple threads to make the process faster.
11
+ However, it is important to consider that making parallel requests to the same server may not be ideal, for instance if the server internally uses a limited number of threads to handle requests.
12
+
13
+ The option to rechunk the data is available, which can be useful when the data is stored on a platform that does not support having may small files or many file on the same directory.
14
+ However keep in mind that rechunking has a huge impact on the performance when reading the data:
15
+ The chunk pattern for the source dataset has been defined for good reasons, and changing it is very likey to have a negative impact on the performance.
16
+
17
+ .. warning::
18
+
19
+ When resuming the copying process (using ``--resume``), calling the script with the same arguments for ``--block-size`` and ``--rechunk`` is recommended.
20
+ Using different values for these arguments to resume copying the same dataset may lead to unexpected behavior.
21
+
22
+
23
+ .. argparse::
24
+ :module: anemoi.datasets.__main__
25
+ :func: create_parser
26
+ :prog: anemoi-datasets
27
+ :path: copy
@@ -0,0 +1,11 @@
1
+ create
2
+ ======
3
+
4
+ Use this command to create a dataset from a recipe file.
5
+ The syntax of the recipe file is described in :doc:`building datasets <../building/introduction>`.
6
+
7
+ .. argparse::
8
+ :module: anemoi.datasets.__main__
9
+ :func: create_parser
10
+ :prog: anemoi-datasets
11
+ :path: create
@@ -0,0 +1,27 @@
1
+ inspect
2
+ =======
3
+
4
+
5
+ Anemoi datasets are stored in a zarr format and can be located on a local file system or on a remote server.
6
+ The `inspect` command is used to inspect the contents of a dataset.
7
+ This command will output the metadata of the dataset, including the variables, dimensions, and attributes.
8
+
9
+ .. code:: console
10
+
11
+ $ anemoi-datasets inspect dataset.zarr
12
+
13
+
14
+ which will output something like the following. The output should be self-explanatory.
15
+
16
+ .. literalinclude:: ../building/yaml/building1.txt
17
+ :language: console
18
+
19
+ *********************
20
+ Command line usage
21
+ *********************
22
+
23
+ .. argparse::
24
+ :module: anemoi.datasets.__main__
25
+ :func: create_parser
26
+ :prog: anemoi-datasets
27
+ :path: inspect
@@ -0,0 +1,28 @@
1
+ Introduction
2
+ ============
3
+
4
+ When you install the `anemoi-datasets` package, this will also install command line tool
5
+ called ``anemoi-datasets`` which can be used to manage the zarr datasets.
6
+
7
+ The tool can provide help with the ``--help`` options:
8
+
9
+ .. code-block:: bash
10
+
11
+ % anemoi-datasets --help
12
+
13
+ The commands are:
14
+
15
+ .. toctree::
16
+ :maxdepth: 1
17
+
18
+ compare
19
+ copy
20
+ create
21
+ inspect
22
+ scan
23
+
24
+ .. argparse::
25
+ :module: anemoi.datasets.__main__
26
+ :func: create_parser
27
+ :prog: anemoi-datasets
28
+ :nosubcommands:
@@ -0,0 +1,10 @@
1
+ scan
2
+ ====
3
+
4
+ Use this command to scan for GRIB files
5
+
6
+ .. argparse::
7
+ :module: anemoi.datasets.__main__
8
+ :func: create_parser
9
+ :prog: anemoi-datasets
10
+ :path: scan
@@ -16,11 +16,11 @@
16
16
 
17
17
  import datetime
18
18
  import os
19
+ import sys
19
20
 
20
21
  read_the_docs_build = os.environ.get("READTHEDOCS", None) == "True"
21
22
 
22
- # top = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
23
- # sys.path.insert(0, top)
23
+ sys.path.insert(0, os.path.join(os.path.abspath(".."), "src"))
24
24
 
25
25
 
26
26
  source_suffix = ".rst"
@@ -45,7 +45,12 @@ else:
45
45
  copyright = "%s, ECMWF" % (years,)
46
46
 
47
47
 
48
- release = "0.1.0"
48
+ try:
49
+ from anemoi.datasets._version import __version__
50
+
51
+ release = __version__
52
+ except ImportError:
53
+ release = "0.0.0"
49
54
 
50
55
 
51
56
  # -- General configuration ---------------------------------------------------
@@ -59,6 +64,9 @@ extensions = [
59
64
  "nbsphinx",
60
65
  "sphinx.ext.graphviz",
61
66
  "sphinx.ext.intersphinx",
67
+ "sphinx.ext.autodoc",
68
+ "sphinx.ext.napoleon",
69
+ "sphinxarg.ext",
62
70
  ]
63
71
 
64
72
  # Add any paths that contain templates here, relative to this directory.
@@ -112,3 +120,5 @@ html_css_files = ["style.css"]
112
120
 
113
121
 
114
122
  todo_include_todos = not read_the_docs_build
123
+
124
+ autodoc_member_order = "bysource" # Keep file order
@@ -2,6 +2,7 @@
2
2
  sphinx
3
3
  sphinx_rtd_theme
4
4
  nbsphinx
5
+ sphinx_argparse
5
6
 
6
7
  # Also requires `brew install pandoc` on Mac
7
8
  pandoc
@@ -5,7 +5,7 @@ config = {
5
5
  "ensemble": [
6
6
  "/path/to/dataset1.zarr",
7
7
  {"dataset": "dataset_name", "end": 2010},
8
- {"dataset": "s3://path/to/dataset3.zarr", "begin": 2000, "end": 2010},
8
+ {"dataset": "s3://path/to/dataset3.zarr", "start": 2000, "end": 2010},
9
9
  ],
10
10
  "frequency": "24h",
11
11
  },
@@ -2,4 +2,4 @@ from anemoi.datasets import open_dataset
2
2
 
3
3
  ds1 = open_dataset("/path/to/dataset.zarr")
4
4
 
5
- ds2 = open_dataset(ds1, frequency="24h", begin="2000", end="2010")
5
+ ds2 = open_dataset(ds1, frequency="24h", start="2000", end="2010")
@@ -0,0 +1,121 @@
1
+ #!/usr/bin/env python
2
+ # (C) Copyright 2024 ECMWF.
3
+ #
4
+ # This software is licensed under the terms of the Apache Licence Version 2.0
5
+ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6
+ # In applying this licence, ECMWF does not waive the privileges and immunities
7
+ # granted to it by virtue of its status as an intergovernmental organisation
8
+ # nor does it submit to any jurisdiction.
9
+
10
+ # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
11
+
12
+ [build-system]
13
+ requires = [
14
+ "setuptools>=60",
15
+ "setuptools-scm>=8",
16
+ ]
17
+
18
+ [project]
19
+ name = "anemoi-datasets"
20
+
21
+ description = "A package to hold various functions to support training of ML models on ECMWF data."
22
+ keywords = [
23
+ "ai",
24
+ "datasets",
25
+ "tools",
26
+ ]
27
+
28
+ license = { file = "LICENSE" }
29
+ authors = [
30
+ { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" },
31
+ ]
32
+
33
+ requires-python = ">=3.9"
34
+
35
+ classifiers = [
36
+ "Development Status :: 4 - Beta",
37
+ "Intended Audience :: Developers",
38
+ "License :: OSI Approved :: Apache Software License",
39
+ "Operating System :: OS Independent",
40
+ "Programming Language :: Python :: 3 :: Only",
41
+ "Programming Language :: Python :: 3.9",
42
+ "Programming Language :: Python :: 3.10",
43
+ "Programming Language :: Python :: 3.11",
44
+ "Programming Language :: Python :: 3.12",
45
+ "Programming Language :: Python :: Implementation :: CPython",
46
+ "Programming Language :: Python :: Implementation :: PyPy",
47
+ ]
48
+
49
+ dynamic = [
50
+ "version",
51
+ ]
52
+ dependencies = [
53
+ "anemoi-utils[provenance]>=0.3",
54
+ "numpy",
55
+ "pyyaml",
56
+ "semantic-version",
57
+ "tqdm",
58
+ "zarr<=2.17",
59
+ ]
60
+
61
+ optional-dependencies.all = [
62
+ "anemoi-utils[provenance]>=0.3",
63
+ "boto3",
64
+ "climetlab>=0.22.1",
65
+ "earthkit-meteo",
66
+ "ecmwflibs>=0.6.3",
67
+ "numpy",
68
+ "pyproj",
69
+ "pyyaml",
70
+ "requests",
71
+ "s3fs",
72
+ "semantic-version",
73
+ "tqdm",
74
+ "zarr<=2.17",
75
+ ]
76
+ optional-dependencies.create = [
77
+ "climetlab>=0.22.1", # "earthkit-data"
78
+ "earthkit-meteo",
79
+ "ecmwflibs>=0.6.3",
80
+ "pyproj",
81
+ ]
82
+ optional-dependencies.dev = [
83
+ "boto3",
84
+ "climetlab>=0.22.1", # "earthkit-data"
85
+ "earthkit-meteo",
86
+ "ecmwflibs>=0.6.3",
87
+ "nbsphinx",
88
+ "pandoc",
89
+ "pyproj",
90
+ "requests",
91
+ "s3fs",
92
+ "sphinx",
93
+ "sphinx-argparse",
94
+ "sphinx-rtd-theme",
95
+ ]
96
+ optional-dependencies.docs = [
97
+ "nbsphinx",
98
+ "pandoc",
99
+ "sphinx",
100
+ "sphinx-argparse",
101
+ "sphinx-rtd-theme",
102
+ ]
103
+ optional-dependencies.remote = [
104
+ "boto3",
105
+ "requests",
106
+ "s3fs",
107
+ ]
108
+ urls.Documentation = "https://anemoi-datasets.readthedocs.io/"
109
+ urls.Homepage = "https://github.com/ecmwf/anemoi-datasets/"
110
+ urls.Issues = "https://github.com/ecmwf/anemoi-datasets/issues"
111
+ # Changelog = "https://github.com/ecmwf/anemoi-datasets/CHANGELOG.md"
112
+ urls.Repository = "https://github.com/ecmwf/anemoi-datasets/"
113
+ scripts.anemoi-datasets = "anemoi.datasets.__main__:main"
114
+
115
+ [tool.setuptools.package-data]
116
+ "anemoi.datasets.data" = [
117
+ "*.css",
118
+ ]
119
+
120
+ [tool.setuptools_scm]
121
+ version_file = "src/anemoi/datasets/_version.py"
@@ -0,0 +1,28 @@
1
+ #!/usr/bin/env python
2
+ # (C) Copyright 2024 ECMWF.
3
+ #
4
+ # This software is licensed under the terms of the Apache Licence Version 2.0
5
+ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6
+ # In applying this licence, ECMWF does not waive the privileges and immunities
7
+ # granted to it by virtue of its status as an intergovernmental organisation
8
+ # nor does it submit to any jurisdiction.
9
+ #
10
+
11
+ from anemoi.utils.cli import cli_main
12
+ from anemoi.utils.cli import make_parser
13
+
14
+ from . import __version__
15
+ from .commands import COMMANDS
16
+
17
+
18
+ # For read-the-docs
19
+ def create_parser():
20
+ return make_parser(__doc__, COMMANDS)
21
+
22
+
23
+ def main():
24
+ cli_main(__version__, __doc__, COMMANDS)
25
+
26
+
27
+ if __name__ == "__main__":
28
+ main()