anemoi-datasets 0.2.1__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.pre-commit-config.yaml +16 -5
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/PKG-INFO +41 -32
- anemoi_datasets-0.3.1/docs/building/handling-missing-values.rst +20 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/accumulations.rst +5 -5
- anemoi_datasets-0.3.1/docs/building/sources/yaml/hindcasts.yaml +2 -0
- anemoi_datasets-0.2.1/docs/building/sources/yaml/perturbations.yaml → anemoi_datasets-0.3.1/docs/building/sources/yaml/recentre.yaml +2 -2
- anemoi_datasets-0.3.1/docs/building/sources.rst +29 -0
- anemoi_datasets-0.3.1/docs/building/statistics.rst +65 -0
- anemoi_datasets-0.3.1/docs/cli/compare.rst +18 -0
- anemoi_datasets-0.3.1/docs/cli/copy.rst +27 -0
- anemoi_datasets-0.3.1/docs/cli/create.rst +11 -0
- anemoi_datasets-0.3.1/docs/cli/inspect.rst +27 -0
- anemoi_datasets-0.3.1/docs/cli/introduction.rst +28 -0
- anemoi_datasets-0.3.1/docs/cli/scan.rst +10 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/conf.py +13 -3
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/requirements.txt +1 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_complex.py +1 -1
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_other.py +1 -1
- anemoi_datasets-0.3.1/pyproject.toml +121 -0
- anemoi_datasets-0.3.1/src/anemoi/datasets/__main__.py +28 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/_version.py +2 -2
- anemoi_datasets-0.3.1/src/anemoi/datasets/commands/__init__.py +24 -0
- anemoi_datasets-0.3.1/src/anemoi/datasets/commands/copy.py +305 -0
- anemoi_datasets-0.3.1/src/anemoi/datasets/commands/create.py +33 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/inspect/__init__.py +1 -5
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/__init__.py +3 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/loaders.py +31 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/persistent.py +1 -3
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/statistics/__init__.py +2 -2
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/statistics/summary.py +1 -4
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/writer.py +4 -3
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/indexing.py +1 -3
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/misc.py +5 -6
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/stores.py +2 -6
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/unchecked.py +1 -6
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/PKG-INFO +41 -32
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/SOURCES.txt +3 -1
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/requires.txt +23 -15
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_data.py +1 -3
- anemoi_datasets-0.2.1/docs/building/handling-missing-values.rst +0 -17
- anemoi_datasets-0.2.1/docs/building/sources.rst +0 -18
- anemoi_datasets-0.2.1/docs/building/statistics.rst +0 -30
- anemoi_datasets-0.2.1/docs/cli/compare.rst +0 -9
- anemoi_datasets-0.2.1/docs/cli/copy.rst +0 -7
- anemoi_datasets-0.2.1/docs/cli/create.rst +0 -9
- anemoi_datasets-0.2.1/docs/cli/inspect.rst +0 -7
- anemoi_datasets-0.2.1/docs/cli/introduction.rst +0 -23
- anemoi_datasets-0.2.1/pyproject.toml +0 -101
- anemoi_datasets-0.2.1/src/anemoi/datasets/__main__.py +0 -72
- anemoi_datasets-0.2.1/src/anemoi/datasets/commands/__init__.py +0 -78
- anemoi_datasets-0.2.1/src/anemoi/datasets/commands/copy.py +0 -247
- anemoi_datasets-0.2.1/src/anemoi/datasets/commands/create.py +0 -22
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.github/workflows/python-publish.yml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.gitignore +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.readthedocs.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/.vscode/spellright.dict +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/LICENSE +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/README.md +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/Makefile +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/_static/logo.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/_static/style.css +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/_templates/.gitkeep +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/apply-fmt.sh +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/empty.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/noop.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/rename.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/rotate_winds.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/select.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters/unrotate_winds.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/filters.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/handling-missing-dates.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/introduction.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/naming-variables.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/operations.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/accumulations1.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/accumulations2.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/forcings.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/forcings.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/grib.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/hindcasts.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/mars.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/mars1.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/mars2.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/netcdf.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/netcdf.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/opendap.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/opendap.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/recentre.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib1.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib2.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib3.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/sources/yaml/grib4.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/syntax.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/syntax.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/Makefile +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building1.txt +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building1.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building2.txt +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building2.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building3.txt +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/building3.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/concat.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/hindcasts.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/input.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/missing_dates.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/nan.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/building/yaml/pipe.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/check-index.sh +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/images.pptx +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/index.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/installing.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/overview.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/overview_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/matrix.excalidraw +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/matrix.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/overview.excalidraw +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/overview.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/recipe.excalidraw +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/schemas/recipe.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/test.ipynb +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/area1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/area2_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/chain_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/combine_example.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/concat1.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/cutout_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/drop_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/end_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/ensembles1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/frequency_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/grids1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/join1.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching0_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching2_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching3_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/matching4_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/misc1.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/misc2.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/missing_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_cloud.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_combine1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_combine2_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_dict_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_first_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_list_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_name.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_path.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/open_yaml_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/rename_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/reorder1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/reorder2_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/select1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/select2_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/shuffle_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/some_attributes_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/start_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/statistics_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/subset_example.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/thinning_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/zip1_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/code/zip2_.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/combining.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/configuration.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/configuration.toml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/grids.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/area-1.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/concat.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-1.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-2.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-3.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/cutout-4.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/join.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/overlay.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/thinning-after.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/images/thinning-before.png +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/introduction.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/matching.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/methods.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/miscellaneous.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/opening.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/other.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/selecting.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/statistics.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/docs/using/subsetting.rst +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/setup.cfg +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/compare.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/inspect/zarr.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/commands/scan.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/compute/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/compute/recentre.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/check.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/chunks.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/config.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/empty.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/noop.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/rename.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/rotate_winds.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/filters/unrotate_winds.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/accumulations.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/constants.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/empty.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/forcings.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/grib.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/hindcasts.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/mars.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/netcdf.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/opendap.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/recentre.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/source.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/functions/sources/tendencies.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/input.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/patch.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/size.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/template.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/utils.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/create/zarr.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/concat.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/dataset.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/debug.css +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/debug.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/ensemble.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/forewards.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/grids.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/join.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/masked.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/select.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/statistics.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/data/subset.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/dates/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/dates/groups.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/grids.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi/datasets/utils/__init__.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/dependency_links.txt +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/entry_points.txt +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/src/anemoi_datasets.egg-info/top_level.txt +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/concat.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/data_sources.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/join.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/missing.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/nan.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/pipe.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/recentre.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create/test_create.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create-perturbations-full.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/create-shift.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/requirements.txt +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_chunks.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_dates.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tests/test_indexing.py +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/.gitignore +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/examples/Makefile +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/examples/an-oper-2023-2023-2p5-6h-v1.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/Makefile +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/grids.ipynb +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/grids1.yaml +0 -0
- {anemoi_datasets-0.2.1 → anemoi_datasets-0.3.1}/tools/grids/grids2.yaml +0 -0
|
@@ -13,19 +13,19 @@ repos:
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
16
|
-
rev: v4.
|
|
16
|
+
rev: v4.6.0
|
|
17
17
|
hooks:
|
|
18
18
|
- id: check-yaml # Check YAML files for syntax errors only
|
|
19
19
|
args: [--unsafe, --allow-multiple-documents]
|
|
20
20
|
- id: debug-statements # Check for debugger imports and py37+ breakpoint()
|
|
21
21
|
- id: end-of-file-fixer # Ensure files end in a newline
|
|
22
22
|
- id: trailing-whitespace # Trailing whitespace checker
|
|
23
|
-
|
|
23
|
+
- id: no-commit-to-branch # Prevent committing to main / master
|
|
24
24
|
- id: check-added-large-files # Check for large files added to git
|
|
25
25
|
- id: check-merge-conflict # Check for files that contain merge conflict
|
|
26
26
|
|
|
27
27
|
- repo: https://github.com/psf/black-pre-commit-mirror
|
|
28
|
-
rev: 24.
|
|
28
|
+
rev: 24.4.2
|
|
29
29
|
hooks:
|
|
30
30
|
- id: black
|
|
31
31
|
args: [--line-length=120]
|
|
@@ -41,7 +41,7 @@ repos:
|
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
44
|
-
rev: v0.
|
|
44
|
+
rev: v0.4.6
|
|
45
45
|
hooks:
|
|
46
46
|
- id: ruff
|
|
47
47
|
exclude: '(dev/.*|.*_)\.py$'
|
|
@@ -51,7 +51,6 @@ repos:
|
|
|
51
51
|
- --exit-non-zero-on-fix
|
|
52
52
|
- --preview
|
|
53
53
|
|
|
54
|
-
|
|
55
54
|
- repo: https://github.com/sphinx-contrib/sphinx-lint
|
|
56
55
|
rev: v0.9.1
|
|
57
56
|
hooks:
|
|
@@ -62,9 +61,21 @@ repos:
|
|
|
62
61
|
rev: v0.0.14
|
|
63
62
|
hooks:
|
|
64
63
|
- id: rstfmt
|
|
64
|
+
exclude: 'cli/.*' # Because we use argparse
|
|
65
65
|
|
|
66
66
|
- repo: https://github.com/b8raoult/pre-commit-docconvert
|
|
67
67
|
rev: "0.1.4"
|
|
68
68
|
hooks:
|
|
69
69
|
- id: docconvert
|
|
70
70
|
args: ["numpy"]
|
|
71
|
+
|
|
72
|
+
- repo: https://github.com/b8raoult/optional-dependencies-all
|
|
73
|
+
rev: "0.0.2"
|
|
74
|
+
hooks:
|
|
75
|
+
- id: optional-dependencies-all
|
|
76
|
+
args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs"]
|
|
77
|
+
|
|
78
|
+
- repo: https://github.com/tox-dev/pyproject-fmt
|
|
79
|
+
rev: "2.1.3"
|
|
80
|
+
hooks:
|
|
81
|
+
- id: pyproject-fmt
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: anemoi-datasets
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: A package to hold various functions to support training of ML models on ECMWF data.
|
|
5
5
|
Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
|
|
6
6
|
License: Apache License
|
|
@@ -205,60 +205,69 @@ License: Apache License
|
|
|
205
205
|
See the License for the specific language governing permissions and
|
|
206
206
|
limitations under the License.
|
|
207
207
|
|
|
208
|
-
Project-URL: Homepage, https://github.com/ecmwf/anemoi-datasets/
|
|
209
208
|
Project-URL: Documentation, https://anemoi-datasets.readthedocs.io/
|
|
210
|
-
Project-URL:
|
|
209
|
+
Project-URL: Homepage, https://github.com/ecmwf/anemoi-datasets/
|
|
211
210
|
Project-URL: Issues, https://github.com/ecmwf/anemoi-datasets/issues
|
|
212
|
-
|
|
211
|
+
Project-URL: Repository, https://github.com/ecmwf/anemoi-datasets/
|
|
212
|
+
Keywords: ai,datasets,tools
|
|
213
213
|
Classifier: Development Status :: 4 - Beta
|
|
214
214
|
Classifier: Intended Audience :: Developers
|
|
215
215
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
216
|
-
Classifier:
|
|
216
|
+
Classifier: Operating System :: OS Independent
|
|
217
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
217
218
|
Classifier: Programming Language :: Python :: 3.9
|
|
218
219
|
Classifier: Programming Language :: Python :: 3.10
|
|
219
220
|
Classifier: Programming Language :: Python :: 3.11
|
|
221
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
220
222
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
221
223
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
222
|
-
Classifier: Operating System :: OS Independent
|
|
223
224
|
Requires-Python: >=3.9
|
|
224
225
|
License-File: LICENSE
|
|
225
|
-
Requires-Dist: anemoi-utils[provenance]>=0.
|
|
226
|
-
Requires-Dist: zarr<=2.17.0
|
|
227
|
-
Requires-Dist: pyyaml
|
|
226
|
+
Requires-Dist: anemoi-utils[provenance]>=0.3
|
|
228
227
|
Requires-Dist: numpy
|
|
229
|
-
Requires-Dist:
|
|
228
|
+
Requires-Dist: pyyaml
|
|
230
229
|
Requires-Dist: semantic-version
|
|
231
|
-
|
|
232
|
-
Requires-Dist:
|
|
233
|
-
Requires-Dist: requests; extra == "remote"
|
|
234
|
-
Requires-Dist: s3fs; extra == "remote"
|
|
235
|
-
Provides-Extra: create
|
|
236
|
-
Requires-Dist: climetlab>=0.22.1; extra == "create"
|
|
237
|
-
Requires-Dist: earthkit-meteo; extra == "create"
|
|
238
|
-
Requires-Dist: pyproj; extra == "create"
|
|
239
|
-
Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
|
|
240
|
-
Provides-Extra: docs
|
|
241
|
-
Requires-Dist: sphinx; extra == "docs"
|
|
242
|
-
Requires-Dist: sphinx_rtd_theme; extra == "docs"
|
|
243
|
-
Requires-Dist: nbsphinx; extra == "docs"
|
|
244
|
-
Requires-Dist: pandoc; extra == "docs"
|
|
230
|
+
Requires-Dist: tqdm
|
|
231
|
+
Requires-Dist: zarr<=2.17
|
|
245
232
|
Provides-Extra: all
|
|
233
|
+
Requires-Dist: anemoi-utils[provenance]>=0.3; extra == "all"
|
|
246
234
|
Requires-Dist: boto3; extra == "all"
|
|
247
|
-
Requires-Dist: requests; extra == "all"
|
|
248
|
-
Requires-Dist: s3fs; extra == "all"
|
|
249
235
|
Requires-Dist: climetlab>=0.22.1; extra == "all"
|
|
250
236
|
Requires-Dist: earthkit-meteo; extra == "all"
|
|
251
|
-
Requires-Dist: pyproj; extra == "all"
|
|
252
237
|
Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
|
|
238
|
+
Requires-Dist: numpy; extra == "all"
|
|
239
|
+
Requires-Dist: pyproj; extra == "all"
|
|
240
|
+
Requires-Dist: pyyaml; extra == "all"
|
|
241
|
+
Requires-Dist: requests; extra == "all"
|
|
242
|
+
Requires-Dist: s3fs; extra == "all"
|
|
243
|
+
Requires-Dist: semantic-version; extra == "all"
|
|
244
|
+
Requires-Dist: tqdm; extra == "all"
|
|
245
|
+
Requires-Dist: zarr<=2.17; extra == "all"
|
|
246
|
+
Provides-Extra: create
|
|
247
|
+
Requires-Dist: climetlab>=0.22.1; extra == "create"
|
|
248
|
+
Requires-Dist: earthkit-meteo; extra == "create"
|
|
249
|
+
Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
|
|
250
|
+
Requires-Dist: pyproj; extra == "create"
|
|
253
251
|
Provides-Extra: dev
|
|
254
252
|
Requires-Dist: boto3; extra == "dev"
|
|
255
|
-
Requires-Dist: requests; extra == "dev"
|
|
256
|
-
Requires-Dist: s3fs; extra == "dev"
|
|
257
253
|
Requires-Dist: climetlab>=0.22.1; extra == "dev"
|
|
258
254
|
Requires-Dist: earthkit-meteo; extra == "dev"
|
|
259
|
-
Requires-Dist: pyproj; extra == "dev"
|
|
260
255
|
Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
|
|
261
|
-
Requires-Dist: sphinx; extra == "dev"
|
|
262
|
-
Requires-Dist: sphinx_rtd_theme; extra == "dev"
|
|
263
256
|
Requires-Dist: nbsphinx; extra == "dev"
|
|
264
257
|
Requires-Dist: pandoc; extra == "dev"
|
|
258
|
+
Requires-Dist: pyproj; extra == "dev"
|
|
259
|
+
Requires-Dist: requests; extra == "dev"
|
|
260
|
+
Requires-Dist: s3fs; extra == "dev"
|
|
261
|
+
Requires-Dist: sphinx; extra == "dev"
|
|
262
|
+
Requires-Dist: sphinx-argparse; extra == "dev"
|
|
263
|
+
Requires-Dist: sphinx-rtd-theme; extra == "dev"
|
|
264
|
+
Provides-Extra: docs
|
|
265
|
+
Requires-Dist: nbsphinx; extra == "docs"
|
|
266
|
+
Requires-Dist: pandoc; extra == "docs"
|
|
267
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
268
|
+
Requires-Dist: sphinx-argparse; extra == "docs"
|
|
269
|
+
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
270
|
+
Provides-Extra: remote
|
|
271
|
+
Requires-Dist: boto3; extra == "remote"
|
|
272
|
+
Requires-Dist: requests; extra == "remote"
|
|
273
|
+
Requires-Dist: s3fs; extra == "remote"
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#########################
|
|
2
|
+
Handling missing values
|
|
3
|
+
#########################
|
|
4
|
+
|
|
5
|
+
When handling data for machine learning models, missing values (`NaNs`)
|
|
6
|
+
can pose a challenge, as models require complete data to operate
|
|
7
|
+
effectively and may crash otherwise. Ideally, we anticipate having
|
|
8
|
+
complete data in all fields.
|
|
9
|
+
|
|
10
|
+
However, there are scenarios where `NaNs` naturally occur, such as with
|
|
11
|
+
variables only relevant on land or at sea. This happens for sea surface
|
|
12
|
+
temperature (`sst`), for example. In such cases, the default behavior is
|
|
13
|
+
to reject data with `NaNs` as invalid. To accommodate `NaNs` and
|
|
14
|
+
accurately compute statistics based on them, you can include the
|
|
15
|
+
``allow_nans`` key in the configuration.
|
|
16
|
+
|
|
17
|
+
Here's an example of how to implement it:
|
|
18
|
+
|
|
19
|
+
.. literalinclude:: yaml/nan.yaml
|
|
20
|
+
:language: yaml
|
|
@@ -24,9 +24,9 @@ dataset is unknown, the package assumes that the fields to use are
|
|
|
24
24
|
accumulated since the beginning of the forecast, over a 6h period.
|
|
25
25
|
|
|
26
26
|
The user can specify the desired accumulation period with the
|
|
27
|
-
``accumulation_period`` parameter. If its value is a single
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
``accumulation_period`` parameter. If its value is a single integer, the
|
|
28
|
+
source will attempt to accumulate the variables over that period. This
|
|
29
|
+
does not always mean that the data used is accumulated from the
|
|
30
30
|
beginning of the forecast, but the most recent data available will be
|
|
31
31
|
used:
|
|
32
32
|
|
|
@@ -44,13 +44,13 @@ used:
|
|
|
44
44
|
If the of ``accumulation_period`` value is a pair of integers `[step1,
|
|
45
45
|
step2]`, the algorithm is different. The source will compute the
|
|
46
46
|
accumulation between the `step1` and `step2` previous forecast that
|
|
47
|
-
|
|
47
|
+
validate at the given date at `step2`. For example, if the accumulation
|
|
48
48
|
period is `[6, 12]`, and the valid date is 2020-10-10 18:00, the source
|
|
49
49
|
will use the forecast of 2020-10-10 06:00 and the steps 6h and 12h.
|
|
50
50
|
|
|
51
51
|
Please note that ``accumulation_period=6`` and ``accumulation_period=[0,
|
|
52
52
|
6]`` are not equivalent. In the first case, the source can use return an
|
|
53
|
-
accumulation
|
|
53
|
+
accumulation between step 1h and step 7h if it is the most appropriate
|
|
54
54
|
data available, while in the second case, the source will always return
|
|
55
55
|
the accumulation between step 0h and step 6h, if available.
|
|
56
56
|
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
.. _sources:
|
|
2
|
+
|
|
3
|
+
#########
|
|
4
|
+
Sources
|
|
5
|
+
#########
|
|
6
|
+
|
|
7
|
+
The source is a software component that given a list of dates and
|
|
8
|
+
variables will return the corresponding fields.
|
|
9
|
+
|
|
10
|
+
A `source` is responsible for reading data from the source and
|
|
11
|
+
converting it to a set of fields. A `source` is also responsible for
|
|
12
|
+
handling the metadata of the data, such as the variables names, and
|
|
13
|
+
more.
|
|
14
|
+
|
|
15
|
+
A example of source is ECMWF’s MARS archive, a collection of GRIB or
|
|
16
|
+
NetCDF files, etc.
|
|
17
|
+
|
|
18
|
+
The following `sources` are currently available:
|
|
19
|
+
|
|
20
|
+
.. toctree::
|
|
21
|
+
:maxdepth: 1
|
|
22
|
+
|
|
23
|
+
sources/mars
|
|
24
|
+
sources/grib
|
|
25
|
+
sources/netcdf
|
|
26
|
+
sources/opendap
|
|
27
|
+
sources/forcings
|
|
28
|
+
sources/accumulations
|
|
29
|
+
sources/recentre
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
.. _gathering_statistics:
|
|
2
|
+
|
|
3
|
+
######################
|
|
4
|
+
Gathering statistics
|
|
5
|
+
######################
|
|
6
|
+
|
|
7
|
+
*Anemoi* will collect statistics about each variables in the dataset as
|
|
8
|
+
it is created. These statistics are intended to be used to normalise the
|
|
9
|
+
data during training.
|
|
10
|
+
|
|
11
|
+
The statistics are stored in the :doc:`statistics attribute
|
|
12
|
+
<../using/statistics>` of the dataset. The computed statistics include
|
|
13
|
+
`minimum, maximum, mean, standard deviation`.
|
|
14
|
+
|
|
15
|
+
************************
|
|
16
|
+
Statistics dates range
|
|
17
|
+
************************
|
|
18
|
+
|
|
19
|
+
By defaults, the statistics are not computed on the whole dataset, but
|
|
20
|
+
on a subset of dates. This usually is done to avoid any data leakage
|
|
21
|
+
from the validation and test sets to the training set.
|
|
22
|
+
|
|
23
|
+
The dates subset used to compute the statistics is defined using the
|
|
24
|
+
following algorithm:
|
|
25
|
+
|
|
26
|
+
- If the dataset covers 20 years or more, the last 3 years are
|
|
27
|
+
excluded.
|
|
28
|
+
- If the dataset covers 10 years or more, the last year is excluded.
|
|
29
|
+
- Otherwise, 80% of the dataset is used.
|
|
30
|
+
|
|
31
|
+
You can override this behaviour by setting either the `start` parameter
|
|
32
|
+
or the `end` parameters in the `statistics` config.
|
|
33
|
+
|
|
34
|
+
Example configuration gathering statistics from 2000 to 2020 :
|
|
35
|
+
|
|
36
|
+
.. code:: yaml
|
|
37
|
+
|
|
38
|
+
statistics:
|
|
39
|
+
start: 2000
|
|
40
|
+
end: 2020
|
|
41
|
+
|
|
42
|
+
Example configuration gathering statistics from the beginning of the
|
|
43
|
+
dataset period to 2020 :
|
|
44
|
+
|
|
45
|
+
.. code:: yaml
|
|
46
|
+
|
|
47
|
+
statistics:
|
|
48
|
+
end: 2020
|
|
49
|
+
|
|
50
|
+
Example configuration gathering statistics using only 2020 data :
|
|
51
|
+
|
|
52
|
+
.. code:: yaml
|
|
53
|
+
|
|
54
|
+
statistics:
|
|
55
|
+
start: 2020
|
|
56
|
+
end: 2020
|
|
57
|
+
|
|
58
|
+
**************************
|
|
59
|
+
Data with missing values
|
|
60
|
+
**************************
|
|
61
|
+
|
|
62
|
+
If the dataset contains missing values (known as `NaNs`), an error will
|
|
63
|
+
be raised when trying to compute the statistics. To allow `NaNs` in the
|
|
64
|
+
dataset, you can set the `allow_nans` as described :doc:`here
|
|
65
|
+
</building/handling-missing-values>`.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
compare
|
|
2
|
+
=======
|
|
3
|
+
|
|
4
|
+
Use this command to compare two datasets.
|
|
5
|
+
|
|
6
|
+
The command will run a quick comparison of the two datasets and output a summary of the differences.
|
|
7
|
+
|
|
8
|
+
.. warning::
|
|
9
|
+
|
|
10
|
+
This command will not compare the data in the datasets, only some of the metadata.
|
|
11
|
+
Subsequent versions of this command may include more detailed comparisons.
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
.. argparse::
|
|
15
|
+
:module: anemoi.datasets.__main__
|
|
16
|
+
:func: create_parser
|
|
17
|
+
:prog: anemoi-datasets
|
|
18
|
+
:path: compare
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
copy
|
|
2
|
+
====
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
Copying a dataset from one location to another can be error-prone and time-consuming.
|
|
6
|
+
This command-line script allows for incremental copying.
|
|
7
|
+
When the copying process fails, it can be resumed.
|
|
8
|
+
It can be used to copy files from a local directory to a remote server, from a remote server to a local directory as long as there is a zarr backend to read and write the data.
|
|
9
|
+
|
|
10
|
+
The script uses multiple threads to make the process faster.
|
|
11
|
+
However, it is important to consider that making parallel requests to the same server may not be ideal, for instance if the server internally uses a limited number of threads to handle requests.
|
|
12
|
+
|
|
13
|
+
The option to rechunk the data is available, which can be useful when the data is stored on a platform that does not support having may small files or many file on the same directory.
|
|
14
|
+
However keep in mind that rechunking has a huge impact on the performance when reading the data:
|
|
15
|
+
The chunk pattern for the source dataset has been defined for good reasons, and changing it is very likey to have a negative impact on the performance.
|
|
16
|
+
|
|
17
|
+
.. warning::
|
|
18
|
+
|
|
19
|
+
When resuming the copying process (using ``--resume``), calling the script with the same arguments for ``--block-size`` and ``--rechunk`` is recommended.
|
|
20
|
+
Using different values for these arguments to resume copying the same dataset may lead to unexpected behavior.
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
.. argparse::
|
|
24
|
+
:module: anemoi.datasets.__main__
|
|
25
|
+
:func: create_parser
|
|
26
|
+
:prog: anemoi-datasets
|
|
27
|
+
:path: copy
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
create
|
|
2
|
+
======
|
|
3
|
+
|
|
4
|
+
Use this command to create a dataset from a recipe file.
|
|
5
|
+
The syntax of the recipe file is described in :doc:`building datasets <../building/introduction>`.
|
|
6
|
+
|
|
7
|
+
.. argparse::
|
|
8
|
+
:module: anemoi.datasets.__main__
|
|
9
|
+
:func: create_parser
|
|
10
|
+
:prog: anemoi-datasets
|
|
11
|
+
:path: create
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
inspect
|
|
2
|
+
=======
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
Anemoi datasets are stored in a zarr format and can be located on a local file system or on a remote server.
|
|
6
|
+
The `inspect` command is used to inspect the contents of a dataset.
|
|
7
|
+
This command will output the metadata of the dataset, including the variables, dimensions, and attributes.
|
|
8
|
+
|
|
9
|
+
.. code:: console
|
|
10
|
+
|
|
11
|
+
$ anemoi-datasets inspect dataset.zarr
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
which will output something like the following. The output should be self-explanatory.
|
|
15
|
+
|
|
16
|
+
.. literalinclude:: ../building/yaml/building1.txt
|
|
17
|
+
:language: console
|
|
18
|
+
|
|
19
|
+
*********************
|
|
20
|
+
Command line usage
|
|
21
|
+
*********************
|
|
22
|
+
|
|
23
|
+
.. argparse::
|
|
24
|
+
:module: anemoi.datasets.__main__
|
|
25
|
+
:func: create_parser
|
|
26
|
+
:prog: anemoi-datasets
|
|
27
|
+
:path: inspect
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Introduction
|
|
2
|
+
============
|
|
3
|
+
|
|
4
|
+
When you install the `anemoi-datasets` package, this will also install command line tool
|
|
5
|
+
called ``anemoi-datasets`` which can be used to manage the zarr datasets.
|
|
6
|
+
|
|
7
|
+
The tool can provide help with the ``--help`` options:
|
|
8
|
+
|
|
9
|
+
.. code-block:: bash
|
|
10
|
+
|
|
11
|
+
% anemoi-datasets --help
|
|
12
|
+
|
|
13
|
+
The commands are:
|
|
14
|
+
|
|
15
|
+
.. toctree::
|
|
16
|
+
:maxdepth: 1
|
|
17
|
+
|
|
18
|
+
compare
|
|
19
|
+
copy
|
|
20
|
+
create
|
|
21
|
+
inspect
|
|
22
|
+
scan
|
|
23
|
+
|
|
24
|
+
.. argparse::
|
|
25
|
+
:module: anemoi.datasets.__main__
|
|
26
|
+
:func: create_parser
|
|
27
|
+
:prog: anemoi-datasets
|
|
28
|
+
:nosubcommands:
|
|
@@ -16,11 +16,11 @@
|
|
|
16
16
|
|
|
17
17
|
import datetime
|
|
18
18
|
import os
|
|
19
|
+
import sys
|
|
19
20
|
|
|
20
21
|
read_the_docs_build = os.environ.get("READTHEDOCS", None) == "True"
|
|
21
22
|
|
|
22
|
-
|
|
23
|
-
# sys.path.insert(0, top)
|
|
23
|
+
sys.path.insert(0, os.path.join(os.path.abspath(".."), "src"))
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
source_suffix = ".rst"
|
|
@@ -45,7 +45,12 @@ else:
|
|
|
45
45
|
copyright = "%s, ECMWF" % (years,)
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
try:
|
|
49
|
+
from anemoi.datasets._version import __version__
|
|
50
|
+
|
|
51
|
+
release = __version__
|
|
52
|
+
except ImportError:
|
|
53
|
+
release = "0.0.0"
|
|
49
54
|
|
|
50
55
|
|
|
51
56
|
# -- General configuration ---------------------------------------------------
|
|
@@ -59,6 +64,9 @@ extensions = [
|
|
|
59
64
|
"nbsphinx",
|
|
60
65
|
"sphinx.ext.graphviz",
|
|
61
66
|
"sphinx.ext.intersphinx",
|
|
67
|
+
"sphinx.ext.autodoc",
|
|
68
|
+
"sphinx.ext.napoleon",
|
|
69
|
+
"sphinxarg.ext",
|
|
62
70
|
]
|
|
63
71
|
|
|
64
72
|
# Add any paths that contain templates here, relative to this directory.
|
|
@@ -112,3 +120,5 @@ html_css_files = ["style.css"]
|
|
|
112
120
|
|
|
113
121
|
|
|
114
122
|
todo_include_todos = not read_the_docs_build
|
|
123
|
+
|
|
124
|
+
autodoc_member_order = "bysource" # Keep file order
|
|
@@ -5,7 +5,7 @@ config = {
|
|
|
5
5
|
"ensemble": [
|
|
6
6
|
"/path/to/dataset1.zarr",
|
|
7
7
|
{"dataset": "dataset_name", "end": 2010},
|
|
8
|
-
{"dataset": "s3://path/to/dataset3.zarr", "
|
|
8
|
+
{"dataset": "s3://path/to/dataset3.zarr", "start": 2000, "end": 2010},
|
|
9
9
|
],
|
|
10
10
|
"frequency": "24h",
|
|
11
11
|
},
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# (C) Copyright 2024 ECMWF.
|
|
3
|
+
#
|
|
4
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
5
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
6
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
7
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
8
|
+
# nor does it submit to any jurisdiction.
|
|
9
|
+
|
|
10
|
+
# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
|
|
11
|
+
|
|
12
|
+
[build-system]
|
|
13
|
+
requires = [
|
|
14
|
+
"setuptools>=60",
|
|
15
|
+
"setuptools-scm>=8",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[project]
|
|
19
|
+
name = "anemoi-datasets"
|
|
20
|
+
|
|
21
|
+
description = "A package to hold various functions to support training of ML models on ECMWF data."
|
|
22
|
+
keywords = [
|
|
23
|
+
"ai",
|
|
24
|
+
"datasets",
|
|
25
|
+
"tools",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
license = { file = "LICENSE" }
|
|
29
|
+
authors = [
|
|
30
|
+
{ name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" },
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
requires-python = ">=3.9"
|
|
34
|
+
|
|
35
|
+
classifiers = [
|
|
36
|
+
"Development Status :: 4 - Beta",
|
|
37
|
+
"Intended Audience :: Developers",
|
|
38
|
+
"License :: OSI Approved :: Apache Software License",
|
|
39
|
+
"Operating System :: OS Independent",
|
|
40
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
41
|
+
"Programming Language :: Python :: 3.9",
|
|
42
|
+
"Programming Language :: Python :: 3.10",
|
|
43
|
+
"Programming Language :: Python :: 3.11",
|
|
44
|
+
"Programming Language :: Python :: 3.12",
|
|
45
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
46
|
+
"Programming Language :: Python :: Implementation :: PyPy",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
dynamic = [
|
|
50
|
+
"version",
|
|
51
|
+
]
|
|
52
|
+
dependencies = [
|
|
53
|
+
"anemoi-utils[provenance]>=0.3",
|
|
54
|
+
"numpy",
|
|
55
|
+
"pyyaml",
|
|
56
|
+
"semantic-version",
|
|
57
|
+
"tqdm",
|
|
58
|
+
"zarr<=2.17",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
optional-dependencies.all = [
|
|
62
|
+
"anemoi-utils[provenance]>=0.3",
|
|
63
|
+
"boto3",
|
|
64
|
+
"climetlab>=0.22.1",
|
|
65
|
+
"earthkit-meteo",
|
|
66
|
+
"ecmwflibs>=0.6.3",
|
|
67
|
+
"numpy",
|
|
68
|
+
"pyproj",
|
|
69
|
+
"pyyaml",
|
|
70
|
+
"requests",
|
|
71
|
+
"s3fs",
|
|
72
|
+
"semantic-version",
|
|
73
|
+
"tqdm",
|
|
74
|
+
"zarr<=2.17",
|
|
75
|
+
]
|
|
76
|
+
optional-dependencies.create = [
|
|
77
|
+
"climetlab>=0.22.1", # "earthkit-data"
|
|
78
|
+
"earthkit-meteo",
|
|
79
|
+
"ecmwflibs>=0.6.3",
|
|
80
|
+
"pyproj",
|
|
81
|
+
]
|
|
82
|
+
optional-dependencies.dev = [
|
|
83
|
+
"boto3",
|
|
84
|
+
"climetlab>=0.22.1", # "earthkit-data"
|
|
85
|
+
"earthkit-meteo",
|
|
86
|
+
"ecmwflibs>=0.6.3",
|
|
87
|
+
"nbsphinx",
|
|
88
|
+
"pandoc",
|
|
89
|
+
"pyproj",
|
|
90
|
+
"requests",
|
|
91
|
+
"s3fs",
|
|
92
|
+
"sphinx",
|
|
93
|
+
"sphinx-argparse",
|
|
94
|
+
"sphinx-rtd-theme",
|
|
95
|
+
]
|
|
96
|
+
optional-dependencies.docs = [
|
|
97
|
+
"nbsphinx",
|
|
98
|
+
"pandoc",
|
|
99
|
+
"sphinx",
|
|
100
|
+
"sphinx-argparse",
|
|
101
|
+
"sphinx-rtd-theme",
|
|
102
|
+
]
|
|
103
|
+
optional-dependencies.remote = [
|
|
104
|
+
"boto3",
|
|
105
|
+
"requests",
|
|
106
|
+
"s3fs",
|
|
107
|
+
]
|
|
108
|
+
urls.Documentation = "https://anemoi-datasets.readthedocs.io/"
|
|
109
|
+
urls.Homepage = "https://github.com/ecmwf/anemoi-datasets/"
|
|
110
|
+
urls.Issues = "https://github.com/ecmwf/anemoi-datasets/issues"
|
|
111
|
+
# Changelog = "https://github.com/ecmwf/anemoi-datasets/CHANGELOG.md"
|
|
112
|
+
urls.Repository = "https://github.com/ecmwf/anemoi-datasets/"
|
|
113
|
+
scripts.anemoi-datasets = "anemoi.datasets.__main__:main"
|
|
114
|
+
|
|
115
|
+
[tool.setuptools.package-data]
|
|
116
|
+
"anemoi.datasets.data" = [
|
|
117
|
+
"*.css",
|
|
118
|
+
]
|
|
119
|
+
|
|
120
|
+
[tool.setuptools_scm]
|
|
121
|
+
version_file = "src/anemoi/datasets/_version.py"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# (C) Copyright 2024 ECMWF.
|
|
3
|
+
#
|
|
4
|
+
# This software is licensed under the terms of the Apache Licence Version 2.0
|
|
5
|
+
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
|
|
6
|
+
# In applying this licence, ECMWF does not waive the privileges and immunities
|
|
7
|
+
# granted to it by virtue of its status as an intergovernmental organisation
|
|
8
|
+
# nor does it submit to any jurisdiction.
|
|
9
|
+
#
|
|
10
|
+
|
|
11
|
+
from anemoi.utils.cli import cli_main
|
|
12
|
+
from anemoi.utils.cli import make_parser
|
|
13
|
+
|
|
14
|
+
from . import __version__
|
|
15
|
+
from .commands import COMMANDS
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# For read-the-docs
|
|
19
|
+
def create_parser():
|
|
20
|
+
return make_parser(__doc__, COMMANDS)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def main():
|
|
24
|
+
cli_main(__version__, __doc__, COMMANDS)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
if __name__ == "__main__":
|
|
28
|
+
main()
|