anemoi-datasets 0.3.5__tar.gz → 0.3.7__tar.gz

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/.github/workflows/python-publish.yml

@@ -41,8 +41,7 @@ jobs:
 
     - name: Install
       run: |
-        pip install pytest
-        pip install -e .[all,test]
+        pip install -e .[all,tests]
         pip freeze
 
     - name: Tests

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/.pre-commit-config.yaml

@@ -37,7 +37,8 @@ repos:
   rev: v0.4.6
   hooks:
   - id: ruff
-    exclude: '(dev/.*|.*_)\.py$'
+    # Next line if for documenation cod snippets
+    exclude: '^(dev/.*|[A-Za-z].*_)\.py$'
     args:
     - --line-length=120
     - --fix
@@ -59,10 +60,10 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.2"
+  rev: "0.0.6"
   hooks:
   - id: optional-dependencies-all
-    args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs,test"]
+    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
   rev: "2.1.3"
   hooks:

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.3.5
+Version: 0.3.7
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,28 +223,21 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-utils[provenance]>=0.3
+Requires-Dist: anemoi-utils[provenance]>=0.3.4
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
 Requires-Dist: zarr<=2.17
 Provides-Extra: all
-Requires-Dist: anemoi-utils[provenance]>=0.3; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: climetlab>=0.23.2; extra == "all"
 Requires-Dist: earthkit-meteo; extra == "all"
 Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
 Requires-Dist: entrypoints; extra == "all"
-Requires-Dist: numpy; extra == "all"
 Requires-Dist: pyproj; extra == "all"
-Requires-Dist: pytest; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: requests; extra == "all"
 Requires-Dist: s3fs; extra == "all"
-Requires-Dist: semantic-version; extra == "all"
-Requires-Dist: tqdm; extra == "all"
-Requires-Dist: zarr<=2.17; extra == "all"
 Provides-Extra: create
 Requires-Dist: climetlab>=0.23.2; extra == "create"
 Requires-Dist: earthkit-meteo; extra == "create"
@@ -253,11 +246,14 @@ Requires-Dist: entrypoints; extra == "create"
 Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
 Requires-Dist: boto3; extra == "dev"
+Requires-Dist: climetlab>=0.23.2; extra == "dev"
 Requires-Dist: earthkit-meteo; extra == "dev"
 Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
+Requires-Dist: entrypoints; extra == "dev"
 Requires-Dist: nbsphinx; extra == "dev"
 Requires-Dist: pandoc; extra == "dev"
 Requires-Dist: pyproj; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
 Requires-Dist: requests; extra == "dev"
 Requires-Dist: s3fs; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
@@ -273,5 +269,5 @@ Provides-Extra: remote
 Requires-Dist: boto3; extra == "remote"
 Requires-Dist: requests; extra == "remote"
 Requires-Dist: s3fs; extra == "remote"
-Provides-Extra: test
-Requires-Dist: pytest; extra == "test"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/pyproject.toml

@@ -50,7 +50,7 @@ dynamic = [
   "version",
 ]
 dependencies = [
-  "anemoi-utils[provenance]>=0.3",
+  "anemoi-utils[provenance]>=0.3.4",
   "numpy",
   "pyyaml",
   "semantic-version",
@@ -59,24 +59,17 @@ dependencies = [
 ]
 
 optional-dependencies.all = [
-  "anemoi-utils[provenance]>=0.3",
   "boto3",
   "climetlab>=0.23.2",
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
   "entrypoints",
-  "numpy",
   "pyproj",
-  "pytest",
-  "pyyaml",
   "requests",
   "s3fs",
-  "semantic-version",
-  "tqdm",
-  "zarr<=2.17",
 ]
 optional-dependencies.create = [
-  "climetlab>=0.23.2", # "earthkit-data"
+  "climetlab>=0.23.2",
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
   "entrypoints",
@@ -85,12 +78,14 @@ optional-dependencies.create = [
 
 optional-dependencies.dev = [
   "boto3",
-
+  "climetlab>=0.23.2",
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
+  "entrypoints",
   "nbsphinx",
   "pandoc",
   "pyproj",
+  "pytest",
   "requests",
   "s3fs",
   "sphinx",
@@ -109,7 +104,7 @@ optional-dependencies.remote = [
   "requests",
   "s3fs",
 ]
-optional-dependencies.test = [
+optional-dependencies.tests = [
   "pytest",
 ]
 urls.Documentation = "https://anemoi-datasets.readthedocs.io/"

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/__init__.py

@@ -9,6 +9,7 @@ from ._version import __version__
 from .data import MissingDateError
 from .data import add_dataset_path
 from .data import add_named_dataset
+from .data import list_dataset_names
 from .data import open_dataset
 
 __all__ = [
@@ -16,4 +17,5 @@ __all__ = [
     "MissingDateError",
     "add_dataset_path",
     "add_named_dataset",
+    "list_dataset_names",
 ]

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.5'
-__version_tuple__ = version_tuple = (0, 3, 5)
+__version__ = version = '0.3.7'
+__version_tuple__ = version_tuple = (0, 3, 7)

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/commands/compare.py

@@ -14,6 +14,8 @@ from . import Command
 
 
 class Compare(Command):
+    """Compare two datasets. This command compares the variables in two datasets and prints the mean of the common variables. It does not compare the data itself (yet)."""
+
     def add_arguments(self, command_parser):
         command_parser.add_argument("dataset1")
         command_parser.add_argument("dataset2")

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/commands/copy.py

@@ -12,6 +12,8 @@ from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import as_completed
 
 import tqdm
+from anemoi.utils.s3 import download
+from anemoi.utils.s3 import upload
 
 from . import Command
 
@@ -22,26 +24,48 @@ try:
 except AttributeError:
     isatty = False
 
-"""
 
-~/.aws/credentials
+class S3Downloader:
+    def __init__(self, source, target, transfers, overwrite, resume, progress, **kwargs):
+        self.source = source
+        self.target = target
+        self.transfers = transfers
+        self.overwrite = overwrite
+        self.resume = resume
+        self.progress = progress
 
-[default]
-endpoint_url = https://object-store.os-api.cci1.ecmwf.int
-aws_access_key_id=xxx
-aws_secret_access_key=xxxx
+    def run(self):
+        download(
+            self.source + "/" if not self.source.endswith("/") else self.source,
+            self.target,
+            overwrite=self.overwrite,
+            ignore_existing=self.resume,
+            threads=self.transfers,
+            show_progress=self.progress,
+        )
 
-Then:
 
-anemoi-datasets copy aifs-ea-an-oper-0001-mars-o96-1979-2022-1h-v3.zarr/
-    s3://ml-datasets/stable/aifs-ea-an-oper-0001-mars-o96-1979-2022-1h-v3.zarr
+class S3Uploader:
+    def __init__(self, source, target, transfers, overwrite, resume, progress, **kwargs):
+        self.source = source
+        self.target = target
+        self.transfers = transfers
+        self.overwrite = overwrite
+        self.resume = resume
+        self.progress = progress
 
-zinfo https://object-store.os-api.cci1.ecmwf.int/
-    ml-datasets/stable/aifs-ea-an-oper-0001-mars-o96-1979-2022-1h-v3.zarr
-"""
+    def run(self):
+        upload(
+            self.source,
+            self.target,
+            overwrite=self.overwrite,
+            ignore_existing=self.resume,
+            threads=self.transfers,
+            show_progress=self.progress,
+        )
 
 
-class Copier:
+class DefaultCopier:
     def __init__(self, source, target, transfers, block_size, overwrite, resume, progress, nested, rechunk, **kwargs):
         self.source = source
         self.target = target
@@ -295,7 +319,29 @@ class CopyMixin:
         command_parser.add_argument("target", help="Target location.")
 
     def run(self, args):
-        Copier(**vars(args)).run()
+        if args.source == args.target:
+            raise ValueError("Source and target are the same.")
+
+        kwargs = vars(args)
+
+        if args.overwrite and args.resume:
+            raise ValueError("Cannot use --overwrite and --resume together.")
+
+        source_in_s3 = args.source.startswith("s3://")
+        target_in_s3 = args.target.startswith("s3://")
+
+        copier = None
+
+        if args.rechunk or (source_in_s3 and target_in_s3):
+            copier = DefaultCopier(**kwargs)
+        else:
+            if source_in_s3:
+                copier = S3Downloader(**kwargs)
+
+            if target_in_s3:
+                copier = S3Uploader(**kwargs)
+
+        copier.run()
 
 
 class Copy(CopyMixin, Command):

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/compute/recentre.py

@@ -29,7 +29,7 @@ CLIP_VARIABLES = (
     "swl1",
 )
 
-SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset")
+SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset", "time", "date", "step")
 
 
 def check_compatible(f1, f2, centre_field_as_mars, ensemble_field_as_mars):

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/create/functions/filters/rename.py

@@ -13,6 +13,14 @@ from climetlab.indexing.fieldset import FieldArray
 
 
 class RenamedFieldMapping:
+    """Rename a field based on the value of another field.
+
+    Args:
+        field (Field): The field to be renamed.
+        what (str): The name of the field that will be used to rename the field.
+        renaming (dict): A dictionary mapping the values of 'what' to the new names.
+    """
+
     def __init__(self, field, what, renaming):
         self.field = field
         self.what = what
@@ -29,6 +37,12 @@ class RenamedFieldMapping:
 
 
 class RenamedFieldFormat:
+    """Rename a field based on a format string.
+
+    Args:
+        format (str): A string that defines the new name of the field.
+    """
+
     def __init__(self, field, format):
         self.field = field
         self.format = format

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/create/loaders.py

@@ -177,11 +177,14 @@ class InitialiserLoader(Loader):
                     return
                 if isinstance(obj, (dict, DictObj)):
                     if "grid" in obj:
+                        previous = obj["grid"]
                         obj["grid"] = "20./20."
-                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']}")
+                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']} instead of {previous}")
                     if "number" in obj:
-                        obj["number"] = obj["number"][0:3]
-                        LOG.warn(f"Running in test mode. Setting number to {obj['number']}")
+                        if isinstance(obj["number"], (list, tuple)):
+                            previous = obj["number"]
+                            obj["number"] = previous[0:3]
+                            LOG.warn(f"Running in test mode. Setting number to {obj['number']} instead of {previous}")
                     for k, v in obj.items():
                         set_to_test_mode(v)
 

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/data/__init__.py

@@ -30,3 +30,10 @@ def open_dataset(*args, **kwargs):
     ds.arguments = {"args": args, "kwargs": kwargs}
     ds._check()
     return ds
+
+
+def list_dataset_names(*args, **kwargs):
+    ds = _open_dataset(*args, **kwargs)
+    names = set()
+    ds.get_dataset_names(names)
+    return sorted(names)

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/data/dataset.py

@@ -226,3 +226,6 @@ class Dataset:
     @property
     def label(self):
         return self.__class__.__name__.lower()
+
+    def get_dataset_names(self, names):
+        raise NotImplementedError(self)

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/data/forwards.py

@@ -103,6 +103,9 @@ class Forwards(Dataset):
             f"subclass_metadata_specific() must be implemented in derived class {self.__class__.__name__}"
         )
 
+    def get_dataset_names(self, names):
+        self.forward.get_dataset_names(names)
+
 
 class Combined(Forwards):
     def __init__(self, datasets):
@@ -193,6 +196,10 @@ class Combined(Forwards):
             offset += len(d)
         return result
 
+    def get_dataset_names(self, names):
+        for d in self.datasets:
+            d.get_dataset_names(names)
+
 
 class GivenAxis(Combined):
     """Given a given axis, combine the datasets along that axis."""

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/data/masked.py

@@ -70,8 +70,12 @@ class Thinning(Masked):
         self.thinning = thinning
         self.method = method
 
-        latitudes = forward.latitudes.reshape(forward.field_shape)
-        longitudes = forward.longitudes.reshape(forward.field_shape)
+        shape = forward.field_shape
+        if len(shape) != 2:
+            raise ValueError("Thinning only works latitude/longitude fields")
+
+        latitudes = forward.latitudes.reshape(shape)
+        longitudes = forward.longitudes.reshape(shape)
         latitudes = latitudes[::thinning, ::thinning].flatten()
         longitudes = longitudes[::thinning, ::thinning].flatten()
 

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/data/misc.py

@@ -133,9 +133,9 @@ def _as_date(d, dates, last):
 
         if "-" in d and ":" in d:
             date, time = d.replace(" ", "T").split("T")
-            year, month, day = date.split("-")
-            hour, minute, second = time.split(":")
-            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour}:{minute}:{second}")
+            year, month, day = [int(_) for _ in date.split("-")]
+            hour, minute, second = [int(_) for _ in time.split(":")]
+            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}")
 
         if "-" in d:
             assert ":" not in d

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/data/statistics.py

@@ -39,3 +39,7 @@ class Statistics(Forwards):
 
     def tree(self):
         return Node(self, [self.forward.tree()])
+
+    def get_dataset_names(self, names):
+        super().get_dataset_names(names)
+        self._statistic.get_dataset_names(names)

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi/datasets/data/stores.py

@@ -6,6 +6,7 @@
 # nor does it submit to any jurisdiction.
 
 import logging
+import os
 import warnings
 from functools import cached_property
 
@@ -235,7 +236,11 @@ class Zarr(Dataset):
 
     @property
     def field_shape(self):
-        return tuple(self.z.attrs["field_shape"])
+        try:
+            return tuple(self.z.attrs["field_shape"])
+        except KeyError:
+            LOG.warning("No 'field_shape' in %r, assuming 1D fields", self)
+            return (self.shape[-1],)
 
     @property
     def frequency(self):
@@ -288,6 +293,10 @@ class Zarr(Dataset):
     def tree(self):
         return Node(self, [], path=self.path)
 
+    def get_dataset_names(self, names):
+        name, _ = os.path.splitext(os.path.basename(self.path))
+        names.add(name)
+
 
 class ZarrWithMissingDates(Zarr):
     def __init__(self, path):

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi_datasets.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.3.5
+Version: 0.3.7
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,28 +223,21 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-utils[provenance]>=0.3
+Requires-Dist: anemoi-utils[provenance]>=0.3.4
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
 Requires-Dist: zarr<=2.17
 Provides-Extra: all
-Requires-Dist: anemoi-utils[provenance]>=0.3; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: climetlab>=0.23.2; extra == "all"
 Requires-Dist: earthkit-meteo; extra == "all"
 Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
 Requires-Dist: entrypoints; extra == "all"
-Requires-Dist: numpy; extra == "all"
 Requires-Dist: pyproj; extra == "all"
-Requires-Dist: pytest; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: requests; extra == "all"
 Requires-Dist: s3fs; extra == "all"
-Requires-Dist: semantic-version; extra == "all"
-Requires-Dist: tqdm; extra == "all"
-Requires-Dist: zarr<=2.17; extra == "all"
 Provides-Extra: create
 Requires-Dist: climetlab>=0.23.2; extra == "create"
 Requires-Dist: earthkit-meteo; extra == "create"
@@ -253,11 +246,14 @@ Requires-Dist: entrypoints; extra == "create"
 Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
 Requires-Dist: boto3; extra == "dev"
+Requires-Dist: climetlab>=0.23.2; extra == "dev"
 Requires-Dist: earthkit-meteo; extra == "dev"
 Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
+Requires-Dist: entrypoints; extra == "dev"
 Requires-Dist: nbsphinx; extra == "dev"
 Requires-Dist: pandoc; extra == "dev"
 Requires-Dist: pyproj; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
 Requires-Dist: requests; extra == "dev"
 Requires-Dist: s3fs; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
@@ -273,5 +269,5 @@ Provides-Extra: remote
 Requires-Dist: boto3; extra == "remote"
 Requires-Dist: requests; extra == "remote"
 Requires-Dist: s3fs; extra == "remote"
-Provides-Extra: test
-Requires-Dist: pytest; extra == "test"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"

{anemoi_datasets-0.3.5 → anemoi_datasets-0.3.7}/src/anemoi_datasets.egg-info/requires.txt

@@ -1,4 +1,4 @@
-anemoi-utils[provenance]>=0.3
+anemoi-utils[provenance]>=0.3.4
 numpy
 pyyaml
 semantic-version
@@ -6,21 +6,14 @@ tqdm
 zarr<=2.17
 
 [all]
-anemoi-utils[provenance]>=0.3
 boto3
 climetlab>=0.23.2
 earthkit-meteo
 ecmwflibs>=0.6.3
 entrypoints
-numpy
 pyproj
-pytest
-pyyaml
 requests
 s3fs
-semantic-version
-tqdm
-zarr<=2.17
 
 [create]
 climetlab>=0.23.2
@@ -31,11 +24,14 @@ pyproj
 
 [dev]
 boto3
+climetlab>=0.23.2
 earthkit-meteo
 ecmwflibs>=0.6.3
+entrypoints
 nbsphinx
 pandoc
 pyproj
+pytest
 requests
 s3fs
 sphinx
@@ -54,5 +50,5 @@ boto3
 requests
 s3fs
 
-[test]
+[tests]
 pytest