anemoi-datasets 0.3.3__tar.gz → 0.3.6__tar.gz

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/.github/workflows/python-publish.yml

@@ -41,9 +41,7 @@ jobs:
 
     - name: Install
       run: |
-        pip install pytest
-        pip install -e .[all]
-        pip install -r tests/requirements.txt
+        pip install -e .[all,tests]
         pip freeze
 
     - name: Tests

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/.pre-commit-config.yaml

@@ -1,17 +1,14 @@
 repos:
-
 # Empty notebookds
 - repo: local
   hooks:
-    - id: clear-notebooks-output
-      name: clear-notebooks-output
-      files: tools/.*\.ipynb$
-      stages: [commit]
-      language: python
-      entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
-      additional_dependencies: [jupyter]
-
-
+  - id: clear-notebooks-output
+    name: clear-notebooks-output
+    files: tools/.*\.ipynb$
+    stages: [commit]
+    language: python
+    entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
+    additional_dependencies: [jupyter]
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.6.0
   hooks:
@@ -23,13 +20,11 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
-
 - repo: https://github.com/psf/black-pre-commit-mirror
   rev: 24.4.2
   hooks:
-    - id: black
-      args: [--line-length=120]
-
+  - id: black
+    args: [--line-length=120]
 - repo: https://github.com/pycqa/isort
   rev: 5.13.2
   hooks:
@@ -38,44 +33,38 @@ repos:
     - -l 120
     - --force-single-line-imports
     - --profile black
-
-
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.4.6
   hooks:
   - id: ruff
-    exclude: '(dev/.*|.*_)\.py$'
+    # Next line if for documenation cod snippets
+    exclude: '^(dev/.*|[A-Za-z].*_)\.py$'
     args:
     - --line-length=120
     - --fix
     - --exit-non-zero-on-fix
     - --preview
-
 - repo: https://github.com/sphinx-contrib/sphinx-lint
   rev: v0.9.1
   hooks:
-    - id: sphinx-lint
-
+  - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
 - repo: https://github.com/dzhu/rstfmt
   rev: v0.0.14
   hooks:
-    - id: rstfmt
-      exclude: 'cli/.*' # Because we use argparse
-
+  - id: rstfmt
+    exclude: 'cli/.*' # Because we use argparse
 - repo: https://github.com/b8raoult/pre-commit-docconvert
   rev: "0.1.5"
   hooks:
   - id: docconvert
     args: ["numpy"]
-
 - repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.2"
+  rev: "0.0.6"
   hooks:
   - id: optional-dependencies-all
-    args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs"]
-
+    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
   rev: "2.1.3"
   hooks:
-    - id: pyproject-fmt
+  - id: pyproject-fmt

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.3.3
+Version: 0.3.6
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -230,20 +230,14 @@ Requires-Dist: semantic-version
 Requires-Dist: tqdm
 Requires-Dist: zarr<=2.17
 Provides-Extra: all
-Requires-Dist: anemoi-utils[provenance]>=0.3; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: climetlab>=0.23.2; extra == "all"
 Requires-Dist: earthkit-meteo; extra == "all"
 Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
 Requires-Dist: entrypoints; extra == "all"
-Requires-Dist: numpy; extra == "all"
 Requires-Dist: pyproj; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: requests; extra == "all"
 Requires-Dist: s3fs; extra == "all"
-Requires-Dist: semantic-version; extra == "all"
-Requires-Dist: tqdm; extra == "all"
-Requires-Dist: zarr<=2.17; extra == "all"
 Provides-Extra: create
 Requires-Dist: climetlab>=0.23.2; extra == "create"
 Requires-Dist: earthkit-meteo; extra == "create"
@@ -252,11 +246,14 @@ Requires-Dist: entrypoints; extra == "create"
 Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
 Requires-Dist: boto3; extra == "dev"
+Requires-Dist: climetlab>=0.23.2; extra == "dev"
 Requires-Dist: earthkit-meteo; extra == "dev"
 Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
+Requires-Dist: entrypoints; extra == "dev"
 Requires-Dist: nbsphinx; extra == "dev"
 Requires-Dist: pandoc; extra == "dev"
 Requires-Dist: pyproj; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
 Requires-Dist: requests; extra == "dev"
 Requires-Dist: s3fs; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
@@ -272,3 +269,5 @@ Provides-Extra: remote
 Requires-Dist: boto3; extra == "remote"
 Requires-Dist: requests; extra == "remote"
 Requires-Dist: s3fs; extra == "remote"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/docs/building/sources/forcings.yaml

@@ -5,7 +5,7 @@ input:
       args2: value2
     - forcings:
       template: ${input.join.0.source1}
-      params:
+      param:
         - insolation
         - cos_julian_day
         - sin_julian_day

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/pyproject.toml

@@ -59,36 +59,33 @@ dependencies = [
 ]
 
 optional-dependencies.all = [
-  "anemoi-utils[provenance]>=0.3",
   "boto3",
   "climetlab>=0.23.2",
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
   "entrypoints",
-  "numpy",
   "pyproj",
-  "pyyaml",
   "requests",
   "s3fs",
-  "semantic-version",
-  "tqdm",
-  "zarr<=2.17",
 ]
 optional-dependencies.create = [
-  "climetlab>=0.23.2", # "earthkit-data"
+  "climetlab>=0.23.2",
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
   "entrypoints",
   "pyproj",
 ]
+
 optional-dependencies.dev = [
   "boto3",
-
+  "climetlab>=0.23.2",
   "earthkit-meteo",
   "ecmwflibs>=0.6.3",
+  "entrypoints",
   "nbsphinx",
   "pandoc",
   "pyproj",
+  "pytest",
   "requests",
   "s3fs",
   "sphinx",
@@ -107,6 +104,9 @@ optional-dependencies.remote = [
   "requests",
   "s3fs",
 ]
+optional-dependencies.tests = [
+  "pytest",
+]
 urls.Documentation = "https://anemoi-datasets.readthedocs.io/"
 urls.Homepage = "https://github.com/ecmwf/anemoi-datasets/"
 urls.Issues = "https://github.com/ecmwf/anemoi-datasets/issues"

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.3'
-__version_tuple__ = version_tuple = (0, 3, 3)
+__version__ = version = '0.3.6'
+__version_tuple__ = version_tuple = (0, 3, 6)

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/commands/compare.py

@@ -14,6 +14,8 @@ from . import Command
 
 
 class Compare(Command):
+    """Compare two datasets. This command compares the variables in two datasets and prints the mean of the common variables. It does not compare the data itself (yet)."""
+
     def add_arguments(self, command_parser):
         command_parser.add_argument("dataset1")
         command_parser.add_argument("dataset2")

anemoi_datasets-0.3.3/src/anemoi/datasets/commands/inspect/zarr.py → anemoi_datasets-0.3.6/src/anemoi/datasets/commands/inspect.py

@@ -24,6 +24,9 @@ from anemoi.utils.text import table
 
 from anemoi.datasets import open_dataset
 from anemoi.datasets.data.stores import open_zarr
+from anemoi.datasets.data.stores import zarr_lookup
+
+from . import Command
 
 LOG = logging.getLogger(__name__)
 
@@ -78,46 +81,12 @@ class Version:
         self.metadata = metadata
         self.version = version
         self.dataset = None
-        # try:
         self.dataset = open_dataset(self.path)
-        # except Exception as e:
-        #     LOG.error("Error opening dataset '%s': %s", self.path, e)
 
     def describe(self):
         print(f"📦 Path          : {self.path}")
         print(f"🔢 Format version: {self.version}")
 
-    def probe(self):
-        if "cos_local_time" not in self.name_to_index:
-            print("⚠️ probe: no cos_local_time")
-            return
-
-        try:
-            lon = self.longitudes
-        except AttributeError:
-            print("⚠️ probe: no longitudes")
-            return
-        # print(json.dumps(self.metadata, indent=4))
-        cos_local_time = self.name_to_index["cos_local_time"]
-        data = self.data
-        start, end, frequency = self.first_date, self.last_date, self.frequency
-        date = start
-        same = 0
-        for i in range(10):
-            field = data[i, cos_local_time]
-            buggy = cos_local_time_bug(lon, date).reshape(field.hape)
-            diff = np.abs(field - buggy)
-            if np.max(diff) < 1e-5:
-                same += 1
-            date += datetime.timedelta(hours=frequency)
-            if date > end:
-                break
-        if same > 1:
-            print("❌ probe: cos_local_time is buggy")
-            return
-
-        print("✅ probe: cos_local_time is fixed")
-
     @property
     def name_to_index(self):
         return find(self.metadata, "name_to_index")
@@ -586,29 +555,35 @@ VERSIONS = {
 }
 
 
-class InspectZarr:
-    """Inspect a checkpoint or zarr file."""
+class InspectZarr(Command):
+    """Inspect a zarr dataset."""
 
-    def inspect_zarr(self, path, **kwargs):
-        version = self._info(path)
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("path", metavar="DATASET")
+        command_parser.add_argument("--detailed", action="store_true")
+
+        command_parser.add_argument("--progress", action="store_true")
+        command_parser.add_argument("--statistics", action="store_true")
+        command_parser.add_argument("--size", action="store_true", help="Print size")
 
-        # try:
-        #     with open("/tmp/probe.json", "w") as f:
-        #         json.dump(version.metadata, f, indent=4, sort_keys=True)
-        # except Exception:
-        #     pass
+    def run(self, args):
+        self.inspect_zarr(**vars(args))
+
+    def inspect_zarr(self, path, progress=False, statistics=False, detailed=False, size=False, **kwargs):
+        version = self._info(path)
 
         dotted_line()
         version.describe()
 
         try:
-            if kwargs.get("probe"):
-                return version.probe()
-            if kwargs.get("progress"):
+            if progress:
                 return version.progress()
-            if kwargs.get("statistics"):
+
+            if statistics:
                 return version.brute_force_statistics()
-            version.info(kwargs.get("detailed"), kwargs.get("size"))
+
+            version.info(detailed, size)
+
         except Exception as e:
             LOG.error("Error inspecting zarr file '%s': %s", path, e)
 
@@ -616,14 +591,7 @@ class InspectZarr:
             raise
 
     def _info(self, path):
-        if path.endswith("/"):
-            path = path[:-1]
-
-        try:
-            z = open_zarr(path)
-        except Exception as e:
-            LOG.error("Error opening zarr file '%s': %s", path, e)
-            raise
+        z = open_zarr(zarr_lookup(path))
 
         metadata = dict(z.attrs)
         version = metadata.get("version", "0.0.0")
@@ -640,3 +608,6 @@ class InspectZarr:
                 candidate = klass
 
         return candidate(path, z, metadata, version)
+
+
+command = InspectZarr

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/compute/recentre.py

@@ -29,7 +29,7 @@ CLIP_VARIABLES = (
     "swl1",
 )
 
-SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset")
+SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset", "time", "date", "step")
 
 
 def check_compatible(f1, f2, centre_field_as_mars, ensemble_field_as_mars):

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/create/functions/filters/rename.py

@@ -13,6 +13,14 @@ from climetlab.indexing.fieldset import FieldArray
 
 
 class RenamedFieldMapping:
+    """Rename a field based on the value of another field.
+
+    Args:
+        field (Field): The field to be renamed.
+        what (str): The name of the field that will be used to rename the field.
+        renaming (dict): A dictionary mapping the values of 'what' to the new names.
+    """
+
     def __init__(self, field, what, renaming):
         self.field = field
         self.what = what
@@ -29,6 +37,12 @@ class RenamedFieldMapping:
 
 
 class RenamedFieldFormat:
+    """Rename a field based on a format string.
+
+    Args:
+        format (str): A string that defines the new name of the field.
+    """
+
     def __init__(self, field, format):
         self.field = field
         self.format = format

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/create/functions/sources/grib.py

@@ -8,6 +8,8 @@
 #
 
 
+import glob
+
 from climetlab import load_source
 from climetlab.utils.patterns import Pattern
 
@@ -22,6 +24,12 @@ def check(ds, paths, **kwargs):
         raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})")
 
 
+def _expand(paths):
+    for path in paths:
+        for p in glob.glob(path):
+            yield p
+
+
 def execute(context, dates, path, *args, **kwargs):
     given_paths = path if isinstance(path, list) else [path]
 
@@ -35,7 +43,7 @@ def execute(context, dates, path, *args, **kwargs):
             if name in kwargs:
                 raise ValueError(f"MARS interpolation parameter '{name}' not supported")
 
-        for path in paths:
+        for path in _expand(paths):
             context.trace("📁", "PATH", path)
             s = load_source("file", path)
             s = s.sel(valid_datetime=dates, **kwargs)

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/create/functions/sources/netcdf.py

@@ -7,10 +7,29 @@
 # nor does it submit to any jurisdiction.
 #
 
+import glob
+
 from climetlab import load_source
 from climetlab.utils.patterns import Pattern
 
 
+def _expand(paths):
+    for path in paths:
+        if path.startswith("file://"):
+            path = path[7:]
+
+        if path.startswith("http://"):
+            yield path
+            continue
+
+        if path.startswith("https://"):
+            yield path
+            continue
+
+        for p in glob.glob(path):
+            yield p
+
+
 def check(what, ds, paths, **kwargs):
     count = 1
     for k, v in kwargs.items():
@@ -32,7 +51,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
 
         levels = kwargs.get("level", kwargs.get("levelist"))
 
-        for path in paths:
+        for path in _expand(paths):
             context.trace(emoji, what.upper(), path)
             s = load_source("opendap", path)
             s = s.sel(

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/create/loaders.py

@@ -177,11 +177,14 @@ class InitialiserLoader(Loader):
                     return
                 if isinstance(obj, (dict, DictObj)):
                     if "grid" in obj:
+                        previous = obj["grid"]
                         obj["grid"] = "20./20."
-                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']}")
+                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']} instead of {previous}")
                     if "number" in obj:
-                        obj["number"] = obj["number"][0:3]
-                        LOG.warn(f"Running in test mode. Setting number to {obj['number']}")
+                        if isinstance(obj["number"], (list, tuple)):
+                            previous = obj["number"]
+                            obj["number"] = previous[0:3]
+                            LOG.warn(f"Running in test mode. Setting number to {obj['number']} instead of {previous}")
                     for k, v in obj.items():
                         set_to_test_mode(v)
 

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/data/concat.py

@@ -12,7 +12,7 @@ import numpy as np
 
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Combined
+from .forwards import Combined
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/data/ensemble.py

@@ -8,7 +8,7 @@
 import logging
 
 from .debug import Node
-from .forewards import GivenAxis
+from .forwards import GivenAxis
 from .misc import _auto_adjust
 from .misc import _open
 

anemoi_datasets-0.3.3/src/anemoi/datasets/data/forewards.py → anemoi_datasets-0.3.6/src/anemoi/datasets/data/forwards.py

@@ -91,12 +91,18 @@ class Forwards(Dataset):
     def metadata_specific(self, **kwargs):
         return super().metadata_specific(
             forward=self.forward.metadata_specific(),
+            **self.subclass_metadata_specific(),
             **kwargs,
         )
 
     def source(self, index):
         return self.forward.source(index)
 
+    def subclass_metadata_specific(self):
+        raise NotImplementedError(
+            f"subclass_metadata_specific() must be implemented in derived class {self.__class__.__name__}"
+        )
+
 
 class Combined(Forwards):
     def __init__(self, datasets):

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/data/grids.py

@@ -12,8 +12,8 @@ import numpy as np
 
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Combined
-from .forewards import GivenAxis
+from .forwards import Combined
+from .forwards import GivenAxis
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -91,7 +91,7 @@ class Concat(Combined):
         return Node(self, [d.tree() for d in self.datasets])
 
 
-class Grids(GivenAxis):
+class GridsBase(GivenAxis):
     def __init__(self, datasets, axis):
         super().__init__(datasets, axis)
         # Shape: (dates, variables, ensemble, 1d-values)
@@ -106,7 +106,7 @@ class Grids(GivenAxis):
         pass
 
 
-class ConcatGrids(Grids):
+class Grids(GridsBase):
     # TODO: select the statistics of the most global grid?
     @property
     def latitudes(self):
@@ -127,7 +127,7 @@ class ConcatGrids(Grids):
         return Node(self, [d.tree() for d in self.datasets], mode="concat")
 
 
-class Cutout(Grids):
+class Cutout(GridsBase):
     def __init__(self, datasets, axis):
         from anemoi.datasets.grids import cutout_mask
 
@@ -220,7 +220,7 @@ def grids_factory(args, kwargs):
     datasets = [_open(e) for e in grids]
     datasets, kwargs = _auto_adjust(datasets, kwargs)
 
-    return ConcatGrids(datasets, axis=axis)._subset(**kwargs)
+    return Grids(datasets, axis=axis)._subset(**kwargs)
 
 
 def cutout_factory(args, kwargs):

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/data/join.py

@@ -13,7 +13,7 @@ import numpy as np
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Combined
+from .forwards import Combined
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/data/masked.py

@@ -14,7 +14,7 @@ from ..grids import cropping_mask
 from .dataset import Dataset
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -83,6 +83,9 @@ class Thinning(Masked):
     def tree(self):
         return Node(self, [self.forward.tree()], thinning=self.thinning, method=self.method)
 
+    def subclass_metadata_specific(self):
+        return dict(thinning=self.thinning, method=self.method)
+
 
 class Cropping(Masked):
     def __init__(self, forward, area):
@@ -104,3 +107,6 @@ class Cropping(Masked):
 
     def tree(self):
         return Node(self, [self.forward.tree()], area=self.area)
+
+    def metadata_specific(self, **kwargs):
+        return super().metadata_specific(area=self.area, **kwargs)

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/data/misc.py

@@ -133,9 +133,9 @@ def _as_date(d, dates, last):
 
         if "-" in d and ":" in d:
             date, time = d.replace(" ", "T").split("T")
-            year, month, day = date.split("-")
-            hour, minute, second = time.split(":")
-            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour}:{minute}:{second}")
+            year, month, day = [int(_) for _ in date.split("-")]
+            hour, minute, second = [int(_) for _ in time.split(":")]
+            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}")
 
         if "-" in d:
             assert ":" not in d

{anemoi_datasets-0.3.3 → anemoi_datasets-0.3.6}/src/anemoi/datasets/data/select.py

@@ -11,7 +11,7 @@ from functools import cached_property
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -88,6 +88,10 @@ class Select(Forwards):
     def tree(self):
         return Node(self, [self.dataset.tree()], **self.title)
 
+    def subclass_metadata_specific(self):
+        # return dict(indices=self.indices)
+        return {}
+
 
 class Rename(Forwards):
     def __init__(self, dataset, rename):
@@ -105,8 +109,8 @@ class Rename(Forwards):
     def name_to_index(self):
         return {k: i for i, k in enumerate(self.variables)}
 
-    def metadata_specific(self, **kwargs):
-        return super().metadata_specific(rename=self.rename, **kwargs)
-
     def tree(self):
         return Node(self, [self.forward.tree()], rename=self.rename)
+
+    def subclass_metadata_specific(self):
+        return dict(rename=self.rename)