anemoi-datasets 0.4.3__tar.gz → 0.4.4__tar.gz

anemoi_datasets-0.4.4/.github/ci-config.yml

@@ -0,0 +1,9 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+  ecmwf/eckit
+  ecmwf/odc
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false # Only for python packages

anemoi_datasets-0.4.4/.github/workflows/changelog-pr-update.yml

@@ -0,0 +1,15 @@
+name: Check Changelog Update on PR
+on:
+  pull_request:
+    types: [assigned, opened, synchronize, reopened, labeled, unlabeled]
+    branches:
+      - main
+      - develop
+jobs:
+  Check-Changelog:
+    name: Check Changelog Action
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: tarides/changelog-check-action@v2
+        with:
+          changelog: CHANGELOG.md

anemoi_datasets-0.4.4/.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'main'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+   # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

anemoi_datasets-0.4.4/.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

anemoi_datasets-0.4.4/.github/workflows/readthedocs-pr-update.yml

@@ -0,0 +1,22 @@
+name: Read the Docs PR Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    # Execute this action only on PRs that touch
+    # documentation files.
+    paths:
+      - "docs/**"
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "anemoi-datasets"

anemoi_datasets-0.4.4/CHANGELOG.md

@@ -0,0 +1,60 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Please add your functional changes to the appropriate section in the PR.
+Keep it human-readable, your future self will thank you!
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Removed
+
+## [0.4.4] Bug fixes
+
+- Bug fix in accumulations() that did not work with webmars
+- Bug fix when using hindcasts input (missing dates on leap years)
+
+## [0.4.0] Minor Release
+
+### Added
+- earthkit-data replaces climetlab
+
+### Removed
+- climetlab
+
+## [0.3.0] Minor Release
+
+### Added
+- hindcast source
+
+### Changed
+- updated documentation
+
+## [0.2.0] Minor Release
+
+### Added
+- statistics tendencies
+
+### Removed
+- CubesFilter
+
+## [0.1.0] Initial Release
+
+### Added
+- Documentation
+- Initial code release for anemoi-datasets: create datasets for data-driven weather-models
+- open datasets
+- combine datasets
+
+## Git Diffs:
+[0.4.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.3.0...0.4.0
+[0.3.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.2.0...0.3.0
+[0.2.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.1.0...0.2.0
+[0.1.0]: https://github.com/ecmwf/anemoi-models/releases/tag/0.1.0

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.4.3
+Version: 0.4.4
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

anemoi_datasets-0.4.4/docs/building/sources/xarray-kerchunk.py

@@ -0,0 +1,30 @@
+import json
+
+import fsspec
+import tqdm
+from kerchunk.combine import MultiZarrToZarr
+from kerchunk.hdf import SingleHdf5ToZarr
+
+fs = fsspec.filesystem("s3", anon=True)
+
+pattern = "s3://nsf-ncar-era5/e5.oper.an.pl/202403/e5.oper.an.pl.*.ll025sc.2024????00_2024????23.nc"
+
+
+jsons = []
+
+for file in tqdm.tqdm(fs.glob(pattern)):
+    with fs.open(file, "rb", anon=True) as f:
+        h5chunks = SingleHdf5ToZarr(f, file)
+        jsons.append(h5chunks.translate())
+
+
+mzz = MultiZarrToZarr(
+    jsons,
+    remote_protocol="s3",
+    remote_options={"anon": True},
+    concat_dims=["time"],
+    identical_dims=["latitude", "longitude"],
+)
+
+with open("combined.json", "w") as f:
+    json.dump(mzz.translate(), f)

anemoi_datasets-0.4.4/docs/building/sources/xarray-kerchunk.rst

@@ -0,0 +1,17 @@
+#################
+ xarray-kerchunk
+#################
+
+.. literalinclude:: xarray-kerchunk.yaml
+   :language: yaml
+
+The code below is inspired by the `kerchunk tutorial`_, and makes use of
+a subset of the `ERA5 dataset available on AWS`_. You may need to
+install the relevant packages before running the code below.
+
+.. literalinclude:: xarray-kerchunk.py
+   :language: python
+
+.. _era5 dataset available on aws: https://registry.opendata.aws/ecmwf-era5/
+
+.. _kerchunk tutorial: https://fsspec.github.io/kerchunk/tutorial.html

anemoi_datasets-0.4.4/docs/building/sources/xarray-kerchunk.yaml

@@ -0,0 +1,10 @@
+dates:
+  start: 2024-03-24T00:00:00
+  end: 2024-03-24T03:00:00
+  frequency: 1h
+
+input:
+  xarray-kerchunk:
+    json: combined.json
+    param: T
+    level: [1000, 50]

anemoi_datasets-0.4.4/docs/building/sources/xarray-zarr.rst

@@ -0,0 +1,6 @@
+#############
+ xarray-zarr
+#############
+
+.. literalinclude:: xarray-zarr.yaml
+   :language: yaml

anemoi_datasets-0.4.3/docs/building/sources/xarray.yaml → anemoi_datasets-0.4.4/docs/building/sources/xarray-zarr.yaml

@@ -1,3 +1,3 @@
 input:
-  xarray:
+  xarray-zarr:
     url: https://...

anemoi_datasets-0.4.4/docs/building/sources/zenodo.rst

@@ -0,0 +1,8 @@
+########
+ zenodo
+########
+
+https://zenodo.org/records/6470725
+
+.. literalinclude:: zenodo.yaml
+   :language: yaml

anemoi_datasets-0.4.4/docs/building/sources/zenodo.yaml

@@ -0,0 +1,9 @@
+dates:
+  start: 2021-06-28
+  end: 2021-06-29
+
+input:
+  zenodo:
+    record_id: 6470725
+    file_key: "{param}_INCA_202106280700.nc"
+    param: [RR,PN, PT]

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/docs/building/sources.rst

@@ -20,11 +20,14 @@ The following `sources` are currently available:
 .. toctree::
    :maxdepth: 1
 
-   sources/mars
+   sources/accumulations
+   sources/forcings
    sources/grib
+   sources/hindcasts
+   sources/mars
    sources/netcdf
-   sources/xarray
    sources/opendap
-   sources/forcings
-   sources/accumulations
    sources/recentre
+   sources/xarray-kerchunk
+   sources/xarray-zarr
+   sources/zenodo

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.3'
-__version_tuple__ = version_tuple = (0, 4, 3)
+__version__ = version = '0.4.4'
+__version_tuple__ = version_tuple = (0, 4, 4)

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/compute/recentre.py

@@ -64,7 +64,7 @@ def recentre(
 
     keys = ["param", "level", "valid_datetime", "date", "time", "step", "number"]
 
-    number_list = members.unique_values("number")["number"]
+    number_list = members.unique_values("number", progress_bar=False)["number"]
     n_numbers = len(number_list)
 
     assert None not in number_list

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/create/functions/sources/accumulations.py

@@ -15,11 +15,10 @@ import earthkit.data as ekd
 import numpy as np
 from earthkit.data.core.temporary import temp_file
 from earthkit.data.readers.grib.output import new_grib_output
-from earthkit.data.utils.availability import Availability
 
 from anemoi.datasets.create.utils import to_datetime_list
 
-from .mars import use_grib_paramid
+from .mars import mars
 
 LOG = logging.getLogger(__name__)
 
@@ -307,26 +306,11 @@ def _compute_accumulations(
     for date, time, steps in mars_date_time_steps:
         for p in param:
             for n in number:
-                requests.append(
-                    patch(
-                        {
-                            "param": p,
-                            "date": date,
-                            "time": time,
-                            "step": sorted(steps),
-                            "number": n,
-                        }
-                    )
-                )
-
-    compressed = Availability(requests)
-    ds = ekd.from_source("empty")
-    for r in compressed.iterate():
-        request.update(r)
-        if context.use_grib_paramid and "param" in request:
-            request = use_grib_paramid(request)
-        print("🌧️", request)
-        ds = ds + ekd.from_source("mars", **request)
+                r = dict(request, param=p, date=date, time=time, step=sorted(steps), number=n)
+
+                requests.append(patch(r))
+
+    ds = mars(context, dates, *requests, request_already_using_valid_datetime=True)
 
     accumulations = {}
     for a in [AccumulationClass(out, frequency=frequency, **r) for r in requests]:

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/create/functions/sources/hindcasts.py

@@ -7,20 +7,16 @@
 # nor does it submit to any jurisdiction.
 #
 import datetime
+import logging
+
+from earthkit.data.core.fieldlist import MultiFieldList
 
 from anemoi.datasets.create.functions.sources.mars import mars
 
+LOGGER = logging.getLogger(__name__)
 DEBUG = True
 
 
-def _member(field):
-    # Bug in eccodes has number=0 randomly
-    number = field.metadata("number")
-    if number is None:
-        number = 0
-    return number
-
-
 def _to_list(x):
     if isinstance(x, (list, tuple)):
         return x
@@ -63,9 +59,19 @@ class HindcastCompute:
 def use_reference_year(reference_year, request):
     request = request.copy()
     hdate = request.pop("date")
-    date = datetime.datetime(reference_year, hdate.month, hdate.day)
+
+    if hdate.year >= reference_year:
+        return None, False
+
+    try:
+        date = datetime.datetime(reference_year, hdate.month, hdate.day)
+    except ValueError:
+        if hdate.month == 2 and hdate.day == 29:
+            return None, False
+        raise
+
     request.update(date=date.strftime("%Y-%m-%d"), hdate=hdate.strftime("%Y-%m-%d"))
-    return request
+    return request, True
 
 
 def hindcasts(context, dates, **request):
@@ -89,9 +95,18 @@ def hindcasts(context, dates, **request):
     requests = []
     for d in dates:
         req = c.compute_hindcast(d)
-        req = use_reference_year(reference_year, req)
+        req, ok = use_reference_year(reference_year, req)
+        if ok:
+            requests.append(req)
+
+    # print("HINDCASTS requests", reference_year, base_times, available_steps)
+    # print("HINDCASTS dates", compress_dates(dates))
+
+    if len(requests) == 0:
+        # print("HINDCASTS no requests")
+        return MultiFieldList([])
 
-        requests.append(req)
+    # print("HINDCASTS requests", requests)
 
     return mars(
         context,

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/create/functions/sources/tendencies.py

@@ -56,7 +56,7 @@ def tendencies(dates, time_increment, **kwargs):
 
     ds = mars(dates=all_dates, **kwargs)
 
-    dates_in_data = ds.unique_values("valid_datetime")["valid_datetime"]
+    dates_in_data = ds.unique_values("valid_datetime", progress_bar=False)["valid_datetime"]
     for d in all_dates:
         assert d.isoformat() in dates_in_data, d
 

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/create/input.py

@@ -277,6 +277,9 @@ class Result:
         if len(args) == 1 and isinstance(args[0], (list, tuple)):
             args = args[0]
 
+        # print("Executing", self.action_path)
+        # print("Dates:", compress_dates(self.dates))
+
         names = []
         for a in args:
             if isinstance(a, str):
@@ -287,12 +290,12 @@ class Result:
         print(f"Building a {len(names)}D hypercube using", names)
 
         ds = ds.order_by(*args, remapping=remapping, patches=patches)
-        user_coords = ds.unique_values(*names, remapping=remapping, patches=patches)
+        user_coords = ds.unique_values(*names, remapping=remapping, patches=patches, progress_bar=False)
 
         print()
         print("Number of unique values found for each coordinate:")
         for k, v in user_coords.items():
-            print(f"  {k:20}:", len(v))
+            print(f"  {k:20}:", len(v), shorten_list(v, max_length=10))
         print()
         user_shape = tuple(len(v) for k, v in user_coords.items())
         print("Shape of the hypercube           :", user_shape)
@@ -305,13 +308,18 @@ class Result:
 
         remapping = build_remapping(remapping, patches)
         expected = set(itertools.product(*user_coords.values()))
+        extra = set()
 
         if math.prod(user_shape) > len(ds):
             print(f"This means that all the fields in the datasets do not exists for all combinations of {names}.")
 
             for f in ds:
                 metadata = remapping(f.metadata)
-                expected.remove(tuple(metadata(n) for n in names))
+                key = tuple(metadata(n, default=None) for n in names)
+                if key in expected:
+                    expected.remove(key)
+                else:
+                    extra.add(key)
 
             print("Missing fields:")
             print()
@@ -321,7 +329,35 @@ class Result:
                     print("...", len(expected) - i - 1, "more")
                     break
 
+            print("Extra fields:")
+            print()
+            for i, f in enumerate(sorted(extra)):
+                print(" ", f)
+                if i >= 9 and len(extra) > 10:
+                    print("...", len(extra) - i - 1, "more")
+                    break
+
             print()
+            print("Missing values:")
+            per_name = defaultdict(set)
+            for e in expected:
+                for n, v in zip(names, e):
+                    per_name[n].add(v)
+
+            for n, v in per_name.items():
+                print(" ", n, len(v), shorten_list(sorted(v), max_length=10))
+            print()
+
+            print("Extra values:")
+            per_name = defaultdict(set)
+            for e in extra:
+                for n, v in zip(names, e):
+                    per_name[n].add(v)
+
+            for n, v in per_name.items():
+                print(" ", n, len(v), shorten_list(sorted(v), max_length=10))
+            print()
+
             print("To solve this issue, you can:")
             print(
                 "  - Provide a better selection, like 'step: 0' or 'level: 1000' to "

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/create/loaders.py

@@ -17,6 +17,7 @@ import numpy as np
 import tqdm
 import zarr
 from anemoi.utils.config import DotDict
+from anemoi.utils.dates import as_datetime
 from anemoi.utils.humanize import seconds_to_human
 
 from anemoi.datasets import MissingDateError
@@ -24,6 +25,7 @@ from anemoi.datasets import open_dataset
 from anemoi.datasets.create.persistent import build_storage
 from anemoi.datasets.data.misc import as_first_date
 from anemoi.datasets.data.misc import as_last_date
+from anemoi.datasets.dates import compress_dates
 from anemoi.datasets.dates.groups import Groups
 
 from .check import DatasetName
@@ -442,14 +444,24 @@ class ContentLoader(Loader):
         dates = result.dates
 
         cube = result.get_cube()
-        assert cube.extended_user_shape[0] == len(dates), (
-            cube.extended_user_shape[0],
-            len(dates),
-        )
-
         shape = cube.extended_user_shape
         dates_in_data = cube.user_coords["valid_datetime"]
 
+        if cube.extended_user_shape[0] != len(dates):
+            print(f"Cube shape does not match the number of dates {cube.extended_user_shape[0]}, {len(dates)}")
+            print("Requested dates", compress_dates(dates))
+            print("Cube dates", compress_dates(dates_in_data))
+
+            a = set(as_datetime(_) for _ in dates)
+            b = set(as_datetime(_) for _ in dates_in_data)
+
+            print("Missing dates", compress_dates(a - b))
+            print("Extra dates", compress_dates(b - a))
+
+            raise ValueError(
+                f"Cube shape does not match the number of dates {cube.extended_user_shape[0]}, {len(dates)}"
+            )
+
         LOG.debug(f"Loading {shape=} in {self.data_array.shape=}")
 
         def check_dates_in_data(lst, lst2):

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/dates/__init__.py

@@ -9,6 +9,45 @@
 import datetime
 import warnings
 
+from anemoi.utils.dates import as_datetime
+
+
+def _compress_dates(dates):
+    dates = sorted(dates)
+    if len(dates) < 3:
+        yield dates
+        return
+
+    prev = first = dates.pop(0)
+    curr = dates.pop(0)
+    delta = curr - prev
+    while curr - prev == delta:
+        prev = curr
+        if not dates:
+            break
+        curr = dates.pop(0)
+
+    yield (first, prev, delta)
+    if dates:
+        yield from _compress_dates([curr] + dates)
+
+
+def compress_dates(dates):
+    dates = [as_datetime(_) for _ in dates]
+    result = []
+
+    for n in _compress_dates(dates):
+        if isinstance(n, list):
+            result.extend([str(_) for _ in n])
+        else:
+            result.append(" ".join([str(n[0]), "to", str(n[1]), "by", str(n[2])]))
+
+    return result
+
+
+def print_dates(dates):
+    print(compress_dates(dates))
+
 
 def no_time_zone(date):
     return date.replace(tzinfo=None)
@@ -30,6 +69,27 @@ def normalize_date(x):
     return x
 
 
+def extend(x):
+
+    if isinstance(x, (list, tuple)):
+        for y in x:
+            yield from extend(y)
+        return
+
+    if isinstance(x, str):
+        if "/" in x:
+            start, end, step = x.split("/")
+            start = normalize_date(start)
+            end = normalize_date(end)
+            step = frequency_to_hours(step)
+            while start <= end:
+                yield start
+                start += datetime.timedelta(hours=step)
+            return
+
+    yield normalize_date(x)
+
+
 class Dates:
     """Base class for date generation.
 
@@ -59,7 +119,7 @@ class Dates:
     def __init__(self, missing=None):
         if not missing:
             missing = []
-        self.missing = [normalize_date(x) for x in missing]
+        self.missing = list(extend(missing))
         if set(self.missing) - set(self.values):
             warnings.warn(f"Missing dates {self.missing} not in list.")
 
@@ -145,3 +205,9 @@ class StartEndDates(Dates):
             "end": self.end.isoformat(),
             "frequency": f"{self.frequency}h",
         }
+
+
+if __name__ == "__main__":
+    print_dates([datetime.datetime(2023, 1, 1, 0, 0)])
+    s = StartEndDates(start="2023-01-01 00:00", end="2023-01-02 00:00", frequency=1)
+    print_dates(list(s))

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi/datasets/dates/groups.py

@@ -9,7 +9,7 @@
 import itertools
 
 from anemoi.datasets.dates import Dates
-from anemoi.datasets.dates import no_time_zone
+from anemoi.datasets.dates import normalize_date
 
 
 class Groups:
@@ -67,7 +67,7 @@ class Groups:
 
 class Filter:
     def __init__(self, missing):
-        self.missing = [no_time_zone(m) for m in missing]
+        self.missing = [normalize_date(m) for m in missing]
 
     def __call__(self, dates):
         return [d for d in dates if d not in self.missing]

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.4}/src/anemoi_datasets.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.4.3
+Version: 0.4.4
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License