anemoi-datasets 0.5.10__tar.gz → 0.5.11__tar.gz

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.5.10
+Version: 0.5.11
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -224,8 +224,8 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-transform>=0.0.5
-Requires-Dist: anemoi-utils[provenance]>=0.4.2
+Requires-Dist: anemoi-transform>=0.1
+Requires-Dist: anemoi-utils[provenance]>=0.4.9
 Requires-Dist: cfunits
 Requires-Dist: numpy
 Requires-Dist: pyyaml

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/pyproject.toml

@@ -49,8 +49,8 @@ dynamic = [
   "version",
 ]
 dependencies = [
-  "anemoi-transform>=0.0.5",
-  "anemoi-utils[provenance]>=0.4.2",
+  "anemoi-transform>=0.1",
+  "anemoi-utils[provenance]>=0.4.9",
   "cfunits",
   "numpy",
   "pyyaml",

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.10'
-__version_tuple__ = version_tuple = (0, 5, 10)
+__version__ = version = '0.5.11'
+__version_tuple__ = version_tuple = (0, 5, 11)

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/__init__.py

@@ -19,7 +19,6 @@ from functools import cached_property
 import cftime
 import numpy as np
 import tqdm
-from anemoi.utils.config import DotDict as DotDict
 from anemoi.utils.dates import as_datetime
 from anemoi.utils.dates import frequency_to_string
 from anemoi.utils.dates import frequency_to_timedelta
@@ -327,7 +326,7 @@ class HasElementForDataMixin:
         self.output = build_output(config.output, parent=self)
 
         self.input = build_input_(main_config=config, output_config=self.output)
-        LOG.info(self.input)
+        LOG.info("%s", self.input)
 
 
 def build_input_(main_config, output_config):

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/functions/filters/rename.py

@@ -44,7 +44,6 @@ class RenamedFieldMapping:
 
     def __repr__(self) -> str:
         return repr(self.field)
-        return f"{self.field} -> {self.what} -> {self.renaming}"
 
 
 class RenamedFieldFormat:

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/functions/sources/grib.py

@@ -9,11 +9,14 @@
 
 
 import glob
+import logging
 
 from earthkit.data import from_source
 from earthkit.data.indexing.fieldlist import FieldArray
 from earthkit.data.utils.patterns import Pattern
 
+LOG = logging.getLogger(__name__)
+
 
 def _load(context, name, record):
     ds = None
@@ -141,4 +144,7 @@ def execute(context, dates, path, latitudes=None, longitudes=None, *args, **kwar
     if geography is not None:
         ds = FieldArray([AddGrid(_, geography) for _ in ds])
 
+    if len(ds) == 0:
+        LOG.warning(f"No fields found for {dates} in {given_paths} (kwargs={kwargs})")
+
     return ds

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/functions/sources/xarray/__init__.py

@@ -12,7 +12,6 @@ import logging
 from earthkit.data.core.fieldlist import MultiFieldList
 
 from anemoi.datasets.data.stores import name_to_zarr_store
-from anemoi.datasets.utils.fields import NewMetadataField as NewMetadataField
 
 from .. import iterate_patterns
 from .fieldlist import XarrayFieldList

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/functions/sources/xarray/field.py

@@ -10,6 +10,7 @@
 
 import datetime
 import logging
+from functools import cached_property
 
 from earthkit.data.core.fieldlist import Field
 from earthkit.data.core.fieldlist import math
@@ -46,8 +47,6 @@ class XArrayField(Field):
             This is actually a nD object, but the first dimensions are always 1.
             The other two dimensions are latitude and longitude.
         """
-        super().__init__(owner.array_backend)
-
         self.owner = owner
         self.selection = selection
 
@@ -86,7 +85,8 @@ class XArrayField(Field):
 
         return values  # .reshape(self.shape)
 
-    def _make_metadata(self):
+    @cached_property
+    def _metadata(self):
         return XArrayMetadata(self)
 
     def grid_points(self):

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/functions/sources/xarray/fieldlist.py

@@ -14,10 +14,8 @@ import logging
 import yaml
 from earthkit.data.core.fieldlist import FieldList
 
-from .coordinates import is_scalar as is_scalar
 from .field import EmptyFieldList
 from .flavour import CoordinateGuesser
-from .metadata import XArrayMetadata as XArrayMetadata
 from .time import Time
 from .variable import FilteredVariable
 from .variable import Variable

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/functions/sources/xarray/variable.py

@@ -13,7 +13,6 @@ import math
 from functools import cached_property
 
 import numpy as np
-from earthkit.data.utils.array import ensure_backend
 
 from .field import XArrayField
 
@@ -30,7 +29,6 @@ class Variable:
         grid,
         time,
         metadata,
-        array_backend=None,
     ):
         self.ds = ds
         self.variable = variable
@@ -48,7 +46,6 @@ class Variable:
         self.by_name = {c.variable.name: c for c in coordinates}
 
         self.length = math.prod(self.shape)
-        self.array_backend = ensure_backend(array_backend)
 
     @property
     def name(self):

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/action.py

@@ -10,14 +10,8 @@
 import logging
 from copy import deepcopy
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from earthkit.data.core.order import build_remapping
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .context import Context
 from .misc import is_function
 

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/context.py

@@ -10,14 +10,8 @@
 import logging
 import textwrap
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from anemoi.utils.humanize import plural
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .trace import step
 from .trace import trace
 

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/data_sources.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .action import Action
 from .action import action_factory
 from .misc import _tidy

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/empty.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .misc import assert_fieldlist
 from .result import Result
 from .trace import trace_datasource

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/filter.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from ..functions import import_function
 from .function import FunctionContext
 from .misc import _tidy

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/function.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from ..functions import import_function
 from .action import Action
 from .misc import _tidy

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/misc.py

@@ -10,15 +10,9 @@
 import logging
 from functools import wraps
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from earthkit.data.core.fieldlist import MultiFieldList
 from earthkit.data.indexing.fieldlist import FieldList
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from ..functions import import_function
 
 LOG = logging.getLogger(__name__)

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/pipe.py

@@ -7,6 +7,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+import json
 import logging
 
 from .action import Action
@@ -20,7 +21,11 @@ LOG = logging.getLogger(__name__)
 class PipeAction(Action):
     def __init__(self, context, action_path, *configs):
         super().__init__(context, action_path, *configs)
-        assert len(configs) > 1, configs
+        if len(configs) <= 1:
+            raise ValueError(
+                f"PipeAction requires at least two actions, got {len(configs)}\n{json.dumps(configs, indent=2)}"
+            )
+
         current = action_factory(configs[0], context, action_path + ["0"])
         for i, c in enumerate(configs[1:]):
             current = step_factory(c, context, action_path + [str(i + 1)], previous_step=current)

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/repeated_dates.py

@@ -12,12 +12,11 @@ import logging
 from collections import defaultdict
 
 import numpy as np
+from anemoi.transform.fields import new_field_with_valid_datetime
+from anemoi.transform.fields import new_fieldlist_from_list
 from anemoi.utils.dates import as_datetime
 from anemoi.utils.dates import frequency_to_timedelta
 
-from anemoi.datasets.fields import FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField
-
 from .action import Action
 from .action import action_factory
 from .join import JoinResult
@@ -116,9 +115,10 @@ class DateMapperClosest(DateMapper):
 
 
 class DateMapperClimatology(DateMapper):
-    def __init__(self, source, year, day):
+    def __init__(self, source, year, day, hour=None):
         self.year = year
         self.day = day
+        self.hour = hour
 
     def transform(self, group_of_dates):
         from anemoi.datasets.dates.groups import GroupOfDates
@@ -130,6 +130,8 @@ class DateMapperClimatology(DateMapper):
         new_dates = defaultdict(list)
         for date in dates:
             new_date = date.replace(year=self.year, day=self.day)
+            if self.hour is not None:
+                new_date = new_date.replace(hour=self.hour, minute=0, second=0)
             new_dates[new_date].append(date)
 
         for date, dates in new_dates.items():
@@ -185,9 +187,12 @@ class DateMapperResult(Result):
 
         for field in self.source_results.datasource:
             for date in self.original_group_of_dates:
-                result.append(NewValidDateTimeField(field, date))
+                result.append(new_field_with_valid_datetime(field, date))
+
+        if not result:
+            raise ValueError("repeated_dates: no input data found")
 
-        return FieldArray(result)
+        return new_fieldlist_from_list(result)
 
 
 class RepeatedDatesAction(Action):

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/result.py

@@ -15,16 +15,10 @@ from collections import defaultdict
 from functools import cached_property
 
 import numpy as np
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from anemoi.utils.humanize import seconds_to_human
 from anemoi.utils.humanize import shorten_list
 from earthkit.data.core.order import build_remapping
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .trace import trace
 from .trace import trace_datasource
 

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/create/input/step.py

@@ -10,13 +10,6 @@
 import logging
 from copy import deepcopy
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .action import Action
 from .context import Context
 from .misc import is_function

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi/datasets/dates/groups.py

@@ -28,7 +28,7 @@ class GroupOfDates:
         assert isinstance(provider, DatesProvider), type(provider)
         assert isinstance(dates, list)
 
-        self.dates = dates
+        self.dates = [as_datetime(_) for _ in dates]
         self.provider = provider
         self.partial_ok = partial_ok
 

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi_datasets.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.5.10
+Version: 0.5.11
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -224,8 +224,8 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-transform>=0.0.5
-Requires-Dist: anemoi-utils[provenance]>=0.4.2
+Requires-Dist: anemoi-transform>=0.1
+Requires-Dist: anemoi-utils[provenance]>=0.4.9
 Requires-Dist: cfunits
 Requires-Dist: numpy
 Requires-Dist: pyyaml

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi_datasets.egg-info/SOURCES.txt

@@ -197,7 +197,6 @@ docs/using/images/thinning-before.png
 src/anemoi/datasets/__init__.py
 src/anemoi/datasets/__main__.py
 src/anemoi/datasets/_version.py
-src/anemoi/datasets/fields.py
 src/anemoi/datasets/grids.py
 src/anemoi/datasets/testing.py
 src/anemoi/datasets/commands/__init__.py

{anemoi_datasets-0.5.10 → anemoi_datasets-0.5.11}/src/anemoi_datasets.egg-info/requires.txt

@@ -1,5 +1,5 @@
-anemoi-transform>=0.0.5
-anemoi-utils[provenance]>=0.4.2
+anemoi-transform>=0.1
+anemoi-utils[provenance]>=0.4.9
 cfunits
 numpy
 pyyaml

anemoi_datasets-0.5.10/src/anemoi/datasets/fields.py

@@ -1,68 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-from earthkit.data.indexing.fieldlist import FieldArray
-
-
-def list_to_fieldlist(fields):
-    return FieldArray(fields)
-
-
-def empty_fieldlist():
-    return FieldArray([])
-
-
-class WrappedField:
-    def __init__(self, field):
-        self._field = field
-
-    def __getattr__(self, name):
-        return getattr(self._field, name)
-
-    def __repr__(self) -> str:
-        return repr(self._field)
-
-
-class NewDataField(WrappedField):
-    def __init__(self, field, data):
-        super().__init__(field)
-        self._data = data
-        self.shape = data.shape
-
-    def to_numpy(self, flatten=False, dtype=None, index=None):
-        data = self._data
-        if dtype is not None:
-            data = data.astype(dtype)
-        if flatten:
-            data = data.flatten()
-        if index is not None:
-            data = data[index]
-        return data
-
-
-class NewMetadataField(WrappedField):
-    def __init__(self, field, **kwargs):
-        super().__init__(field)
-        self._metadata = kwargs
-
-    def metadata(self, *args, **kwargs):
-        if len(args) == 1 and args[0] in self._metadata:
-            return self._metadata[args[0]]
-        return self._field.metadata(*args, **kwargs)
-
-
-class NewValidDateTimeField(NewMetadataField):
-    def __init__(self, field, valid_datetime):
-        date = valid_datetime.date().strftime("%Y%m%d")
-        time = valid_datetime.time().strftime("%H%M")
-
-        self.valid_datetime = valid_datetime
-
-        super().__init__(field, date=date, time=time, step=0, valid_datetime=valid_datetime.isoformat())