anemoi-datasets 0.5.25__py3-none-any.whl → 0.5.27__py3-none-any.whl

anemoi/datasets/create/filters/rename.py

@@ -1,205 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import re
-from typing import Any
-from typing import Dict
-from typing import Optional
-
-import earthkit.data as ekd
-from earthkit.data.indexing.fieldlist import FieldArray
-
-from .legacy import legacy_filter
-
-
-class RenamedFieldMapping:
-    """Rename a field based on the value of another field.
-
-    Parameters
-    ----------
-    field : Any
-        The field to be renamed.
-    what : str
-        The name of the field that will be used to rename the field.
-    renaming : dict of dict of str
-        A dictionary mapping the values of 'what' to the new names.
-    """
-
-    def __init__(self, field: Any, what: str, renaming: Dict[str, Dict[str, str]]) -> None:
-        """Initialize a RenamedFieldMapping instance.
-
-        Parameters
-        ----------
-        field : Any
-            The field to be renamed.
-        what : str
-            The name of the field that will be used to rename the field.
-        renaming : dict of dict of str
-            A dictionary mapping the values of 'what' to the new names.
-        """
-        self.field = field
-        self.what = what
-        self.renaming = {}
-        for k, v in renaming.items():
-            self.renaming[k] = {str(a): str(b) for a, b in v.items()}
-
-    def metadata(self, key: Optional[str] = None, **kwargs: Any) -> Any:
-        """Get metadata from the original field, with the option to rename the parameter.
-
-        Parameters
-        ----------
-        key : str, optional
-            The metadata key.
-        **kwargs : Any
-            Additional keyword arguments.
-
-        Returns
-        -------
-        Any
-            The metadata value.
-        """
-        if key is None:
-            return self.field.metadata(**kwargs)
-
-        value = self.field.metadata(key, **kwargs)
-        if key == self.what:
-            return self.renaming.get(self.what, {}).get(value, value)
-
-        return value
-
-    def __getattr__(self, name: str) -> Any:
-        """Get an attribute from the original field.
-
-        Parameters
-        ----------
-        name : str
-            The name of the attribute.
-
-        Returns
-        -------
-        Any
-            The attribute value.
-        """
-        return getattr(self.field, name)
-
-    def __repr__(self) -> str:
-        """Get the string representation of the original field.
-
-        Returns
-        -------
-        str
-            The string representation of the original field.
-        """
-        return repr(self.field)
-
-
-class RenamedFieldFormat:
-    """Rename a field based on a format string.
-
-    Parameters
-    ----------
-    field : Any
-        The field to be renamed.
-    what : str
-        The name of the field that will be used to rename the field.
-    format : str
-        The format string for renaming.
-    """
-
-    def __init__(self, field: Any, what: str, format: str) -> None:
-        """Initialize a RenamedFieldFormat instance.
-
-        Parameters
-        ----------
-        field : Any
-            The field to be renamed.
-        what : str
-            The name of the field that will be used to rename the field.
-        format : str
-            The format string for renaming.
-        """
-        self.field = field
-        self.what = what
-        self.format = format
-        self.bits = re.findall(r"{(\w+)}", format)
-
-    def metadata(self, *args: Any, **kwargs: Any) -> Any:
-        """Get metadata from the original field, with the option to rename the parameter using a format string.
-
-        Parameters
-        ----------
-        *args : Any
-            Positional arguments.
-        **kwargs : Any
-            Additional keyword arguments.
-
-        Returns
-        -------
-        Any
-            The metadata value.
-        """
-        value = self.field.metadata(*args, **kwargs)
-        if args:
-            assert len(args) == 1
-            if args[0] == self.what:
-                bits = {b: self.field.metadata(b, **kwargs) for b in self.bits}
-                return self.format.format(**bits)
-        return value
-
-    def __getattr__(self, name: str) -> Any:
-        """Get an attribute from the original field.
-
-        Parameters
-        ----------
-        name : str
-            The name of the attribute.
-
-        Returns
-        -------
-        Any
-            The attribute value.
-        """
-        return getattr(self.field, name)
-
-    def __repr__(self) -> str:
-        """Get the string representation of the original field.
-
-        Returns
-        -------
-        str
-            The string representation of the original field.
-        """
-        return repr(self.field)
-
-
-@legacy_filter(__file__)
-def execute(context: Any, input: ekd.FieldList, what: str = "param", **kwargs: Any) -> ekd.FieldList:
-    """Rename fields based on the value of another field or a format string.
-
-    Parameters
-    ----------
-    context : Any
-        The context in which the function is executed.
-    input : List[Any]
-        List of input fields.
-    what : str, optional
-        The field to be used for renaming. Defaults to "param".
-    **kwargs : Any
-        Additional keyword arguments for renaming.
-
-    Returns
-    -------
-    ekd.FieldList
-        Array of renamed fields.
-    """
-
-    if what in kwargs and isinstance(kwargs[what], str):
-        return FieldArray([RenamedFieldFormat(fs, what, kwargs[what]) for fs in input])
-
-    return FieldArray([RenamedFieldMapping(fs, what, kwargs) for fs in input])

anemoi/datasets/create/filters/rotate_winds.py

@@ -1,105 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-from collections import defaultdict
-from typing import Any
-from typing import Dict
-from typing import Optional
-
-import earthkit.data as ekd
-import tqdm
-from anemoi.transform.fields import new_field_from_numpy
-from anemoi.transform.fields import new_fieldlist_from_list
-from anemoi.utils.humanize import plural
-from earthkit.geo.rotate import rotate_vector
-
-from .legacy import legacy_filter
-
-
-@legacy_filter(__file__)
-def execute(
-    context: Any,
-    input: ekd.FieldList,
-    x_wind: str,
-    y_wind: str,
-    source_projection: Optional[str] = None,
-    target_projection: str = "+proj=longlat",
-) -> ekd.FieldList:
-    """Rotate wind components from one projection to another.
-
-    Parameters
-    ----------
-    context : Any
-        The context in which the function is executed.
-    input : ekd.FieldList
-        List of input fields.
-    x_wind : str
-        X wind component parameter.
-    y_wind : str
-        Y wind component parameter.
-    source_projection : Optional[str], optional
-        Source projection, by default None.
-    target_projection : str, optional
-        Target projection, by default "+proj=longlat".
-
-    Returns
-    -------
-    ekd.FieldList
-        Array of fields with rotated wind components.
-    """
-    from pyproj import CRS
-
-    context.trace("🔄", "Rotating winds (extracting winds from ", plural(len(input), "field"))
-
-    result = []
-
-    wind_params: tuple[str, str] = (x_wind, y_wind)
-    wind_pairs: Dict[tuple, Dict[str, Any]] = defaultdict(dict)
-
-    for f in input:
-        key = f.metadata(namespace="mars")
-        param = key.pop("param")
-
-        if param not in wind_params:
-            result.append(f)
-            continue
-
-        key = tuple(key.items())
-
-        if param in wind_pairs[key]:
-            raise ValueError(f"Duplicate wind component {param} for {key}")
-
-        wind_pairs[key][param] = f
-
-    context.trace("🔄", "Rotating", plural(len(wind_pairs), "wind"), "(speed will likely include data download)")
-
-    for _, pairs in tqdm.tqdm(list(wind_pairs.items())):
-        if len(pairs) != 2:
-            raise ValueError("Missing wind component")
-
-        x = pairs[x_wind]
-        y = pairs[y_wind]
-
-        assert x.grid_mapping == y.grid_mapping
-
-        lats, lons = x.grid_points()
-        x_new, y_new = rotate_vector(
-            lats,
-            lons,
-            x.to_numpy(flatten=True),
-            y.to_numpy(flatten=True),
-            (source_projection if source_projection is not None else CRS.from_cf(x.grid_mapping)),
-            target_projection,
-        )
-
-        result.append(new_field_from_numpy(x, x_new))
-        result.append(new_field_from_numpy(y, y_new))
-
-    return new_fieldlist_from_list(result)

anemoi/datasets/create/filters/single_level_dewpoint_to_relative_humidity.py

@@ -1,78 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-from collections import defaultdict
-from typing import Any
-from typing import Dict
-
-import earthkit.data as ekd
-from anemoi.transform.fields import new_field_from_numpy
-from anemoi.transform.fields import new_fieldlist_from_list
-from earthkit.meteo import thermo
-
-from .legacy import legacy_filter
-
-
-@legacy_filter(__file__)
-def execute(context: Any, input: ekd.FieldList, t: str, td: str, rh: str = "d") -> ekd.FieldList:
-    """Convert dewpoint on single levels to relative humidity.
-
-    Parameters
-    ----------
-    context : Any
-        The context in which the function is executed.
-    input : List[Any]
-        List of input fields.
-    t : str
-        Temperature parameter.
-    td : str
-        Dewpoint parameter.
-    rh : str, optional
-        Relative humidity parameter. Defaults to "d".
-
-    Returns
-    -------
-    ekd.FieldList
-        Array of fields with relative humidity.
-    """
-    result = []
-    params: tuple[str, str] = (t, td)
-    pairs: Dict[tuple, Dict[str, Any]] = defaultdict(dict)
-
-    # Gather all necessary fields
-    for f in input:
-        key = f.metadata(namespace="mars")
-        param = key.pop("param")
-        if param in params:
-            key = tuple(key.items())
-
-            if param in pairs[key]:
-                raise ValueError(f"Duplicate field {param} for {key}")
-
-            pairs[key][param] = f
-            if param == t:
-                result.append(f)
-        # all other parameters
-        else:
-            result.append(f)
-
-    for keys, values in pairs.items():
-        # some checks
-
-        if len(values) != 2:
-            raise ValueError("Missing fields")
-
-        t_values = values[t].to_numpy(flatten=True)
-        td_values = values[td].to_numpy(flatten=True)
-        # actual conversion from td --> rh
-        rh_values = thermo.relative_humidity_from_dewpoint(t=t_values, td=td_values)
-        result.append(new_field_from_numpy(values[td], rh_values, param=rh))
-
-    return new_fieldlist_from_list(result)

anemoi/datasets/create/filters/single_level_relative_humidity_to_dewpoint.py

@@ -1,84 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-from collections import defaultdict
-from typing import Any
-from typing import Dict
-from typing import Tuple
-
-import earthkit.data as ekd
-from anemoi.transform.fields import new_field_from_numpy
-from anemoi.transform.fields import new_fieldlist_from_list
-from earthkit.data.indexing.fieldlist import FieldArray
-from earthkit.meteo import thermo
-
-from .legacy import legacy_filter
-
-EPS = 1.0e-4
-
-
-@legacy_filter(__file__)
-def execute(context: Any, input: ekd.FieldList, t: str, rh: str, td: str = "d") -> FieldArray:
-    """Convert relative humidity on single levels to dewpoint.
-
-    Parameters
-    ----------
-    context : Any
-        The context in which the function is executed.
-    input : List[Any]
-        List of input fields.
-    t : str
-        Temperature parameter.
-    rh : str
-        Relative humidity parameter.
-    td : str, optional
-        Dewpoint parameter. Defaults to "d".
-
-    Returns
-    -------
-    FieldArray
-        Array of fields with dewpoint.
-    """
-    result = []
-    params: Tuple[str, str] = (t, rh)
-    pairs: Dict[Tuple[Any, ...], Dict[str, Any]] = defaultdict(dict)
-
-    # Gather all necessary fields
-    for f in input:
-        key = f.metadata(namespace="mars")
-        param = key.pop("param")
-        if param in params:
-            key = tuple(key.items())
-
-            if param in pairs[key]:
-                raise ValueError(f"Duplicate field {param} for {key}")
-
-            pairs[key][param] = f
-            if param == t:
-                result.append(f)
-        # all other parameters
-        else:
-            result.append(f)
-
-    for keys, values in pairs.items():
-        # some checks
-
-        if len(values) != 2:
-            raise ValueError("Missing fields")
-
-        t_values = values[t].to_numpy(flatten=True)
-        rh_values = values[rh].to_numpy(flatten=True)
-        # Prevent 0 % Relative humidity which cannot be converted to dewpoint
-        # Seems to happen over Egypt in the CERRA dataset
-        rh_values[rh_values == 0] = EPS
-        # actual conversion from rh --> td
-        td_values = thermo.dewpoint_from_relative_humidity(t=t_values, r=rh_values)
-        result.append(new_field_from_numpy(values[rh], td_values, param=td))
-
-    return new_fieldlist_from_list(result)

anemoi/datasets/create/filters/single_level_relative_humidity_to_specific_humidity.py

@@ -1,163 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-from typing import Any
-from typing import Dict
-from typing import List
-
-import earthkit.data as ekd
-import numpy as np
-from anemoi.transform.fields import new_field_from_numpy
-from anemoi.transform.fields import new_fieldlist_from_list
-from earthkit.meteo import thermo
-
-from .legacy import legacy_filter
-from .single_level_specific_humidity_to_relative_humidity import AutoDict
-from .single_level_specific_humidity_to_relative_humidity import pressure_at_height_level
-
-
-@legacy_filter(__file__)
-def execute(
-    context: Any,
-    input: List[Any],
-    height: float,
-    t: str,
-    rh: str,
-    sp: str,
-    new_name: str = "2q",
-    **kwargs: Dict[str, Any],
-) -> ekd.FieldList:
-    """Convert the single (height) level relative humidity to specific humidity.
-
-    Parameters
-    ----------
-    context : Any
-        The context in which the function is executed.
-    input : List[Any]
-        List of input fields.
-    height : float
-        The height level.
-    t : str
-        Temperature parameter name.
-    rh : str
-        Relative humidity parameter name.
-    sp : str
-        Surface pressure parameter name.
-    new_name : str, optional
-        The new name for the specific humidity field, by default "2q".
-    **kwargs : Dict[str, Any]
-        Additional keyword arguments.
-
-    Returns
-    -------
-    ekd.FieldList
-        The resulting field list with specific humidity fields.
-
-    Raises
-    ------
-    KeyError
-        If mandatory keys are missing.
-    ValueError
-        If there are duplicate fields or missing fields.
-    """
-    result = []
-
-    MANDATORY_KEYS = ["A", "B"]
-    OPTIONAL_KEYS = ["t_ml", "q_ml"]
-    MISSING_KEYS = []
-    DEFAULTS = dict(t_ml="t", q_ml="q")
-
-    for key in OPTIONAL_KEYS:
-        if key not in kwargs:
-            print(f"key {key} not found in yaml-file, using default key: {DEFAULTS[key]}")
-            kwargs[key] = DEFAULTS[key]
-
-    for key in MANDATORY_KEYS:
-        if key not in kwargs:
-            MISSING_KEYS.append(key)
-
-    if MISSING_KEYS:
-        raise KeyError(f"Following keys are missing: {', '.join(MISSING_KEYS)}")
-
-    single_level_params = (t, rh, sp)
-    model_level_params = (kwargs["t_ml"], kwargs["q_ml"])
-
-    needed_fields = AutoDict()
-
-    # Gather all necessary fields
-    for f in input:
-        key = f.metadata(namespace="mars")
-        param = key.pop("param")
-        # check single level parameters
-        if param in single_level_params:
-            levtype = key.pop("levtype")
-            key = tuple(key.items())
-
-            if param in needed_fields[key][levtype]:
-                raise ValueError(f"Duplicate single level field {param} for {key}")
-
-            needed_fields[key][levtype][param] = f
-            if param == rh:
-                if kwargs.get("keep_rh", False):
-                    result.append(f)
-            else:
-                result.append(f)
-
-        # check model level parameters
-        elif param in model_level_params:
-            levtype = key.pop("levtype")
-            levelist = key.pop("levelist")
-            key = tuple(key.items())
-
-            if param in needed_fields[key][levtype][levelist]:
-                raise ValueError(f"Duplicate model level field {param} for {key} at level {levelist}")
-
-            needed_fields[key][levtype][levelist][param] = f
-
-        # all other parameters
-        else:
-            result.append(f)
-
-    for _, values in needed_fields.items():
-        # some checks
-        if len(values["sfc"]) != 3:
-            raise ValueError("Missing surface fields")
-
-        rh_sl = values["sfc"][rh].to_numpy(flatten=True)
-        t_sl = values["sfc"][t].to_numpy(flatten=True)
-        sp_sl = values["sfc"][sp].to_numpy(flatten=True)
-
-        nlevels = len(kwargs["A"]) - 1
-        if len(values["ml"]) != nlevels:
-            raise ValueError("Missing model levels")
-
-        for key in values["ml"].keys():
-            if len(values["ml"][key]) != 2:
-                raise ValueError(f"Missing field on level {key}")
-
-        # create 3D arrays for upper air fields
-        levels = list(values["ml"].keys())
-        levels.sort()
-        t_ml = []
-        q_ml = []
-        for level in levels:
-            t_ml.append(values["ml"][level][kwargs["t_ml"]].to_numpy(flatten=True))
-            q_ml.append(values["ml"][level][kwargs["q_ml"]].to_numpy(flatten=True))
-
-        t_ml = np.stack(t_ml)
-        q_ml = np.stack(q_ml)
-
-        # actual conversion from rh --> q_v
-        p_sl = pressure_at_height_level(height, q_ml, t_ml, sp_sl, np.array(kwargs["A"]), np.array(kwargs["B"]))
-        q_sl = thermo.specific_humidity_from_relative_humidity(t_sl, rh_sl, p_sl)
-
-        result.append(new_field_from_numpy(values["sfc"][rh], q_sl, param=new_name))
-
-    return new_fieldlist_from_list(result)