climate-ref-core 0.16.2__tar.gz → 0.17.0__tar.gz

{climate_ref_core-0.16.2 → climate_ref_core-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: climate-ref-core
-Version: 0.16.2
+Version: 0.17.0
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0
@@ -26,12 +26,13 @@ Requires-Dist: loguru>=0.7.0
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: pandas>=2.1.0
 Requires-Dist: pooch<2,>=1.8.0
+Requires-Dist: psutil>=5.9.8
 Requires-Dist: pyarrow>=17.0.0
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: requests
 Requires-Dist: rich
-Requires-Dist: setuptools<81
+Requires-Dist: setuptools<84
 Requires-Dist: typing-extensions
 Provides-Extra: aws
 Requires-Dist: boto3>=1.34; extra == 'aws'

{climate_ref_core-0.16.2 → climate_ref_core-0.17.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref-core"
-version = "0.16.2"
+version = "0.17.0"
 description = "Core library for the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [
@@ -37,8 +37,9 @@ dependencies = [
     "pooch>=1.8.0,<2",
     "environs>=11",
     "pyyaml>=6.0.2",
+    "psutil>=5.9.8",
     # Not used directly, but required to support some installations
-    "setuptools<81",
+    "setuptools<84",
 
     # ESGF data fetching
     "intake-esgf>=2025.7.16",
@@ -65,6 +66,7 @@ aws = [
 [dependency-groups]
 dev = [
     "types-requests",
+    "types-psutil",
 ]
 
 [build-system]

{climate_ref_core-0.16.2 → climate_ref_core-0.17.0}/src/climate_ref_core/cmip6_to_cmip7.py

@@ -543,14 +543,14 @@ def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> x
     if offset_months == 0:
         return ds
 
-    calendar = last.calendar  # type: ignore[attr-defined]
+    calendar = last.calendar
 
     def _days_in_month(year: int, month: int) -> int:
         # Last day of ``month`` = day before the first of the following month,
         # computed via cftime arithmetic so it respects the dataset's calendar
         # (noleap, 360_day, etc.).
         next_year, next_month = (year + 1, 1) if month == _MONTHS_PER_YEAR else (year, month + 1)
-        first_of_next = cftime.datetime(next_year, next_month, 1, calendar=calendar)  # type: ignore[call-arg]
+        first_of_next = cftime.datetime(next_year, next_month, 1, calendar=calendar)
         last_of_month = first_of_next - timedelta(days=1)  # type: ignore[operator]
         return int(last_of_month.day)  # type: ignore[attr-defined]
 
@@ -562,9 +562,7 @@ def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> x
         # can land a day-31 (or leap Feb-29) label on a shorter month, which cftime
         # would reject. Our monthly data is mid-month so this is normally a no-op.
         day = min(t.day, _days_in_month(year, month))
-        return cftime.datetime(  # type: ignore[call-arg]
-            year, month, day, t.hour, t.minute, t.second, t.microsecond, calendar=calendar
-        )
+        return cftime.datetime(year, month, day, t.hour, t.minute, t.second, t.microsecond, calendar=calendar)
 
     ds = ds.copy(deep=False)
     shifted = np.array([_shift(t) for t in time_values])

climate_ref_core-0.17.0/src/climate_ref_core/cmor.py

@@ -0,0 +1,118 @@
+"""
+CMOR controlled vocabularies shared across source types.
+"""
+
+# CMIP6 CMOR table -> CMIP6 ``frequency`` CV value.
+#
+# A CMOR table name is a realm prefix plus a frequency suffix, but the reduction is not a plain
+# suffix strip: ``Oclim`` is a monthly climatology (``monC``), ``E1hrClimMon`` is ``1hrCM``, the
+# zonal-mean tables (``AERmonZ``, ``EmonZ``, ``EdayZ``, ``E6hrZ``) keep the frequency of their
+# non-zonal counterpart, and the ``Pt`` (point-sampled) tables map to distinct ``*Pt``
+# frequencies. So the mapping is enumerated rather than derived.
+_MIP_TABLE_FREQUENCIES: dict[str, str] = {
+    "3hr": "3hr",
+    "6hrLev": "6hr",
+    "6hrPlev": "6hr",
+    "6hrPlevPt": "6hrPt",
+    "AERday": "day",
+    "AERhr": "1hr",
+    "AERmon": "mon",
+    "AERmonZ": "mon",
+    "Amon": "mon",
+    "CF3hr": "3hr",
+    "CFday": "day",
+    "CFmon": "mon",
+    "CFsubhr": "subhrPt",
+    "day": "day",
+    "E1hr": "1hr",
+    "E1hrClimMon": "1hrCM",
+    "E3hr": "3hr",
+    "E3hrPt": "3hrPt",
+    "E6hrZ": "6hr",
+    "Eday": "day",
+    "EdayZ": "day",
+    "Efx": "fx",
+    "Emon": "mon",
+    "EmonZ": "mon",
+    "Esubhr": "subhrPt",
+    "Eyr": "yr",
+    "IfxAnt": "fx",
+    "IfxGre": "fx",
+    "ImonAnt": "mon",
+    "ImonGre": "mon",
+    "IyrAnt": "yr",
+    "IyrGre": "yr",
+    "LImon": "mon",
+    "Lmon": "mon",
+    "OIclim": "monC",
+    "OIday": "day",
+    "OImon": "mon",
+    "Oclim": "monC",
+    "Oday": "day",
+    "Odec": "dec",
+    "Ofx": "fx",
+    "Omon": "mon",
+    "Oyr": "yr",
+    "SIday": "day",
+    "SImon": "mon",
+    "fx": "fx",
+}
+
+# The CMIP6 ``frequency`` CV. Values already in this set pass through ``frequency_from_mip_table``
+# untouched, which is what lets a layout that stores a frequency rather than a table
+# use the same call site.
+_FREQUENCIES: frozenset[str] = frozenset(
+    {
+        "1hr",
+        "1hrCM",
+        "1hrPt",
+        "3hr",
+        "3hrPt",
+        "6hr",
+        "6hrPt",
+        "day",
+        "dec",
+        "fx",
+        "mon",
+        "monC",
+        "monPt",
+        "subhrPt",
+        "yr",
+        "yrPt",
+    }
+)
+
+
+def frequency_from_mip_table(value: str) -> str:
+    """
+    Reduce a CMOR MIP table name to its CMIP6 ``frequency`` CV value.
+
+    Some datasets record ``frequency`` and not the MIP table.
+    This maps a table (e.g. ``Amon`` -> ``mon``) onto the same axis.
+
+    A value that is already a valid frequency is returned unchanged,
+    so a caller reading a layout that stores ``mon`` or ``day`` can use this without branching.
+    ``day`` and ``fx`` are both a table name and a frequency, and map to themselves either way.
+
+    Parameters
+    ----------
+    value
+        A CMOR MIP table name (``Amon``, ``Omon``, ``SIday``) or an existing frequency (``mon``).
+
+    Returns
+    -------
+    :
+        The corresponding CMIP6 frequency.
+
+    Raises
+    ------
+    ValueError
+        If the value is neither a known MIP table nor a known frequency. Failing loudly is
+        deliberate: silently defaulting would let a mis-parsed path collapse two datasets that
+        differ only by frequency onto one ``instance_id``.
+    """
+    if value in _MIP_TABLE_FREQUENCIES:
+        return _MIP_TABLE_FREQUENCIES[value]
+    if value in _FREQUENCIES:
+        return value
+    raise ValueError(f"Unknown MIP table or frequency: {value!r}")

{climate_ref_core-0.16.2 → climate_ref_core-0.17.0}/src/climate_ref_core/constraints.py

@@ -2,6 +2,7 @@
 Dataset selection constraints
 """
 
+import re
 from collections import defaultdict
 from collections.abc import Mapping
 from functools import total_ordering
@@ -632,3 +633,61 @@ class AddParentDataset:
             },
         }
         return cls(parent_facet_map=parent_facet_options[source_type])
+
+
+_MEMBER_RE = re.compile(r"r(\d+)i(\d+)p(\d+)f(\d+)")
+
+
+def _member_sort_key(member: str) -> tuple[int, tuple[int, ...] | str]:
+    """
+    Sort key that orders CMIP ensemble members numerically.
+
+    Members matching ``r<r>i<i>p<p>f<f>`` sort first, ordered by their numeric
+    (realization, initialization, physics, forcing) indices, so ``r1i1p1f1`` precedes
+    both ``r2i1p1f1`` and ``r10i1p1f1``. Members that do not match sort afterwards by
+    their raw string, keeping the ordering deterministic.
+    """
+    match = _MEMBER_RE.fullmatch(str(member))
+    if match:
+        return (0, tuple(int(value) for value in match.groups()))
+    return (1, str(member))
+
+
+@frozen
+class SelectFirstMember:
+    """
+    Keep a single ensemble member per group.
+
+    Within each group defined by ``group_by``, only the datasets belonging to a single
+    ensemble member are retained: the member with the lowest CMIP indices (``r1i1p1f1``
+    when present, see :func:`_member_sort_key`).
+
+    Some diagnostic scripts expect exactly one member per model and error when given
+    several (e.g. ``None or too many matching files``). The REF otherwise passes every
+    ingested member of a model into a single execution, so this constraint reduces each
+    model to one representative member.
+    """
+
+    member_facet: str = "member_id"
+    """The facet identifying the ensemble member (``member_id`` for CMIP6, ``variant_label`` for CMIP7)."""
+
+    group_by: tuple[str, ...] = field(converter=_to_tuple, default=("source_id",))
+    """The facets defining each model within which a single member is kept."""
+
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Keep only the first ensemble member within each group.
+        """
+        if group.empty or self.member_facet not in group.columns:
+            return group
+
+        keep = pd.Series(False, index=group.index)
+        for _, subgroup in group.groupby(list(self.group_by)):
+            members = subgroup[self.member_facet].dropna().unique()
+            if len(members) == 0:
+                # No member information to select on; keep the subgroup unchanged.
+                keep.loc[subgroup.index] = True
+                continue
+            first = min(members, key=_member_sort_key)
+            keep.loc[subgroup.index[subgroup[self.member_facet] == first]] = True
+        return group[keep]

climate_ref_core-0.17.0/src/climate_ref_core/data.py

@@ -0,0 +1,367 @@
+"""
+Resolution of the data files that are distributed with the REF
+
+Several files the REF depends on are shipped inside the wheels:
+the controlled vocabulary, the grey list, and the dataset registry manifests.
+Some of them also have a newer copy available over the network.
+
+`LayeredResource` resolves one logical file across three layers:
+an operator override, a cached download, and the copy shipped in the package.
+The packaged copy is always present and always readable,
+so resolution never requires the network or a writable filesystem.
+
+Packaged files are read through `importlib.resources` rather than being converted
+to a filesystem path, so an installation that is not unpacked on disk still works.
+"""
+
+import enum
+import importlib.resources
+import os
+import pathlib
+from collections.abc import Generator
+from contextlib import ExitStack, contextmanager
+from typing import Protocol
+
+import platformdirs
+from attrs import field, frozen
+
+from climate_ref_core.exceptions import RefException
+
+_RESOURCE_ERRORS = (ModuleNotFoundError, FileNotFoundError, OSError)
+
+
+class DataResourceError(RefException):
+    """Raised when a data file cannot be resolved or read."""
+
+
+def resolve_cache_dir(cache_name: str) -> pathlib.Path:
+    """
+    Resolve a cache directory used to hold downloaded data
+
+    If the ``REF_DATASET_CACHE_DIR`` environment variable is set, use that as the root.
+    Otherwise, fall back to the OS cache under ``climate_ref``.
+
+    Parameters
+    ----------
+    cache_name
+        Subdirectory name within the cache root.
+
+    Returns
+    -------
+        The resolved cache directory path.
+    """
+    if env_cache_dir := os.environ.get("REF_DATASET_CACHE_DIR"):
+        cache_dir = pathlib.Path(os.path.expandvars(env_cache_dir))
+        try:
+            cache_dir = cache_dir.expanduser()
+        except RuntimeError:
+            # An unresolvable ``~user`` is left as-is rather than failing the caller.
+            pass
+    else:
+        cache_dir = platformdirs.user_cache_path("climate_ref")
+
+    return cache_dir / cache_name
+
+
+class Resource(Protocol):
+    """
+    Something a data file can be read from
+
+    This is the contract `LayeredResource` resolves to,
+    and the narrowest one its consumers need.
+    """
+
+    def read_text(self, encoding: str = "utf-8") -> str:
+        """Read the file as text."""
+
+    def describe(self) -> str:
+        """Describe where the file is read from, for a log or error message."""
+
+
+@frozen
+class PackagedResource:
+    """
+    A data file shipped inside an installed package
+
+    The file is always accessed through `importlib.resources`.
+    It is never converted to a filesystem path unless a caller explicitly asks for one
+    via `as_path`, which extracts it to a temporary location when needed.
+    """
+
+    package: str
+    """Import name of the package that contains the file, for example ``climate_ref_core.pycmec``."""
+
+    resource: str
+    """Name of the file within that package, for example ``cv_cmip7_aft.yaml``."""
+
+    def _traversable(self) -> importlib.resources.abc.Traversable:
+        """
+        Locate the file within the installed package.
+
+        Existence is checked here rather than at each accessor,
+        because `importlib.resources.as_file` happily hands back a path that does not exist.
+
+        Returns
+        -------
+        :
+            The located file.
+
+        Raises
+        ------
+        DataResourceError
+            If the file is not present in the installed package.
+        """
+        missing = DataResourceError(
+            f"Could not read {self} from the installed package. "
+            "This usually means the package was built without its data files."
+        )
+        try:
+            located = importlib.resources.files(self.package) / self.resource
+            if not located.is_file():
+                raise missing
+        except _RESOURCE_ERRORS as exc:
+            raise missing from exc
+        return located
+
+    def exists(self) -> bool:
+        """
+        Check whether the file is present in the installed package
+
+        Returns
+        -------
+        :
+            True if the file can be read.
+        """
+        try:
+            self._traversable()
+        except DataResourceError:
+            return False
+        return True
+
+    def read_text(self, encoding: str = "utf-8") -> str:
+        """
+        Read the file as text
+
+        Parameters
+        ----------
+        encoding
+            Text encoding of the file.
+
+        Returns
+        -------
+        :
+            The contents of the file.
+        """
+        traversable = self._traversable()
+        try:
+            return traversable.read_text(encoding=encoding)
+        except _RESOURCE_ERRORS as exc:
+            raise DataResourceError(f"Could not read {self} from the installed package.") from exc
+
+    @contextmanager
+    def as_path(self) -> Generator[pathlib.Path]:
+        """
+        Expose the file as a filesystem path for the duration of the context
+
+        Prefer `read_text`.
+        Use this only for third-party APIs that insist on a path.
+        The path may point at a temporary file that is removed on exit,
+        so it must not be retained beyond the context.
+
+        Yields
+        ------
+        :
+            A path that exists for the duration of the context.
+        """
+        traversable = self._traversable()
+        with ExitStack() as stack:
+            try:
+                path = stack.enter_context(importlib.resources.as_file(traversable))
+            except _RESOURCE_ERRORS as exc:
+                raise DataResourceError(f"Could not materialise {self} as a filesystem path.") from exc
+            # Only the lookup is guarded. An exception raised in the caller's `with`
+            # block propagates back in through this yield and must not be relabelled.
+            yield path
+
+    def describe(self) -> str:
+        """
+        Describe where the file is read from
+
+        Returns
+        -------
+        :
+            The package and file name.
+        """
+        return str(self)
+
+    def __str__(self) -> str:
+        return f"{self.package}/{self.resource}"
+
+
+@frozen
+class FileResource:
+    """
+    A data file at a plain filesystem path
+    """
+
+    path: pathlib.Path
+
+    def exists(self) -> bool:
+        """
+        Check whether the file exists
+
+        Returns
+        -------
+        :
+            True if the path points at a readable file.
+        """
+        return self.path.is_file()
+
+    def read_text(self, encoding: str = "utf-8") -> str:
+        """
+        Read the file as text
+
+        Parameters
+        ----------
+        encoding
+            Text encoding of the file.
+
+        Returns
+        -------
+        :
+            The contents of the file.
+        """
+        try:
+            return self.path.read_text(encoding=encoding)
+        except OSError as exc:
+            raise DataResourceError(f"Could not read {self}.") from exc
+
+    @contextmanager
+    def as_path(self) -> Generator[pathlib.Path]:
+        """
+        Expose the file as a filesystem path
+
+        Yields
+        ------
+        :
+            The path itself, which already exists on disk.
+        """
+        yield self.path
+
+    def describe(self) -> str:
+        """
+        Describe where the file is read from
+
+        Returns
+        -------
+        :
+            The path.
+        """
+        return str(self)
+
+    def __str__(self) -> str:
+        return str(self.path)
+
+
+class ResourceOrigin(enum.Enum):
+    """
+    Which layer a resolved data file came from
+    """
+
+    override = "override"
+    """An explicit path supplied by the operator via configuration or an environment variable."""
+
+    cache = "cache"
+    """A copy downloaded into the local cache."""
+
+    package = "package"
+    """The copy shipped inside the installed package."""
+
+
+@frozen
+class LayeredResource:
+    """
+    A data file resolved across an override, a cache, and the packaged copy
+
+    Resolution happens on every access rather than being fixed at construction,
+    so a cache that is populated after the object is built is picked up
+    without the object being rebuilt.
+    """
+
+    packaged: PackagedResource
+    """The copy shipped in the package. This is the floor, and is always available."""
+
+    override: pathlib.Path | None = field(default=None)
+    """An explicit path supplied by the operator. Takes precedence over everything else."""
+
+    cache: pathlib.Path | None = field(default=None)
+    """A local cache location. Used only when the file is actually present there."""
+
+    def resolve(self) -> tuple[ResourceOrigin, Resource]:
+        """
+        Determine which layer supplies the file
+
+        Returns
+        -------
+        :
+            The origin, and the resource that supplies the file.
+
+        Raises
+        ------
+        DataResourceError
+            If an override was supplied but does not point at a readable file.
+        """
+        if self.override is not None:
+            if not self.override.is_file():
+                raise DataResourceError(
+                    f"The configured file {self.override} does not exist. "
+                    "Point it at an existing file, or remove the setting to use the copy "
+                    f"shipped with the REF ({self.packaged})."
+                )
+            return ResourceOrigin.override, FileResource(self.override)
+
+        if self.cache is not None and self.cache.is_file():
+            return ResourceOrigin.cache, FileResource(self.cache)
+
+        return ResourceOrigin.package, self.packaged
+
+    @property
+    def origin(self) -> ResourceOrigin:
+        """
+        The layer that currently supplies the file
+        """
+        return self.resolve()[0]
+
+    def describe(self) -> str:
+        """
+        Describe where the file is being read from
+
+        This never raises, so it is safe to use when building a log message
+        for a resolution that has itself failed.
+
+        Returns
+        -------
+        :
+            A short description suitable for a log message.
+        """
+        try:
+            origin, source = self.resolve()
+        except DataResourceError:
+            return f"{self.override} (missing)"
+        return f"{source} ({origin.value})"
+
+    def read_text(self, encoding: str = "utf-8") -> str:
+        """
+        Read the file as text from whichever layer supplies it
+
+        Parameters
+        ----------
+        encoding
+            Text encoding of the file.
+
+        Returns
+        -------
+        :
+            The contents of the file.
+        """
+        return self.resolve()[1].read_text(encoding=encoding)