climate-ref-core 0.18.2__tar.gz → 0.18.3__tar.gz

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: climate-ref-core
-Version: 0.18.2
+Version: 0.18.3
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref-core"
-version = "0.18.2"
+version = "0.18.3"
 description = "Core library for the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/src/climate_ref_core/cmip6_to_cmip7.py

@@ -403,6 +403,7 @@ def make_tracking_id(file_id: str) -> str:
 
 def convert_cmip6_to_cmip7_attrs(
     cmip6_attrs: dict[str, Any],
+    file_id: str | None = None,
 ) -> dict[str, Any]:
     """
     Convert CMIP6 global attributes to CMIP7 format.
@@ -413,6 +414,9 @@ def convert_cmip6_to_cmip7_attrs(
     ----------
     cmip6_attrs
         Dictionary of CMIP6 global attributes. Must contain ``table_id``.
+    file_id
+        What identifies the file being written, used to derive its ``tracking_id`` (see
+        :func:`make_tracking_id`).
 
     Returns
     -------
@@ -490,7 +494,7 @@ def convert_cmip6_to_cmip7_attrs(
     attrs["cmip6_compound_name"] = dreq_entry.cmip6_compound_name
 
     # Add tracking_id with CMIP7 handle prefix
-    attrs["tracking_id"] = f"hdl:21.14107/{uuid.uuid4()}"
+    attrs["tracking_id"] = make_tracking_id(file_id) if file_id else f"hdl:21.14107/{uuid.uuid4()}"
 
     # Add creation_date in ISO format
     attrs["creation_date"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -581,6 +585,15 @@ def repeat_final_year_to(ds: xr.Dataset, end_year: int, end_month: int = 12) ->
         raise ValueError(f"The series must end in December to repeat whole years, ends {last}")
 
     bounds_name = ds["time"].attrs.get("bounds")
+    if bounds_name and bounds_name in ds.variables:
+        # The repeated blocks below carry their restamped bounds as a NumPy array of cftime
+        # objects. If the source bounds arrived lazily -- a file opened with ``chunks=`` gets
+        # Dask-backed ones -- the concat at the end mixes the two, and Dask refuses to chunk
+        # the NumPy side because it cannot size object dtype. The array is one row per
+        # timestep, so materialise it here rather than make every caller open the file a
+        # particular way. Assigned to a shallow copy, leaving the caller's dataset lazy.
+        ds = ds.copy()
+        ds[bounds_name] = ds[bounds_name].compute()
 
     def _restamp(t: cftime.datetime, shift: int) -> cftime.datetime:
         year = t.year + shift
@@ -621,6 +634,7 @@ def convert_cmip6_dataset(
     ds: xr.Dataset,
     inplace: bool = False,
     extend_historical_to: tuple[int, int] | None = None,
+    file_id: str | None = None,
 ) -> xr.Dataset:
     """
     Convert a CMIP6 xarray Dataset to CMIP7 format in-memory.
@@ -644,6 +658,9 @@ def convert_cmip6_dataset(
         via :func:`repeat_final_year_to`, fabricating CMIP7 coverage for years without real
         data. Defaults to ``None`` (time axis untouched), so existing conversions are
         byte-identical.
+    file_id
+        What identifies the file being written, so that converting it again gives it the
+        same ``tracking_id``. See :func:`convert_cmip6_to_cmip7_attrs`.
 
     Returns
     -------
@@ -673,7 +690,7 @@ def convert_cmip6_dataset(
     ds = ds.rename({variable_id: dreq_entry.variable_id})
 
     # Convert global attributes
-    ds.attrs = convert_cmip6_to_cmip7_attrs(ds.attrs)
+    ds.attrs = convert_cmip6_to_cmip7_attrs(ds.attrs, file_id=file_id)
 
     return ds
 

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/src/climate_ref_core/esgf/__init__.py

@@ -5,7 +5,11 @@ This module provides classes for searching and fetching datasets from ESGF
 (Earth System Grid Federation) and other data registries.
 """
 
-from climate_ref_core.esgf.base import ESGFRequest, IntakeESGFMixin, enable_ceda_solr_index
+from climate_ref_core.esgf.base import (
+    ESGFRequest,
+    IntakeESGFMixin,
+    enable_ceda_solr_index,
+)
 from climate_ref_core.esgf.cmip6 import CMIP6Request
 from climate_ref_core.esgf.cmip7 import CMIP7Request
 from climate_ref_core.esgf.fetcher import ESGFFetcher

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/src/climate_ref_core/esgf/base.py

@@ -5,12 +5,15 @@ This module provides the infrastructure for fetching datasets from ESGF
 using the intake-esgf package.
 """
 
-from typing import Any, Protocol, runtime_checkable
+import copy
+from collections.abc import Mapping, Sequence
+from typing import Any, Protocol, Self, cast, runtime_checkable
 
 import intake_esgf
 import pandas as pd
 from intake_esgf import ESGFCatalog
 from intake_esgf.exceptions import NoSearchResults
+from loguru import logger
 
 from climate_ref_core.exceptions import DatasetResolutionError
 
@@ -59,6 +62,69 @@ class ESGFRequest(Protocol):
         """
         ...
 
+    def pin_to_datasets(self, datasets: Sequence[Mapping[str, Any]]) -> "ESGFRequest":
+        """
+        Return a copy of this request that limits its resolution to the given datasets.
+
+        Parameters
+        ----------
+        datasets
+            The catalog records of the datasets recorded for this request's source type.
+
+        Returns
+        -------
+        :
+            The pinned request, or this request unchanged when the records do not carry
+            what the pin needs. Pinning is all-or-nothing, so a pinned request never
+            silently resolves a subset of what was recorded.
+        """
+        ...
+
+
+def facet_pins_from_datasets(
+    datasets: Sequence[Mapping[str, Any]],
+    fields: Mapping[str, str],
+    slug: str,
+) -> tuple[dict[str, str], ...] | None:
+    """
+    Read one facet set per recorded dataset, for pins that a search cannot ask for by id.
+
+    Parameters
+    ----------
+    datasets
+        The catalog records to pin to.
+    fields
+        The record field each search facet is spelled as, keyed by facet name.
+    slug
+        Slug of the request being pinned, for the warning when a record falls short.
+
+    Returns
+    -------
+    :
+        The facet sets, deduplicated because a dataset is recorded once per file it
+        holds, or ``None`` if any record is missing one of the facets.
+    """
+    pins = []
+    for dataset in datasets:
+        pin = {
+            facet: str(dataset[field])
+            for facet, field in fields.items()
+            if dataset.get(field) not in (None, "")
+        }
+        if len(pin) != len(fields):
+            logger.warning(
+                "Recorded datasets are missing facets needed to identify them; "
+                f"resolving request {slug} from its declared facets instead"
+            )
+            return None
+        pins.append(pin)
+
+    if not pins:
+        return None
+
+    unique = dict.fromkeys(tuple(sorted(pin.items())) for pin in pins)
+    return tuple(dict(facets) for facets in unique)
+
 
 def _deduplicate_datasets(datasets: pd.DataFrame) -> pd.DataFrame:
     """
@@ -102,15 +168,67 @@ class IntakeESGFMixin:
     - facets: dict[str, str | tuple[str, ...]]
     - remove_ensembles: bool
     - time_span: tuple[str, str] | None
+
+    A request can also be pinned to the datasets an existing test case catalog records,
+    see :meth:`pin_to_datasets`.
     """
 
+    slug: str
     facets: dict[str, str | tuple[str, ...]]
     remove_ensembles: bool
     time_span: tuple[str, str] | None
 
+    pinned_instance_ids: tuple[str, ...] | None = None
+    """Ids of the datasets this request is pinned to."""
+
+    def pin_to_datasets(self, datasets: Sequence[Mapping[str, Any]]) -> Self:
+        """
+        Return a copy of this request that only resolves the given datasets.
+
+        The ids a catalog records are the ones ESGF publishes the datasets under, so they
+        are used in the search instead of the facets if possible.
+
+        Parameters
+        ----------
+        datasets
+            The catalog records of the datasets recorded for this request's source type.
+
+        Returns
+        -------
+        :
+            The pinned request, or this request unchanged if any record has no id.
+            Pinning is all-or-nothing, so a pinned request never silently resolves a
+            subset of what was recorded.
+        """
+        instance_ids = [dataset.get("instance_id") for dataset in datasets]
+        if not instance_ids or None in instance_ids:
+            logger.warning(
+                f"Recorded datasets have no instance_id; "
+                f"resolving request {self.slug} from its declared facets instead"
+            )
+            return self
+
+        pinned = copy.copy(self)
+        # A dataset is recorded once per file it holds
+        pinned.pinned_instance_ids = tuple(dict.fromkeys(cast(list[str], instance_ids)))
+        return pinned
+
     def fetch_datasets(self) -> pd.DataFrame:
         """Fetch dataset metadata from ESGF."""
         facets: dict[str, Any] = dict(self.facets)
+
+        if self.pinned_instance_ids:
+            # Ask for the pinned datasets by id as the declared facets may now
+            # resolve to something else. The project is kept because it selects how the
+            # index is read rather than which datasets match.
+            logger.info(f"Resolving {len(self.pinned_instance_ids)} pinned datasets for {self.slug}")
+            facets = {
+                **({"project": facets["project"]} if "project" in facets else {}),
+                "instance_id": list(self.pinned_instance_ids),
+                # A pin holds even once a newer version of the dataset has been published
+                "latest": [True, False],
+            }
+
         if self.time_span:
             facets["file_start"] = self.time_span[0]
             facets["file_end"] = self.time_span[1]
@@ -133,7 +251,8 @@ class IntakeESGFMixin:
                 msg = f"ESGF search returned no results for facets: {facets}"
                 raise DatasetResolutionError(msg) from None
 
-            if self.remove_ensembles:
+            if self.remove_ensembles and not self.pinned_instance_ids:
+                # Pinned ids already name one ensemble member each
                 cat.remove_ensembles()
 
             path_dict = cat.to_path_dict(prefer_streaming=False, minimal_keys=False, quiet=True)

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/src/climate_ref_core/esgf/cmip7.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import re
 import uuid
+from collections.abc import Mapping, Sequence
 from pathlib import Path
 from typing import Any, ClassVar, NamedTuple
 
@@ -53,32 +54,61 @@ _MANDATORY_CONVERTED_ATTRS = (
 _LOSSY_DECIMAL_DIGITS = 3
 
 
+def _write_encoding(ds: xr.Dataset) -> dict[str, dict[str, Any]]:
+    """
+    Per-variable NetCDF encoding for a converted dataset.
+
+    Everything is compressed. Values are also rounded to :data:`_LOSSY_DECIMAL_DIGITS`,
+    except where that would quantise the data away: gpp and o3 magnitudes sit below the
+    rounding floor, and ozone additionally needs lossless hybrid-coordinate coefficients.
+    """
+    lossless_dataset = ds.attrs.get("variable_id") == "o3"
+
+    encoding: dict[str, dict[str, Any]] = {}
+    for var in ds.data_vars:
+        var_encoding: dict[str, Any] = {"zlib": True, "complevel": 5}
+        if not lossless_dataset and str(var) != "gpp":
+            var_encoding["least_significant_digit"] = _LOSSY_DECIMAL_DIGITS
+        encoding[str(var)] = var_encoding
+    return encoding
+
+
+def _quantised_variables(ds: xr.Dataset) -> list[str]:
+    """Name the variables stored with the lossy decimal quantisation the writer applies."""
+    return [
+        str(name)
+        for name in ds.data_vars
+        if ds[name].encoding.get("least_significant_digit") == _LOSSY_DECIMAL_DIGITS
+    ]
+
+
 def _invalid_conversion_reason(path: Path) -> str | None:
-    """Return why a cached conversion cannot satisfy the complete CMIP7 parser."""
+    """Return why a cached conversion cannot be reused, or ``None`` if it can."""
     try:
         with xr.open_dataset(path, decode_times=False) as ds:
-            missing = [name for name in _MANDATORY_CONVERTED_ATTRS if not ds.attrs.get(name)]
-            if missing:
-                return f"missing mandatory attributes: {', '.join(missing)}"
-            variable_id = str(ds.attrs["variable_id"])
-            if variable_id not in ds.variables:
-                return f"missing data variable: {variable_id}"
-            if variable_id == "o3":
-                quantised = [
-                    str(name)
-                    for name in ds.data_vars
-                    if ds[name].encoding.get("least_significant_digit") == _LOSSY_DECIMAL_DIGITS
-                ]
-                if quantised:
-                    return f"ozone dataset uses destructive decimal quantisation: {', '.join(quantised)}"
-            if "time" in ds.variables and ds.sizes.get("time", 0):
-                time = ds["time"]
-                if not time.attrs.get("units"):
-                    return "time coordinate has no units"
-                # Reading both endpoints catches a truncated file.
-                time.isel(time=[0, -1]).load()
+            return _conversion_defect(ds)
     except (OSError, ValueError) as e:
         return f"cannot read metadata: {e}"
+
+
+def _conversion_defect(ds: xr.Dataset) -> str | None:
+    """Return what is wrong with an opened cached conversion, or ``None`` if nothing is."""
+    missing = [name for name in _MANDATORY_CONVERTED_ATTRS if not ds.attrs.get(name)]
+    if missing:
+        return f"missing mandatory attributes: {', '.join(missing)}"
+
+    variable_id = str(ds.attrs["variable_id"])
+    if variable_id not in ds.variables:
+        return f"missing data variable: {variable_id}"
+    if variable_id == "o3" and (quantised := _quantised_variables(ds)):
+        return f"ozone dataset uses destructive decimal quantisation: {', '.join(quantised)}"
+
+    if "time" in ds.variables and ds.sizes.get("time", 0):
+        time = ds["time"]
+        if not time.attrs.get("units"):
+            return "time coordinate has no units"
+        # Reading both endpoints catches a truncated file.
+        time.isel(time=[0, -1]).load()
     return None
 
 
@@ -229,7 +259,7 @@ def _convert_file_to_cmip7(
                     return output_file
                 logger.info(f"Rebuilding invalid CMIP7 conversion ({invalid_reason}): {output_file}")
 
-            ds_cmip7 = convert_cmip6_dataset(source_ds)
+            ds_cmip7 = convert_cmip6_dataset(source_ds, file_id=str(output_file.relative_to(cache_dir)))
 
             # Ensure version and sanitized activity_id are in the file attributes
             # so that parse_cmip7_file can extract them for instance_id construction
@@ -246,17 +276,7 @@ def _convert_file_to_cmip7(
                 _load_time_coordinates(ds_cmip7)
                 suppress_bounds_coordinates(ds_cmip7)
 
-                # gpp and o3 magnitudes sit below the quantisation floor.
-                # Ozone also needs lossless hybrid-coordinate coefficients.
-                lossless_dataset = ds_cmip7.attrs.get("variable_id") == "o3"
-                encoding: dict[str, dict[str, Any]] = {}
-                for var in ds_cmip7.data_vars:
-                    var_encoding: dict[str, Any] = {"zlib": True, "complevel": 5}
-                    if not lossless_dataset and str(var) != "gpp":
-                        var_encoding["least_significant_digit"] = _LOSSY_DECIMAL_DIGITS
-                    encoding[str(var)] = var_encoding
-
-                ds_cmip7.to_netcdf(temporary_file, encoding=encoding)
+                ds_cmip7.to_netcdf(temporary_file, encoding=_write_encoding(ds_cmip7))
                 temporary_file.replace(output_file)
             except PermissionError:
                 # If we can't write but file exists (race condition or permission issue), use it
@@ -348,6 +368,24 @@ class CMIP7Request:
         # Create corresponding CMIP6 facets
         self._cmip6_facets = self._convert_to_cmip6_facets(facets)
 
+    def pin_to_datasets(self, datasets: Sequence[Mapping[str, Any]]) -> CMIP7Request:
+        """
+        Return this request unchanged: CMIP7 requests are not pinned yet.
+
+        CMIP7 data cannot be fetched from ESGF at the moment.
+
+        Parameters
+        ----------
+        datasets
+            The catalog records of the datasets recorded for this request's source type.
+
+        Returns
+        -------
+        :
+            This request, unchanged.
+        """
+        return self
+
     def _convert_to_cmip6_facets(self, cmip7_facets: dict[str, Any]) -> dict[str, Any]:
         """Convert CMIP7 facets to CMIP6 facets for fetching."""
         cmip6_facets = {}
@@ -464,6 +502,15 @@ class CMIP7Request:
             if converted_files:
                 cmip7_row["files"] = converted_files
                 converted_rows.append(cmip7_row)
+            elif paths:
+                # Every file failed to convert above, so this dataset drops out of the
+                # result. Say so here, while the reason is still next to it in the log:
+                # downstream all that can be seen is a dataset that never arrived.
+                dataset = row_dict.get("key") or cmip7_row.get("variable_id", "<unknown dataset>")
+                logger.error(
+                    f"None of the {len(paths)} file(s) for {dataset} could be converted, "
+                    f"so it is missing from the data fetched for {self.slug}"
+                )
 
         if not converted_rows:
             logger.warning(f"No files converted for request: {self.slug}")

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/src/climate_ref_core/esgf/registry.py

@@ -7,15 +7,17 @@ This module provides request classes for fetching datasets from pooch registries
 
 from __future__ import annotations
 
+import copy
 import re
-from collections.abc import Callable, Collection, Mapping
-from typing import Any
+from collections.abc import Callable, Collection, Mapping, Sequence
+from typing import Any, ClassVar
 
 import pandas as pd
 from loguru import logger
 
 from climate_ref_core.dataset_registry import dataset_registry_manager
 from climate_ref_core.datasets import select_latest_version
+from climate_ref_core.esgf.base import facet_pins_from_datasets
 
 type FacetValue = str | int | Collection[str | int]
 """
@@ -40,6 +42,10 @@ def _parse_obs4ref_key(key: str) -> dict[str, Any]:
     Where filename is:
     {variable_id}_{frequency}_{source_id}_{inst_short}_{grid_label}_{time_range}.nc
 
+    ``inst_short`` is the short name the file is published under (e.g. ``PCMDI``) and is
+    reported as ``institution_short``; ``institution_id`` is the institution directory of
+    the key, which is the dataset's own. Some keys carry a placeholder (``NR``) there.
+
     Parameters
     ----------
     key
@@ -101,7 +107,10 @@ def _parse_pmp_climatology_key(key: str) -> dict[str, Any]:
     PMP_obs4MIPsClims/{variable_id}/{grid_label}/{version}/{filename}
 
     Where filename is:
-    {variable_id}_mon_{source_id}_{institution_id}_{grid_label}_{time_range}_AC_{version}_{resolution}.nc
+    {variable_id}_mon_{source_id}_{institution_short}_{grid_label}_{time_range}_AC_{version}_{resolution}.nc
+
+    ``institution_short`` is the short name the file is published under (e.g. ``PCMDI``), which
+    is not the dataset's ``institution_id`` (``ECMWF`` for that example).
 
     Parameters
     ----------
@@ -121,13 +130,13 @@ def _parse_pmp_climatology_key(key: str) -> dict[str, Any]:
 
     _, _variable_id_dir, _grid_label, _version, filename = parts
 
-    # Parse filename: {var}_mon_{source_id}_{inst_id}_{grid}_{time}_AC_{ver}_{res}.nc
-    # source_id and institution_id may both contain hyphens (e.g. "GPCP-3-3", "NASA-GISS");
+    # Parse filename: {var}_mon_{source_id}_{inst_short}_{grid}_{time}_AC_{ver}_{res}.nc
+    # source_id and inst_short may both contain hyphens (e.g. "GPCP-3-3", "NASA-GISS");
     # the literal "_" separators keep tokenisation unambiguous.
     filename_pattern = re.compile(
         r"^(?P<variable_id>[a-z]+)_mon_"
         r"(?P<source_id>[A-Za-z0-9-]+)_"
-        r"(?P<institution_id>[A-Za-z0-9-]+)_"
+        r"(?P<institution_short>[A-Za-z0-9-]+)_"
         r"(?P<grid_label>[a-z]+)_"
         r"(?P<time_range>\d+-\d+)_AC_"
         r"(?P<version>v\d+)_"
@@ -227,6 +236,34 @@ class RegistryRequest:
     ```
     """
 
+    pinned_facet_fields: ClassVar[dict[str, str]] = {
+        "source_id": "source_id",
+        "variable_id": "variable_id",
+        "version": "version",
+    }
+    """
+    Facets a pin matches registry keys on, and the recorded facet each is spelled as.
+
+    A registry key carries no ``instance_id`` to ask for, so a pin is matched against the
+    facets parsed out of the key. A catalog, though, records the facets read from the
+    *file*, and the two disagree wherever a registry describes a dataset differently from
+    how the data describes itself. Only facets that agree can be pinned on, which rules
+    out three that otherwise look like obvious candidates:
+
+    * ``grid_label``: PMP climatologies are published under ``gr`` because they are
+      regridded, while the files themselves still report the ``gn`` they were derived from.
+    * ``institution_id``: PMP keys do not carry one (only the short name the file is
+      published under, e.g. ``PCMDI`` for data whose institution is ``ECMWF``), and some
+      obs4REF keys carry a placeholder (``NR``) in its place.
+    * ``frequency``: the PMP key parser does not report it at all.
+
+    The three that remain agree across both registries, and identify a dataset within
+    one: no two keys in either registry share them.
+    """
+
+    pinned_facets: tuple[dict[str, str], ...] | None = None
+    """Facet sets to keep, one per pinned dataset."""
+
     def __init__(  # noqa: PLR0913
         self,
         slug: str,
@@ -244,6 +281,33 @@ class RegistryRequest:
         self.time_span = time_span
         self.key_parser = key_parser
 
+    def pin_to_datasets(self, datasets: Sequence[Mapping[str, Any]]) -> RegistryRequest:
+        """
+        Return a copy of this request that only resolves the given datasets.
+
+        Registry data drifts like published data does: a newly added version of a dataset
+        would otherwise be picked up by :func:`select_latest_version`. A pinned request
+        keeps the recorded version instead.
+
+        Parameters
+        ----------
+        datasets
+            The catalog records of the datasets recorded for this request's source type.
+
+        Returns
+        -------
+        :
+            The pinned request, or this request unchanged if any record is missing one of
+            the facets a pin needs.
+        """
+        pins = facet_pins_from_datasets(datasets, self.pinned_facet_fields, self.slug)
+        if pins is None:
+            return self
+
+        pinned = copy.copy(self)
+        pinned.pinned_facets = pins
+        return pinned
+
     def __repr__(self) -> str:
         return (
             f"RegistryRequest(slug={self.slug!r}, registry_name={self.registry_name!r}, "
@@ -292,8 +356,12 @@ class RegistryRequest:
             if not metadata:
                 continue
 
-            # Check if it matches the requested facets
-            if not _matches_facets(metadata, self.facets):
+            if self.pinned_facets is not None:
+                # The pinned datasets, not the declared facets, decide what is fetched.
+                # Matching before the fetch below keeps superseded versions from downloading.
+                if not any(_matches_facets(metadata, pin) for pin in self.pinned_facets):
+                    continue
+            elif not _matches_facets(metadata, self.facets):
                 continue
 
             # Fetch the file (downloads if not cached)
@@ -321,7 +389,8 @@ class RegistryRequest:
         # Filter to only the latest version for each unique dataset.
         # Datasets are identified by source_id, variable_id, and grid_label.
         # Versions are compared numerically (so v10 > v2) via select_latest_version.
-        if "version" in result.columns:
+        # A pinned request skips this: its pins already name the version of each dataset.
+        if self.pinned_facets is None and "version" in result.columns:
             group_by_cols = [
                 col for col in ("source_id", "variable_id", "grid_label") if col in result.columns
             ]

{climate_ref_core-0.18.2 → climate_ref_core-0.18.3}/src/climate_ref_core/testing.py

@@ -393,6 +393,50 @@ def load_datasets_from_yaml(path: Path, paths_file: Path) -> ExecutionDatasetCol
     return ExecutionDatasetCollection(collections)
 
 
+def load_catalog_datasets(path: Path) -> dict[str, tuple[dict[str, Any], ...]]:
+    """
+    Read the datasets recorded in a catalog YAML file.
+
+    These are the exact datasets a test case's regression baseline was built from, so a
+    request can be pinned to them and fetch that data again instead of re-resolving the
+    facets it declares (see
+    :meth:`~climate_ref_core.esgf.PinnableRequest.pin_to_datasets`).
+
+    Parameters
+    ----------
+    path
+        Path to the catalog YAML file
+
+    Returns
+    -------
+    :
+        The records of each source type, keyed by :class:`SourceDatasetType` name (e.g.
+        ``"CMIP6"``), which is how a request names its source type. A dataset spanning
+        several files is returned once. Source types without any recorded datasets are
+        left out.
+    """
+    with open(path) as f:
+        data = yaml.safe_load(f) or {}
+
+    datasets: dict[str, tuple[dict[str, Any], ...]] = {}
+    for source_type_str, source_data in data.items():
+        if source_type_str == "_metadata":
+            continue
+
+        slug_column = source_data.get("slug_column", "instance_id")
+        # A dataset spanning several files is recorded once per file
+        recorded: dict[str, dict[str, Any]] = {}
+        for dataset in source_data.get("datasets", []):
+            slug = dataset.get(slug_column)
+            if slug:
+                recorded.setdefault(slug, dataset)
+
+        if recorded:
+            datasets[SourceDatasetType(source_type_str).name] = tuple(recorded.values())
+
+    return datasets
+
+
 def validate_catalog_paths(path: Path, paths_file: Path) -> ExecutionDatasetCollection:
     """Load a catalog and check that every row resolves to a local file.
 
@@ -614,8 +658,9 @@ def save_datasets_to_yaml(
     one per file. Paths are keyed by `{instance_id}::{filename}` to support
     multiple files per dataset.
 
-    By default, the catalog is only written if the content has changed
-    (detected via hash comparison). Use `force=True` to always write.
+    By default, the catalog is only written if its content would change -- which covers
+    the datasets selected and the metadata recorded for them. Use `force=True` to always
+    write.
 
     The paths sidecar is regenerated on every save, even when the catalog content is
     unchanged. Local cache contents can change independently of the version-controlled
@@ -638,11 +683,14 @@ def save_datasets_to_yaml(
         True if the catalog was (re)written, False if the catalog was left unchanged
         (the paths sidecar may still have been regenerated).
     """
-    new_hash = datasets.hash
-
     data, paths_map = _serialise_datasets(datasets)
+    content = yaml.dump(data, default_flow_style=False, sort_keys=False)
 
-    if not force and get_catalog_hash(path) == new_hash:
+    # Compared as content rather than by the stored hash: that hash identifies which
+    # datasets were selected, deliberately ignoring their metadata, so going by it alone
+    # would leave a catalog whose datasets are unchanged but whose recorded metadata is
+    # stale. The hash is part of the content, so a different selection still writes.
+    if not force and path.exists() and path.read_text() == content:
         # Keep the tracked catalog byte-identical, but always refresh its machine-local
         # paths. An existing sidecar may be partial or point at an old cache location.
         _write_paths_file(paths_file, paths_map)
@@ -650,8 +698,7 @@ def save_datasets_to_yaml(
         return False
 
     path.parent.mkdir(parents=True, exist_ok=True)
-    with open(path, "w") as f:
-        yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+    path.write_text(content)
     _write_paths_file(paths_file, paths_map)
     logger.info(f"Saved catalog to {path} (paths: {paths_file})")
     return True