climate-ref-core 0.17.2__tar.gz → 0.18.0__tar.gz

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: climate-ref-core
-Version: 0.17.2
+Version: 0.18.0
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref-core"
-version = "0.17.2"
+version = "0.18.0"
 description = "Core library for the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/src/climate_ref_core/cmip6_to_cmip7.py

@@ -12,6 +12,8 @@ Regenerate with:
 The variable_id and table_id attributes are used to map the CMIP6 variable to its CMIP7 equivalent,
 and to determine the appropriate branding suffix and realm for the CMIP7 format.
 Grid information is not converted so the grid_labels may not be valid for CMIP7.
+
+This is a temporary work around until we have better CMIP7 coverage.
 """
 
 from __future__ import annotations
@@ -20,19 +22,17 @@ import json
 import re
 import uuid
 from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
+from datetime import UTC, datetime
 from importlib import resources
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import attrs
 import cftime
 import numpy as np
 import pandas as pd
+import xarray as xr
 from loguru import logger
 
-if TYPE_CHECKING:
-    import xarray as xr
-
 
 def suppress_bounds_coordinates(ds: xr.Dataset) -> xr.Dataset:
     """
@@ -381,6 +381,26 @@ class CMIP7Metadata:
     license_id: str = "CC-BY-4.0"
 
 
+def make_tracking_id(file_id: str) -> str:
+    """
+    Return the CMIP7 handle for a file, derived from ``file_id`` rather than minted at random.
+
+    A file therefore keeps its handle when a catalog is regenerated,
+    so the regeneration diff shows what actually moved.
+
+    Parameters
+    ----------
+    file_id
+        Something that identifies the file, such as its ``instance_id`` and filename.
+
+    Returns
+    -------
+    str
+        The handle.
+    """
+    return f"hdl:21.14107/{uuid.uuid5(uuid.NAMESPACE_URL, f'hdl:21.14107/{file_id}')}"
+
+
 def convert_cmip6_to_cmip7_attrs(
     cmip6_attrs: dict[str, Any],
 ) -> dict[str, Any]:
@@ -485,29 +505,37 @@ def convert_cmip6_to_cmip7_attrs(
     return attrs
 
 
+_MONTHS_PER_YEAR = 12
+
+
 def _month_index(t: Any) -> int:
     """Return the absolute month index (``year * 12 + month - 1``) of a time value."""
     if isinstance(t, cftime.datetime):
-        return t.year * 12 + t.month - 1
+        return t.year * _MONTHS_PER_YEAR + t.month - 1
     ts = pd.Timestamp(t)
-    return ts.year * 12 + ts.month - 1
+    return ts.year * _MONTHS_PER_YEAR + ts.month - 1
 
 
-_MONTHS_PER_YEAR = 12
+def months_to_extend(last: Any, end_year: int, end_month: int) -> int:
+    """
+    Return how many months short of ``end_year``-``end_month`` a series ending at ``last`` falls.
+
+    Zero or less means the series already reaches that month.
+    """
+    return end_year * _MONTHS_PER_YEAR + (end_month - 1) - _month_index(last)
 
 
-def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> xr.Dataset:
+def repeat_final_year_to(ds: xr.Dataset, end_year: int, end_month: int = 12) -> xr.Dataset:
     """
-    Relabel a monthly time axis so the final timestep lands on ``end_year``-``end_month``.
+    Extend a monthly series to ``end_year``-``end_month`` by repeating its final year.
 
-    The whole ``time`` coordinate (and any ``time_bnds``) is shifted by a constant
-    whole-month offset, so monthly spacing, calendar, and cftime types are preserved
-    and only the labels move. This is used to fabricate CMIP7 ``historical`` coverage
-    that reaches years for which no real data exists (e.g. 2002-2021 for the fire
-    diagnostic), without altering the underlying data values.
+    The real timesteps keep their real dates and values.
+    Any missing years are tiled from the last year.
+    This gives CMIP7 ``historical`` coverage for years the CMIP6 source never ran (e.g. 2015-2021)
+    without moving the years that did.
 
-    Only monthly data with a ``cftime`` time axis is supported; datasets without a
-    ``time`` coordinate (fixed-frequency, e.g. ``sftlf``) are returned unchanged.
+    Only monthly data with a ``cftime`` time axis is supported.
+    Datasets without a ``time`` coordinate (fixed-frequency, e.g. ``sftlf``) are returned unchanged.
 
     Parameters
     ----------
@@ -521,7 +549,15 @@ def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> x
     Returns
     -------
     xr.Dataset
-        A shallow copy with the relabelled time axis.
+        A dataset whose time axis runs to the requested end.
+
+    Raises
+    ------
+    TypeError
+        If the time axis is not ``cftime``.
+    ValueError
+        If ``end_month`` is outside 1-12, the series does not end in December,
+        or its final calendar year is incomplete.
     """
     if not 1 <= end_month <= _MONTHS_PER_YEAR:
         raise ValueError(f"end_month must be in 1..12, got {end_month}")
@@ -533,52 +569,52 @@ def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> x
     last = time_values[-1]
     if not isinstance(last, cftime.datetime):
         raise TypeError(
-            "shift_time_axis_end requires a cftime time axis; "
-            f"got {type(last).__name__}. Decode with use_cftime=True."
+            "repeat_final_year_to requires a cftime time axis. "
+            f"Got {type(last).__name__}. Decode with use_cftime=True."
         )
 
-    # Whole-month offset that moves the final label onto the requested end.
-    target_index = end_year * 12 + (end_month - 1)
-    offset_months = target_index - _month_index(last)
-    if offset_months == 0:
+    months_to_add = months_to_extend(last, end_year, end_month)
+    if months_to_add <= 0:
         return ds
 
-    calendar = last.calendar
-
-    def _days_in_month(year: int, month: int) -> int:
-        # Last day of ``month`` = day before the first of the following month,
-        # computed via cftime arithmetic so it respects the dataset's calendar
-        # (noleap, 360_day, etc.).
-        next_year, next_month = (year + 1, 1) if month == _MONTHS_PER_YEAR else (year, month + 1)
-        first_of_next = cftime.datetime(next_year, next_month, 1, calendar=calendar)
-        last_of_month = first_of_next - timedelta(days=1)  # type: ignore[operator]
-        return int(last_of_month.day)  # type: ignore[attr-defined]
-
-    def _shift(t: cftime.datetime) -> cftime.datetime:
-        total = _month_index(t) + offset_months
-        year, month = divmod(total, 12)
-        month += 1
-        # Clamp the day to the target month/calendar: a non-multiple-of-12 offset
-        # can land a day-31 (or leap Feb-29) label on a shorter month, which cftime
-        # would reject. Our monthly data is mid-month so this is normally a no-op.
-        day = min(t.day, _days_in_month(year, month))
-        return cftime.datetime(year, month, day, t.hour, t.minute, t.second, t.microsecond, calendar=calendar)
-
-    ds = ds.copy(deep=False)
-    shifted = np.array([_shift(t) for t in time_values])
-    new_time = ds["time"].copy(data=shifted)
-    new_time.encoding = dict(ds["time"].encoding)
-    ds = ds.assign_coords(time=new_time)
-
-    # Shift the matching time bounds, if present, so the axis stays self-consistent.
+    if last.month != _MONTHS_PER_YEAR:
+        raise ValueError(f"The series must end in December to repeat whole years, ends {last}")
+
     bounds_name = ds["time"].attrs.get("bounds")
-    if bounds_name and bounds_name in ds:
-        bnds_values = ds[bounds_name].values
-        shifted_bnds = np.array([[_shift(v) for v in row] for row in bnds_values])
-        ds[bounds_name] = ds[bounds_name].copy(data=shifted_bnds)
 
-    logger.debug(f"Shifted time axis by {offset_months} months so it ends {end_year:04d}-{end_month:02d}")
-    return ds
+    def _restamp(t: cftime.datetime, shift: int) -> cftime.datetime:
+        year = t.year + shift
+        try:
+            return t.replace(year=year)  # type: ignore[attr-defined,no-any-return]
+        except ValueError:
+            # A leap day has no counterpart in a common year, so take the day before.
+            return t.replace(year=year, day=t.day - 1)  # type: ignore[attr-defined,no-any-return]
+
+    def _relabel(block: xr.Dataset, shift: int) -> xr.Dataset:
+        relabelled = block.copy()
+        if bounds_name and bounds_name in relabelled:
+            bnds = [[_restamp(b, shift) for b in row] for row in block[bounds_name].values]
+            # Assign as (dims, data) so the new bounds are not aligned against the old time axis.
+            relabelled[bounds_name] = (block[bounds_name].dims, np.array(bnds))
+        return relabelled.assign_coords(time=[_restamp(t, shift) for t in block["time"].values])
+
+    final_year = ds.sel(time=str(last.year))
+    if len(final_year["time"]) != _MONTHS_PER_YEAR:
+        raise ValueError(f"A full final year is needed to repeat, got {len(final_year['time'])} months")
+
+    repeats = [_relabel(final_year, year - last.year) for year in range(last.year + 1, end_year + 1)]
+
+    extended = xr.concat([ds, *repeats], dim="time", data_vars="minimal", coords="minimal")
+    extended["time"].attrs = dict(ds["time"].attrs)
+    extended["time"].encoding = dict(ds["time"].encoding)
+
+    # The repeats are whole years, so drop anything past the requested month.
+    extended = extended.sel(time=slice(None, f"{end_year:04d}-{end_month:02d}"))
+
+    logger.debug(
+        f"Repeated {last.year} for {months_to_add} months so the series ends {end_year:04d}-{end_month:02d}"
+    )
+    return extended
 
 
 def convert_cmip6_dataset(
@@ -604,10 +640,10 @@ def convert_cmip6_dataset(
     inplace
         If True, modify the dataset in place; otherwise return a copy
     extend_historical_to
-        Opt-in ``(end_year, end_month)``. When set, the ``time`` axis is relabelled
-        via :func:`shift_time_axis_end` so the series ends on that month, fabricating
-        CMIP7 coverage for years without real data. Defaults to ``None`` (time axis
-        untouched), so existing conversions are byte-identical.
+        Opt-in ``(end_year, end_month)``. When set, the series is padded out to that month
+        via :func:`repeat_final_year_to`, fabricating CMIP7 coverage for years without real
+        data. Defaults to ``None`` (time axis untouched), so existing conversions are
+        byte-identical.
 
     Returns
     -------
@@ -619,7 +655,7 @@ def convert_cmip6_dataset(
 
     if extend_historical_to is not None:
         end_year, end_month = extend_historical_to
-        ds = shift_time_axis_end(ds, end_year=end_year, end_month=end_month)
+        ds = repeat_final_year_to(ds, end_year=end_year, end_month=end_month)
 
     # Determine the primary variable (skip coordinates/bounds)
     data_vars = [str(v) for v in ds.data_vars if not str(v).endswith("_bnds") and v not in ds.coords]

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/src/climate_ref_core/diagnostics.py

@@ -443,6 +443,14 @@ class DataRequirement:
     This is effectively an AND operation.
     """
 
+    fallback_source_types: tuple[SourceDatasetType, ...] = field(factory=tuple)
+    """
+    Source types that may supply this requirement when ``source_type`` does not hold the dataset.
+
+    A dataset present under ``source_type`` always wins,
+    and the data is delivered under ``source_type`` whichever collection it came from.
+    """
+
     def apply_filters(self, data_catalog: pd.DataFrame) -> pd.DataFrame:
         """
         Apply filters to a DataFrame-based data catalog.

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/src/climate_ref_core/esgf/cmip7.py

@@ -7,9 +7,11 @@ a request class that fetches CMIP6 data and converts it to CMIP7 format.
 
 from __future__ import annotations
 
+import re
 from pathlib import Path
-from typing import Any, ClassVar
+from typing import Any, ClassVar, NamedTuple
 
+import cftime
 import pandas as pd
 import xarray as xr
 from loguru import logger
@@ -21,7 +23,8 @@ from climate_ref_core.cmip6_to_cmip7 import (
     format_cmip7_time_range,
     get_dreq_entry,
     get_frequency_from_table,
-    shift_time_axis_end,
+    months_to_extend,
+    repeat_final_year_to,
     suppress_bounds_coordinates,
 )
 from climate_ref_core.data import resolve_cache_dir
@@ -36,6 +39,65 @@ def _get_cmip7_cache_dir() -> Path:
     return cache_dir
 
 
+class LatestFile(NamedTuple):
+    """The file holding a dataset's final timestep, and that timestep."""
+
+    path: Path
+    end: cftime.datetime
+
+
+def _latest_file(files: list[Path]) -> LatestFile | None:
+    """
+    Find the file holding a dataset's final timestep.
+
+    Only that file is extended, so a dataset split across several files keeps its earlier
+    chunks where they are instead of every chunk being padded out to the same end.
+
+    Parameters
+    ----------
+    files
+        The dataset's CMIP6 source files. Files without a time axis are skipped.
+
+    Returns
+    -------
+    LatestFile | None
+        The file ending last and its final timestep, or ``None`` when none of them carry a time axis.
+    """
+    time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+
+    latest: LatestFile | None = None
+    for path in files:
+        with xr.open_dataset(path, decode_times=time_coder) as ds:
+            if "time" not in ds.coords or len(ds["time"]) == 0:
+                continue
+            end = ds["time"].values[-1]
+            if latest is None or end > latest.end:
+                latest = LatestFile(path, end)
+    return latest
+
+
+def _bump_version(version: str) -> str:
+    """
+    Bump a DRS version so fabricated data does not share an ``instance_id`` with the real thing.
+
+    A version that is not already ``v<digits>`` is read as ``v0``,
+    which is what the ingest falls back to when it cannot parse one out of the path.
+
+    Parameters
+    ----------
+    version
+        The version the dataset would otherwise be published under.
+
+    Returns
+    -------
+    str
+        The next version.
+    """
+    match = re.fullmatch(r"v(\d+)", version)
+    current = int(match.group(1)) if match else 0
+    return f"v{current + 1}"
+
+
 def _convert_file_to_cmip7(
     cmip6_path: Path,
     cmip7_facets: dict[str, Any],
@@ -51,8 +113,8 @@ def _convert_file_to_cmip7(
     cmip7_facets
         CMIP7 facets for the output path
     extend_historical_to
-        Opt-in ``(end_year, end_month)`` passed to :func:`convert_cmip6_dataset`
-        to relabel the time axis so historical coverage reaches that month.
+        Opt-in ``(end_year, end_month)``.
+        The series is padded out to that month by repeating its final year, so historical coverage reaches it.
         Defaults to ``None`` (time axis untouched).
 
     Returns
@@ -65,7 +127,7 @@ def _convert_file_to_cmip7(
     # CMIP6 activity_id can contain multiple activities separated by spaces
     # (e.g. "C4MIP CDRMIP"). Use only the first activity for the DRS path.
     activity_id = str(cmip7_facets.get("activity_id", "CMIP")).split()[0]
-    version = str(cmip7_facets.get("version", "v1"))
+    version = str(cmip7_facets.get("version", "v0"))
 
     # Build CMIP7 DRS path using the standard MIP-DRS7 path builder.
     # Provide defaults for fields that may not be in facets.
@@ -85,12 +147,12 @@ def _convert_file_to_cmip7(
 
     time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
     with xr.open_dataset(cmip6_path, decode_times=time_coder) as ds:
-        # When fabricating extended historical coverage, relabel the time axis first
-        # so both the filename time range and the written data reflect the new dates.
+        # When fabricating extended historical coverage, pad the series first so both the
+        # filename time range and the written data reflect the added months.
         source_ds = ds
         if extend_historical_to is not None:
             end_year, end_month = extend_historical_to
-            source_ds = shift_time_axis_end(ds, end_year=end_year, end_month=end_month)
+            source_ds = repeat_final_year_to(ds, end_year=end_year, end_month=end_month)
 
         frequency = str(cmip7_facets.get("frequency", "mon"))
         time_range = format_cmip7_time_range(source_ds, frequency)
@@ -199,11 +261,11 @@ class CMIP7Request:
         time_span
             Optional time range filter (start, end) in YYYY-MM format
         extend_historical_to
-            Opt-in ``(end_year, end_month)``. When set, each converted CMIP7 file has
-            its time axis relabelled so historical coverage ends on that month, letting
-            us fabricate CMIP7 data for years without real CMIP6 source data (e.g. the
-            fire diagnostic's 2002-2021 window). Defaults to ``None`` (time axis
-            untouched), so other CMIP7 conversions are unchanged.
+            Opt-in ``(end_year, end_month)``. When set, a converted CMIP7 file that stops
+            short has its final year repeated until it ends on that month, letting us
+            fabricate CMIP7 data for years without real CMIP6 source data (e.g. the fire
+            diagnostic's 2002-2021 window). Files that already reach it are untouched, as
+            are all conversions when this defaults to ``None``.
         """
         self.slug = slug
         self.facets = facets
@@ -302,21 +364,30 @@ class CMIP7Request:
             cmip7_row = self._convert_to_cmip7_metadata(row_dict)
 
             # Get file paths and convert them
-            files = row_dict.get("files", [])
+            paths = [Path(file_path) for file_path in row_dict.get("files", [])]
+            latest = None
+            if self.extend_historical_to is not None:
+                found = _latest_file([path for path in paths if path.exists()])
+                if found is not None and months_to_extend(found.end, *self.extend_historical_to) > 0:
+                    # if modified bump the version so they don't collide
+                    latest = found.path
+                    cmip7_row["version"] = _bump_version(str(cmip7_row.get("version", "")))
+
             converted_files = []
-            for file_path in files:
-                cmip6_path = Path(file_path)
+            for cmip6_path in paths:
                 if cmip6_path.exists():
                     try:
                         cmip7_path = _convert_file_to_cmip7(
-                            cmip6_path, cmip7_row, extend_historical_to=self.extend_historical_to
+                            cmip6_path,
+                            cmip7_row,
+                            extend_historical_to=self.extend_historical_to if cmip6_path == latest else None,
                         )
                         converted_files.append(str(cmip7_path))
                     except Exception as e:
                         logger.exception(f"Failed to convert {cmip6_path.name}: {e}")
                         continue
                 else:
-                    logger.warning(f"CMIP6 file not found: {file_path}")
+                    logger.warning(f"CMIP6 file not found: {cmip6_path}")
 
             if converted_files:
                 cmip7_row["files"] = converted_files

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/src/climate_ref_core/reference_data.py

@@ -141,21 +141,14 @@ def source_ids_by_registry(
         parser = _registry_key_parser(entry.source_type)
         if parser is None:
             continue
-        # obs4REF data is ingested under the obs4MIPs source type, so a registry declaring
-        # obs4REF supplies requirements written against either.
-        source_types = {entry.source_type.value}
-        if entry.source_type is SourceDatasetType.obs4REF:
-            source_types.add(SourceDatasetType.obs4MIPs.value)
-
         for key in entry.registry.registry:
             metadata: Mapping[str, Any] = parser(key)
             source_id = metadata.get("source_id")
             if not source_id:
                 continue
-            for source_type in source_types:
-                names = found[(source_type, source_id)]
-                if name not in names:
-                    names.append(name)
+            names = found[(entry.source_type.value, source_id)]
+            if name not in names:
+                names.append(name)
     return dict(found)
 
 
@@ -182,6 +175,8 @@ def collect_required_reference_data(
 
     variables: dict[tuple[str, str], set[str]] = defaultdict(set)
     diagnostics: dict[tuple[str, str], set[DiagnosticReference]] = defaultdict(set)
+    # Source types a requirement may be met from, its own first
+    suppliers: dict[tuple[str, str], list[str]] = defaultdict(list)
 
     for provider in providers:
         for diagnostic in summarize_provider(provider).diagnostics:
@@ -198,11 +193,20 @@ def collect_required_reference_data(
                         key = (requirement.source_type, source_id)
                         variables[key].update(requirement.variables)
                         diagnostics[key].add(reference)
+                        for source_type in (requirement.source_type, *requirement.fallback_source_types):
+                            if source_type not in suppliers[key]:
+                                suppliers[key].append(source_type)
 
     datasets = []
     for (source_type, source_id), variable_ids in variables.items():
-        registry_names = registry_of.get((source_type, source_id), [])
-        registry_name = registry_names[0] if registry_names else None
+        registry_name = next(
+            (
+                registry_of[(supplier, source_id)][0]
+                for supplier in suppliers[(source_type, source_id)]
+                if registry_of.get((supplier, source_id))
+            ),
+            None,
+        )
         datasets.append(
             ReferenceDataset(
                 source_type=source_type,

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/src/climate_ref_core/regression/__init__.py

@@ -29,6 +29,7 @@ from climate_ref_core.regression.gate import (
 )
 from climate_ref_core.regression.manifest import (
     COMMITTED_BUNDLE_FILES,
+    COMMITTED_DIRNAME,
     SCHEMA_VERSION,
     Manifest,
     NativeEntry,
@@ -37,6 +38,10 @@ from climate_ref_core.regression.manifest import (
     sha256_file,
     verify_committed_integrity,
 )
+from climate_ref_core.regression.report_store import (
+    ReportStore,
+    build_report_store,
+)
 from climate_ref_core.regression.store import (
     NativeStore,
     NativeStoreUnavailableError,
@@ -46,6 +51,7 @@ from climate_ref_core.regression.store import (
 
 __all__ = [
     "COMMITTED_BUNDLE_FILES",
+    "COMMITTED_DIRNAME",
     "SCHEMA_VERSION",
     "Action",
     "GateDecision",
@@ -53,11 +59,13 @@ __all__ = [
     "NativeEntry",
     "NativeStore",
     "NativeStoreUnavailableError",
+    "ReportStore",
     "S3WriteConfig",
     "Tolerance",
     "assert_bundle_regression",
     "build_native_snapshot",
     "build_native_store",
+    "build_report_store",
     "compare_json_content",
     "compute_committed_digests",
     "decide_coupling",

{climate_ref_core-0.17.2 → climate_ref_core-0.18.0}/src/climate_ref_core/regression/manifest.py

@@ -62,6 +62,10 @@ def _validate_digest(digest: str) -> str:
     return digest
 
 
+COMMITTED_DIRNAME = "regression"
+"""The directory a test case's committed bundle sits in, both in git and in an output slot."""
+
+
 COMMITTED_BUNDLE_FILES: tuple[str, ...] = (
     "series.json",
     "diagnostic.json",