climate-ref-core 0.18.0__tar.gz → 0.18.2__tar.gz

{climate_ref_core-0.18.0 → climate_ref_core-0.18.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: climate-ref-core
-Version: 0.18.0
+Version: 0.18.2
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0
@@ -19,6 +19,7 @@ Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.12
 Requires-Dist: attrs>=23.2.0
 Requires-Dist: cattrs>=24.1
+Requires-Dist: dask[array]>=2024.10.0
 Requires-Dist: environs>=11
 Requires-Dist: fastprogress==1.0.5
 Requires-Dist: intake-esgf>=2025.7.16
@@ -32,7 +33,7 @@ Requires-Dist: pydantic>=2.10.6
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: requests
 Requires-Dist: rich
-Requires-Dist: setuptools<84
+Requires-Dist: setuptools<85
 Requires-Dist: typing-extensions
 Provides-Extra: aws
 Requires-Dist: boto3>=1.34; extra == 'aws'

{climate_ref_core-0.18.0 → climate_ref_core-0.18.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref-core"
-version = "0.18.0"
+version = "0.18.2"
 description = "Core library for the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [
@@ -39,11 +39,12 @@ dependencies = [
     "pyyaml>=6.0.2",
     "psutil>=5.9.8",
     # Not used directly, but required to support some installations
-    "setuptools<84",
+    "setuptools<85",
 
     # ESGF data fetching
     "intake-esgf>=2025.7.16",
     "pyarrow>=17.0.0",
+    "dask[array]>=2024.10.0",
 
     # SPEC 0000 constraints
     # We follow [SPEC-0000](https://scientific-python.org/specs/spec-0000/)

{climate_ref_core-0.18.0 → climate_ref_core-0.18.2}/src/climate_ref_core/esgf/cmip7.py

@@ -8,10 +8,12 @@ a request class that fetches CMIP6 data and converts it to CMIP7 format.
 from __future__ import annotations
 
 import re
+import uuid
 from pathlib import Path
 from typing import Any, ClassVar, NamedTuple
 
 import cftime
+import dask
 import pandas as pd
 import xarray as xr
 from loguru import logger
@@ -30,6 +32,66 @@ from climate_ref_core.cmip6_to_cmip7 import (
 from climate_ref_core.data import resolve_cache_dir
 from climate_ref_core.esgf.cmip6 import CMIP6Request
 
+_MANDATORY_CONVERTED_ATTRS = (
+    "activity_id",
+    "institution_id",
+    "source_id",
+    "experiment_id",
+    "variant_label",
+    "variable_id",
+    "grid_label",
+    "frequency",
+    "region",
+    "branding_suffix",
+    "branded_variable",
+    "mip_era",
+    "realm",
+    "nominal_resolution",
+    "license_id",
+    "tracking_id",
+)
+_LOSSY_DECIMAL_DIGITS = 3
+
+
+def _invalid_conversion_reason(path: Path) -> str | None:
+    """Return why a cached conversion cannot satisfy the complete CMIP7 parser."""
+    try:
+        with xr.open_dataset(path, decode_times=False) as ds:
+            missing = [name for name in _MANDATORY_CONVERTED_ATTRS if not ds.attrs.get(name)]
+            if missing:
+                return f"missing mandatory attributes: {', '.join(missing)}"
+            variable_id = str(ds.attrs["variable_id"])
+            if variable_id not in ds.variables:
+                return f"missing data variable: {variable_id}"
+            if variable_id == "o3":
+                quantised = [
+                    str(name)
+                    for name in ds.data_vars
+                    if ds[name].encoding.get("least_significant_digit") == _LOSSY_DECIMAL_DIGITS
+                ]
+                if quantised:
+                    return f"ozone dataset uses destructive decimal quantisation: {', '.join(quantised)}"
+            if "time" in ds.variables and ds.sizes.get("time", 0):
+                time = ds["time"]
+                if not time.attrs.get("units"):
+                    return "time coordinate has no units"
+                # Reading both endpoints catches a truncated file.
+                time.isel(time=[0, -1]).load()
+    except (OSError, ValueError) as e:
+        return f"cannot read metadata: {e}"
+    return None
+
+
+def _load_time_coordinates(ds: xr.Dataset) -> None:
+    """Materialise the small CF time variables before a Dask-backed NetCDF write."""
+    time_variables = {"time"}
+    if "time" in ds:
+        bounds_name = ds["time"].attrs.get("bounds")
+        if bounds_name:
+            time_variables.add(str(bounds_name))
+    for name in time_variables.intersection(ds.variables):
+        ds[name].load()
+
 
 def _get_cmip7_cache_dir() -> Path:
     """Get the cache directory for converted CMIP7 files."""
@@ -146,57 +208,67 @@ def _convert_file_to_cmip7(
     logger.info(f"Converting to CMIP7: {cmip6_path.name}")
 
     time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
-    with xr.open_dataset(cmip6_path, decode_times=time_coder) as ds:
-        # When fabricating extended historical coverage, pad the series first so both the
-        # filename time range and the written data reflect the added months.
-        source_ds = ds
-        if extend_historical_to is not None:
-            end_year, end_month = extend_historical_to
-            source_ds = repeat_final_year_to(ds, end_year=end_year, end_month=end_month)
-
-        frequency = str(cmip7_facets.get("frequency", "mon"))
-        time_range = format_cmip7_time_range(source_ds, frequency)
-        output_file = drs_path / create_cmip7_filename(cmip7_facets, time_range=time_range)
-
-        if output_file.exists():
-            logger.debug(f"Using cached CMIP7 file: {output_file}")
-            return output_file
-
-        ds_cmip7 = convert_cmip6_dataset(source_ds)
-
-        # Ensure version and sanitized activity_id are in the file attributes
-        # so that parse_cmip7_file can extract them for instance_id construction
-        ds_cmip7.attrs["version"] = version
-        ds_cmip7.attrs["activity_id"] = activity_id
-
-        try:
-            logger.info(f"Writing translated CMIP7 file: {output_file}")
-            suppress_bounds_coordinates(ds_cmip7)
-
-            # Apply lossy compression - these are converted files used for
-            # verification only, so precision loss is acceptable.
-            # gpp magnitudes (~1e-8 kg m-2 s-1) sit far below the fixed
-            # least_significant_digit=3 (~1e-3) precision floor, which would round the entire field to zero.
-            # Keep gpp lossless; other variables tolerate it.
-            encoding: dict[str, dict[str, Any]] = {}
-            for var in ds_cmip7.data_vars:
-                var_encoding: dict[str, Any] = {"zlib": True, "complevel": 5}
-
-                lossless_variables = {"gpp"}
-                if str(var) not in lossless_variables:
-                    var_encoding["least_significant_digit"] = 3
-                encoding[str(var)] = var_encoding
-
-            ds_cmip7.to_netcdf(output_file, encoding=encoding)
-        except PermissionError:
-            # If we can't write but file exists (race condition or permission issue), use it
+    # Some CMIP fields are several GiB, so bound chunk size and write concurrency.
+    with dask.config.set({"array.chunk-size": "64MiB", "scheduler": "single-threaded"}):
+        with xr.open_dataset(cmip6_path, decode_times=time_coder, chunks="auto") as ds:
+            # When fabricating extended historical coverage, pad the series first so both the
+            # filename time range and the written data reflect the added months.
+            source_ds = ds
+            if extend_historical_to is not None:
+                end_year, end_month = extend_historical_to
+                source_ds = repeat_final_year_to(ds, end_year=end_year, end_month=end_month)
+
+            frequency = str(cmip7_facets.get("frequency", "mon"))
+            time_range = format_cmip7_time_range(source_ds, frequency)
+            output_file = drs_path / create_cmip7_filename(cmip7_facets, time_range=time_range)
+
             if output_file.exists():
-                logger.debug(f"Using existing CMIP7 file (could not overwrite): {output_file}")
-                return output_file
-            # Clear the cache directory hint for the user
-            logger.error(f"Permission denied writing to {output_file}")
-            logger.error(f"Try clearing the cache: rm -rf {_get_cmip7_cache_dir()}")
-            raise
+                invalid_reason = _invalid_conversion_reason(output_file)
+                if invalid_reason is None:
+                    logger.debug(f"Using cached CMIP7 file: {output_file}")
+                    return output_file
+                logger.info(f"Rebuilding invalid CMIP7 conversion ({invalid_reason}): {output_file}")
+
+            ds_cmip7 = convert_cmip6_dataset(source_ds)
+
+            # Ensure version and sanitized activity_id are in the file attributes
+            # so that parse_cmip7_file can extract them for instance_id construction
+            ds_cmip7.attrs["version"] = version
+            ds_cmip7.attrs["activity_id"] = activity_id
+
+            temporary_file = output_file.with_name(
+                f".{output_file.stem}.{uuid.uuid4().hex}.tmp{output_file.suffix}"
+            )
+
+            try:
+                logger.info(f"Writing translated CMIP7 file: {output_file}")
+                # Xarray cannot CF-encode a Dask-backed cftime bounds array.
+                _load_time_coordinates(ds_cmip7)
+                suppress_bounds_coordinates(ds_cmip7)
+
+                # gpp and o3 magnitudes sit below the quantisation floor.
+                # Ozone also needs lossless hybrid-coordinate coefficients.
+                lossless_dataset = ds_cmip7.attrs.get("variable_id") == "o3"
+                encoding: dict[str, dict[str, Any]] = {}
+                for var in ds_cmip7.data_vars:
+                    var_encoding: dict[str, Any] = {"zlib": True, "complevel": 5}
+                    if not lossless_dataset and str(var) != "gpp":
+                        var_encoding["least_significant_digit"] = _LOSSY_DECIMAL_DIGITS
+                    encoding[str(var)] = var_encoding
+
+                ds_cmip7.to_netcdf(temporary_file, encoding=encoding)
+                temporary_file.replace(output_file)
+            except PermissionError:
+                # If we can't write but file exists (race condition or permission issue), use it
+                if output_file.exists() and _invalid_conversion_reason(output_file) is None:
+                    logger.debug(f"Using existing CMIP7 file (could not overwrite): {output_file}")
+                    return output_file
+                # Clear the cache directory hint for the user
+                logger.error(f"Permission denied writing to {output_file}")
+                logger.error(f"Try clearing the cache: rm -rf {_get_cmip7_cache_dir()}")
+                raise
+            finally:
+                temporary_file.unlink(missing_ok=True)
 
     return output_file
 

{climate_ref_core-0.18.0 → climate_ref_core-0.18.2}/src/climate_ref_core/providers.py

@@ -534,7 +534,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
         Environment variables overriding the process environment
         for commands run in the conda environment.
     pip_packages
-        Pip packages to install (as URLs) with ``--no-deps``
+        Pip packages to install (as URLs) with ``--no-deps --force-reinstall``
         after creating the conda environment.
 
     """
@@ -567,7 +567,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
     def prefix(self, path: Path) -> None:
         self._prefix = path
 
-    def _launch_env(self) -> dict[str, str]:
+    def _launch_env(self, *, use_lockfiles: bool = True) -> dict[str, str]:
         """
         Environment for a command launched in the conda environment.
 
@@ -575,10 +575,20 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
         so settings applied to a worker reach every command it launches,
         with the provider's overrides on top.
         HOME falls back to the conda prefix so micromamba has a writable directory.
+
+        Parameters
+        ----------
+        use_lockfiles
+            Whether micromamba should take its file locks.
+            Disable this for commands that only read the environment.
+            An explicit ``MAMBA_USE_LOCKFILES`` in the process environment
+            or in `env_overrides` always wins.
         """
         env = {**os.environ, **self.env_overrides}
         if "HOME" not in env:
             env["HOME"] = str(self.prefix)
+        if not use_lockfiles:
+            env.setdefault("MAMBA_USE_LOCKFILES", "false")
         return env
 
     @property
@@ -708,6 +718,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
                     "pip",
                     "install",
                     "--no-deps",
+                    "--force-reinstall",
                     pkg,
                 ]
                 logger.debug(f"Running {' '.join(cmd)}")
@@ -753,7 +764,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 text=True,
-                env=self._launch_env(),
+                env=self._launch_env(use_lockfiles=False),
             )
             logger.info("Command output: \n" + res.stdout)
             logger.info("Command execution successful")

{climate_ref_core-0.18.0 → climate_ref_core-0.18.2}/src/climate_ref_core/testing.py

@@ -393,6 +393,71 @@ def load_datasets_from_yaml(path: Path, paths_file: Path) -> ExecutionDatasetCol
     return ExecutionDatasetCollection(collections)
 
 
+def validate_catalog_paths(path: Path, paths_file: Path) -> ExecutionDatasetCollection:
+    """Load a catalog and check that every row resolves to a local file.
+
+    Loading remains permissive because callers also use catalogs for metadata-only
+    operations. Fetch and execution entry points can call this function when they
+    require usable input files.
+
+    Parameters
+    ----------
+    path
+        Path to the catalog YAML file.
+    paths_file
+        Path to the machine-local paths sidecar.
+
+    Returns
+    -------
+    :
+        The loaded datasets, so callers need not parse the catalog a second time.
+
+    Raises
+    ------
+    DatasetResolutionError
+        If the sidecar is absent for a non-empty catalog, incomplete, or points to missing files.
+    """
+    from climate_ref_core.exceptions import DatasetResolutionError  # noqa: PLC0415
+
+    try:
+        datasets = load_datasets_from_yaml(path, paths_file)
+    except Exception as e:
+        raise DatasetResolutionError(f"Could not load catalog paths for {path}: {e}") from e
+
+    has_rows = any(len(collection.datasets) for collection in datasets.values())
+    if has_rows and not paths_file.exists():
+        raise DatasetResolutionError(
+            f"Paths file is missing for catalog {path}: {paths_file}. "
+            "Run `ref test-cases fetch` to rebuild it."
+        )
+
+    missing_entries: list[str] = []
+    missing_files: list[str] = []
+    for source_type, collection in datasets.items():
+        for _, dataset in collection.datasets.iterrows():
+            instance_id = dataset.get(collection.slug_column)
+            filename = dataset.get("filename")
+            key = f"{instance_id}::{filename}" if instance_id and filename else instance_id
+            resolved_path = dataset.get("path")
+            label = f"{source_type.value}:{key or '<unknown dataset>'}"
+            if not isinstance(resolved_path, (str, Path)) or not str(resolved_path):
+                missing_entries.append(label)
+            elif not Path(resolved_path).is_file():
+                missing_files.append(f"{label} -> {resolved_path}")
+
+    if missing_entries or missing_files:
+        problems = []
+        if missing_entries:
+            problems.append(f"missing path entries: {', '.join(missing_entries)}")
+        if missing_files:
+            problems.append(f"files not found: {', '.join(missing_files)}")
+        raise DatasetResolutionError(
+            f"Catalog paths are incomplete for {path} ({'. '.join(problems)}). "
+            "Run `ref test-cases fetch` to rebuild the paths file."
+        )
+    return datasets
+
+
 def get_catalog_hash(path: Path) -> str | None:
     """
     Get the hash stored in an existing catalog file.
@@ -493,17 +558,25 @@ def _serialise_datasets(
         filtered_records = []
         for record in datasets_records:
             instance_id = record.get(slug_column)
-            if instance_id and "path" in record:  # pragma: no branch
-                file_path = record.pop("path")
-                filename = Path(file_path).name
-                # Store filename in record for matching when loading
-                record["filename"] = filename
-                # Use composite key to support multiple files per instance_id
-                paths_map[f"{instance_id}::{filename}"] = file_path
-                # Sanitize and sort fields within each record alphabetically
-                sanitized_record = {k: _sanitize_for_yaml(v) for k, v in record.items()}
-                sorted_record = dict(sorted(sanitized_record.items()))
-                filtered_records.append(sorted_record)
+            if not isinstance(instance_id, str) or not instance_id:
+                raise ValueError(
+                    f"Dataset is missing the required {slug_column!r} identity field and cannot be saved"
+                )
+            file_path = record.get("path")
+            if not isinstance(file_path, (str, Path)) or not str(file_path):
+                raise ValueError(
+                    f"Dataset {instance_id!r} is missing the required 'path' field and cannot be saved"
+                )
+            record.pop("path")
+            filename = Path(file_path).name
+            # Store filename in record for matching when loading
+            record["filename"] = filename
+            # Use composite key to support multiple files per instance_id
+            paths_map[f"{instance_id}::{filename}"] = str(file_path)
+            # Sanitize and sort fields within each record alphabetically
+            sanitized_record = {k: _sanitize_for_yaml(v) for k, v in record.items()}
+            sorted_record = dict(sorted(sanitized_record.items()))
+            filtered_records.append(sorted_record)
 
         # Sort records by instance_id, then by filename for stability
         filtered_records.sort(key=lambda r: (r.get(slug_column, ""), r.get("filename", "")))
@@ -544,11 +617,9 @@ def save_datasets_to_yaml(
     By default, the catalog is only written if the content has changed
     (detected via hash comparison). Use `force=True` to always write.
 
-    The paths sidecar is (re)generated whenever it is **missing**, even when the catalog
-    content is unchanged: on a machine with a cold cache the version-controlled
-    `catalog.yaml` exists but the sidecar does not, and `run`/`mint` need it to resolve
-    inputs. In that case the catalog itself is left untouched (so a plain
-    `ref test-cases fetch` is enough, no `--force` required) and only the sidecar is written.
+    The paths sidecar is regenerated on every save, even when the catalog content is
+    unchanged. Local cache contents can change independently of the version-controlled
+    catalog, so retaining an existing sidecar can leave missing or stale paths behind.
 
     Parameters
     ----------
@@ -569,19 +640,15 @@ def save_datasets_to_yaml(
     """
     new_hash = datasets.hash
 
+    data, paths_map = _serialise_datasets(datasets)
+
     if not force and get_catalog_hash(path) == new_hash:
-        # Catalog content is unchanged. Still regenerate the paths sidecar if it is missing
-        # (e.g. a cold dataset cache) so run/mint can resolve inputs, but leave the
-        # version-controlled catalog untouched to avoid spurious diffs.
-        if paths_file.exists():
-            logger.info(f"Catalog unchanged, skipping write: {path}")
-        else:
-            _, paths_map = _serialise_datasets(datasets)
-            _write_paths_file(paths_file, paths_map)
-            logger.info(f"Catalog unchanged; regenerated missing paths file: {paths_file}")
+        # Keep the tracked catalog byte-identical, but always refresh its machine-local
+        # paths. An existing sidecar may be partial or point at an old cache location.
+        _write_paths_file(paths_file, paths_map)
+        logger.info(f"Catalog unchanged, refreshed paths file: {paths_file}")
         return False
 
-    data, paths_map = _serialise_datasets(datasets)
     path.parent.mkdir(parents=True, exist_ok=True)
     with open(path, "w") as f:
         yaml.dump(data, f, default_flow_style=False, sort_keys=False)