climate-ref-esmvaltool 0.6.5__tar.gz → 0.6.6__tar.gz

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref-esmvaltool
-Version: 0.6.5
+Version: 0.6.6
 Summary: ESMValTool diagnostic provider for the Rapid Evaluation Framework
 Author-email: ESMValTool development team <esmvaltool-dev@listserv.dfn.de>, Jared Lewis <jared.lewis@climate-resource.com>
 License-Expression: Apache-2.0

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref-esmvaltool"
-version = "0.6.5"
+version = "0.6.6"
 description = "ESMValTool diagnostic provider for the Rapid Evaluation Framework"
 readme = "README.md"
 authors = [
@@ -23,8 +23,8 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
 ]
 dependencies = [
-    "pooch >= 1.8",
     "climate-ref-core",
+    "pooch >= 1.8",
     "pyyaml",
     "xarray >= 2023.3.0",
 ]

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/src/climate_ref_esmvaltool/dataset_registry/data.txt

@@ -73,6 +73,9 @@ ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rlut_200003-202311
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rlutcs_200003-202311.nc e70e3273092edf01527970693271641fc6474d1974887d7d272e7d656bab83c2
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rsut_200003-202311.nc e31e648886c4fa9c09686672a06ab18fbba687ff0d6de2891616d4c8b74e215d
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rsutcs_200003-202311.nc eb96edd9274670aa705eab2a6d1ee0cca11e01ac17096706463e032b58e6be47
+ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_clivi_198201-201612.nc 13bc6e3a46397386a14a36776fdd6bdbf5c45147c8dc695d4a7387883d449775
+ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_clt_198201-201612.nc 4a430d77dbe9164dba2d1fdef4bb89ea9358b6d10023f3fbee50917422446ed0
+ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_clwvi_198201-201612.nc e0ffa31369d9552be16b920110b24013d31e116b180ee3f70b3d0aaa5281eff0
 ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_rlut_198201-201612.nc 075144d673a9f2ff49fbe59e701535bf80c04908797a9dca83781000a9b1b7f2
 ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_rlutcs_198201-201612.nc 21f096ecafff659e5c7e3338060425f7194e5d1b39c9510865496e04ecac3d75
 ESMValTool/OBS/Tier2/ESACCI-CLOUD/OBS_ESACCI-CLOUD_sat_AVHRR-AMPM-fv3.0_Amon_rsut_198201-201612.nc f2c3f3afcdc2e730df7985c210a3de89b0d4f83b150e0c3846f7ac3c5fa9c54a
@@ -156,5 +159,6 @@ ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201101-20111
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201201-201212.nc 86187c3d1174053f2cba6dad010af49ceab77d368aa9314bf53c330b5f2217b9
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201301-201312.nc 8820353570884b2ef182caaffb5986ed6268bbe199fd867f61b56e798ca01f1a
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201401-201412.nc 7102d0db3dc02c5b0eb0cfe3535ee50171007ef5b43eb9aae1220ac21b0b98e9
+ESMValTool/OBS/Tier3/CALIPSO-ICECLOUD/OBS_CALIPSO-ICECLOUD_sat_1-00_Amon_cli_200701-201512.nc 977824810e8f9dbe7df278c59397ffc3f78491a9eb5a0b70e6b28ac66db8e12d
 ESMValTool/OBS/Tier2/TROPFLUX/OBS6_TROPFLUX_reanaly_v1_Amon_tauu_197901-201812.nc bf313e661b42341d5090038b501ed1ff09e58201009c3fccfe45b78e116fdd78
 ESMValTool/OBS/Tier2/TROPFLUX/OBS6_TROPFLUX_reanaly_v1_Omon_tos_197901-201812.nc 5f10a5a2aa47f5d21378ad3178bf8e4b577b0ed72ef8402dc04f5ff6fc99ec07

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/src/climate_ref_esmvaltool/diagnostics/__init__.py

@@ -1,10 +1,23 @@
 """ESMValTool diagnostics."""
 
 from climate_ref_esmvaltool.diagnostics.climate_at_global_warming_levels import ClimateAtGlobalWarmingLevels
+from climate_ref_esmvaltool.diagnostics.climate_drivers_for_fire import ClimateDriversForFire
 from climate_ref_esmvaltool.diagnostics.cloud_radiative_effects import CloudRadiativeEffects
+from climate_ref_esmvaltool.diagnostics.cloud_scatterplots import (
+    CloudScatterplotCliTa,
+    CloudScatterplotCliviLwcre,
+    CloudScatterplotCltSwcre,
+    CloudScatterplotClwviPr,
+    CloudScatterplotsReference,
+)
 from climate_ref_esmvaltool.diagnostics.ecs import EquilibriumClimateSensitivity
 from climate_ref_esmvaltool.diagnostics.enso import ENSOBasicClimatology, ENSOCharacteristics
 from climate_ref_esmvaltool.diagnostics.example import GlobalMeanTimeseries
+from climate_ref_esmvaltool.diagnostics.regional_historical_changes import (
+    RegionalHistoricalAnnualCycle,
+    RegionalHistoricalTimeSeries,
+    RegionalHistoricalTrend,
+)
 from climate_ref_esmvaltool.diagnostics.sea_ice_area_basic import SeaIceAreaBasic
 from climate_ref_esmvaltool.diagnostics.sea_ice_sensitivity import SeaIceSensitivity
 from climate_ref_esmvaltool.diagnostics.tcr import TransientClimateResponse
@@ -13,11 +26,20 @@ from climate_ref_esmvaltool.diagnostics.zec import ZeroEmissionCommitment
 
 __all__ = [
     "ClimateAtGlobalWarmingLevels",
+    "ClimateDriversForFire",
     "CloudRadiativeEffects",
+    "CloudScatterplotCliTa",
+    "CloudScatterplotCliviLwcre",
+    "CloudScatterplotCltSwcre",
+    "CloudScatterplotClwviPr",
+    "CloudScatterplotsReference",
     "ENSOBasicClimatology",
     "ENSOCharacteristics",
     "EquilibriumClimateSensitivity",
     "GlobalMeanTimeseries",
+    "RegionalHistoricalAnnualCycle",
+    "RegionalHistoricalTimeSeries",
+    "RegionalHistoricalTrend",
     "SeaIceAreaBasic",
     "SeaIceSensitivity",
     "TransientClimateResponse",

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/src/climate_ref_esmvaltool/diagnostics/base.py

@@ -1,9 +1,11 @@
+import fnmatch
 from abc import abstractmethod
 from collections.abc import Iterable
 from pathlib import Path
 from typing import ClassVar
 
 import pandas
+import xarray as xr
 import yaml
 from loguru import logger
 
@@ -14,6 +16,7 @@ from climate_ref_core.diagnostics import (
     ExecutionDefinition,
     ExecutionResult,
 )
+from climate_ref_core.metric_values.typing import SeriesMetricValue
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput, OutputCV
 from climate_ref_esmvaltool.recipe import load_recipe, prepare_climate_data
@@ -27,7 +30,10 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
 
     @staticmethod
     @abstractmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """
         Update the base recipe for the run.
 
@@ -67,9 +73,9 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
         """
         return CMECMetric.model_validate(metric_args), CMECOutput.model_validate(output_args)
 
-    def build_cmd(self, definition: ExecutionDefinition) -> Iterable[str]:
+    def write_recipe(self, definition: ExecutionDefinition) -> Path:
         """
-        Build the command to run an ESMValTool recipe.
+        Update the ESMValTool recipe for the diagnostic and write it to file.
 
         Parameters
         ----------
@@ -79,22 +85,42 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
         Returns
         -------
         :
-            The result of running the diagnostic.
+            The path to the written recipe.
         """
-        input_files = definition.datasets[SourceDatasetType.CMIP6].datasets
+        input_files = {
+            project: dataset_collection.datasets
+            for project, dataset_collection in definition.datasets.items()
+        }
         recipe = load_recipe(self.base_recipe)
         self.update_recipe(recipe, input_files)
 
         recipe_path = definition.to_output_path("recipe.yml")
         with recipe_path.open("w", encoding="utf-8") as file:
-            yaml.dump(recipe, file)
+            yaml.safe_dump(recipe, file, sort_keys=False)
+        return recipe_path
+
+    def build_cmd(self, definition: ExecutionDefinition) -> Iterable[str]:
+        """
+        Build the command to run an ESMValTool recipe.
+
+        Parameters
+        ----------
+        definition
+            A description of the information needed for this execution of the diagnostic
 
+        Returns
+        -------
+        :
+            The result of running the diagnostic.
+        """
+        recipe_path = self.write_recipe(definition)
         climate_data = definition.to_output_path("climate_data")
 
-        prepare_climate_data(
-            definition.datasets[SourceDatasetType.CMIP6].datasets,
-            climate_data_dir=climate_data,
-        )
+        for metric_dataset in definition.datasets.values():
+            prepare_climate_data(
+                metric_dataset.datasets,
+                climate_data_dir=climate_data,
+            )
 
         config = {
             "drs": {
@@ -130,7 +156,7 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
                 {
                     "OBS": str(data_dir / "OBS"),
                     "OBS6": str(data_dir / "OBS"),
-                    "native6": str(data_dir / "RAWOBS"),
+                    "native6": str(data_dir / "native6"),
                 }
             )
             config["rootpath"]["obs4MIPs"] = [  # type: ignore[index]
@@ -141,7 +167,7 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
         config_dir = definition.to_output_path("config")
         config_dir.mkdir()
         with (config_dir / "config.yml").open("w", encoding="utf-8") as file:
-            yaml.dump(config, file)
+            yaml.safe_dump(config, file)
 
         return [
             "esmvaltool",
@@ -173,6 +199,12 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
         output_args = CMECOutput.create_template()
 
         # Add the plots and data files
+        variable_attributes = (
+            "long_name",
+            "standard_name",
+            "units",
+        )
+        series = []
         plot_suffixes = {".png", ".jpg", ".pdf", ".ps"}
         for metadata_file in result_dir.glob("run/*/*/diagnostic_provenance.yml"):
             metadata = yaml.safe_load(metadata_file.read_text(encoding="utf-8"))
@@ -188,6 +220,41 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
                     OutputCV.LONG_NAME.value: caption,
                     OutputCV.DESCRIPTION.value: "",
                 }
+                for series_def in definition.diagnostic.series:
+                    if fnmatch.fnmatch(str(relative_path), f"executions/*/{series_def.file_pattern}"):
+                        dataset = xr.open_dataset(
+                            filename, decode_times=xr.coders.CFDatetimeCoder(use_cftime=True)
+                        )
+                        dataset = dataset.sel(series_def.sel)
+                        attributes = {
+                            attr: dataset.attrs[attr]
+                            for attr in series_def.attributes
+                            if attr in dataset.attrs
+                        }
+                        attributes["caption"] = caption
+                        attributes["values_name"] = series_def.values_name
+                        attributes["index_name"] = series_def.index_name
+                        for attr in variable_attributes:
+                            if attr in dataset[series_def.values_name].attrs:
+                                attributes[f"value_{attr}"] = dataset[series_def.values_name].attrs[attr]
+                            if attr in dataset[series_def.index_name].attrs:
+                                attributes[f"index_{attr}"] = dataset[series_def.index_name].attrs[attr]
+                        index = dataset[series_def.index_name].values.tolist()
+                        if hasattr(index[0], "calendar"):
+                            attributes["calendar"] = index[0].calendar
+                        if hasattr(index[0], "isoformat"):
+                            # Convert time objects to strings.
+                            index = [v.isoformat() for v in index]
+
+                        series.append(
+                            SeriesMetricValue(
+                                dimensions=series_def.dimensions,
+                                values=dataset[series_def.values_name].values.tolist(),
+                                index=index,
+                                index_name=series_def.index_name,
+                                attributes=attributes,
+                            )
+                        )
 
         # Add the index.html file
         index_html = f"{result_dir}/index.html"
@@ -218,4 +285,5 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
             definition,
             cmec_output_bundle=output_bundle,
             cmec_metric_bundle=metric_bundle,
+            series=series,
         )

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/src/climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py

@@ -65,13 +65,16 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic):
     facets = ()
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Set up the datasets
         diagnostics = recipe["diagnostics"]
         for diagnostic in diagnostics.values():
             diagnostic.pop("additional_datasets")
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         datasets = recipe_variables["tas"]["additional_datasets"]
         datasets = [ds for ds in datasets if ds["exp"] != "historical"]
         for dataset in datasets:

climate_ref_esmvaltool-0.6.6/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py

@@ -0,0 +1,68 @@
+import pandas
+
+from climate_ref_core.constraints import (
+    AddSupplementaryDataset,
+    RequireFacets,
+    RequireOverlappingTimerange,
+)
+from climate_ref_core.datasets import FacetFilter, SourceDatasetType
+from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
+from climate_ref_esmvaltool.recipe import dataframe_to_recipe
+from climate_ref_esmvaltool.types import Recipe
+
+
+class ClimateDriversForFire(ESMValToolDiagnostic):
+    """
+    Calculate diagnostics regarding climate drivers for fire.
+    """
+
+    name = "Climate drivers for fire"
+    slug = "climate-drivers-for-fire"
+    base_recipe = "ref/recipe_ref_fire.yml"
+
+    variables = (
+        "cVeg",
+        "hurs",
+        "pr",
+        "tas",
+        "tasmax",
+        "treeFrac",
+        "vegFrac",
+    )
+    data_requirements = (
+        DataRequirement(
+            source_type=SourceDatasetType.CMIP6,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "variable_id": variables,
+                        "frequency": "mon",
+                        "experiment_id": "historical",
+                    }
+                ),
+            ),
+            group_by=("source_id", "member_id", "grid_label"),
+            constraints=(
+                RequireFacets("variable_id", variables),
+                RequireOverlappingTimerange(group_by=("instance_id",)),
+                AddSupplementaryDataset.from_defaults("sftlf", SourceDatasetType.CMIP6),
+            ),
+        ),
+    )
+    facets = ()
+
+    @staticmethod
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
+        """Update the recipe."""
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
+        dataset = recipe_variables["cVeg"]["additional_datasets"][0]
+        dataset.pop("mip")
+        dataset.pop("timerange")
+        dataset["start_year"] = 2013
+        dataset["end_year"] = 2014
+        recipe["datasets"] = [dataset]
+        recipe["diagnostics"]["fire_evaluation"]["scripts"]["fire_evaluation"]["remove_confire_files"] = True

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py

@@ -53,9 +53,9 @@ class CloudRadiativeEffects(ESMValToolDiagnostic):
     )
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(recipe: Recipe, input_files: dict[SourceDatasetType, pandas.DataFrame]) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
 
         # Select a timerange covered by all datasets.

climate_ref_esmvaltool-0.6.6/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py

@@ -0,0 +1,188 @@
+from functools import partial
+
+import pandas
+
+from climate_ref_core.constraints import (
+    AddSupplementaryDataset,
+    RequireContiguousTimerange,
+    RequireFacets,
+    RequireOverlappingTimerange,
+)
+from climate_ref_core.datasets import FacetFilter, SourceDatasetType
+from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
+from climate_ref_esmvaltool.recipe import dataframe_to_recipe
+from climate_ref_esmvaltool.types import Recipe
+
+
+def get_cmip6_data_requirements(variables: tuple[str, ...]) -> tuple[DataRequirement, ...]:
+    """Create a data requirement for CMIP6 data."""
+    return (
+        DataRequirement(
+            source_type=SourceDatasetType.CMIP6,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "variable_id": variables,
+                        "experiment_id": "historical",
+                    },
+                ),
+            ),
+            group_by=("source_id", "experiment_id", "member_id", "frequency", "grid_label"),
+            constraints=(
+                RequireFacets("variable_id", variables),
+                RequireContiguousTimerange(group_by=("instance_id",)),
+                RequireOverlappingTimerange(group_by=("instance_id",)),
+                # TODO: Add a RequireTimeRange constraint to match reference datasets?
+                AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
+            ),
+        ),
+    )
+
+
+def update_recipe(
+    recipe: Recipe,
+    input_files: dict[SourceDatasetType, pandas.DataFrame],
+    var_x: str,
+    var_y: str,
+) -> None:
+    """Update the recipe."""
+    recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6], equalize_timerange=True)
+    diagnostics = recipe["diagnostics"]
+    diagnostic_name = f"plot_joint_{var_x}_{var_y}_model"
+    diagnostic = diagnostics.pop(diagnostic_name)
+    diagnostics.clear()
+    diagnostics[diagnostic_name] = diagnostic
+    datasets = next(iter(recipe_variables.values()))["additional_datasets"]
+    diagnostic["additional_datasets"] = datasets
+    suptitle = "CMIP6 {dataset} {ensemble} {grid} {timerange}".format(**datasets[0])
+    diagnostic["scripts"]["plot"]["suptitle"] = suptitle
+    diagnostic["scripts"]["plot"]["plot_filename"] = (
+        f"jointplot_{var_x}_{var_y}_{suptitle.replace(' ', '_').replace('/', '-')}"
+    )
+
+
+class CloudScatterplotCltSwcre(ESMValToolDiagnostic):
+    """
+    Scatterplot of clt vs swcre.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (clt vs swcre)"
+    slug = "cloud-scatterplots-clt-swcre"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("clt", "rsut", "rsutcs"))
+    update_recipe = partial(update_recipe, var_x="clt", var_y="swcre")
+
+
+class CloudScatterplotClwviPr(ESMValToolDiagnostic):
+    """
+    Scatterplot of clwvi vs pr.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (clwvi vs pr)"
+    slug = "cloud-scatterplots-clwvi-pr"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("clwvi", "pr"))
+    update_recipe = partial(update_recipe, var_x="clwvi", var_y="pr")
+
+
+class CloudScatterplotCliviLwcre(ESMValToolDiagnostic):
+    """
+    Scatterplot of clivi vs lwcre.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (clivi vs lwcre)"
+    slug = "cloud-scatterplots-clivi-lwcre"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("clivi", "rlut", "rlutcs"))
+    update_recipe = partial(update_recipe, var_x="clivi", var_y="lwcre")
+
+
+class CloudScatterplotCliTa(ESMValToolDiagnostic):
+    """
+    Scatterplot of cli vs ta.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (cli vs ta)"
+    slug = "cloud-scatterplots-cli-ta"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("cli", "ta"))
+    update_recipe = partial(update_recipe, var_x="cli", var_y="ta")
+
+
+class CloudScatterplotsReference(ESMValToolDiagnostic):
+    """
+    Reference scatterplots of two cloud-relevant variables.
+    """
+
+    name = "Reference scatterplots of two cloud-relevant variables"
+    slug = "cloud-scatterplots-reference"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = (
+        DataRequirement(
+            source_type=SourceDatasetType.obs4MIPs,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "source_id": ("ERA-5",),
+                        "variable_id": ("ta",),
+                    },
+                ),
+            ),
+            group_by=("instance_id",),
+            constraints=(RequireContiguousTimerange(group_by=("instance_id",)),),
+            # TODO: Add obs4MIPs datasets once available and working:
+            #
+            # obs4MIPs datasets with issues:
+            # - GPCP-V2.3: pr
+            # - CERES-EBAF-4-2: rlut, rlutcs, rsut, rsutcs
+            #
+            # Unsure if available on obs4MIPs:
+            # - AVHRR-AMPM-fv3.0: clivi, clwvi
+            # - ESACCI-CLOUD: clt
+            # - CALIPSO-ICECLOUD: cli
+            #
+            # Related issues:
+            # - https://github.com/Climate-REF/climate-ref/issues/260
+            # - https://github.com/esMValGroup/esMValCore/issues/2712
+            # - https://github.com/esMValGroup/esMValCore/issues/2711
+            # - https://github.com/sciTools/iris/issues/6411
+        ),
+    )
+
+    @staticmethod
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
+        """Update the recipe."""
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.obs4MIPs])
+        recipe["diagnostics"] = {k: v for k, v in recipe["diagnostics"].items() if k.endswith("_ref")}
+
+        era5_dataset = recipe_variables["ta"]["additional_datasets"][0]
+        era5_dataset["timerange"] = "2007/2015"  # Use the same timerange as for the other variable.
+        era5_dataset["alias"] = era5_dataset["dataset"]
+        diagnostic = recipe["diagnostics"]["plot_joint_cli_ta_ref"]
+        diagnostic["variables"]["ta"]["additional_datasets"] = [era5_dataset]
+        suptitle = "CALIPSO-ICECLOUD / {dataset} {timerange}".format(**era5_dataset)
+        diagnostic["scripts"]["plot"]["suptitle"] = suptitle
+        diagnostic["scripts"]["plot"]["plot_filename"] = (
+            f"jointplot_cli_ta_{suptitle.replace(' ', '_').replace('/', '-')}"
+        )
+
+        # Use the correct obs4MIPs dataset name for dataset that cannot be ingested
+        # https://github.com/Climate-REF/climate-ref/issues/260.
+        diagnostic = recipe["diagnostics"]["plot_joint_clwvi_pr_ref"]
+        diagnostic["variables"]["pr"]["additional_datasets"] = [
+            {
+                "dataset": "GPCP-V2.3",
+                "project": "obs4MIPs",
+                "alias": "GPCP-SG",
+                "timerange": "1992/2016",
+            }
+        ]

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/src/climate_ref_esmvaltool/diagnostics/ecs.py

@@ -61,7 +61,10 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Only run the diagnostic that computes ECS for a single model.
         recipe["diagnostics"] = {
@@ -88,21 +91,11 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
         # Prepare updated datasets section in recipe. It contains two
         # datasets, one for the "abrupt-4xCO2" and one for the "piControl"
         # experiment.
-        recipe_variables = dataframe_to_recipe(input_files)
-        recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
-
-        # Select a timerange covered by all datasets.
-        start_times, end_times = [], []
-        for variable in recipe_variables.values():
-            for dataset in variable["additional_datasets"]:
-                start, end = dataset["timerange"].split("/")
-                start_times.append(start)
-                end_times.append(end)
-        timerange = f"{max(start_times)}/{min(end_times)}"
-
-        datasets = recipe_variables["tas"]["additional_datasets"]
-        for dataset in datasets:
-            dataset["timerange"] = timerange
+        recipe_variables = dataframe_to_recipe(
+            input_files[SourceDatasetType.CMIP6],
+            equalize_timerange=True,
+        )
+        recipe["datasets"] = recipe_variables["tas"]["additional_datasets"]
 
         # Remove keys from the recipe that are only used for YAML anchors
         keys_to_remove = [
@@ -116,8 +109,6 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
         for key in keys_to_remove:
             recipe.pop(key, None)
 
-        recipe["datasets"] = datasets
-
     @staticmethod
     def format_result(
         result_dir: Path,

{climate_ref_esmvaltool-0.6.5 → climate_ref_esmvaltool-0.6.6}/src/climate_ref_esmvaltool/diagnostics/enso.py

@@ -54,9 +54,12 @@ class ENSOBasicClimatology(ESMValToolDiagnostic):
     facets = ()
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe.pop("datasets")
         for diagnostic in recipe["diagnostics"].values():
             for variable in diagnostic["variables"].values():
@@ -97,9 +100,12 @@ class ENSOCharacteristics(ESMValToolDiagnostic):
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe["datasets"] = recipe_variables["tos"]["additional_datasets"]
         # TODO: update the observational data requirement once available on ESGF.
         # Observations - use only one per run