climate-ref-esmvaltool 0.6.5__py3-none-any.whl → 0.7.0__py3-none-any.whl

climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py

@@ -0,0 +1,200 @@
+from functools import partial
+
+import pandas
+
+from climate_ref_core.constraints import (
+    AddSupplementaryDataset,
+    PartialDateTime,
+    RequireFacets,
+    RequireTimerange,
+)
+from climate_ref_core.datasets import FacetFilter, SourceDatasetType
+from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
+from climate_ref_esmvaltool.recipe import dataframe_to_recipe
+from climate_ref_esmvaltool.types import Recipe
+
+
+def get_cmip6_data_requirements(variables: tuple[str, ...]) -> tuple[DataRequirement, ...]:
+    """Create a data requirement for CMIP6 data."""
+    return (
+        DataRequirement(
+            source_type=SourceDatasetType.CMIP6,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "variable_id": variables,
+                        "experiment_id": "historical",
+                        "table_id": "Amon",
+                    },
+                ),
+            ),
+            group_by=("source_id", "experiment_id", "member_id", "frequency", "grid_label"),
+            constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1996, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
+                RequireFacets("variable_id", variables),
+                AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
+            ),
+        ),
+    )
+
+
+def update_recipe(
+    recipe: Recipe,
+    input_files: dict[SourceDatasetType, pandas.DataFrame],
+    var_x: str,
+    var_y: str,
+) -> None:
+    """Update the recipe."""
+    recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
+    diagnostics = recipe["diagnostics"]
+    diagnostic_name = f"plot_joint_{var_x}_{var_y}_model"
+    diagnostic = diagnostics.pop(diagnostic_name)
+    diagnostics.clear()
+    diagnostics[diagnostic_name] = diagnostic
+    recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
+    datasets = next(iter(recipe_variables.values()))["additional_datasets"]
+    for dataset in datasets:
+        dataset["timerange"] = "1996/2014"
+    diagnostic["additional_datasets"] = datasets
+    suptitle = "CMIP6 {dataset} {ensemble} {grid} {timerange}".format(**datasets[0])
+    diagnostic["scripts"]["plot"]["suptitle"] = suptitle
+    diagnostic["scripts"]["plot"]["plot_filename"] = (
+        f"jointplot_{var_x}_{var_y}_{suptitle.replace(' ', '_').replace('/', '-')}"
+    )
+
+
+class CloudScatterplotCltSwcre(ESMValToolDiagnostic):
+    """
+    Scatterplot of clt vs swcre.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (clt vs swcre)"
+    slug = "cloud-scatterplots-clt-swcre"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("clt", "rsut", "rsutcs"))
+    update_recipe = partial(update_recipe, var_x="clt", var_y="swcre")
+
+
+class CloudScatterplotClwviPr(ESMValToolDiagnostic):
+    """
+    Scatterplot of clwvi vs pr.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (clwvi vs pr)"
+    slug = "cloud-scatterplots-clwvi-pr"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("clwvi", "pr"))
+    update_recipe = partial(update_recipe, var_x="clwvi", var_y="pr")
+
+
+class CloudScatterplotCliviLwcre(ESMValToolDiagnostic):
+    """
+    Scatterplot of clivi vs lwcre.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (clivi vs lwcre)"
+    slug = "cloud-scatterplots-clivi-lwcre"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("clivi", "rlut", "rlutcs"))
+    update_recipe = partial(update_recipe, var_x="clivi", var_y="lwcre")
+
+
+class CloudScatterplotCliTa(ESMValToolDiagnostic):
+    """
+    Scatterplot of cli vs ta.
+    """
+
+    name = "Scatterplots of two cloud-relevant variables (cli vs ta)"
+    slug = "cloud-scatterplots-cli-ta"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = get_cmip6_data_requirements(("cli", "ta"))
+    update_recipe = partial(update_recipe, var_x="cli", var_y="ta")
+
+
+class CloudScatterplotsReference(ESMValToolDiagnostic):
+    """
+    Reference scatterplots of two cloud-relevant variables.
+    """
+
+    name = "Reference scatterplots of two cloud-relevant variables"
+    slug = "cloud-scatterplots-reference"
+    base_recipe = "ref/recipe_ref_scatterplot.yml"
+    facets = ()
+    data_requirements = (
+        DataRequirement(
+            source_type=SourceDatasetType.obs4MIPs,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "source_id": ("ERA-5",),
+                        "variable_id": ("ta",),
+                    },
+                ),
+            ),
+            group_by=("instance_id",),
+            constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(2007, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
+            ),
+            # TODO: Add obs4MIPs datasets once available and working:
+            #
+            # obs4MIPs datasets with issues:
+            # - GPCP-V2.3: pr
+            # - CERES-EBAF-4-2: rlut, rlutcs, rsut, rsutcs
+            #
+            # Unsure if available on obs4MIPs:
+            # - AVHRR-AMPM-fv3.0: clivi, clwvi
+            # - ESACCI-CLOUD: clt
+            # - CALIPSO-ICECLOUD: cli
+            #
+            # Related issues:
+            # - https://github.com/Climate-REF/climate-ref/issues/260
+            # - https://github.com/esMValGroup/esMValCore/issues/2712
+            # - https://github.com/esMValGroup/esMValCore/issues/2711
+            # - https://github.com/sciTools/iris/issues/6411
+        ),
+    )
+
+    @staticmethod
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
+        """Update the recipe."""
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.obs4MIPs])
+        recipe["diagnostics"] = {k: v for k, v in recipe["diagnostics"].items() if k.endswith("_ref")}
+
+        era5_dataset = recipe_variables["ta"]["additional_datasets"][0]
+        era5_dataset["timerange"] = "2007/2015"  # Use the same timerange as for the other variable.
+        era5_dataset["alias"] = era5_dataset["dataset"]
+        diagnostic = recipe["diagnostics"]["plot_joint_cli_ta_ref"]
+        diagnostic["variables"]["ta"]["additional_datasets"] = [era5_dataset]
+        suptitle = "CALIPSO-ICECLOUD / {dataset} {timerange}".format(**era5_dataset)
+        diagnostic["scripts"]["plot"]["suptitle"] = suptitle
+        diagnostic["scripts"]["plot"]["plot_filename"] = (
+            f"jointplot_cli_ta_{suptitle.replace(' ', '_').replace('/', '-')}"
+        )
+
+        # Use the correct obs4MIPs dataset name for dataset that cannot be ingested
+        # https://github.com/Climate-REF/climate-ref/issues/260.
+        diagnostic = recipe["diagnostics"]["plot_joint_clwvi_pr_ref"]
+        diagnostic["variables"]["pr"]["additional_datasets"] = [
+            {
+                "dataset": "GPCP-V2.3",
+                "project": "obs4MIPs",
+                "alias": "GPCP-SG",
+                "timerange": "1992/2016",
+            }
+        ]

climate_ref_esmvaltool/diagnostics/ecs.py

@@ -11,6 +11,7 @@ from climate_ref_core.constraints import (
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
@@ -45,23 +46,46 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
                     facets={
                         "variable_id": variables,
                         "experiment_id": experiments,
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", variables),
-                RequireFacets("experiment_id", experiments),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets(
+                    "variable_id",
+                    required_facets=variables,
+                    group_by=("source_id", "member_id", "grid_label", "experiment_id"),
+                ),
+                RequireFacets(
+                    "experiment_id",
+                    required_facets=experiments,
+                    group_by=("source_id", "member_id", "grid_label", "variable_id"),
+                ),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    series = (
+        SeriesDefinition(
+            file_pattern="ecs/calculate/ecs_regression_*.nc",
+            dimensions={
+                "statistic": ("global annual mean anomaly of rtnt vs tas"),
+            },
+            values_name="rtnt_anomaly",
+            index_name="tas_anomaly",
+            attributes=[],
+        ),
+    )
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Only run the diagnostic that computes ECS for a single model.
         recipe["diagnostics"] = {
@@ -88,21 +112,11 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
         # Prepare updated datasets section in recipe. It contains two
         # datasets, one for the "abrupt-4xCO2" and one for the "piControl"
         # experiment.
-        recipe_variables = dataframe_to_recipe(input_files)
-        recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
-
-        # Select a timerange covered by all datasets.
-        start_times, end_times = [], []
-        for variable in recipe_variables.values():
-            for dataset in variable["additional_datasets"]:
-                start, end = dataset["timerange"].split("/")
-                start_times.append(start)
-                end_times.append(end)
-        timerange = f"{max(start_times)}/{min(end_times)}"
-
-        datasets = recipe_variables["tas"]["additional_datasets"]
-        for dataset in datasets:
-            dataset["timerange"] = timerange
+        recipe_variables = dataframe_to_recipe(
+            input_files[SourceDatasetType.CMIP6],
+            equalize_timerange=True,
+        )
+        recipe["datasets"] = recipe_variables["tas"]["additional_datasets"]
 
         # Remove keys from the recipe that are only used for YAML anchors
         keys_to_remove = [
@@ -116,8 +130,6 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
         for key in keys_to_remove:
             recipe.pop(key, None)
 
-        recipe["datasets"] = datasets
-
     @staticmethod
     def format_result(
         result_dir: Path,

climate_ref_esmvaltool/diagnostics/enso.py

@@ -11,6 +11,7 @@ from climate_ref_core.constraints import (
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
@@ -27,36 +28,104 @@ class ENSOBasicClimatology(ESMValToolDiagnostic):
     slug = "enso-basic-climatology"
     base_recipe = "ref/recipe_enso_basicclimatology.yml"
 
-    variables = (
-        "pr",
-        "tos",
-        "tauu",
-    )
     data_requirements = (
         DataRequirement(
             source_type=SourceDatasetType.CMIP6,
             filters=(
                 FacetFilter(
                     facets={
-                        "variable_id": variables,
+                        "variable_id": ("pr", "tauu"),
+                        "experiment_id": "historical",
+                        "table_id": "Amon",
+                    },
+                ),
+                FacetFilter(
+                    facets={
+                        "variable_id": "tos",
                         "experiment_id": "historical",
+                        "table_id": "Omon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", variables),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets(
+                    "variable_id",
+                    (
+                        "pr",
+                        "tauu",
+                        "tos",
+                    ),
+                ),
             ),
         ),
     )
     facets = ()
 
+    series = (
+        tuple(
+            SeriesDefinition(
+                file_pattern=f"diagnostic_metrics/plot_script/{{source_id}}_eq_{var_name}_bias.nc",
+                dimensions={
+                    "statistic": (
+                        f"zonal bias in the time-mean {var_name} structure across the equatorial Pacific"
+                    ),
+                },
+                values_name="tos" if var_name == "sst" else var_name,
+                index_name="lon",
+                attributes=[],
+            )
+            for var_name in ["pr", "sst", "tauu"]
+        )
+        + tuple(
+            SeriesDefinition(
+                file_pattern=f"diagnostic_metrics/plot_script/{{source_id}}_eq_{var_name}_seacycle.nc",
+                dimensions={
+                    "statistic": (
+                        "zonal bias in the amplitude of the mean seasonal cycle of "
+                        f"{var_name} in the equatorial Pacific"
+                    ),
+                },
+                values_name="tos" if var_name == "sst" else var_name,
+                index_name="lon",
+                attributes=[],
+            )
+            for var_name in ["pr", "sst", "tauu"]
+        )
+        + (
+            SeriesDefinition(
+                file_pattern="diagnostic_metrics/plot_script/{source_id}_pr_double.nc",
+                dimensions={
+                    "statistic": ("meridional bias in the time-mean pr structure across the eastern Pacific"),
+                },
+                values_name="pr",
+                index_name="lat",
+                attributes=[],
+            ),
+            SeriesDefinition(
+                file_pattern="diagnostic_metrics/plot_script/*_pr_double_seacycle.nc",
+                dimensions={
+                    "statistic": (
+                        "meridional bias in the amplitude of the mean seasonal "
+                        "pr cycle in the eastern Pacific"
+                    ),
+                },
+                values_name="pr",
+                index_name="lat",
+                attributes=[],
+            ),
+        )
+    )
+
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe.pop("datasets")
         for diagnostic in recipe["diagnostics"].values():
             for variable in diagnostic["variables"].values():
@@ -82,24 +151,31 @@ class ENSOCharacteristics(ESMValToolDiagnostic):
                     facets={
                         "variable_id": "tos",
                         "experiment_id": "historical",
+                        "table_id": "Omon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", ("tos",)),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
                 AddSupplementaryDataset.from_defaults("areacello", SourceDatasetType.CMIP6),
+                RequireFacets("variable_id", ("tos", "areacello")),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    # ENSO pattern and lifecycle are series, but the ESMValTool diagnostic
+    # script does not save the values used in the figure.
+    series = tuple()
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe["datasets"] = recipe_variables["tos"]["additional_datasets"]
         # TODO: update the observational data requirement once available on ESGF.
         # Observations - use only one per run

climate_ref_esmvaltool/diagnostics/example.py

@@ -3,6 +3,7 @@ import pandas
 from climate_ref_core.constraints import AddSupplementaryDataset, RequireContiguousTimerange
 from climate_ref_core.datasets import FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import Recipe
@@ -21,17 +22,30 @@ class GlobalMeanTimeseries(ESMValToolDiagnostic):
         DataRequirement(
             source_type=SourceDatasetType.CMIP6,
             filters=(FacetFilter(facets={"variable_id": ("tas",)}),),
-            group_by=("instance_id",),
+            group_by=("source_id", "experiment_id", "member_id", "table_id", "variable_id", "grid_label"),
             constraints=(
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
+
     facets = ()
+    series = (
+        SeriesDefinition(
+            file_pattern="timeseries/script1/*.nc",
+            dimensions={"statistic": "tas annual global mean"},
+            values_name="tas",
+            index_name="time",
+            attributes=[],
+        ),
+    )
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Clear unwanted elements from the recipe.
         recipe["datasets"].clear()
@@ -40,7 +54,7 @@ class GlobalMeanTimeseries(ESMValToolDiagnostic):
         variables.clear()
 
         # Prepare updated variables section in recipe.
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
         for variable in recipe_variables.values():
             variable["preprocessor"] = "annual_mean_global"