climate-ref-esmvaltool 0.6.6__py3-none-any.whl → 0.8.0__py3-none-any.whl

climate_ref_esmvaltool/diagnostics/base.py

@@ -4,6 +4,8 @@ from collections.abc import Iterable
 from pathlib import Path
 from typing import ClassVar
 
+import netCDF4
+import numpy as np
 import pandas
 import xarray as xr
 import yaml
@@ -23,6 +25,18 @@ from climate_ref_esmvaltool.recipe import load_recipe, prepare_climate_data
 from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
 
 
+def mask_fillvalues(array: np.ndarray) -> np.ma.MaskedArray:  # type: ignore[type-arg]
+    """Convert netCDF4 fill values in an array to a mask."""
+    # Workaround for https://github.com/pydata/xarray/issues/2742
+    defaults = {np.dtype(k): v for k, v in netCDF4.default_fillvals.items()}
+    return np.ma.masked_equal(array, defaults[array.dtype])  # type: ignore[no-untyped-call,no-any-return]
+
+
+def fillvalues_to_nan(array: np.ndarray) -> np.ndarray:  # type: ignore[type-arg]
+    """Convert netCDF4 fill values in an array to NaN."""
+    return mask_fillvalues(array).filled(np.nan)  # type: ignore[no-untyped-call,no-any-return]
+
+
 class ESMValToolDiagnostic(CommandLineDiagnostic):
     """ESMValTool Diagnostic base class."""
 
@@ -93,10 +107,11 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
         }
         recipe = load_recipe(self.base_recipe)
         self.update_recipe(recipe, input_files)
-
+        recipe_txt = yaml.safe_dump(recipe, sort_keys=False)
+        logger.info(f"Using ESMValTool recipe:\n{recipe_txt}")
         recipe_path = definition.to_output_path("recipe.yml")
         with recipe_path.open("w", encoding="utf-8") as file:
-            yaml.safe_dump(recipe, file, sort_keys=False)
+            file.write(recipe_txt)
         return recipe_path
 
     def build_cmd(self, definition: ExecutionDefinition) -> Iterable[str]:
@@ -166,8 +181,10 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
 
         config_dir = definition.to_output_path("config")
         config_dir.mkdir()
+        config_txt = yaml.safe_dump(config)
+        logger.info(f"Using ESMValTool configuration:\n{config_txt}")
         with (config_dir / "config.yml").open("w", encoding="utf-8") as file:
-            yaml.safe_dump(config, file)
+            file.write(config_txt)
 
         return [
             "esmvaltool",
@@ -198,15 +215,21 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
         metric_args = CMECMetric.create_template()
         output_args = CMECOutput.create_template()
 
+        # Input selectors for the datasets used in the diagnostic.
+        # TODO: Better handling of multiple source types
+        if SourceDatasetType.CMIP6 in definition.datasets:
+            input_selectors = definition.datasets[SourceDatasetType.CMIP6].selector_dict()
+        elif SourceDatasetType.obs4MIPs in definition.datasets:
+            input_selectors = definition.datasets[SourceDatasetType.obs4MIPs].selector_dict()
+        else:
+            input_selectors = {}
+
         # Add the plots and data files
-        variable_attributes = (
-            "long_name",
-            "standard_name",
-            "units",
-        )
         series = []
         plot_suffixes = {".png", ".jpg", ".pdf", ".ps"}
-        for metadata_file in result_dir.glob("run/*/*/diagnostic_provenance.yml"):
+        # Sort metadata files for stable processing
+        metadata_files = sorted(result_dir.glob("run/*/*/diagnostic_provenance.yml"))
+        for metadata_file in metadata_files:
             metadata = yaml.safe_load(metadata_file.read_text(encoding="utf-8"))
             for filename in metadata:
                 caption = metadata[filename].get("caption", "")
@@ -220,41 +243,11 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
                     OutputCV.LONG_NAME.value: caption,
                     OutputCV.DESCRIPTION.value: "",
                 }
-                for series_def in definition.diagnostic.series:
-                    if fnmatch.fnmatch(str(relative_path), f"executions/*/{series_def.file_pattern}"):
-                        dataset = xr.open_dataset(
-                            filename, decode_times=xr.coders.CFDatetimeCoder(use_cftime=True)
-                        )
-                        dataset = dataset.sel(series_def.sel)
-                        attributes = {
-                            attr: dataset.attrs[attr]
-                            for attr in series_def.attributes
-                            if attr in dataset.attrs
-                        }
-                        attributes["caption"] = caption
-                        attributes["values_name"] = series_def.values_name
-                        attributes["index_name"] = series_def.index_name
-                        for attr in variable_attributes:
-                            if attr in dataset[series_def.values_name].attrs:
-                                attributes[f"value_{attr}"] = dataset[series_def.values_name].attrs[attr]
-                            if attr in dataset[series_def.index_name].attrs:
-                                attributes[f"index_{attr}"] = dataset[series_def.index_name].attrs[attr]
-                        index = dataset[series_def.index_name].values.tolist()
-                        if hasattr(index[0], "calendar"):
-                            attributes["calendar"] = index[0].calendar
-                        if hasattr(index[0], "isoformat"):
-                            # Convert time objects to strings.
-                            index = [v.isoformat() for v in index]
-
-                        series.append(
-                            SeriesMetricValue(
-                                dimensions=series_def.dimensions,
-                                values=dataset[series_def.values_name].values.tolist(),
-                                index=index,
-                                index_name=series_def.index_name,
-                                attributes=attributes,
-                            )
-                        )
+                series.extend(
+                    self._extract_series_from_file(
+                        definition, filename, relative_path, caption=caption, input_selectors=input_selectors
+                    )
+                )
 
         # Add the index.html file
         index_html = f"{result_dir}/index.html"
@@ -278,7 +271,6 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
 
         # Add the extra information from the groupby operations
         if len(metric_bundle.DIMENSIONS[MetricCV.JSON_STRUCTURE.value]):
-            input_selectors = definition.datasets[SourceDatasetType.CMIP6].selector_dict()
             metric_bundle = metric_bundle.prepend_dimensions(input_selectors)
 
         return ExecutionResult.build_from_output_bundle(
@@ -287,3 +279,58 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
             cmec_metric_bundle=metric_bundle,
             series=series,
         )
+
+    def _extract_series_from_file(
+        self,
+        definition: ExecutionDefinition,
+        filename: Path,
+        relative_path: Path,
+        caption: str,
+        input_selectors: dict[str, str],
+    ) -> list[SeriesMetricValue]:
+        """
+        Extract series data from a file if it matches any of the series definitions.
+        """
+        variable_attributes = (
+            "long_name",
+            "standard_name",
+            "units",
+        )
+
+        series = []
+        for series_def in definition.diagnostic.series:
+            if fnmatch.fnmatch(
+                str(relative_path),
+                f"executions/*/{series_def.file_pattern.format(**input_selectors)}",
+            ):
+                dataset = xr.open_dataset(filename, decode_times=xr.coders.CFDatetimeCoder(use_cftime=True))
+                dataset = dataset.sel(series_def.sel)
+                attributes = {
+                    attr: dataset.attrs[attr] for attr in series_def.attributes if attr in dataset.attrs
+                }
+                attributes["caption"] = caption
+                attributes["values_name"] = series_def.values_name
+                attributes["index_name"] = series_def.index_name
+                for attr in variable_attributes:
+                    if attr in dataset[series_def.values_name].attrs:
+                        attributes[f"value_{attr}"] = dataset[series_def.values_name].attrs[attr]
+                    if attr in dataset[series_def.index_name].attrs:
+                        attributes[f"index_{attr}"] = dataset[series_def.index_name].attrs[attr]
+                # TODO: Handle masked values in the index
+                index = dataset[series_def.index_name].values.tolist()
+                if hasattr(index[0], "calendar"):
+                    attributes["calendar"] = index[0].calendar
+                if hasattr(index[0], "isoformat"):
+                    # Convert time objects to strings.
+                    index = [v.isoformat() for v in index]
+
+                series.append(
+                    SeriesMetricValue(
+                        dimensions={**input_selectors, **series_def.dimensions},
+                        values=fillvalues_to_nan(dataset[series_def.values_name].values).tolist(),
+                        index=index,
+                        index_name=series_def.index_name,
+                        attributes=attributes,
+                    )
+                )
+        return series

climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py

@@ -2,8 +2,9 @@ import pandas
 
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
-    RequireContiguousTimerange,
+    PartialDateTime,
     RequireFacets,
+    RequireTimerange,
 )
 from climate_ref_core.datasets import FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
@@ -26,6 +27,14 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic):
         "tas",
     )
 
+    matching_facets = (
+        "source_id",
+        "member_id",
+        "grid_label",
+        "table_id",
+        "variable_id",
+    )
+
     data_requirements = (
         DataRequirement(
             source_type=SourceDatasetType.CMIP6,
@@ -39,25 +48,32 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic):
                             "ssp370",
                             "ssp585",
                         ),
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("experiment_id",),
             constraints=(
-                RequireFacets("variable_id", variables),
                 AddSupplementaryDataset(
                     supplementary_facets={"experiment_id": "historical"},
-                    matching_facets=(
-                        "source_id",
-                        "member_id",
-                        "grid_label",
-                        "table_id",
-                        "variable_id",
-                    ),
+                    matching_facets=matching_facets,
                     optional_matching_facets=tuple(),
                 ),
-                RequireFacets("experiment_id", ("historical",)),
-                RequireContiguousTimerange(group_by=("instance_id",)),
+                RequireTimerange(
+                    group_by=matching_facets,
+                    start=PartialDateTime(year=1850, month=1),
+                    end=PartialDateTime(year=2100, month=12),
+                ),
+                RequireFacets(
+                    "experiment_id",
+                    required_facets=("historical",),
+                    group_by=matching_facets,
+                ),
+                RequireFacets(
+                    "variable_id",
+                    required_facets=variables,
+                    group_by=("experiment_id", "source_id", "member_id", "grid_label", "table_id"),
+                ),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
@@ -74,13 +90,20 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic):
         diagnostics = recipe["diagnostics"]
         for diagnostic in diagnostics.values():
             diagnostic.pop("additional_datasets")
-        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
+        recipe_variables = dataframe_to_recipe(
+            input_files[SourceDatasetType.CMIP6],
+            group_by=(
+                "source_id",
+                "member_id",
+                "grid_label",
+                "table_id",
+                "variable_id",
+            ),
+        )
         datasets = recipe_variables["tas"]["additional_datasets"]
         datasets = [ds for ds in datasets if ds["exp"] != "historical"]
         for dataset in datasets:
             dataset.pop("timerange")
-            dataset["activity"] = ["CMIP", dataset["activity"]]
-            dataset["exp"] = ["historical", dataset["exp"]]
         recipe["datasets"] = datasets
 
         # Specify the timeranges

climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py

@@ -2,8 +2,9 @@ import pandas
 
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
+    PartialDateTime,
     RequireFacets,
-    RequireOverlappingTimerange,
+    RequireTimerange,
 )
 from climate_ref_core.datasets import FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
@@ -21,32 +22,53 @@ class ClimateDriversForFire(ESMValToolDiagnostic):
     slug = "climate-drivers-for-fire"
     base_recipe = "ref/recipe_ref_fire.yml"
 
-    variables = (
-        "cVeg",
-        "hurs",
-        "pr",
-        "tas",
-        "tasmax",
-        "treeFrac",
-        "vegFrac",
-    )
     data_requirements = (
         DataRequirement(
             source_type=SourceDatasetType.CMIP6,
             filters=(
                 FacetFilter(
-                    facets={
-                        "variable_id": variables,
-                        "frequency": "mon",
+                    {
+                        "variable_id": ("hurs", "pr", "tas", "tasmax"),
+                        "experiment_id": "historical",
+                        "table_id": "Amon",
+                    }
+                ),
+                FacetFilter(
+                    {
+                        "variable_id": ("cVeg", "treeFrac"),
                         "experiment_id": "historical",
+                        "table_id": "Lmon",
+                    }
+                ),
+                FacetFilter(
+                    {
+                        "variable_id": "vegFrac",
+                        "experiment_id": "historical",
+                        "table_id": "Emon",
                     }
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", variables),
-                RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(2013, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
                 AddSupplementaryDataset.from_defaults("sftlf", SourceDatasetType.CMIP6),
+                RequireFacets(
+                    "variable_id",
+                    (
+                        "cVeg",
+                        "hurs",
+                        "pr",
+                        "tas",
+                        "tasmax",
+                        "sftlf",
+                        "treeFrac",
+                        "vegFrac",
+                    ),
+                ),
             ),
         ),
     )

climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py

@@ -2,12 +2,14 @@ import pandas
 
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
-    RequireContiguousTimerange,
+    PartialDateTime,
     RequireFacets,
     RequireOverlappingTimerange,
+    RequireTimerange,
 )
 from climate_ref_core.datasets import FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import Recipe
@@ -22,8 +24,6 @@ class CloudRadiativeEffects(ESMValToolDiagnostic):
     slug = "cloud-radiative-effects"
     base_recipe = "ref/recipe_ref_cre.yml"
 
-    facets = ()
-
     variables = (
         "rlut",
         "rlutcs",
@@ -37,40 +37,59 @@ class CloudRadiativeEffects(ESMValToolDiagnostic):
                 FacetFilter(
                     facets={
                         "variable_id": variables,
-                        "experiment_id": ("historical",),
+                        "experiment_id": "historical",
+                        "table_id": "Amon",
                     }
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", variables),
-                RequireContiguousTimerange(group_by=("instance_id",)),
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1996, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets("variable_id", variables),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
         # TODO: Use CERES-EBAF, ESACCI-CLOUD, and ISCCP-FH from obs4MIPs once available.
     )
 
+    facets = ()
+    series = tuple(
+        SeriesDefinition(
+            file_pattern=f"plot_profiles/plot/variable_vs_lat_{var_name}_*.nc",
+            sel={"dim0": 0},  # Select the model.
+            dimensions={"variable_id": var_name, "statistic": "zonal mean"},
+            values_name=var_name,
+            index_name="lat",
+            attributes=[],
+        )
+        for var_name in ["lwcre", "swcre"]
+    ) + tuple(
+        SeriesDefinition(
+            file_pattern=f"plot_profiles/plot/variable_vs_lat_{var_name}_*.nc",
+            sel={"dim0": i},  # Select the observation.
+            dimensions={"variable_id": var_name, "statistic": "zonal mean", "reference_source_id": source_id},
+            values_name=var_name,
+            index_name="lat",
+            attributes=[],
+        )
+        for var_name in ["lwcre", "swcre"]
+        for i, source_id in enumerate(
+            ["CERES-EBAF-Ed4.2", "ESACCI-CLOUD-AVHRR-AMPM-fv3.0", "ISCCP-FH"], start=1
+        )
+    )
+
     @staticmethod
     def update_recipe(recipe: Recipe, input_files: dict[SourceDatasetType, pandas.DataFrame]) -> None:
         """Update the recipe."""
         recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
 
-        # Select a timerange covered by all datasets.
-        start_times, end_times = [], []
-        for variable in recipe_variables.values():
-            for dataset in variable["additional_datasets"]:
-                start, end = dataset["timerange"].split("/")
-                start_times.append(start)
-                end_times.append(end)
-        start_time = max(start_times)
-        start_time = max(start_time, "20010101T000000")  # Earliest observational dataset availability
-        timerange = f"{start_time}/{min(end_times)}"
-
         datasets = recipe_variables["rsut"]["additional_datasets"]
         for dataset in datasets:
             dataset.pop("timerange")
         recipe["datasets"] = datasets
-        recipe["timerange_for_models"] = timerange

climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py

@@ -4,9 +4,9 @@ import pandas
 
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
-    RequireContiguousTimerange,
+    PartialDateTime,
     RequireFacets,
-    RequireOverlappingTimerange,
+    RequireTimerange,
 )
 from climate_ref_core.datasets import FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
@@ -25,15 +25,18 @@ def get_cmip6_data_requirements(variables: tuple[str, ...]) -> tuple[DataRequire
                     facets={
                         "variable_id": variables,
                         "experiment_id": "historical",
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "experiment_id", "member_id", "frequency", "grid_label"),
             constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1996, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
                 RequireFacets("variable_id", variables),
-                RequireContiguousTimerange(group_by=("instance_id",)),
-                RequireOverlappingTimerange(group_by=("instance_id",)),
-                # TODO: Add a RequireTimeRange constraint to match reference datasets?
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
@@ -47,13 +50,16 @@ def update_recipe(
     var_y: str,
 ) -> None:
     """Update the recipe."""
-    recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6], equalize_timerange=True)
+    recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
     diagnostics = recipe["diagnostics"]
     diagnostic_name = f"plot_joint_{var_x}_{var_y}_model"
     diagnostic = diagnostics.pop(diagnostic_name)
     diagnostics.clear()
     diagnostics[diagnostic_name] = diagnostic
+    recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
     datasets = next(iter(recipe_variables.values()))["additional_datasets"]
+    for dataset in datasets:
+        dataset["timerange"] = "1996/2014"
     diagnostic["additional_datasets"] = datasets
     suptitle = "CMIP6 {dataset} {ensemble} {grid} {timerange}".format(**datasets[0])
     diagnostic["scripts"]["plot"]["suptitle"] = suptitle
@@ -135,7 +141,13 @@ class CloudScatterplotsReference(ESMValToolDiagnostic):
                 ),
             ),
             group_by=("instance_id",),
-            constraints=(RequireContiguousTimerange(group_by=("instance_id",)),),
+            constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(2007, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
+            ),
             # TODO: Add obs4MIPs datasets once available and working:
             #
             # obs4MIPs datasets with issues:

climate_ref_esmvaltool/diagnostics/ecs.py

@@ -11,9 +11,10 @@ from climate_ref_core.constraints import (
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
-from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
+from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic, fillvalues_to_nan
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
 
@@ -45,20 +46,40 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
                     facets={
                         "variable_id": variables,
                         "experiment_id": experiments,
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", variables),
-                RequireFacets("experiment_id", experiments),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets(
+                    "variable_id",
+                    required_facets=variables,
+                    group_by=("source_id", "member_id", "grid_label", "experiment_id"),
+                ),
+                RequireFacets(
+                    "experiment_id",
+                    required_facets=experiments,
+                    group_by=("source_id", "member_id", "grid_label", "variable_id"),
+                ),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    series = (
+        SeriesDefinition(
+            file_pattern="ecs/calculate/ecs_regression_*.nc",
+            dimensions={
+                "statistic": ("global annual mean anomaly of rtnt vs tas"),
+            },
+            values_name="rtnt_anomaly",
+            index_name="tas_anomaly",
+            attributes=[],
+        ),
+    )
 
     @staticmethod
     def update_recipe(
@@ -118,9 +139,9 @@ class EquilibriumClimateSensitivity(ESMValToolDiagnostic):
     ) -> tuple[CMECMetric, CMECOutput]:
         """Format the result."""
         ecs_ds = xarray.open_dataset(result_dir / "work" / "ecs" / "calculate" / "ecs.nc")
-        ecs = float(ecs_ds["ecs"].values[0])
+        ecs = float(fillvalues_to_nan(ecs_ds["ecs"].values)[0])
         lambda_ds = xarray.open_dataset(result_dir / "work" / "ecs" / "calculate" / "lambda.nc")
-        lambda_ = float(lambda_ds["lambda"].values[0])
+        lambda_ = float(fillvalues_to_nan(lambda_ds["lambda"].values)[0])
 
         # Update the diagnostic bundle arguments with the computed diagnostics.
         metric_args[MetricCV.DIMENSIONS.value] = {