climate-ref-esmvaltool 0.6.5__py3-none-any.whl → 0.7.0__py3-none-any.whl

climate_ref_esmvaltool/diagnostics/tcr.py

@@ -11,6 +11,7 @@ from climate_ref_core.constraints import (
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
@@ -39,22 +40,37 @@ class TransientClimateResponse(ESMValToolDiagnostic):
                     facets={
                         "variable_id": ("tas",),
                         "experiment_id": experiments,
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("experiment_id", experiments),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets("experiment_id", experiments),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    series = (
+        SeriesDefinition(
+            file_pattern="tcr/calculate/{source_id}*.nc",
+            dimensions={
+                "statistic": "global annual mean tas anomaly relative to linear fit of piControl run",
+            },
+            values_name="tas_anomaly",
+            index_name="time",
+            attributes=[],
+        ),
+    )
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Only run the diagnostic that computes TCR for a single model.
         recipe["diagnostics"] = {
@@ -77,21 +93,11 @@ class TransientClimateResponse(ESMValToolDiagnostic):
         # Prepare updated datasets section in recipe. It contains two
         # datasets, one for the "1pctCO2" and one for the "piControl"
         # experiment.
-        recipe_variables = dataframe_to_recipe(input_files)
-        recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"}
-
-        # Select a timerange covered by all datasets.
-        start_times, end_times = [], []
-        for variable in recipe_variables.values():
-            for dataset in variable["additional_datasets"]:
-                start, end = dataset["timerange"].split("/")
-                start_times.append(start)
-                end_times.append(end)
-        timerange = f"{max(start_times)}/{min(end_times)}"
-
-        datasets = recipe_variables["tas"]["additional_datasets"]
-        for dataset in datasets:
-            dataset["timerange"] = timerange
+        recipe_variables = dataframe_to_recipe(
+            input_files[SourceDatasetType.CMIP6],
+            equalize_timerange=True,
+        )
+        recipe["datasets"] = recipe_variables["tas"]["additional_datasets"]
 
         # Remove keys from the recipe that are only used for YAML anchors
         keys_to_remove = [
@@ -102,8 +108,6 @@ class TransientClimateResponse(ESMValToolDiagnostic):
         for key in keys_to_remove:
             recipe.pop(key, None)
 
-        recipe["datasets"] = datasets
-
     @staticmethod
     def format_result(
         result_dir: Path,

climate_ref_esmvaltool/diagnostics/tcre.py

@@ -27,10 +27,6 @@ class TransientClimateResponseEmissions(ESMValToolDiagnostic):
     slug = "transient-climate-response-emissions"
     base_recipe = "recipe_tcre.yml"
 
-    experiments = (
-        "esm-1pctCO2",
-        "esm-piControl",
-    )
     variables = (
         "tas",
         "fco2antt",
@@ -42,37 +38,42 @@ class TransientClimateResponseEmissions(ESMValToolDiagnostic):
                 FacetFilter(
                     facets={
                         "variable_id": variables,
-                        "frequency": "mon",
-                        "experiment_id": experiments,
+                        "experiment_id": "esm-1pctCO2",
+                        "table_id": "Amon",
                     },
                 ),
                 FacetFilter(
                     facets={
-                        "variable_id": "fco2antt",
+                        "variable_id": "tas",
                         "experiment_id": "esm-piControl",
+                        "table_id": "Amon",
                     },
-                    keep=False,
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("experiment_id", experiments),
-                RequireFacets("variable_id", variables),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets("experiment_id", ("esm-1pctCO2", "esm-piControl")),
+                RequireFacets("variable_id", variables),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    # TODO: the ESMValTool diagnostic script does not save the data for the timeseries.
+    series = tuple()
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Prepare updated datasets section in recipe. It contains three
         # datasets, "tas" and "fco2antt" for the "esm-1pctCO2" and just "tas"
         # for the "esm-piControl" experiment.
-        recipe_variables = dataframe_to_recipe(input_files)
+        recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])
         tas_esm_1pctCO2 = next(
             ds for ds in recipe_variables["tas"]["additional_datasets"] if ds["exp"] == "esm-1pctCO2"
         )

climate_ref_esmvaltool/diagnostics/zec.py

@@ -11,6 +11,7 @@ from climate_ref_core.constraints import (
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
@@ -39,27 +40,43 @@ class ZeroEmissionCommitment(ESMValToolDiagnostic):
                     facets={
                         "variable_id": ("tas",),
                         "experiment_id": experiments,
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("experiment_id", experiments),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets("experiment_id", experiments),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    series = (
+        SeriesDefinition(
+            file_pattern="work/zec/zec/zec.nc",
+            sel={"dim0": 0},
+            dimensions={
+                "statistic": "zec",
+            },
+            values_name="zec",
+            index_name="time",
+            attributes=[],
+        ),
+    )
 
     @staticmethod
-    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+    def update_recipe(
+        recipe: Recipe,
+        input_files: dict[SourceDatasetType, pandas.DataFrame],
+    ) -> None:
         """Update the recipe."""
         # Prepare updated datasets section in recipe. It contains two
         # datasets, one for the "esm-1pct-brch-1000PgC" and one for the "piControl"
         # experiment.
-        datasets = dataframe_to_recipe(input_files)["tas"]["additional_datasets"]
+        datasets = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6])["tas"]["additional_datasets"]
         base_dataset = next(ds for ds in datasets if ds["exp"] == "1pctCO2")
         dataset = next(ds for ds in datasets if ds["exp"] == "esm-1pct-brch-1000PgC")
         start = dataset["timerange"].split("/")[0]

climate_ref_esmvaltool/recipe.py

@@ -12,6 +12,7 @@ from climate_ref_esmvaltool.types import Recipe
 if TYPE_CHECKING:
     import pandas as pd
 
+
 FACETS = {
     "CMIP6": {
         "activity": "activity_id",
@@ -23,6 +24,13 @@ FACETS = {
         "mip": "table_id",
         "short_name": "variable_id",
     },
+    "obs4MIPs": {
+        "dataset": "source_id",
+        "frequency": "frequency",
+        "grid": "grid_label",
+        "institute": "institution_id",
+        "short_name": "variable_id",
+    },
 }
 
 
@@ -79,42 +87,65 @@ def as_facets(
 
     """
     facets = {}
-    first_row = group.iloc[0]
-    project = first_row.instance_id.split(".", 2)[0]
+    project = group.iloc[0].instance_id.split(".", 2)[0]
     facets["project"] = project
     for esmvaltool_name, ref_name in FACETS[project].items():
-        facets[esmvaltool_name] = getattr(first_row, ref_name)
+        values = group[ref_name].unique().tolist()
+        facets[esmvaltool_name] = values if len(values) > 1 else values[0]
     timerange = as_timerange(group)
     if timerange is not None:
         facets["timerange"] = timerange
     return facets
 
 
-def dataframe_to_recipe(files: pd.DataFrame) -> dict[str, Any]:
+def dataframe_to_recipe(
+    files: pd.DataFrame,
+    group_by: tuple[str, ...] = ("instance_id",),
+    equalize_timerange: bool = False,
+) -> dict[str, Any]:
     """Convert the datasets dataframe to a recipe "variables" section.
 
     Parameters
     ----------
     files
         The pandas dataframe describing the input files.
+    group_by
+        The columns to group the input files by.
+    equalize_timerange
+        If True, use the timerange that is covered by all datasets.
 
     Returns
     -------
         A "variables" section that can be used in an ESMValTool recipe.
     """
     variables: dict[str, Any] = {}
-    # TODO: refine to make it possible to combine historical and scenario runs.
-    for _, group in files.groupby("instance_id"):
+    for _, group in files.groupby(list(group_by)):
         facets = as_facets(group)
         short_name = facets.pop("short_name")
         if short_name not in variables:
             variables[short_name] = {"additional_datasets": []}
         variables[short_name]["additional_datasets"].append(facets)
+
+    if equalize_timerange:
+        # Select a timerange covered by all datasets.
+        start_times, end_times = [], []
+        for variable in variables.values():
+            for dataset in variable["additional_datasets"]:
+                if "timerange" in dataset:
+                    start, end = dataset["timerange"].split("/")
+                    start_times.append(start)
+                    end_times.append(end)
+        timerange = f"{max(start_times)}/{min(end_times)}"
+        for variable in variables.values():
+            for dataset in variable["additional_datasets"]:
+                if "timerange" in dataset:
+                    dataset["timerange"] = timerange
+
     return variables
 
 
-_ESMVALTOOL_COMMIT = "8f56863a70ba4df76ec501ba0372c571a0af6cf9"
-_ESMVALTOOL_VERSION = f"2.13.0.dev120+g{_ESMVALTOOL_COMMIT[:9]}"
+_ESMVALTOOL_COMMIT = "2c438d0e0cc8904790294c72450eb7f06552c52a"
+_ESMVALTOOL_VERSION = f"2.13.0.dev148+g{_ESMVALTOOL_COMMIT[:9]}"
 
 _RECIPES = pooch.create(
     path=pooch.os_cache("climate_ref_esmvaltool"),
@@ -142,7 +173,16 @@ def load_recipe(recipe: str) -> Recipe:
         The loaded recipe.
     """
     filename = _RECIPES.fetch(recipe)
-    return yaml.safe_load(Path(filename).read_text(encoding="utf-8"))  # type: ignore[no-any-return]
+
+    def normalize(obj: Any) -> Any:
+        # Ensure objects in the recipe are not shared.
+        if isinstance(obj, dict):
+            return {k: normalize(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [normalize(item) for item in obj]
+        return obj
+
+    return normalize(yaml.safe_load(Path(filename).read_text(encoding="utf-8")))  # type: ignore[no-any-return]
 
 
 def prepare_climate_data(datasets: pd.DataFrame, climate_data_dir: Path) -> None:
@@ -165,6 +205,11 @@ def prepare_climate_data(datasets: pd.DataFrame, climate_data_dir: Path) -> None
         if not isinstance(row.path, str):  # pragma: no branch
             msg = f"Invalid path encountered in {row}"
             raise ValueError(msg)
-        tgt = climate_data_dir.joinpath(*row.instance_id.split(".")) / Path(row.path).name
+        if row.instance_id.startswith("obs4MIPs."):
+            version = row.instance_id.split(".")[-1]
+            subdirs: list[str] = ["obs4MIPs", row.source_id, version]  # type: ignore[list-item]
+        else:
+            subdirs = row.instance_id.split(".")
+        tgt = climate_data_dir.joinpath(*subdirs) / Path(row.path).name
         tgt.parent.mkdir(parents=True, exist_ok=True)
         tgt.symlink_to(row.path)

climate_ref_esmvaltool/recipes.txt

@@ -1,11 +1,16 @@
-examples/recipe_python.yml ab3f06d269bb2c1368f4dc39da9bcb232fb2adb1fa556ba769e6c16294ffb4a3
-recipe_calculate_gwl_exceedance_stats.yml 5aa266abc9a8029649b689a2b369a47623b0935d609354332ff4148994642d6b
-recipe_ecs.yml             0cc57034fcb64e32015b4ff949ece5df8cdb8c6f493618b50ceded119fb37918
-recipe_seaice_sensitivity.yml f7247c076e161c582d422947c8155f3ca98549e6f2e4c3b1c76414786d7e50c5
-recipe_tcr.yml             35f9ef035a4e71aff5cac5dd26c49da2162fc00291bf3b0bd16b661b7b2f606b
-recipe_tcre.yml            48fc9e3baf541bbcef7491853ea3a774053771dca33352b41466425faeaa38af
-recipe_zec.yml             b0af7f789b7610ab3f29a6617124aa40c40866ead958204fc199eaf82863de51
-ref/recipe_enso_basicclimatology.yml 9ea7deb7ee668e39ac44618b96496d898bd82285c22dcee4fce4695e0c9fa82b
-ref/recipe_enso_characteristics.yml 34c2518b138068ac96d212910b979d54a8fcedee2c0089b5acd56a42c41dc3e4
-ref/recipe_ref_cre.yml     4375f262479c3b3e1b348b71080a6d758e195bda76516a591182045a3a29aa32
-ref/recipe_ref_sea_ice_area_basic.yml 7d01a8527880663ca28284772f83a8356d9972fb4f022a4000e50a56ce044b09
+examples/recipe_python.yml                  ab3f06d269bb2c1368f4dc39da9bcb232fb2adb1fa556ba769e6c16294ffb4a3
+recipe_calculate_gwl_exceedance_stats.yml   5aa266abc9a8029649b689a2b369a47623b0935d609354332ff4148994642d6b
+recipe_ecs.yml                              0cc57034fcb64e32015b4ff949ece5df8cdb8c6f493618b50ceded119fb37918
+recipe_seaice_sensitivity.yml               f7247c076e161c582d422947c8155f3ca98549e6f2e4c3b1c76414786d7e50c5
+recipe_tcr.yml                              35f9ef035a4e71aff5cac5dd26c49da2162fc00291bf3b0bd16b661b7b2f606b
+recipe_tcre.yml                             48fc9e3baf541bbcef7491853ea3a774053771dca33352b41466425faeaa38af
+recipe_zec.yml                              b0af7f789b7610ab3f29a6617124aa40c40866ead958204fc199eaf82863de51
+ref/recipe_enso_basicclimatology.yml        9ea7deb7ee668e39ac44618b96496d898bd82285c22dcee4fce4695e0c9fa82b
+ref/recipe_enso_characteristics.yml         34c2518b138068ac96d212910b979d54a8fcedee2c0089b5acd56a42c41dc3e4
+ref/recipe_ref_annual_cycle_region.yml      64ebc687789dad6c45a2361b45218cb5a0ad0e38c516840c65fc7e8bf7b5ace7
+ref/recipe_ref_cre.yml                      4375f262479c3b3e1b348b71080a6d758e195bda76516a591182045a3a29aa32
+ref/recipe_ref_fire.yml                     2ad82effaca4e742d8abe6a0aa07bb46e1e92ef0d2d240760f7623b0ba045926
+ref/recipe_ref_sea_ice_area_basic.yml       7d01a8527880663ca28284772f83a8356d9972fb4f022a4000e50a56ce044b09
+ref/recipe_ref_scatterplot.yml              b99d1736e16256d161847b025811d7088ad9f892d4887fb009fa99c4079135a0
+ref/recipe_ref_timeseries_region.yml        86f36e442021caba201601d8cf4624f8ce6715ce421670a467c792db2910db22
+ref/recipe_ref_trend_regions.yml            18fe246a51474bd12172ab1ba141efac999a247de7774822f77ae6ef144645fe