climate-ref-esmvaltool 0.6.6__py3-none-any.whl → 0.8.0__py3-none-any.whl

climate_ref_esmvaltool/diagnostics/enso.py

@@ -11,6 +11,7 @@ from climate_ref_core.constraints import (
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
@@ -27,32 +28,117 @@ class ENSOBasicClimatology(ESMValToolDiagnostic):
     slug = "enso-basic-climatology"
     base_recipe = "ref/recipe_enso_basicclimatology.yml"
 
-    variables = (
-        "pr",
-        "tos",
-        "tauu",
-    )
     data_requirements = (
         DataRequirement(
             source_type=SourceDatasetType.CMIP6,
             filters=(
                 FacetFilter(
                     facets={
-                        "variable_id": variables,
+                        "variable_id": ("pr", "tauu"),
                         "experiment_id": "historical",
+                        "table_id": "Amon",
+                    },
+                ),
+                FacetFilter(
+                    facets={
+                        "variable_id": "tos",
+                        "experiment_id": "historical",
+                        "table_id": "Omon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", variables),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets(
+                    "variable_id",
+                    (
+                        "pr",
+                        "tauu",
+                        "tos",
+                    ),
+                ),
             ),
         ),
     )
     facets = ()
 
+    series = (
+        tuple(
+            SeriesDefinition(
+                file_pattern=f"diagnostic_metrics/plot_script/{source_id}_eq_{var_name}_bias.nc",
+                dimensions=(
+                    {
+                        "statistic": (
+                            f"zonal bias in the time-mean {var_name} structure across the equatorial Pacific"
+                        ),
+                    }
+                    | ({} if source_id == "{source_id}" else {"reference_source_id": source_id})
+                ),
+                values_name="tos" if var_name == "sst" else var_name,
+                index_name="lon",
+                attributes=[],
+            )
+            for var_name in ("pr", "sst", "tauu")
+            for source_id in ("{source_id}", "GPCP-V2.3", "TROPFLUX")
+        )
+        + tuple(
+            SeriesDefinition(
+                file_pattern=f"diagnostic_metrics/plot_script/{{source_id}}_eq_{var_name}_seacycle.nc",
+                dimensions=(
+                    {
+                        "statistic": (
+                            "zonal bias in the amplitude of the mean seasonal cycle of "
+                            f"{var_name} in the equatorial Pacific"
+                        ),
+                    }
+                    | ({} if source_id == "{source_id}" else {"reference_source_id": source_id})
+                ),
+                values_name="tos" if var_name == "sst" else var_name,
+                index_name="lon",
+                attributes=[],
+            )
+            for var_name in ("pr", "sst", "tauu")
+            for source_id in ("{source_id}", "GPCP-V2.3", "TROPFLUX")
+        )
+        + tuple(
+            SeriesDefinition(
+                file_pattern="diagnostic_metrics/plot_script/{source_id}_pr_double.nc",
+                dimensions=(
+                    {
+                        "statistic": (
+                            "meridional bias in the time-mean pr structure across the eastern Pacific"
+                        ),
+                    }
+                    | ({} if source_id == "{source_id}" else {"reference_source_id": source_id})
+                ),
+                values_name="pr",
+                index_name="lat",
+                attributes=[],
+            )
+            for source_id in ("{source_id}", "GPCP-V2.3")
+        )
+        + tuple(
+            SeriesDefinition(
+                file_pattern="diagnostic_metrics/plot_script/*_pr_double_seacycle.nc",
+                dimensions=(
+                    {
+                        "statistic": (
+                            "meridional bias in the amplitude of the mean seasonal "
+                            "pr cycle in the eastern Pacific"
+                        ),
+                    }
+                    | ({} if source_id == "{source_id}" else {"reference_source_id": source_id})
+                ),
+                values_name="pr",
+                index_name="lat",
+                attributes=[],
+            )
+            for source_id in ("{source_id}", "GPCP-V2.3")
+        )
+    )
+
     @staticmethod
     def update_recipe(
         recipe: Recipe,
@@ -85,19 +171,23 @@ class ENSOCharacteristics(ESMValToolDiagnostic):
                     facets={
                         "variable_id": "tos",
                         "experiment_id": "historical",
+                        "table_id": "Omon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("variable_id", ("tos",)),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
                 AddSupplementaryDataset.from_defaults("areacello", SourceDatasetType.CMIP6),
+                RequireFacets("variable_id", ("tos", "areacello")),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    # ENSO pattern and lifecycle are series, but the ESMValTool diagnostic
+    # script does not save the values used in the figure.
+    series = tuple()
 
     @staticmethod
     def update_recipe(

climate_ref_esmvaltool/diagnostics/example.py

@@ -17,28 +17,29 @@ class GlobalMeanTimeseries(ESMValToolDiagnostic):
     name = "Global Mean Timeseries"
     slug = "global-mean-timeseries"
     base_recipe = "examples/recipe_python.yml"
-    series = (
-        SeriesDefinition(
-            file_pattern="timeseries/script1/*.nc",
-            dimensions={},
-            values_name="tas",
-            index_name="time",
-            attributes=[],
-        ),
-    )
 
     data_requirements = (
         DataRequirement(
             source_type=SourceDatasetType.CMIP6,
             filters=(FacetFilter(facets={"variable_id": ("tas",)}),),
-            group_by=("instance_id",),
+            group_by=("source_id", "experiment_id", "member_id", "table_id", "variable_id", "grid_label"),
             constraints=(
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
+
     facets = ()
+    series = (
+        SeriesDefinition(
+            file_pattern="timeseries/script1/*.nc",
+            dimensions={"statistic": "tas annual global mean"},
+            values_name="tas",
+            index_name="time",
+            attributes=[],
+        ),
+    )
 
     @staticmethod
     def update_recipe(

climate_ref_esmvaltool/diagnostics/regional_historical_changes.py

@@ -7,15 +7,16 @@ import xarray
 
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
-    RequireContiguousTimerange,
+    PartialDateTime,
     RequireFacets,
+    RequireTimerange,
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
 from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
-from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
+from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic, fillvalues_to_nan
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
 
@@ -86,6 +87,15 @@ def normalize_region(region: str) -> str:
     return region.replace("&", "-and-").replace("/", "-and-")
 
 
+REFERENCE_DATASETS = {
+    "hus": "ERA-5",
+    "pr": "GPCP-V2.3",
+    "psl": "ERA-5",
+    "tas": "HadCRUT5-5.0.1.0-analysis",
+    "ua": "ERA-5",
+}
+
+
 class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic):
     """
     Plot regional historical annual cycle of climate variables.
@@ -102,18 +112,6 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic):
         "tas",
         "ua",
     )
-    series = tuple(
-        SeriesDefinition(
-            file_pattern=f"anncyc-{region}/allplots/*_{var_name}_*.nc",
-            sel={"dim0": 0},  # Select the model and not the observation.
-            dimensions={"region": region},
-            values_name=var_name,
-            index_name="month_number",
-            attributes=[],
-        )
-        for var_name in variables
-        for region in REGIONS
-    )
 
     data_requirements = (
         DataRequirement(
@@ -123,14 +121,18 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic):
                     facets={
                         "variable_id": variables,
                         "experiment_id": "historical",
-                        "frequency": "mon",
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1980, 1),
+                    end=PartialDateTime(2009, 12),
+                ),
                 RequireFacets("variable_id", variables),
-                RequireContiguousTimerange(group_by=("instance_id",)),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
@@ -149,7 +151,14 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic):
                 ),
             ),
             group_by=("source_id",),
-            constraints=(RequireContiguousTimerange(group_by=("instance_id",)),),
+            constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1980, 1),
+                    end=PartialDateTime(2009, 12),
+                ),
+                RequireFacets("variable_id", ("psl", "ua")),
+            ),
             # TODO: Add obs4MIPs datasets once available and working:
             #
             # obs4MIPs dataset that cannot be ingested (https://github.com/Climate-REF/climate-ref/issues/260):
@@ -160,7 +169,28 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic):
             # - HadCRUT5_ground_5.0.1.0-analysis: tas
         ),
     )
+
     facets = ()
+    series = tuple(
+        SeriesDefinition(
+            file_pattern=f"anncyc-{region}/allplots/*_{var_name}_*.nc",
+            sel={"dim0": 0},  # Select the model and not the observation.
+            dimensions=(
+                {
+                    "region": region,
+                    "variable_id": var_name,
+                    "statistic": "mean",
+                }
+                | ({} if i == 0 else {"reference_source_id": REFERENCE_DATASETS[var_name]})
+            ),
+            values_name=var_name,
+            index_name="month_number",
+            attributes=[],
+        )
+        for var_name in variables
+        for region in REGIONS
+        for i in range(2)
+    )
 
     @staticmethod
     def update_recipe(
@@ -211,18 +241,91 @@ class RegionalHistoricalTimeSeries(RegionalHistoricalAnnualCycle):
     name = "Regional historical mean and anomaly of climate variables"
     slug = "regional-historical-timeseries"
     base_recipe = "ref/recipe_ref_timeseries_region.yml"
+
+    variables = (
+        "hus",
+        "pr",
+        "psl",
+        "tas",
+        "ua",
+    )
+
+    data_requirements = (
+        DataRequirement(
+            source_type=SourceDatasetType.CMIP6,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "variable_id": variables,
+                        "experiment_id": "historical",
+                        "table_id": "Amon",
+                    },
+                ),
+            ),
+            group_by=("source_id", "member_id", "grid_label"),
+            constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1980, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
+                RequireFacets("variable_id", variables),
+                AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
+            ),
+        ),
+        DataRequirement(
+            source_type=SourceDatasetType.obs4MIPs,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "variable_id": (
+                            "psl",
+                            "ua",
+                        ),
+                        "source_id": "ERA-5",
+                        "frequency": "mon",
+                    },
+                ),
+            ),
+            group_by=("source_id",),
+            constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1980, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
+            ),
+            # TODO: Add obs4MIPs datasets once available and working:
+            #
+            # obs4MIPs dataset that cannot be ingested (https://github.com/Climate-REF/climate-ref/issues/260):
+            # - GPCP-V2.3: pr
+            #
+            # Not yet available on obs4MIPs:
+            # - ERA5: hus
+            # - HadCRUT5_ground_5.0.1.0-analysis: tas
+        ),
+    )
+
     series = tuple(
         SeriesDefinition(
             file_pattern=f"{diagnostic}-{region}/allplots/*_{var_name}_*.nc",
-            sel={"dim0": 0},  # Select the model and not the observation.
-            dimensions={"region": region},
+            sel={"dim0": i},
+            dimensions=(
+                {
+                    "region": region,
+                    "variable_id": var_name,
+                    "statistic": ("mean" if diagnostic == "timeseries_abs" else "mean anomaly"),
+                }
+                | ({} if i == 0 else {"reference_source_id": REFERENCE_DATASETS[var_name]})
+            ),
             values_name=var_name,
             index_name="time",
             attributes=[],
         )
-        for var_name in RegionalHistoricalAnnualCycle.variables
+        for var_name in variables
         for region in REGIONS
         for diagnostic in ["timeseries_abs", "timeseries"]
+        for i in range(2)
     )
 
 
@@ -249,13 +352,17 @@ class RegionalHistoricalTrend(ESMValToolDiagnostic):
                             "ua",
                         ),
                         "experiment_id": "historical",
-                        "frequency": "mon",
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireContiguousTimerange(group_by=("instance_id",)),
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1980, 1),
+                    end=PartialDateTime(2009, 12),
+                ),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
@@ -275,7 +382,13 @@ class RegionalHistoricalTrend(ESMValToolDiagnostic):
                 ),
             ),
             group_by=("source_id",),
-            constraints=(RequireContiguousTimerange(group_by=("instance_id",)),),
+            constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1980, 1),
+                    end=PartialDateTime(2009, 12),
+                ),
+            ),
             # TODO: Add obs4MIPs datasets once available and working:
             #
             # obs4MIPs dataset that cannot be ingested (https://github.com/Climate-REF/climate-ref/issues/260):
@@ -330,7 +443,9 @@ class RegionalHistoricalTrend(ESMValToolDiagnostic):
             variable_id = next(iter(ds.data_vars.keys()))
             metric_args[MetricCV.DIMENSIONS.value]["variable_id"][variable_id] = {}
             metric_args[MetricCV.RESULTS.value][variable_id] = {}
-            for region_value, trend_value in zip(ds.shape_id.astype(str).values, ds[variable_id].values):
+            for region_value, trend_value in zip(
+                ds.shape_id.astype(str).values, fillvalues_to_nan(ds[variable_id].values)
+            ):
                 region = region_value.strip()
                 trend = float(trend_value)
                 if region not in metric_args[MetricCV.DIMENSIONS.value]["region"]:

climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py

@@ -2,14 +2,27 @@ import pandas
 
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
-    RequireContiguousTimerange,
+    PartialDateTime,
+    RequireFacets,
+    RequireTimerange,
 )
 from climate_ref_core.datasets import FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import Recipe
 
+REGIONS = {
+    "nh": "Northern Hemisphere",
+    "sh": "Southern Hemisphere",
+}
+
+MONTHS = {
+    "nh": "September",
+    "sh": "February",
+}
+
 
 class SeaIceAreaBasic(ESMValToolDiagnostic):
     """
@@ -28,18 +41,59 @@ class SeaIceAreaBasic(ESMValToolDiagnostic):
                     facets={
                         "variable_id": "siconc",
                         "experiment_id": "historical",
+                        "table_id": "SImon",
                     },
                 ),
             ),
-            group_by=("instance_id",),
+            group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireContiguousTimerange(group_by=("instance_id",)),
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1979, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
                 AddSupplementaryDataset.from_defaults("areacello", SourceDatasetType.CMIP6),
+                RequireFacets("variable_id", ("siconc", "areacello")),
             ),
         ),
         # TODO: Use OSI-450-nh and OSI-450-sh from obs4MIPs once available.
     )
     facets = ()
+    series = tuple(
+        SeriesDefinition(
+            file_pattern=f"siarea_min/allplots/timeseries_sea_ice_area_{region}_*.nc",
+            sel={"dim0": i},
+            dimensions=(
+                {
+                    "region": REGIONS[region],
+                    "statistic": f"{MONTHS[region]} sea ice area",
+                }
+                | ({} if i == 0 else {"reference_source_id": f"OSI-450-{region}"})
+            ),
+            values_name="siconc",
+            index_name="time",
+            attributes=[],
+        )
+        for region in REGIONS
+        for i in range(2)
+    ) + tuple(
+        SeriesDefinition(
+            file_pattern=f"siarea_seas/allplots/annual_cycle_sea_ice_area_{region}_*.nc",
+            sel={"dim0": i},
+            dimensions=(
+                {
+                    "region": REGIONS[region],
+                    "statistic": "20-year average seasonal cycle of the sea ice area",
+                }
+                | ({} if i == 0 else {"reference_source_id": f"OSI-450-{region}"})
+            ),
+            values_name="siconc",
+            index_name="month_number",
+            attributes=[],
+        )
+        for region in REGIONS
+        for i in range(2)
+    )
 
     @staticmethod
     def update_recipe(

climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py

@@ -5,8 +5,9 @@ import pandas as pd
 
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
-    RequireContiguousTimerange,
+    PartialDateTime,
     RequireFacets,
+    RequireTimerange,
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
@@ -26,31 +27,44 @@ class SeaIceSensitivity(ESMValToolDiagnostic):
     slug = "sea-ice-sensitivity"
     base_recipe = "recipe_seaice_sensitivity.yml"
 
-    variables = (
-        "siconc",
-        "tas",
-    )
-
     data_requirements = (
         DataRequirement(
             source_type=SourceDatasetType.CMIP6,
             filters=(
                 FacetFilter(
                     facets={
-                        "variable_id": variables,
+                        "variable_id": "siconc",
+                        "experiment_id": "historical",
+                        "table_id": "SImon",
+                    },
+                ),
+                FacetFilter(
+                    facets={
+                        "variable_id": "tas",
                         "experiment_id": "historical",
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("experiment_id",),  # this does nothing, but group_by cannot be empty
             constraints=(
+                RequireTimerange(
+                    group_by=("instance_id",),
+                    start=PartialDateTime(1979, 1),
+                    end=PartialDateTime(2014, 12),
+                ),
+                RequireFacets(
+                    "variable_id",
+                    required_facets=("siconc", "tas"),
+                    group_by=("source_id", "member_id", "grid_label"),
+                ),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
                 AddSupplementaryDataset.from_defaults("areacello", SourceDatasetType.CMIP6),
-                RequireContiguousTimerange(group_by=("instance_id",)),
-                RequireFacets("variable_id", variables),
-                # TODO: Add a constraint to ensure that tas, siconc and areacello
-                # are available for each model or alternatively filter out
-                # incomplete models below.
+                RequireFacets(
+                    "variable_id",
+                    required_facets=("areacello",),
+                    group_by=("source_id", "grid_label"),
+                ),
             ),
         ),
     )

climate_ref_esmvaltool/diagnostics/tcr.py

@@ -11,9 +11,10 @@ from climate_ref_core.constraints import (
 )
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput
-from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
+from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic, fillvalues_to_nan
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
 
@@ -39,19 +40,31 @@ class TransientClimateResponse(ESMValToolDiagnostic):
                     facets={
                         "variable_id": ("tas",),
                         "experiment_id": experiments,
+                        "table_id": "Amon",
                     },
                 ),
             ),
             group_by=("source_id", "member_id", "grid_label"),
             constraints=(
-                RequireFacets("experiment_id", experiments),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 RequireOverlappingTimerange(group_by=("instance_id",)),
+                RequireFacets("experiment_id", experiments),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6),
             ),
         ),
     )
     facets = ("grid_label", "member_id", "source_id", "region", "metric")
+    series = (
+        SeriesDefinition(
+            file_pattern="tcr/calculate/{source_id}*.nc",
+            dimensions={
+                "statistic": "global annual mean tas anomaly relative to linear fit of piControl run",
+            },
+            values_name="tas_anomaly",
+            index_name="time",
+            attributes=[],
+        ),
+    )
 
     @staticmethod
     def update_recipe(
@@ -104,7 +117,7 @@ class TransientClimateResponse(ESMValToolDiagnostic):
     ) -> tuple[CMECMetric, CMECOutput]:
         """Format the result."""
         tcr_ds = xarray.open_dataset(result_dir / "work" / "tcr" / "calculate" / "tcr.nc")
-        tcr = float(tcr_ds["tcr"].values[0])
+        tcr = float(fillvalues_to_nan(tcr_ds["tcr"].values)[0])
 
         # Update the diagnostic bundle arguments with the computed diagnostics.
         metric_args[MetricCV.DIMENSIONS.value] = {