climate-ref-core 0.19.2__tar.gz → 0.20.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/PKG-INFO +2 -2
  2. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/pyproject.toml +2 -2
  3. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/constraints.py +101 -0
  4. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/esgf/__init__.py +2 -2
  5. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/esgf/base.py +52 -25
  6. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/esgf/cmip6.py +1 -34
  7. climate_ref_core-0.20.0/src/climate_ref_core/esgf/cmip7.py +62 -0
  8. climate_ref_core-0.19.2/src/climate_ref_core/esgf/cmip7.py → climate_ref_core-0.20.0/src/climate_ref_core/esgf/cmip7_converted.py +10 -12
  9. climate_ref_core-0.20.0/src/climate_ref_core/esgf/obs4mips.py +56 -0
  10. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/providers.py +30 -29
  11. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/source_types.py +5 -0
  12. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/esgf/test_base.py +2 -28
  13. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/esgf/test_cmip6.py +0 -20
  14. climate_ref_core-0.20.0/tests/unit/esgf/test_cmip7.py +120 -0
  15. climate_ref_core-0.19.2/tests/unit/esgf/test_cmip7.py → climate_ref_core-0.20.0/tests/unit/esgf/test_cmip7_converted.py +93 -91
  16. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/esgf/test_obs4mips.py +36 -13
  17. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_constraints.py +70 -0
  18. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_providers.py +1 -0
  19. climate_ref_core-0.20.0/tests/unit/test_source_types.py +6 -0
  20. climate_ref_core-0.19.2/src/climate_ref_core/esgf/obs4mips.py +0 -82
  21. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/.gitignore +0 -0
  22. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/LICENCE +0 -0
  23. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/NOTICE +0 -0
  24. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/README.md +0 -0
  25. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/__init__.py +0 -0
  26. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/cmip6_to_cmip7.py +0 -0
  27. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/cmor.py +0 -0
  28. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/data/cmip6_cmip7_variable_map.json +0 -0
  29. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/data.py +0 -0
  30. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/dataset_registry.py +0 -0
  31. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/datasets.py +0 -0
  32. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/diagnostics.py +0 -0
  33. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/env.py +0 -0
  34. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/esgf/fetcher.py +0 -0
  35. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/esgf/registry.py +0 -0
  36. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/esmvaltool_reference.py +0 -0
  37. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/exceptions.py +0 -0
  38. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/executor.py +0 -0
  39. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/logging.py +0 -0
  40. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/metric_values/__init__.py +0 -0
  41. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/metric_values/typing.py +0 -0
  42. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/output_files.py +0 -0
  43. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/paths.py +0 -0
  44. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/py.typed +0 -0
  45. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/pycmec/README.md +0 -0
  46. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/pycmec/__init__.py +0 -0
  47. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/pycmec/controlled_vocabulary.py +0 -0
  48. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/pycmec/cv_cmip7_aft.yaml +0 -0
  49. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/pycmec/metric.py +0 -0
  50. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/pycmec/output.py +0 -0
  51. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/reference_data.py +0 -0
  52. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/__init__.py +0 -0
  53. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/_quantise.py +0 -0
  54. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/capture.py +0 -0
  55. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/compare.py +0 -0
  56. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/gate.py +0 -0
  57. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/manifest.py +0 -0
  58. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/report_store.py +0 -0
  59. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/regression/store.py +0 -0
  60. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/resources.py +0 -0
  61. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/serialisation.py +0 -0
  62. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/summary.py +0 -0
  63. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/src/climate_ref_core/testing.py +0 -0
  64. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/integration/test_resource_measurement.py +0 -0
  65. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/esgf/test_fetcher.py +0 -0
  66. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/esgf/test_registry.py +0 -0
  67. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/metric_values/test_typing.py +0 -0
  68. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/cmec_testdata/cmec_metric_sample.json +0 -0
  69. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/cmec_testdata/cmec_output_sample.json +0 -0
  70. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/cmec_testdata/cv_sample.yaml +0 -0
  71. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/cmec_testdata/test_metric_json_schema.yml +0 -0
  72. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/cmec_testdata/test_output_json_schema.yml +0 -0
  73. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/conftest.py +0 -0
  74. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/test_cmec_metric.py +0 -0
  75. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/test_cmec_output.py +0 -0
  76. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/pycmec/test_controlled_vocabulary.py +0 -0
  77. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/regression/test_capture.py +0 -0
  78. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/regression/test_compare.py +0 -0
  79. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/regression/test_gate.py +0 -0
  80. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/regression/test_manifest.py +0 -0
  81. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/regression/test_quantise.py +0 -0
  82. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/regression/test_report_store.py +0 -0
  83. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/regression/test_store.py +0 -0
  84. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_cmip6_to_cmip7.py +0 -0
  85. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_cmor.py +0 -0
  86. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_data.py +0 -0
  87. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_dataset_registry/test_dataset_registry.py +0 -0
  88. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_datasets/dataset_collection_hash.yml +0 -0
  89. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_datasets/dataset_collection_obs4mips_hash.yml +0 -0
  90. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_datasets/execution_dataset_hash.yml +0 -0
  91. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_datasets.py +0 -0
  92. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_diagnostics.py +0 -0
  93. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_esmvaltool_reference.py +0 -0
  94. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_exceptions.py +0 -0
  95. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_executor.py +0 -0
  96. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_logging.py +0 -0
  97. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_output_files.py +0 -0
  98. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_packaging.py +0 -0
  99. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_paths.py +0 -0
  100. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_reference_data.py +0 -0
  101. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_resources.py +0 -0
  102. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_serialisation.py +0 -0
  103. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_summary.py +0 -0
  104. {climate_ref_core-0.19.2 → climate_ref_core-0.20.0}/tests/unit/test_testing.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: climate-ref-core
3
- Version: 0.19.2
3
+ Version: 0.20.0
4
4
  Summary: Core library for the CMIP Rapid Evaluation Framework
5
5
  Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
6
6
  License-Expression: Apache-2.0
@@ -22,7 +22,7 @@ Requires-Dist: cattrs>=24.1
22
22
  Requires-Dist: dask[array]>=2024.10.0
23
23
  Requires-Dist: environs>=11
24
24
  Requires-Dist: fastprogress==1.0.5
25
- Requires-Dist: intake-esgf>=2025.7.16
25
+ Requires-Dist: intake-esgf>=2026.9.28
26
26
  Requires-Dist: loguru>=0.7.0
27
27
  Requires-Dist: numpy>=2.0.0
28
28
  Requires-Dist: pandas>=2.1.0
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "climate-ref-core"
3
- version = "0.19.2"
3
+ version = "0.20.0"
4
4
  description = "Core library for the CMIP Rapid Evaluation Framework"
5
5
  readme = "README.md"
6
6
  authors = [
@@ -42,7 +42,7 @@ dependencies = [
42
42
  "setuptools<85",
43
43
 
44
44
  # ESGF data fetching
45
- "intake-esgf>=2025.7.16",
45
+ "intake-esgf>=2026.9.28",
46
46
  "pyarrow>=17.0.0",
47
47
  "dask[array]>=2024.10.0",
48
48
 
@@ -691,3 +691,104 @@ class SelectFirstMember:
691
691
  first = min(members, key=_member_sort_key)
692
692
  keep.loc[subgroup.index[subgroup[self.member_facet] == first]] = True
693
693
  return group[keep]
694
+
695
+
696
+ _RESOLUTION_KM_RE = re.compile(r"\s*([0-9.]+)\s*km\s*")
697
+
698
+
699
+ @runtime_checkable
700
+ class GridPreference(Protocol):
701
+ """
702
+ A rule for choosing one grid when a variable is published on several.
703
+
704
+ CMIP7 grids are described by the
705
+ [Essential Model Documentation](https://github.com/WCRP-CMIP/Essential-Model-Documentation),
706
+ which has a [grid viewer](https://emd.wcrp-cmip.org/docs/grid_viewer/horizontal/).
707
+ A native grid is likely the one that is not regular lat/lon.
708
+ """
709
+
710
+ def choose(self, grids: pd.DataFrame, data_catalog: pd.DataFrame) -> str:
711
+ """
712
+ Choose the grid to keep.
713
+
714
+ Parameters
715
+ ----------
716
+ grids
717
+ The candidate grids, one row per ``grid_label``.
718
+ Each row holds the facets of the first dataset on that grid.
719
+ data_catalog
720
+ The data catalog of datasets
721
+
722
+ Returns
723
+ -------
724
+ :
725
+ The ``grid_label`` of the chosen grid
726
+ """
727
+ ...
728
+
729
+
730
+ @frozen
731
+ class PreferNativeGrid:
732
+ """
733
+ Prefer the native grid, then the finest resolution.
734
+
735
+ The CMIP6 native grid ``gn`` comes first, then the finest ``nominal_resolution``.
736
+
737
+ CMIP7 grid labels carry no meaning, so for CMIP7 this picks the finest ``nominal_resolution``.
738
+ Resolutions that are missing or not given in km sort last,
739
+ and the grid label breaks any remaining tie.
740
+ """
741
+
742
+ def choose(self, grids: pd.DataFrame, data_catalog: pd.DataFrame) -> str:
743
+ """
744
+ Choose the native or finest grid.
745
+ """
746
+ resolutions = (
747
+ grids["nominal_resolution"]
748
+ if "nominal_resolution" in grids.columns
749
+ else pd.Series(None, index=grids.index)
750
+ )
751
+
752
+ def sort_key(row: int) -> tuple[bool, float, str]:
753
+ label = str(grids["grid_label"].iloc[row])
754
+ match = _RESOLUTION_KM_RE.fullmatch(str(resolutions.iloc[row]))
755
+ resolution = float(match.group(1)) if match else float("inf")
756
+ return (label != "gn", resolution, label)
757
+
758
+ return str(grids["grid_label"].iloc[min(range(len(grids)), key=sort_key)])
759
+
760
+
761
+ @frozen
762
+ class SelectSingleGrid:
763
+ """
764
+ Keep a single grid per variable within each group.
765
+
766
+ A model may publish one variable on several grids, such as its native grid and a regridded copy.
767
+ A group that spans realms cannot be split by ``grid_label``,
768
+ because CMIP7 labels each realm's grid separately, so this constraint drops the extra copies instead.
769
+
770
+ The ``preference`` decides which grid is kept (:class:`PreferNativeGrid` by default).
771
+ """
772
+
773
+ group_by: tuple[str, ...] = field(converter=_to_tuple)
774
+ """The facets defining each variable of a model, within which a single grid is kept."""
775
+
776
+ preference: GridPreference = field(factory=PreferNativeGrid)
777
+ """The rule that chooses which grid to keep."""
778
+
779
+ def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
780
+ """
781
+ Keep only the preferred grid within each group.
782
+ """
783
+ if group.empty or "grid_label" not in group.columns:
784
+ return group
785
+
786
+ keep = pd.Series(False, index=group.index)
787
+ for _, subgroup in group.groupby(list(self.group_by), dropna=False):
788
+ grids = subgroup.dropna(subset=["grid_label"]).drop_duplicates("grid_label")
789
+ if len(grids) <= 1:
790
+ keep.loc[subgroup.index] = True
791
+ continue
792
+ preferred = self.preference.choose(grids, data_catalog)
793
+ keep.loc[subgroup.index[subgroup["grid_label"] == preferred]] = True
794
+ return group[keep]
@@ -8,10 +8,10 @@ This module provides classes for searching and fetching datasets from ESGF
8
8
  from climate_ref_core.esgf.base import (
9
9
  ESGFRequest,
10
10
  IntakeESGFMixin,
11
- enable_ceda_solr_index,
12
11
  )
13
12
  from climate_ref_core.esgf.cmip6 import CMIP6Request
14
13
  from climate_ref_core.esgf.cmip7 import CMIP7Request
14
+ from climate_ref_core.esgf.cmip7_converted import ConvertedCMIP7Request
15
15
  from climate_ref_core.esgf.fetcher import ESGFFetcher
16
16
  from climate_ref_core.esgf.obs4mips import Obs4MIPsRequest
17
17
  from climate_ref_core.esgf.registry import RegistryRequest
@@ -19,10 +19,10 @@ from climate_ref_core.esgf.registry import RegistryRequest
19
19
  __all__ = [
20
20
  "CMIP6Request",
21
21
  "CMIP7Request",
22
+ "ConvertedCMIP7Request",
22
23
  "ESGFFetcher",
23
24
  "ESGFRequest",
24
25
  "IntakeESGFMixin",
25
26
  "Obs4MIPsRequest",
26
27
  "RegistryRequest",
27
- "enable_ceda_solr_index",
28
28
  ]
@@ -7,9 +7,8 @@ using the intake-esgf package.
7
7
 
8
8
  import copy
9
9
  from collections.abc import Mapping, Sequence
10
- from typing import Any, Protocol, Self, cast, runtime_checkable
10
+ from typing import Any, ClassVar, Protocol, Self, cast, runtime_checkable
11
11
 
12
- import intake_esgf
13
12
  import pandas as pd
14
13
  from intake_esgf import ESGFCatalog
15
14
  from intake_esgf.exceptions import NoSearchResults
@@ -18,18 +17,6 @@ from loguru import logger
18
17
  from climate_ref_core.exceptions import DatasetResolutionError
19
18
 
20
19
 
21
- def enable_ceda_solr_index() -> None:
22
- """
23
- Enable the CEDA Solr index as an additional ESGF search index.
24
-
25
- The obs4MIPs record for C3S-GTO-ECV-9-0 (the ozone reference) is missing
26
- from the default ESGF2-US-1.5-Catalog index but is still served by the
27
- CEDA Solr index, so search that index as well.
28
- Remove once the record returns to the 1.5 catalog.
29
- """
30
- intake_esgf.conf["solr_indices"]["esgf.ceda.ac.uk"] = True
31
-
32
-
33
20
  @runtime_checkable
34
21
  class ESGFRequest(Protocol):
35
22
  """
@@ -164,11 +151,7 @@ class IntakeESGFMixin:
164
151
  """
165
152
  Mixin that fetches datasets from ESGF using intake-esgf.
166
153
 
167
- Subclasses must define:
168
- - project: str
169
- - facets: dict[str, str | tuple[str, ...]]
170
- - remove_ensembles: bool
171
- - time_span: tuple[str, str] | None
154
+ Subclasses must define ``source_type`` and ``project``.
172
155
 
173
156
  A request can also be pinned to the datasets an existing test case catalog records,
174
157
  see :meth:`pin_to_datasets`.
@@ -184,6 +167,35 @@ class IntakeESGFMixin:
184
167
  pinned_instance_ids: tuple[str, ...] | None = None
185
168
  """Ids of the datasets this request is pinned to."""
186
169
 
170
+ facet_renames: ClassVar[Mapping[str, str]] = {}
171
+ """REF facet names that the ESGF index spells differently."""
172
+
173
+ def __init__(
174
+ self,
175
+ slug: str,
176
+ facets: dict[str, Any],
177
+ remove_ensembles: bool = False,
178
+ time_span: tuple[str, str] | None = None,
179
+ ):
180
+ """
181
+ Initialize an ESGF request.
182
+
183
+ Parameters
184
+ ----------
185
+ slug
186
+ Unique identifier for this request
187
+ facets
188
+ ESGF search facets (e.g., source_id, variable_id, experiment_id)
189
+ remove_ensembles
190
+ If True, keep only one ensemble member per model
191
+ time_span
192
+ Optional time range filter (start, end) in YYYY-MM format
193
+ """
194
+ self.slug = slug
195
+ self.facets = facets
196
+ self.remove_ensembles = remove_ensembles
197
+ self.time_span = time_span
198
+
187
199
  def pin_to_datasets(self, datasets: Sequence[Mapping[str, Any]]) -> Self:
188
200
  """
189
201
  Return a copy of this request that only resolves the given datasets.
@@ -216,9 +228,20 @@ class IntakeESGFMixin:
216
228
  pinned.pinned_instance_ids = tuple(dict.fromkeys(cast(list[str], instance_ids)))
217
229
  return pinned
218
230
 
231
+ def _pinned_facets(self, instance_ids: tuple[str, ...]) -> dict[str, Any]:
232
+ """Search facets that find the pinned datasets."""
233
+ return {"instance_id": list(instance_ids)}
234
+
235
+ def _select_pinned(self, results: pd.DataFrame) -> pd.DataFrame:
236
+ """Narrow the search results for a pinned request down to the pinned datasets."""
237
+ return results
238
+
219
239
  def fetch_datasets(self) -> pd.DataFrame:
220
240
  """Fetch dataset metadata from ESGF."""
221
- facets: dict[str, Any] = {"project": self.project, **self.facets}
241
+ facets: dict[str, Any] = {
242
+ "project": self.project,
243
+ **{self.facet_renames.get(key, key): value for key, value in self.facets.items()},
244
+ }
222
245
 
223
246
  if self.pinned_instance_ids:
224
247
  # Ask for the pinned datasets by id as the declared facets may now
@@ -227,7 +250,7 @@ class IntakeESGFMixin:
227
250
  logger.info(f"Resolving {len(self.pinned_instance_ids)} pinned datasets for {self.slug}")
228
251
  facets = {
229
252
  "project": facets["project"],
230
- "instance_id": list(self.pinned_instance_ids),
253
+ **self._pinned_facets(self.pinned_instance_ids),
231
254
  # A pin holds even once a newer version of the dataset has been published
232
255
  "latest": [True, False],
233
256
  }
@@ -244,8 +267,6 @@ class IntakeESGFMixin:
244
267
  # intake-esgf assigns Python lists into individual DataFrame cells, which only works for object dtype.
245
268
  # Pandas >= 3.0 defaults strings to the pyarrow-backed dtype, where that assignment raises,
246
269
  # so pin the legacy object-string behaviour for the intake-esgf interaction.
247
- enable_ceda_solr_index()
248
-
249
270
  with pd.option_context("future.infer_string", False):
250
271
  cat = ESGFCatalog() # type: ignore[no-untyped-call]
251
272
  try:
@@ -258,10 +279,16 @@ class IntakeESGFMixin:
258
279
  # Pinned ids already name one ensemble member each
259
280
  cat.remove_ensembles()
260
281
 
261
- path_dict = cat.to_path_dict(prefer_streaming=False, minimal_keys=False, quiet=True)
262
282
  if cat.df is None or cat.df.empty:
263
- raise ValueError("No datasets found for the given ESGF request")
283
+ raise DatasetResolutionError(f"No datasets found for the ESGF request {self.slug}")
284
+ if self.pinned_instance_ids:
285
+ cat.df = self._select_pinned(cat.df)
286
+
287
+ path_dict = cat.to_path_dict(prefer_streaming=False, minimal_keys=False, quiet=True)
264
288
  merged_df = cat.df.merge(pd.Series(path_dict, name="files"), left_on="key", right_index=True)
289
+ if self.pinned_instance_ids and len(merged_df) < len(cat.df):
290
+ missing = sorted(set(cat.df["key"]) - set(path_dict))
291
+ raise DatasetResolutionError(f"Could not retrieve pinned datasets for {self.slug}: {missing}")
265
292
 
266
293
  if self.time_span:
267
294
  merged_df["time_start"] = self.time_span[0]
@@ -4,7 +4,7 @@ CMIP6 dataset request implementation.
4
4
 
5
5
  from __future__ import annotations
6
6
 
7
- from typing import TYPE_CHECKING, Any
7
+ from typing import TYPE_CHECKING
8
8
 
9
9
  from climate_ref_core.esgf.base import IntakeESGFMixin
10
10
 
@@ -83,38 +83,5 @@ class CMIP6Request(IntakeESGFMixin):
83
83
  "data_node",
84
84
  )
85
85
 
86
- def __init__(
87
- self,
88
- slug: str,
89
- facets: dict[str, Any],
90
- remove_ensembles: bool = False,
91
- time_span: tuple[str, str] | None = None,
92
- ):
93
- """
94
- Initialize a CMIP6 request.
95
-
96
- Parameters
97
- ----------
98
- slug
99
- Unique identifier for this request
100
- facets
101
- ESGF search facets (e.g., source_id, variable_id, experiment_id)
102
- remove_ensembles
103
- If True, keep only one ensemble member per model
104
- time_span
105
- Optional time range filter (start, end) in YYYY-MM format
106
- """
107
- self.slug = slug
108
- self.facets = facets
109
- self.remove_ensembles = remove_ensembles
110
- self.time_span = time_span
111
-
112
- for key in self.cmip6_path_items:
113
- if key not in self.available_facets:
114
- raise ValueError(f"Path item {key!r} not in available facets")
115
- for key in self.cmip6_filename_paths:
116
- if key not in self.available_facets:
117
- raise ValueError(f"Filename path {key!r} not in available facets")
118
-
119
86
  def __repr__(self) -> str:
120
87
  return f"CMIP6Request(slug={self.slug!r}, facets={self.facets!r})"
@@ -0,0 +1,62 @@
1
+ """
2
+ CMIP7 dataset request implementation.
3
+ """
4
+
5
+ from collections.abc import Mapping
6
+ from typing import Any, ClassVar
7
+
8
+ import intake_esgf
9
+ import pandas as pd
10
+ from loguru import logger
11
+
12
+ from climate_ref_core.esgf.base import IntakeESGFMixin
13
+ from climate_ref_core.exceptions import DatasetResolutionError
14
+
15
+ STAC_INDICES = ("discovery.east.esgf.io", "discovery.west.esgf.io")
16
+ """The ESGF indices that publish CMIP7 data."""
17
+
18
+
19
+ def _esgf_id(instance_id: str) -> str:
20
+ """Spell a REF CMIP7 ``instance_id`` the way ESGF identifies the dataset."""
21
+ return f"MIP-DRS7.{instance_id}"
22
+
23
+
24
+ class CMIP7Request(IntakeESGFMixin):
25
+ """
26
+ Represents a request for CMIP7 datasets published on ESGF.
27
+ """
28
+
29
+ source_type = "CMIP7"
30
+ project = "CMIP7"
31
+ facet_renames: ClassVar[Mapping[str, str]] = {"branding_suffix": "variable_branding_suffix"}
32
+
33
+ def _pinned_facets(self, instance_ids: tuple[str, ...]) -> dict[str, Any]:
34
+ # The STAC indices cannot be searched by id, only by the id without its version
35
+ base_ids, versions = zip(*(_esgf_id(instance_id).rsplit(".", 1) for instance_id in instance_ids))
36
+ return {"base_id": sorted(set(base_ids)), "version": sorted({v.removeprefix("v") for v in versions})}
37
+
38
+ def _select_pinned(self, results: pd.DataFrame) -> pd.DataFrame:
39
+ wanted = {_esgf_id(instance_id) for instance_id in self.pinned_instance_ids or ()}
40
+ found = results["id"].map(lambda ids: {str(i).split("|")[0] for i in ids} & wanted)
41
+
42
+ # intake-esgf keeps only the newest version it finds of each dataset
43
+ # TODO: fix this upstream in intake-esgf as ESGF has the information
44
+ missing = wanted - set().union(*found)
45
+ if missing:
46
+ raise DatasetResolutionError(
47
+ f"ESGF no longer resolves {len(missing)} pinned datasets for {self.slug}: {sorted(missing)}"
48
+ )
49
+ return results[found.map(bool)]
50
+
51
+ def fetch_datasets(self) -> pd.DataFrame:
52
+ """Fetch dataset metadata from ESGF, warning if no index that publishes CMIP7 is enabled."""
53
+ enabled = intake_esgf.conf["stac_indices"]
54
+
55
+ if not any(enabled.get(index) for index in STAC_INDICES):
56
+ logger.warning(
57
+ f"No CMIP7 index enabled in {intake_esgf.conf.filename}: enable {', '.join(STAC_INDICES)}"
58
+ )
59
+ return super().fetch_datasets()
60
+
61
+ def __repr__(self) -> str:
62
+ return f"CMIP7Request(slug={self.slug!r}, facets={self.facets!r})"
@@ -1,8 +1,8 @@
1
1
  """
2
- CMIP7 dataset request implementation.
2
+ CMIP7 test data fabricated from CMIP6 data.
3
3
 
4
- Since CMIP7 data is not yet available on ESGF, this module provides
5
- a request class that fetches CMIP6 data and converts it to CMIP7 format.
4
+ :class:`ConvertedCMIP7Request` serves test cases whose models have not published CMIP7 data yet.
5
+ It will be removed once every test case can use :class:`~climate_ref_core.esgf.CMIP7Request`.
6
6
  """
7
7
 
8
8
  from __future__ import annotations
@@ -293,12 +293,12 @@ def _convert_file_to_cmip7(
293
293
  return output_file
294
294
 
295
295
 
296
- class CMIP7Request:
296
+ class ConvertedCMIP7Request:
297
297
  """
298
- Represents a CMIP7 dataset request.
298
+ Represents a CMIP7 dataset request fabricated from CMIP6 data.
299
299
 
300
- Since CMIP7 data is not yet available on ESGF, this class fetches
301
- CMIP6 data and converts it to CMIP7 format using convert_cmip6_dataset().
300
+ This class fetches CMIP6 data and converts it to CMIP7 format using convert_cmip6_dataset().
301
+ Use :class:`CMIP7Request` for data that is published on ESGF as CMIP7.
302
302
 
303
303
  The facets use CMIP7 naming conventions (e.g., variant_label instead of member_id).
304
304
  """
@@ -368,11 +368,9 @@ class CMIP7Request:
368
368
  # Create corresponding CMIP6 facets
369
369
  self._cmip6_facets = self._convert_to_cmip6_facets(facets)
370
370
 
371
- def pin_to_datasets(self, datasets: Sequence[Mapping[str, Any]]) -> CMIP7Request:
371
+ def pin_to_datasets(self, datasets: Sequence[Mapping[str, Any]]) -> ConvertedCMIP7Request:
372
372
  """
373
- Return this request unchanged: CMIP7 requests are not pinned yet.
374
-
375
- CMIP7 data cannot be fetched from ESGF at the moment.
373
+ Return this request unchanged: converted CMIP7 data has no ESGF ids to pin to.
376
374
 
377
375
  Parameters
378
376
  ----------
@@ -519,4 +517,4 @@ class CMIP7Request:
519
517
  return pd.DataFrame(converted_rows)
520
518
 
521
519
  def __repr__(self) -> str:
522
- return f"CMIP7Request(slug={self.slug!r}, facets={self.facets!r})"
520
+ return f"ConvertedCMIP7Request(slug={self.slug!r}, facets={self.facets!r})"
@@ -0,0 +1,56 @@
1
+ """
2
+ Obs4MIPs dataset request implementation.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ import intake_esgf
8
+ import pandas as pd
9
+
10
+ from climate_ref_core.esgf.base import IntakeESGFMixin
11
+
12
+
13
+ class Obs4MIPsRequest(IntakeESGFMixin):
14
+ """
15
+ Represents an Obs4MIPs dataset request.
16
+
17
+ These data are fetched from ESGF based on the provided facets.
18
+ """
19
+
20
+ source_type = "obs4MIPs"
21
+ project = "obs4MIPs"
22
+
23
+ obs4mips_path_items = (
24
+ "activity_id",
25
+ "institution_id",
26
+ "source_id",
27
+ "variable_id",
28
+ "grid_label",
29
+ )
30
+
31
+ obs4mips_filename_paths = (
32
+ "variable_id",
33
+ "source_id",
34
+ "grid_label",
35
+ )
36
+
37
+ avail_facets = (
38
+ "activity_id",
39
+ "institution_id",
40
+ "source_id",
41
+ "frequency",
42
+ "variable_id",
43
+ "grid_label",
44
+ "version",
45
+ "data_node",
46
+ "project",
47
+ )
48
+
49
+ def fetch_datasets(self) -> pd.DataFrame:
50
+ """Fetch dataset metadata from ESGF, also searching the CEDA Solr index."""
51
+ # C3S-GTO-ECV-9-0 (the ozone reference) is missing from the default index but CEDA still serves it
52
+ with intake_esgf.conf.set(indices={"esgf.ceda.ac.uk": True}):
53
+ return super().fetch_datasets()
54
+
55
+ def __repr__(self) -> str:
56
+ return f"Obs4MIPsRequest(slug={self.slug!r}, facets={self.facets!r})"
@@ -25,7 +25,7 @@ from typing import Any
25
25
 
26
26
  import requests
27
27
  import yaml
28
- from attrs import evolve
28
+ from attrs import evolve, frozen
29
29
  from loguru import logger
30
30
 
31
31
  from climate_ref_core.constraints import IgnoreFacets
@@ -57,6 +57,19 @@ def _slugify(value: str) -> str:
57
57
  return value.lower().replace(" ", "-")
58
58
 
59
59
 
60
+ @frozen
61
+ class DataIngestRequest:
62
+ """
63
+ A directory of fetched provider data that the application should ingest into its database.
64
+ """
65
+
66
+ source_type: SourceDatasetType
67
+ """Source type whose dataset adapter parses the files"""
68
+
69
+ path: Path
70
+ """Directory containing the files to ingest"""
71
+
72
+
60
73
  class DiagnosticProvider:
61
74
  """
62
75
  The interface for registering and running diagnostics.
@@ -203,15 +216,16 @@ class DiagnosticProvider:
203
216
  self,
204
217
  config: Any,
205
218
  *,
206
- db: Any = None,
207
219
  skip_env: bool = False,
208
220
  skip_data: bool = False,
209
221
  ) -> None:
210
222
  """
211
223
  Perform all setup required before offline execution.
212
224
 
213
- This calls setup_environment, fetch_data, and ingest_data in the correct order.
225
+ This calls setup_environment and fetch_data in the correct order.
214
226
  Override individual hooks for fine-grained control.
227
+ Ingesting the fetched data is left to the application,
228
+ which reads what to ingest from `data_to_ingest`.
215
229
 
216
230
  This method MUST be idempotent - safe to call multiple times.
217
231
 
@@ -219,11 +233,6 @@ class DiagnosticProvider:
219
233
  ----------
220
234
  config
221
235
  The application configuration
222
- db
223
- Optional database instance for data ingestion.
224
-
225
- If None, ingestion is skipped. This allows providers to be set up without the full
226
- climate-ref package (e.g., for environment setup or data fetching only).
227
236
  skip_env
228
237
  If True, skip environment setup (e.g., conda)
229
238
  skip_data
@@ -233,8 +242,6 @@ class DiagnosticProvider:
233
242
  self.setup_environment(config)
234
243
  if not skip_data:
235
244
  self.fetch_data(config)
236
- if db is not None:
237
- self.ingest_data(config, db)
238
245
 
239
246
  def setup_environment(self, config: Any) -> None:
240
247
  """
@@ -276,34 +283,28 @@ class DiagnosticProvider:
276
283
  """
277
284
  pass
278
285
 
279
- def ingest_data(self, config: Any, db: Any) -> None:
286
+ def data_to_ingest(self, config: Any) -> list[DataIngestRequest]:
280
287
  """
281
- Ingest fetched data into the database.
288
+ Describe the fetched data that needs ingesting into the database.
282
289
 
283
- This is called after fetch_data to register any provider-specific
284
- datasets in the database. For example, PMP climatology data needs
285
- to be ingested so it can be used by diagnostics.
290
+ `ref providers setup` ingests each request after fetch_data,
291
+ using `climate_ref.datasets.ingest_provider_data`,
292
+ so the solver can find provider-specific datasets such as the PMP climatologies.
286
293
 
287
- Default implementation does nothing. Override in subclasses
288
- that have data to ingest.
289
-
290
- This method MUST be idempotent.
291
-
292
- Note: This method is only called when a database instance is provided
293
- to setup(). The database and ingestion utilities are part of the
294
- climate-ref package, not climate-ref-core. Provider implementations
295
- should handle ImportError gracefully if they need to import from
296
- climate-ref, allowing the provider to work standalone without
297
- the full climate-ref package installed.
294
+ Default implementation returns no requests.
295
+ Override in subclasses that have data to ingest.
298
296
 
299
297
  Parameters
300
298
  ----------
301
299
  config
302
300
  The application configuration
303
- db
304
- The database instance (from climate-ref package)
301
+
302
+ Returns
303
+ -------
304
+ :
305
+ The directories to ingest
305
306
  """
306
- pass
307
+ return []
307
308
 
308
309
  def validate_setup(self, config: Any) -> bool:
309
310
  """
@@ -33,6 +33,11 @@ class SourceDatasetType(enum.Enum):
33
33
  obs4REF = "obs4ref"
34
34
  ESMValToolReference = "esmvaltool-reference"
35
35
 
36
+ @property
37
+ def is_model_output(self) -> bool:
38
+ """Whether the dataset holds model simulation output rather than observations."""
39
+ return self in (SourceDatasetType.CMIP6, SourceDatasetType.CMIP7)
40
+
36
41
  @classmethod
37
42
  @functools.lru_cache(maxsize=1)
38
43
  def ordered(