climate-ref-core 0.7.0__tar.gz → 0.8.0__tar.gz

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref-core
-Version: 0.7.0
+Version: 0.8.0
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref-core"
-version = "0.7.0"
+version = "0.8.0"
 description = "Core library for the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/src/climate_ref_core/constraints.py

@@ -100,7 +100,7 @@ def apply_constraint(
 
 def _to_tuple(value: None | str | tuple[str, ...]) -> tuple[str, ...]:
     """
-    Clean the value of group_by to a tuple of strings
+    Normalize value to a tuple of strings.
     """
     if value is None:
         return ()
@@ -109,6 +109,13 @@ def _to_tuple(value: None | str | tuple[str, ...]) -> tuple[str, ...]:
     return tuple(value)
 
 
+def _to_tuple_dict(value: dict[str, str | tuple[str, ...]]) -> dict[str, tuple[str, ...]]:
+    """
+    Normalize value to a dict of tuples of strings.
+    """
+    return {k: _to_tuple(v) for k, v in value.items()}
+
+
 @frozen
 class RequireFacets:
     """
@@ -153,6 +160,27 @@ class RequireFacets:
         return group[select]
 
 
+@frozen
+class IgnoreFacets:
+    """
+    A constraint that ignores certain facet values.
+
+    Datasets with these facet values are removed from the selection.
+    """
+
+    facets: dict[str, str | tuple[str, ...]] = field(converter=_to_tuple_dict)
+    """The facet values to ignore."""
+
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Filter out datasets with the ignored facets.
+        """
+        mask = group[list(self.facets)].isin(self.facets).all(axis="columns")
+        if mask.any():
+            logger.debug(f"Ignoring files {', '.join(group.loc[mask, 'path'])} becauseof {self}")
+        return group[~mask]
+
+
 @frozen
 class AddSupplementaryDataset:
     """

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/src/climate_ref_core/metric_values/typing.py

@@ -3,7 +3,8 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Self
 
-from pydantic import BaseModel, model_validator
+import numpy as np
+from pydantic import BaseModel, field_validator, model_validator
 
 Value = float | int
 
@@ -64,20 +65,35 @@ class SeriesMetricValue(BaseModel):
     This is used for presentation purposes and is not used in the controlled vocabulary.
     """
 
-    attributes: dict[str, str | Value] | None = None
+    attributes: dict[str, str | Value | None] | None = None
     """
     Additional unstructured attributes associated with the metric value
     """
 
     @model_validator(mode="after")
-    def validate_index_length(self) -> Self:
-        """Validate that index has the same length as values"""
+    def validate_index(self) -> Self:
+        """Validate that index has the same length as values and contains no NaNs"""
         if len(self.index) != len(self.values):
             raise ValueError(
                 f"Index length ({len(self.index)}) must match values length ({len(self.values)})"
             )
+        for v in self.index:
+            if isinstance(v, float) and not np.isfinite(v):
+                raise ValueError("NaN or Inf values are not allowed in the index")
         return self
 
+    @field_validator("values", mode="before")
+    @classmethod
+    def validate_values(cls, value: Any) -> Any:
+        """
+        Transform None values to NaN in the values field
+        """
+        if not isinstance(value, (list, tuple)):
+            raise ValueError("`values` must be a list or tuple.")
+
+        # Transform None values to NaN
+        return [float("nan") if v is None else v for v in value]
+
     @classmethod
     def dump_to_json(cls, path: Path, series: Sequence["SeriesMetricValue"]) -> None:
         """
@@ -94,7 +110,13 @@ class SeriesMetricValue(BaseModel):
             The series values to dump.
         """
         with open(path, "w") as f:
-            json.dump([s.model_dump() for s in series], f, indent=2)
+            json.dump(
+                [s.model_dump(mode="json") for s in series],
+                f,
+                indent=2,
+                allow_nan=False,
+                sort_keys=True,
+            )
 
     @classmethod
     def load_from_json(
@@ -102,7 +124,7 @@ class SeriesMetricValue(BaseModel):
         path: Path,
     ) -> list["SeriesMetricValue"]:
         """
-        Dump a sequence of SeriesMetricValue to a JSON file.
+        Load a sequence of SeriesMetricValue from a JSON file.
 
         Parameters
         ----------
@@ -115,7 +137,7 @@ class SeriesMetricValue(BaseModel):
         if not isinstance(data, list):
             raise ValueError(f"Expected a list of series values, got {type(data)}")
 
-        return [cls.model_validate(s) for s in data]
+        return [cls.model_validate(s, strict=True) for s in data]
 
 
 class ScalarMetricValue(BaseModel):

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/src/climate_ref_core/providers.py

@@ -16,14 +16,18 @@ import os
 import stat
 import subprocess
 from abc import abstractmethod
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from contextlib import AbstractContextManager
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import requests
+import yaml
+from attrs import evolve
 from loguru import logger
 
+from climate_ref_core.constraints import IgnoreFacets
+from climate_ref_core.datasets import SourceDatasetType
 from climate_ref_core.diagnostics import Diagnostic
 from climate_ref_core.exceptions import InvalidDiagnosticException, InvalidProviderException
 
@@ -74,6 +78,51 @@ class DiagnosticProvider:
         config :
             A configuration.
         """
+        logger.debug(
+            f"Configuring provider {self.slug} using ignore_datasets_file {config.ignore_datasets_file}"
+        )
+        # The format of the configuration file is:
+        # provider:
+        #   diagnostic:
+        #     source_type:
+        #       - facet: value
+        #       - other_facet: [other_value1, other_value2]
+        ignore_datasets_all = yaml.safe_load(config.ignore_datasets_file.read_text(encoding="utf-8")) or {}
+        ignore_datasets = ignore_datasets_all.get(self.slug, {})
+        if unknown_slugs := {slug for slug in ignore_datasets} - {d.slug for d in self.diagnostics()}:
+            logger.warning(
+                f"Unknown diagnostics found in {config.ignore_datasets_file} "
+                f"for provider {self.slug}: {', '.join(sorted(unknown_slugs))}"
+            )
+
+        known_source_types = {s.value for s in iter(SourceDatasetType)}
+        for diagnostic in self.diagnostics():
+            if diagnostic.slug in ignore_datasets:
+                if unknown_source_types := set(ignore_datasets[diagnostic.slug]) - known_source_types:
+                    logger.warning(
+                        f"Unknown source types found in {config.ignore_datasets_file} for "
+                        f"diagnostic '{diagnostic.slug}' by provider {self.slug}: "
+                        f"{', '.join(sorted(unknown_source_types))}"
+                    )
+                data_requirements = (
+                    r if isinstance(r, Sequence) else (r,) for r in diagnostic.data_requirements
+                )
+                diagnostic.data_requirements = tuple(
+                    tuple(
+                        evolve(
+                            data_requirement,
+                            constraints=tuple(
+                                IgnoreFacets(facets)
+                                for facets in ignore_datasets[diagnostic.slug].get(
+                                    data_requirement.source_type.value, []
+                                )
+                            )
+                            + data_requirement.constraints,
+                        )
+                        for data_requirement in requirement_collection
+                    )
+                    for requirement_collection in data_requirements
+                )
 
     def diagnostics(self) -> list[Diagnostic]:
         """
@@ -287,6 +336,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
 
     def configure(self, config: Config) -> None:
         """Configure the provider."""
+        super().configure(config)
         self.prefix = config.paths.software / "conda"
 
     def _install_conda(self, update: bool) -> Path:

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/src/climate_ref_core/pycmec/metric.py

@@ -153,7 +153,7 @@ class MetricResults(RootModel[Any]):
     CMEC diagnostic bundle RESULTS object
     """
 
-    model_config = ConfigDict(strict=True)
+    model_config = ConfigDict(strict=True, allow_inf_nan=False)
     root: dict[str, dict[Any, Any]]
 
     @classmethod
@@ -284,7 +284,7 @@ class CMECMetric(BaseModel):
     Contains the diagnostics calculated during a diagnostic execution, in a standardised format.
     """
 
-    model_config = ConfigDict(strict=True, extra="allow")
+    model_config = ConfigDict(strict=True, extra="allow", allow_inf_nan=False)
 
     DIMENSIONS: MetricDimensions
     """
@@ -342,7 +342,15 @@ class CMECMetric(BaseModel):
         :
             None
         """
-        pathlib.Path(json_file).write_text(self.model_dump_json(indent=2))
+        pathlib.Path(json_file).write_text(
+            json.dumps(
+                self.model_dump(mode="json"),
+                indent=2,
+                allow_nan=False,
+                sort_keys=True,
+            ),
+            encoding="utf-8",
+        )
 
     @classmethod
     @validate_call
@@ -360,7 +368,7 @@ class CMECMetric(BaseModel):
         :
             CMEC Diagnostic object if the file is CMEC-compatible
         """
-        json_str = pathlib.Path(json_file).read_text()
+        json_str = pathlib.Path(json_file).read_text(encoding="utf-8")
         metric_obj = cls.model_validate_json(json_str)
 
         return metric_obj

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/tests/unit/test_constraints.py

@@ -9,6 +9,7 @@ from pandas.testing import assert_frame_equal
 from climate_ref_core.constraints import (
     AddSupplementaryDataset,
     GroupConstraint,
+    IgnoreFacets,
     PartialDateTime,
     RequireContiguousTimerange,
     RequireFacets,
@@ -97,6 +98,58 @@ class TestRequireFacets:
         assert_frame_equal(result, expected)
 
 
+class TestIgnoreFacets:
+    @pytest.mark.parametrize(
+        "data, expected_rows",
+        [
+            (
+                pd.DataFrame(
+                    {
+                        "variable_id": [],
+                        "path": [],
+                    }
+                ),
+                [],
+            ),
+            (
+                pd.DataFrame(
+                    {
+                        "variable_id": ["tas", "pr", "tas", "rsut"],
+                        "path": ["tas.nc", "pr.nc", "tas2.nc", "rsut.nc"],
+                    }
+                ),
+                [1, 3],
+            ),
+        ],
+    )
+    def test_apply_single(self, data, expected_rows):
+        constraint = IgnoreFacets(facets={"variable_id": "tas"})
+        result = constraint.apply(group=data, data_catalog=data.loc[[]])
+        expected = data.iloc[expected_rows]
+        assert_frame_equal(result, expected)
+
+    @pytest.mark.parametrize(
+        "data, expected_rows",
+        [
+            (
+                pd.DataFrame(
+                    {
+                        "variable_id": ["tas", "pr", "tas", "rsut"],
+                        "source_id": ["A", "B", "C", "C"],
+                        "path": ["A_tas.nc", "B_pr.nc", "C_tas.nc", "C_rsut.nc"],
+                    }
+                ),
+                [0, 1, 3],
+            ),
+        ],
+    )
+    def test_apply_multiple(self, data, expected_rows):
+        constraint = IgnoreFacets(facets={"variable_id": ("tas", "pr"), "source_id": "C"})
+        result = constraint.apply(group=data, data_catalog=data.loc[[]])
+        expected = data.iloc[expected_rows]
+        assert_frame_equal(result, expected)
+
+
 class TestAddSupplementaryDataset:
     constraint = AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6)
 

{climate_ref_core-0.7.0 → climate_ref_core-0.8.0}/tests/unit/test_providers.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import subprocess
+import textwrap
 import time
 from contextlib import contextmanager
 from pathlib import Path
@@ -8,12 +9,23 @@ from pathlib import Path
 import pytest
 
 import climate_ref_core.providers
+from climate_ref_core.constraints import IgnoreFacets
 from climate_ref_core.diagnostics import CommandLineDiagnostic, Diagnostic
 from climate_ref_core.exceptions import InvalidDiagnosticException, InvalidProviderException
 from climate_ref_core.providers import CondaDiagnosticProvider, DiagnosticProvider, import_provider
 
 
-class TestMetricsProvider:
+@pytest.fixture
+def mock_config(tmp_path, mocker):
+    """Use a mock config to avoid depending on `climate_ref.config.Config`."""
+    config = mocker.Mock()
+    config.paths.software = tmp_path / "software"
+    config.ignore_datasets_file = tmp_path / "ignore_datasets.yaml"
+    config.ignore_datasets_file.touch()
+    return config
+
+
+class TestDiagnosticProvider:
     def test_provider(self):
         provider = DiagnosticProvider("provider_name", "v0.23")
 
@@ -51,6 +63,62 @@ class TestMetricsProvider:
         result = provider.get("mock")
         assert isinstance(result, Diagnostic)
 
+    def test_configure(self, provider, mock_config):
+        mock_config.ignore_datasets_file.write_text(
+            textwrap.dedent(
+                """
+                mock_provider:
+                  mock:
+                    cmip6:
+                      - source_id: A
+                """
+            ),
+            encoding="utf-8",
+        )
+        provider.configure(mock_config)
+        expected_constraint = IgnoreFacets(facets={"source_id": ("A",)})
+        assert provider.diagnostics()[0].data_requirements[0][0].constraints[0] == expected_constraint
+
+    def test_configure_unknown_diagnostic(self, provider, mock_config, caplog):
+        mock_config.ignore_datasets_file.write_text(
+            textwrap.dedent(
+                """
+                mock_provider:
+                  invalid_diagnostic:
+                    cmip6:
+                      - source_id: A
+                """
+            ),
+            encoding="utf-8",
+        )
+        with caplog.at_level(logging.WARNING):
+            provider.configure(mock_config)
+        expected_msg = (
+            f"Unknown diagnostics found in {mock_config.ignore_datasets_file} "
+            "for provider mock_provider: invalid_diagnostic"
+        )
+        assert expected_msg in caplog.text
+
+    def test_configure_unknown_source_type(self, provider, mock_config, caplog):
+        mock_config.ignore_datasets_file.write_text(
+            textwrap.dedent(
+                """
+                mock_provider:
+                  mock:
+                    invalid_source_type:
+                      - source_id: A
+                """
+            ),
+            encoding="utf-8",
+        )
+        with caplog.at_level(logging.WARNING):
+            provider.configure(mock_config)
+        expected_msg = (
+            f"Unknown source types found in {mock_config.ignore_datasets_file} "
+            "for diagnostic 'mock' by provider mock_provider: invalid_source_type"
+        )
+        assert expected_msg in caplog.text
+
 
 @pytest.mark.parametrize("fqn", ["climate_ref_esmvaltool:provider", "climate_ref_esmvaltool"])
 def test_import_provider(fqn):
@@ -104,7 +172,7 @@ def test_get_micromamba_url(mocker, sysname, machine):
         assert "{" not in result
 
 
-class TestCondaMetricsProvider:
+class TestCondaDiagnosticProvider:
     @pytest.fixture
     def provider(self, tmp_path):
         provider = CondaDiagnosticProvider("provider_name", "v0.23")
@@ -117,9 +185,9 @@ class TestCondaMetricsProvider:
         with pytest.raises(ValueError, match=r"No prefix for conda environments configured.*"):
             provider.prefix
 
-    def test_configure(self, config):
+    def test_configure(self, mock_config):
         provider = CondaDiagnosticProvider("provider_name", "v0.23")
-        provider.configure(config)
+        provider.configure(mock_config)
 
         assert isinstance(provider.prefix, Path)