climate-ref-core 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

climate_ref_core/constraints.py

@@ -100,7 +100,7 @@ def apply_constraint(
 
 def _to_tuple(value: None | str | tuple[str, ...]) -> tuple[str, ...]:
     """
-    Clean the value of group_by to a tuple of strings
+    Normalize value to a tuple of strings.
     """
     if value is None:
         return ()
@@ -109,6 +109,13 @@ def _to_tuple(value: None | str | tuple[str, ...]) -> tuple[str, ...]:
     return tuple(value)
 
 
+def _to_tuple_dict(value: dict[str, str | tuple[str, ...]]) -> dict[str, tuple[str, ...]]:
+    """
+    Normalize value to a dict of tuples of strings.
+    """
+    return {k: _to_tuple(v) for k, v in value.items()}
+
+
 @frozen
 class RequireFacets:
     """
@@ -153,6 +160,27 @@ class RequireFacets:
         return group[select]
 
 
+@frozen
+class IgnoreFacets:
+    """
+    A constraint that ignores certain facet values.
+
+    Datasets with these facet values are removed from the selection.
+    """
+
+    facets: dict[str, str | tuple[str, ...]] = field(converter=_to_tuple_dict)
+    """The facet values to ignore."""
+
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Filter out datasets with the ignored facets.
+        """
+        mask = group[list(self.facets)].isin(self.facets).all(axis="columns")
+        if mask.any():
+            logger.debug(f"Ignoring files {', '.join(group.loc[mask, 'path'])} becauseof {self}")
+        return group[~mask]
+
+
 @frozen
 class AddSupplementaryDataset:
     """

climate_ref_core/metric_values/typing.py

@@ -3,7 +3,8 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Self
 
-from pydantic import BaseModel, model_validator
+import numpy as np
+from pydantic import BaseModel, field_validator, model_validator
 
 Value = float | int
 
@@ -64,20 +65,35 @@ class SeriesMetricValue(BaseModel):
     This is used for presentation purposes and is not used in the controlled vocabulary.
     """
 
-    attributes: dict[str, str | Value] | None = None
+    attributes: dict[str, str | Value | None] | None = None
     """
     Additional unstructured attributes associated with the metric value
     """
 
     @model_validator(mode="after")
-    def validate_index_length(self) -> Self:
-        """Validate that index has the same length as values"""
+    def validate_index(self) -> Self:
+        """Validate that index has the same length as values and contains no NaNs"""
         if len(self.index) != len(self.values):
             raise ValueError(
                 f"Index length ({len(self.index)}) must match values length ({len(self.values)})"
             )
+        for v in self.index:
+            if isinstance(v, float) and not np.isfinite(v):
+                raise ValueError("NaN or Inf values are not allowed in the index")
         return self
 
+    @field_validator("values", mode="before")
+    @classmethod
+    def validate_values(cls, value: Any) -> Any:
+        """
+        Transform None values to NaN in the values field
+        """
+        if not isinstance(value, (list, tuple)):
+            raise ValueError("`values` must be a list or tuple.")
+
+        # Transform None values to NaN
+        return [float("nan") if v is None else v for v in value]
+
     @classmethod
     def dump_to_json(cls, path: Path, series: Sequence["SeriesMetricValue"]) -> None:
         """
@@ -94,7 +110,13 @@ class SeriesMetricValue(BaseModel):
             The series values to dump.
         """
         with open(path, "w") as f:
-            json.dump([s.model_dump() for s in series], f, indent=2)
+            json.dump(
+                [s.model_dump(mode="json") for s in series],
+                f,
+                indent=2,
+                allow_nan=False,
+                sort_keys=True,
+            )
 
     @classmethod
     def load_from_json(
@@ -102,7 +124,7 @@ class SeriesMetricValue(BaseModel):
         path: Path,
     ) -> list["SeriesMetricValue"]:
         """
-        Dump a sequence of SeriesMetricValue to a JSON file.
+        Load a sequence of SeriesMetricValue from a JSON file.
 
         Parameters
         ----------
@@ -115,7 +137,7 @@ class SeriesMetricValue(BaseModel):
         if not isinstance(data, list):
             raise ValueError(f"Expected a list of series values, got {type(data)}")
 
-        return [cls.model_validate(s) for s in data]
+        return [cls.model_validate(s, strict=True) for s in data]
 
 
 class ScalarMetricValue(BaseModel):

climate_ref_core/providers.py

@@ -16,14 +16,18 @@ import os
 import stat
 import subprocess
 from abc import abstractmethod
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from contextlib import AbstractContextManager
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import requests
+import yaml
+from attrs import evolve
 from loguru import logger
 
+from climate_ref_core.constraints import IgnoreFacets
+from climate_ref_core.datasets import SourceDatasetType
 from climate_ref_core.diagnostics import Diagnostic
 from climate_ref_core.exceptions import InvalidDiagnosticException, InvalidProviderException
 
@@ -74,6 +78,51 @@ class DiagnosticProvider:
         config :
             A configuration.
         """
+        logger.debug(
+            f"Configuring provider {self.slug} using ignore_datasets_file {config.ignore_datasets_file}"
+        )
+        # The format of the configuration file is:
+        # provider:
+        #   diagnostic:
+        #     source_type:
+        #       - facet: value
+        #       - other_facet: [other_value1, other_value2]
+        ignore_datasets_all = yaml.safe_load(config.ignore_datasets_file.read_text(encoding="utf-8")) or {}
+        ignore_datasets = ignore_datasets_all.get(self.slug, {})
+        if unknown_slugs := {slug for slug in ignore_datasets} - {d.slug for d in self.diagnostics()}:
+            logger.warning(
+                f"Unknown diagnostics found in {config.ignore_datasets_file} "
+                f"for provider {self.slug}: {', '.join(sorted(unknown_slugs))}"
+            )
+
+        known_source_types = {s.value for s in iter(SourceDatasetType)}
+        for diagnostic in self.diagnostics():
+            if diagnostic.slug in ignore_datasets:
+                if unknown_source_types := set(ignore_datasets[diagnostic.slug]) - known_source_types:
+                    logger.warning(
+                        f"Unknown source types found in {config.ignore_datasets_file} for "
+                        f"diagnostic '{diagnostic.slug}' by provider {self.slug}: "
+                        f"{', '.join(sorted(unknown_source_types))}"
+                    )
+                data_requirements = (
+                    r if isinstance(r, Sequence) else (r,) for r in diagnostic.data_requirements
+                )
+                diagnostic.data_requirements = tuple(
+                    tuple(
+                        evolve(
+                            data_requirement,
+                            constraints=tuple(
+                                IgnoreFacets(facets)
+                                for facets in ignore_datasets[diagnostic.slug].get(
+                                    data_requirement.source_type.value, []
+                                )
+                            )
+                            + data_requirement.constraints,
+                        )
+                        for data_requirement in requirement_collection
+                    )
+                    for requirement_collection in data_requirements
+                )
 
     def diagnostics(self) -> list[Diagnostic]:
         """
@@ -287,6 +336,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
 
     def configure(self, config: Config) -> None:
         """Configure the provider."""
+        super().configure(config)
         self.prefix = config.paths.software / "conda"
 
     def _install_conda(self, update: bool) -> Path:

climate_ref_core/pycmec/metric.py

@@ -153,7 +153,7 @@ class MetricResults(RootModel[Any]):
     CMEC diagnostic bundle RESULTS object
     """
 
-    model_config = ConfigDict(strict=True)
+    model_config = ConfigDict(strict=True, allow_inf_nan=False)
     root: dict[str, dict[Any, Any]]
 
     @classmethod
@@ -284,7 +284,7 @@ class CMECMetric(BaseModel):
     Contains the diagnostics calculated during a diagnostic execution, in a standardised format.
     """
 
-    model_config = ConfigDict(strict=True, extra="allow")
+    model_config = ConfigDict(strict=True, extra="allow", allow_inf_nan=False)
 
     DIMENSIONS: MetricDimensions
     """
@@ -342,7 +342,15 @@ class CMECMetric(BaseModel):
         :
             None
         """
-        pathlib.Path(json_file).write_text(self.model_dump_json(indent=2))
+        pathlib.Path(json_file).write_text(
+            json.dumps(
+                self.model_dump(mode="json"),
+                indent=2,
+                allow_nan=False,
+                sort_keys=True,
+            ),
+            encoding="utf-8",
+        )
 
     @classmethod
     @validate_call
@@ -360,7 +368,7 @@ class CMECMetric(BaseModel):
         :
             CMEC Diagnostic object if the file is CMEC-compatible
         """
-        json_str = pathlib.Path(json_file).read_text()
+        json_str = pathlib.Path(json_file).read_text(encoding="utf-8")
         metric_obj = cls.model_validate_json(json_str)
 
         return metric_obj

{climate_ref_core-0.7.0.dist-info → climate_ref_core-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref-core
-Version: 0.7.0
+Version: 0.8.0
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0

{climate_ref_core-0.7.0.dist-info → climate_ref_core-0.8.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 climate_ref_core/__init__.py,sha256=MtmPThF2F9_2UODEN6rt1x30LDxrHIZ0wyRN_wsHx5I,127
-climate_ref_core/constraints.py,sha256=GsACa7mwVC7XAo58qkx6ViKu-iBC8dWwVTK8el1fbZI,16879
+climate_ref_core/constraints.py,sha256=OlSpXwLRENS4-2LKOhDq-uZ7QznymMwKhtL4Lf6uhso,17761
 climate_ref_core/dataset_registry.py,sha256=sQp2VT9xSVAaWsf0tF4E_VQxuEsvIxU2MZm5uNX1ynw,7172
 climate_ref_core/datasets.py,sha256=cx1L-CDf8uv7_MYtnhx3xu2oYKVeZTK72nLKw0ZImL0,6472
 climate_ref_core/diagnostics.py,sha256=sXDyUwJZWr03Wk8an-hbpLAmQKz6M8jXtmJ6e7duWl4,19689
@@ -7,18 +7,18 @@ climate_ref_core/env.py,sha256=Ph2dejVxTELfP3bL0xES086WLGvV5H6KvsOwCkL6m-k,753
 climate_ref_core/exceptions.py,sha256=7Mkz22P-kbiL-ZevAhlOuQaaeTio6zpwE9YA45OTGvs,1909
 climate_ref_core/executor.py,sha256=9mKVkm0S7ikub3_FP7CrgdC4Qj9ynOi0r_DIfzCDS-0,5459
 climate_ref_core/logging.py,sha256=xO0j7OKkuO9JoMtMTnMc62yLO2mJZmhQKAvj-CojblI,7396
-climate_ref_core/providers.py,sha256=z5oD7EErIWprH5zv8I5yuU0IavEiSsi6SFkYMtiqE7g,13448
+climate_ref_core/providers.py,sha256=klM-pOUAdBcU0lj1a5r-nKPKzq4QojAE_nb9iOZ-y9w,15942
 climate_ref_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 climate_ref_core/metric_values/__init__.py,sha256=aHfwRrqzLOmmaBKf1-4q97DnHb8KwmW0Dhwd79ZQiNQ,634
-climate_ref_core/metric_values/typing.py,sha256=4Qmr1LJQxQd2qkndwOzKTHq-hihiIBfWDZ_vzRWNbsI,3880
+climate_ref_core/metric_values/typing.py,sha256=n4I3kcEBYPhoWL8aPRCBAHjwNLUrDz40c289iLsOMRM,4677
 climate_ref_core/pycmec/README.md,sha256=PzkovlPpsXqFopsYzz5GRvCAipNRGO1Wo-0gc17qr2Y,36
 climate_ref_core/pycmec/__init__.py,sha256=hXvKGEJQWyAp1i-ndr3D4zuYxkRhcR2LfXgFXlhYOk4,28
 climate_ref_core/pycmec/controlled_vocabulary.py,sha256=kgMEvQ1P6EwXC7sFgdC77IQDo8I0DnnQ2CPXXQaavjE,5944
 climate_ref_core/pycmec/cv_cmip7_aft.yaml,sha256=gx5QyW88pZQVUfiYXmsJtJO6AJg6NbIZgdU4vDIa3fE,4390
-climate_ref_core/pycmec/metric.py,sha256=zymXoutnjbdcxvG_fMJugFLLcBrfSPG0XoV-2tA0ujA,18499
+climate_ref_core/pycmec/metric.py,sha256=k6kB_tvXbaixmTPGiUVZyCn5kvyoKwzHEMk7GCNNY4M,18742
 climate_ref_core/pycmec/output.py,sha256=Il4j6sjGrChBioiQS7lQ_CJmofT1BEesmSZbuZDnXN8,6102
-climate_ref_core-0.7.0.dist-info/METADATA,sha256=OLUdTWE6F70mVvm9Uo47DGYdHVrVLtoQvbBuUsFn3GQ,2925
-climate_ref_core-0.7.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-climate_ref_core-0.7.0.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-climate_ref_core-0.7.0.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
-climate_ref_core-0.7.0.dist-info/RECORD,,
+climate_ref_core-0.8.0.dist-info/METADATA,sha256=C3q17g9WyPdI2YBl4SDpLiM1LCCYhAgFoLhq2ccKUOo,2925
+climate_ref_core-0.8.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+climate_ref_core-0.8.0.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+climate_ref_core-0.8.0.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
+climate_ref_core-0.8.0.dist-info/RECORD,,

{climate_ref_core-0.7.0.dist-info → climate_ref_core-0.8.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any