climate-ref-core 0.6.6__tar.gz → 0.8.0__tar.gz

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref-core
-Version: 0.6.6
+Version: 0.8.0
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref-core"
-version = "0.6.6"
+version = "0.8.0"
 description = "Core library for the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/src/climate_ref_core/constraints.py

@@ -6,6 +6,8 @@ import sys
 import warnings
 from collections import defaultdict
 from collections.abc import Mapping
+from datetime import datetime
+from functools import total_ordering
 from typing import Literal, Protocol, runtime_checkable
 
 if sys.version_info < (3, 11):
@@ -15,45 +17,21 @@ else:
 
 import numpy as np
 import pandas as pd
-from attrs import frozen
+from attrs import field, frozen
 from loguru import logger
 
 from climate_ref_core.datasets import SourceDatasetType
-from climate_ref_core.exceptions import ConstraintNotSatisfied
 
 
 @runtime_checkable
-class GroupValidator(Protocol):
+class GroupConstraint(Protocol):
     """
-    A constraint that must be satisfied when executing a given diagnostic run.
-
-    All constraints must be satisfied for a given group to be run.
-    """
-
-    def validate(self, group: pd.DataFrame) -> bool:
-        """
-        Validate if the constraint is satisfied by the dataset.
-
-        This is executed after the apply method to determine if the constraint is satisfied.
-        If the constraint is not satisfied, the group will not be executed.
-
-        Parameters
-        ----------
-        group
-            A group of datasets that is being validated.
-
-        Returns
-        -------
-        :
-            Whether the constraint is satisfied
-        """
-        ...
+    An operation to perform on a group of datasets resulting in a new group of datasets.
 
+    This is applied to a group of datasets representing the inputs to a potential diagnostic execution.
 
-@runtime_checkable
-class GroupOperation(Protocol):
-    """
-    An operation to perform on a group of datasets resulting in a new group of datasets.
+    If the operation results in an empty group, the constraint is considered not satisfied.
+    The group must satisfy all constraints to be processed.
 
     !! warning
 
@@ -90,18 +68,6 @@ class GroupOperation(Protocol):
         ...
 
 
-GroupConstraint = GroupOperation | GroupValidator
-"""
-A constraint that must be satisfied for a group of datasets to be executed.
-
-This is applied to a group of datasets representing the inputs to a potential diagnostic execution.
-The group must satisfy all constraints to be processed.
-
-This can include operations that are applied to a group of datasets which may modify the group,
-but may also include validators that check if the group satisfies a certain condition.
-"""
-
-
 def apply_constraint(
     dataframe: pd.DataFrame,
     constraint: GroupConstraint,
@@ -124,41 +90,95 @@ def apply_constraint(
     :
         The updated group of datasets or None if the constraint was not satisfied
     """
-    try:
-        updated_group = (
-            constraint.apply(dataframe, data_catalog) if isinstance(constraint, GroupOperation) else dataframe
-        )
-
-        valid = constraint.validate(updated_group) if isinstance(constraint, GroupValidator) else True
-        if not valid:
-            logger.debug(f"Constraint {constraint} not satisfied for {dataframe}")
-            raise ConstraintNotSatisfied(f"Constraint {constraint} not satisfied for {dataframe}")
-    except ConstraintNotSatisfied:
+    updated_group = constraint.apply(dataframe, data_catalog)
+    if updated_group.empty:
         logger.debug(f"Constraint {constraint} not satisfied for {dataframe}")
         return None
 
     return updated_group
 
 
+def _to_tuple(value: None | str | tuple[str, ...]) -> tuple[str, ...]:
+    """
+    Normalize value to a tuple of strings.
+    """
+    if value is None:
+        return ()
+    if isinstance(value, str):
+        return (value,)
+    return tuple(value)
+
+
+def _to_tuple_dict(value: dict[str, str | tuple[str, ...]]) -> dict[str, tuple[str, ...]]:
+    """
+    Normalize value to a dict of tuples of strings.
+    """
+    return {k: _to_tuple(v) for k, v in value.items()}
+
+
 @frozen
 class RequireFacets:
     """
-    A constraint that requires a dataset to have certain facets.
+    A constraint that requires datasets to have certain facet values.
     """
 
     dimension: str
-    required_facets: tuple[str, ...]
+    """The name of the facet to filter on."""
+
+    required_facets: tuple[str, ...] = field(converter=_to_tuple)
+    "The required facet values."
+
     operator: Literal["all", "any"] = "all"
+    """Whether all or any of the required facets must be present."""
 
-    def validate(self, group: pd.DataFrame) -> bool:
+    group_by: tuple[str, ...] | None = field(converter=_to_tuple, default=None)
+    """
+    The facets to group the datasets by.
+
+    Each group created by `group_by` must contain at least one dataset where the
+    value of the given dimension is in the list of required facet values.
+
+    For example, if there are multiple models and variables in the selection,
+    `group_by` can be used to make sure that only those models are selected that
+    provide all required variables.
+    """
+
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
         """
-        Check that the required facets are present in the group
+        Filter out groups of datasets that do not provide the required facets
         """
-        if self.dimension not in group:
-            logger.warning(f"Dimension {self.dimension} not present in group {group}")
-            return False
         op = all if self.operator == "all" else any
-        return op(value in group[self.dimension].values for value in self.required_facets)
+        select = pd.Series(True, index=group.index)
+        groups = [group] if not self.group_by else (g[1] for g in group.groupby(list(self.group_by)))
+        for subgroup in groups:
+            if not op(value in subgroup[self.dimension].values for value in self.required_facets):
+                logger.debug(
+                    f"Constraint {self} not satisfied because required facet values "
+                    f"not found for group {', '.join(subgroup['path'])}"
+                )
+                select.loc[subgroup.index] = False
+        return group[select]
+
+
+@frozen
+class IgnoreFacets:
+    """
+    A constraint that ignores certain facet values.
+
+    Datasets with these facet values are removed from the selection.
+    """
+
+    facets: dict[str, str | tuple[str, ...]] = field(converter=_to_tuple_dict)
+    """The facet values to ignore."""
+
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Filter out datasets with the ignored facets.
+        """
+        mask = group[list(self.facets)].isin(self.facets).all(axis="columns")
+        if mask.any():
+            logger.debug(f"Ignoring files {', '.join(group.loc[mask, 'path'])} becauseof {self}")
+        return group[~mask]
 
 
 @frozen
@@ -273,6 +293,123 @@ class AddSupplementaryDataset:
         return cls(supplementary_facets, **kwargs[source_type])
 
 
+@frozen
+@total_ordering
+class PartialDateTime:  # noqa: PLW1641
+    """
+    A partial datetime object that can be used to compare datetimes.
+
+    Only the specified fields are used for comparison.
+    """
+
+    year: int | None = None
+    month: int | None = None
+    day: int | None = None
+    hour: int | None = None
+    minute: int | None = None
+    second: int | None = None
+
+    @property
+    def _attrs(self) -> dict[str, int]:
+        """The attributes that are set."""
+        return {
+            a: v
+            for a in self.__slots__  # type: ignore[attr-defined]
+            if not a.startswith("_") and (v := getattr(self, a)) is not None
+        }
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({', '.join(f'{a}={v}' for a, v in self._attrs.items())})"
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, datetime):
+            msg = (
+                f"Can only compare PartialDateTime with `datetime.datetime` "
+                f"objects, got object {other} of type {type(other)}"
+            )
+            raise TypeError(msg)
+
+        for attr, value in self._attrs.items():
+            other_value = getattr(other, attr)
+            if value != other_value:
+                return False
+        return True
+
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, datetime):
+            msg = (
+                f"Can only compare PartialDateTime with `datetime.datetime` "
+                f"objects, got object {other} of type {type(other)}"
+            )
+            raise TypeError(msg)
+
+        for attr, value in self._attrs.items():
+            other_value = getattr(other, attr)
+            if value != other_value:
+                return value < other_value  # type: ignore[no-any-return]
+        return False
+
+
+@frozen
+class RequireTimerange:
+    """
+    A constraint that requires datasets to have a specific timerange.
+
+    Specify the start and/or end of the required timerange using a precision
+    that matches the frequency of the datasets.
+
+    For example, to ensure that datasets at monthly frequency cover the period
+    from 2000 to 2010, use start=PartialDateTime(year=2000, month=1) and
+    end=PartialDateTime(year=2010, month=12).
+    """
+
+    group_by: tuple[str, ...]
+    """
+    The fields to group the datasets by. Groups that do not cover the timerange
+    will be removed.
+    """
+
+    start: PartialDateTime | None = None
+    """
+    The start time of the required timerange. If None, no start time is required.
+    """
+
+    end: PartialDateTime | None = None
+    """
+    The end time of the required timerange. If None, no end time is required.
+    """
+
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Check that all subgroups of the group have a contiguous timerange.
+        """
+        select = pd.Series(True, index=group.index)
+        for _, subgroup in group.dropna(subset=["start_time", "end_time"]).groupby(list(self.group_by)):
+            start = subgroup["start_time"].min()
+            end = subgroup["end_time"].max()
+            result = True
+            if self.start is not None and start > self.start:
+                logger.debug(
+                    f"Constraint {self} not satisfied because start time {start} "
+                    f"is after required start time for {', '.join(subgroup['path'])}"
+                )
+                result = False
+            if self.end is not None and end < self.end:
+                logger.debug(
+                    f"Constraint {self} not satisfied because end time {end} "
+                    f"is before required end time for {', '.join(subgroup['path'])}"
+                )
+                result = False
+            if result:
+                contiguous_subgroup = RequireContiguousTimerange(group_by=self.group_by).apply(
+                    subgroup, data_catalog
+                )
+                result = len(contiguous_subgroup) == len(subgroup)
+            if not result:
+                select.loc[subgroup.index] = False
+        return group[select]
+
+
 @frozen
 class RequireContiguousTimerange:
     """
@@ -281,11 +418,11 @@ class RequireContiguousTimerange:
 
     group_by: tuple[str, ...]
     """
-    The fields to group the datasets by. Each group must be contiguous in time
-    to fulfill the constraint.
+    The fields to group the datasets by. Groups that are not be contiguous in time
+    are removed.
     """
 
-    def validate(self, group: pd.DataFrame) -> bool:
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
         """
         Check that all subgroups of the group have a contiguous timerange.
         """
@@ -295,11 +432,10 @@ class RequireContiguousTimerange:
             days=31,  # Maximum number of days in a month.
             hours=1,  # Allow for potential rounding errors.
         )
-        group = group.dropna(subset=["start_time", "end_time"])
-        if len(group) < 2:  # noqa: PLR2004
-            return True
 
-        for _, subgroup in group.groupby(list(self.group_by)):
+        select = pd.Series(True, index=group.index)
+
+        for _, subgroup in group.dropna(subset=["start_time", "end_time"]).groupby(list(self.group_by)):
             if len(subgroup) < 2:  # noqa: PLR2004
                 continue
             sorted_group = subgroup.sort_values("start_time", kind="stable")
@@ -327,12 +463,13 @@ class RequireContiguousTimerange:
                 paths = sorted_group["path"]
                 for gap_idx in np.flatnonzero(gap_indices):
                     logger.debug(
-                        f"Constraint {self.__class__.__name__} not satisfied "
-                        f"because gap larger than {max_timedelta} found between "
+                        f"Constraint {self} not satisfied because gap larger "
+                        f"than {max_timedelta} found between "
                         f"{paths.iloc[gap_idx]} and {paths.iloc[gap_idx + 1]}"
                     )
-                return False
-        return True
+                select.loc[subgroup.index] = False
+
+        return group[select]
 
 
 @frozen
@@ -347,17 +484,24 @@ class RequireOverlappingTimerange:
     the groups to fulfill the constraint.
     """
 
-    def validate(self, group: pd.DataFrame) -> bool:
+    def apply(self, group: pd.DataFrame, data_catalog: pd.DataFrame) -> pd.DataFrame:
         """
         Check that all subgroups of the group have an overlapping timerange.
         """
-        group = group.dropna(subset=["start_time", "end_time"])
-        if len(group) < 2:  # noqa: PLR2004
-            return True
-
-        starts = group.groupby(list(self.group_by))["start_time"].min()
-        ends = group.groupby(list(self.group_by))["end_time"].max()
-        return starts.max() < ends.min()  # type: ignore[no-any-return]
+        group_with_time = group.dropna(subset=["start_time", "end_time"])
+        if len(group_with_time) < 2:  # noqa: PLR2004
+            return group
+
+        starts = group_with_time.groupby(list(self.group_by))["start_time"].min()
+        ends = group_with_time.groupby(list(self.group_by))["end_time"].max()
+        result = starts.max() < ends.min()
+        if not result:
+            logger.debug(
+                f"Constraint {self} not satisfied because no overlapping timerange "
+                f"found for groups in {', '.join(group['path'])}"
+            )
+            return group.loc[[]]
+        return group
 
 
 @frozen

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/src/climate_ref_core/datasets.py

@@ -76,12 +76,6 @@ class FacetFilter:
     The result will only contain datasets where for all fields,
     the value of the field is one of the given values.
     """
-    keep: bool = True
-    """
-    Whether to keep or remove datasets that match the filter.
-
-    If true (default), datasets that match the filter will be kept else they will be removed.
-    """
 
 
 def sort_selector(inp: Selector) -> Selector:
@@ -159,6 +153,9 @@ class ExecutionDatasetCollection:
     def __init__(self, collection: dict[SourceDatasetType | str, DatasetCollection]):
         self._collection = {SourceDatasetType(k): v for k, v in collection.items()}
 
+    def __repr__(self) -> str:
+        return f"ExecutionDatasetCollection({self._collection})"
+
     def __contains__(self, key: SourceDatasetType | str) -> bool:
         if isinstance(key, str):
             key = SourceDatasetType(key)

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/src/climate_ref_core/diagnostics.py

@@ -321,7 +321,12 @@ class DataRequirement:
     Filters to apply to the data catalog of datasets.
 
     This is used to reduce the set of datasets to only those that are required by the diagnostic.
-    The filters are applied iteratively to reduce the set of datasets.
+
+    Each FacetFilter contains one or more facet values that must all be satisfied
+    for a dataset to match that filter. The overall selection keeps any dataset
+    that matches at least one of the provided filters.
+
+    If no filters are specified, all datasets in the data catalog are used.
     """
 
     group_by: tuple[str, ...] | None
@@ -361,6 +366,10 @@ class DataRequirement:
         :
             Filtered data catalog
         """
+        if not self.filters or any(not f.facets for f in self.filters):
+            return data_catalog
+
+        select = pd.Series(False, index=data_catalog.index)
         for facet_filter in self.filters:
             values = {}
             for facet, value in facet_filter.facets.items():
@@ -372,11 +381,9 @@ class DataRequirement:
                     )
                 values[facet] = clean_value
 
-            mask = data_catalog[list(values)].isin(values).all(axis="columns")
-            if not facet_filter.keep:
-                mask = ~mask
-            data_catalog = data_catalog[mask]
-        return data_catalog
+            select |= data_catalog[list(values)].isin(values).all(axis="columns")
+
+        return data_catalog[select]
 
 
 @runtime_checkable

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/src/climate_ref_core/logging.py

@@ -9,6 +9,7 @@ import contextlib
 import inspect
 import logging
 import multiprocessing
+import os
 import sys
 from collections.abc import Generator
 from pathlib import Path
@@ -94,7 +95,10 @@ def initialise_logging(level: int | str, format: str, log_directory: str | Path)
     logger.info("Starting REF logging")
     logger.info(f"arguments: {sys.argv}")
 
-    add_log_handler(level=level, format=format, colorize=True)
+    # LOGURU_COLORIZE is the default env var used by loguru to determine if color should be used
+    # We override this to use NO_COLOR which is more widely supported
+    no_color = os.environ.get("NO_COLOR") is not None
+    add_log_handler(level=level, format=format, colorize=not no_color)
 
 
 def capture_logging() -> None:
@@ -150,12 +154,12 @@ def remove_log_handler() -> None:
     """
     if hasattr(logger, "default_handler_id"):
         try:
-            logger.remove(logger.default_handler_id)
+            logger.remove(logger.default_handler_id)  # pyright: ignore[reportAttributeAccessIssue]
         except ValueError:
             # This can happen if the handler has already been removed
             # or if the logger was never configured
             pass
-        del logger.default_handler_id
+        del logger.default_handler_id  # pyright: ignore[reportAttributeAccessIssue]
     else:
         raise AssertionError("No default log handler to remove.")
 

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/src/climate_ref_core/metric_values/typing.py

@@ -3,7 +3,8 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Self
 
-from pydantic import BaseModel, model_validator
+import numpy as np
+from pydantic import BaseModel, field_validator, model_validator
 
 Value = float | int
 
@@ -64,20 +65,35 @@ class SeriesMetricValue(BaseModel):
     This is used for presentation purposes and is not used in the controlled vocabulary.
     """
 
-    attributes: dict[str, str | Value] | None = None
+    attributes: dict[str, str | Value | None] | None = None
     """
     Additional unstructured attributes associated with the metric value
     """
 
     @model_validator(mode="after")
-    def validate_index_length(self) -> Self:
-        """Validate that index has the same length as values"""
+    def validate_index(self) -> Self:
+        """Validate that index has the same length as values and contains no NaNs"""
         if len(self.index) != len(self.values):
             raise ValueError(
                 f"Index length ({len(self.index)}) must match values length ({len(self.values)})"
             )
+        for v in self.index:
+            if isinstance(v, float) and not np.isfinite(v):
+                raise ValueError("NaN or Inf values are not allowed in the index")
         return self
 
+    @field_validator("values", mode="before")
+    @classmethod
+    def validate_values(cls, value: Any) -> Any:
+        """
+        Transform None values to NaN in the values field
+        """
+        if not isinstance(value, (list, tuple)):
+            raise ValueError("`values` must be a list or tuple.")
+
+        # Transform None values to NaN
+        return [float("nan") if v is None else v for v in value]
+
     @classmethod
     def dump_to_json(cls, path: Path, series: Sequence["SeriesMetricValue"]) -> None:
         """
@@ -94,7 +110,13 @@ class SeriesMetricValue(BaseModel):
             The series values to dump.
         """
         with open(path, "w") as f:
-            json.dump([s.model_dump() for s in series], f, indent=2)
+            json.dump(
+                [s.model_dump(mode="json") for s in series],
+                f,
+                indent=2,
+                allow_nan=False,
+                sort_keys=True,
+            )
 
     @classmethod
     def load_from_json(
@@ -102,7 +124,7 @@ class SeriesMetricValue(BaseModel):
         path: Path,
     ) -> list["SeriesMetricValue"]:
         """
-        Dump a sequence of SeriesMetricValue to a JSON file.
+        Load a sequence of SeriesMetricValue from a JSON file.
 
         Parameters
         ----------
@@ -115,7 +137,7 @@ class SeriesMetricValue(BaseModel):
         if not isinstance(data, list):
             raise ValueError(f"Expected a list of series values, got {type(data)}")
 
-        return [cls.model_validate(s) for s in data]
+        return [cls.model_validate(s, strict=True) for s in data]
 
 
 class ScalarMetricValue(BaseModel):

{climate_ref_core-0.6.6 → climate_ref_core-0.8.0}/src/climate_ref_core/providers.py

@@ -16,14 +16,18 @@ import os
 import stat
 import subprocess
 from abc import abstractmethod
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from contextlib import AbstractContextManager
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import requests
+import yaml
+from attrs import evolve
 from loguru import logger
 
+from climate_ref_core.constraints import IgnoreFacets
+from climate_ref_core.datasets import SourceDatasetType
 from climate_ref_core.diagnostics import Diagnostic
 from climate_ref_core.exceptions import InvalidDiagnosticException, InvalidProviderException
 
@@ -74,6 +78,51 @@ class DiagnosticProvider:
         config :
             A configuration.
         """
+        logger.debug(
+            f"Configuring provider {self.slug} using ignore_datasets_file {config.ignore_datasets_file}"
+        )
+        # The format of the configuration file is:
+        # provider:
+        #   diagnostic:
+        #     source_type:
+        #       - facet: value
+        #       - other_facet: [other_value1, other_value2]
+        ignore_datasets_all = yaml.safe_load(config.ignore_datasets_file.read_text(encoding="utf-8")) or {}
+        ignore_datasets = ignore_datasets_all.get(self.slug, {})
+        if unknown_slugs := {slug for slug in ignore_datasets} - {d.slug for d in self.diagnostics()}:
+            logger.warning(
+                f"Unknown diagnostics found in {config.ignore_datasets_file} "
+                f"for provider {self.slug}: {', '.join(sorted(unknown_slugs))}"
+            )
+
+        known_source_types = {s.value for s in iter(SourceDatasetType)}
+        for diagnostic in self.diagnostics():
+            if diagnostic.slug in ignore_datasets:
+                if unknown_source_types := set(ignore_datasets[diagnostic.slug]) - known_source_types:
+                    logger.warning(
+                        f"Unknown source types found in {config.ignore_datasets_file} for "
+                        f"diagnostic '{diagnostic.slug}' by provider {self.slug}: "
+                        f"{', '.join(sorted(unknown_source_types))}"
+                    )
+                data_requirements = (
+                    r if isinstance(r, Sequence) else (r,) for r in diagnostic.data_requirements
+                )
+                diagnostic.data_requirements = tuple(
+                    tuple(
+                        evolve(
+                            data_requirement,
+                            constraints=tuple(
+                                IgnoreFacets(facets)
+                                for facets in ignore_datasets[diagnostic.slug].get(
+                                    data_requirement.source_type.value, []
+                                )
+                            )
+                            + data_requirement.constraints,
+                        )
+                        for data_requirement in requirement_collection
+                    )
+                    for requirement_collection in data_requirements
+                )
 
     def diagnostics(self) -> list[Diagnostic]:
         """
@@ -287,6 +336,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
 
     def configure(self, config: Config) -> None:
         """Configure the provider."""
+        super().configure(config)
         self.prefix = config.paths.software / "conda"
 
     def _install_conda(self, update: bool) -> Path: