climate-ref-core 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

climate_ref_core/constraints.py

@@ -6,7 +6,7 @@ import sys
 import warnings
 from collections import defaultdict
 from collections.abc import Mapping
-from typing import Protocol, runtime_checkable
+from typing import Literal, Protocol, runtime_checkable
 
 if sys.version_info < (3, 11):
     from typing_extensions import Self
@@ -148,6 +148,7 @@ class RequireFacets:
 
     dimension: str
     required_facets: tuple[str, ...]
+    operator: Literal["all", "any"] = "all"
 
     def validate(self, group: pd.DataFrame) -> bool:
         """
@@ -156,7 +157,8 @@ class RequireFacets:
         if self.dimension not in group:
             logger.warning(f"Dimension {self.dimension} not present in group {group}")
             return False
-        return all(value in group[self.dimension].values for value in self.required_facets)
+        op = all if self.operator == "all" else any
+        return op(value in group[self.dimension].values for value in self.required_facets)
 
 
 @frozen
@@ -200,17 +202,27 @@ class AddSupplementaryDataset:
         for facet, values in supplementary_facets.items():
             mask = supplementary_group[facet].isin(values)
             supplementary_group = supplementary_group[mask]
-
-        if not supplementary_group.empty and self.optional_matching_facets:
-            facets = list(self.matching_facets + self.optional_matching_facets)
+        if not supplementary_group.empty:
+            matching_facets = list(self.matching_facets)
+            facets = matching_facets + list(self.optional_matching_facets)
             datasets = group[facets].drop_duplicates()
             indices = set()
             for i in range(len(datasets)):
-                scores = (supplementary_group[facets] == datasets.iloc[i]).sum(axis=1)
-                matches = supplementary_group[scores == scores.max()]
-                # Select the latest version if there are multiple matches
-                matches = matches[matches["version"] == matches["version"].max()]
-                indices.add(matches.index[0])
+                dataset = datasets.iloc[i]
+                # Restrict the supplementary datasets to those that match the main dataset.
+                supplementaries = supplementary_group[
+                    (supplementary_group[matching_facets] == dataset[matching_facets]).all(1)
+                ]
+                if not supplementaries.empty:
+                    # Select the best matching supplementary dataset based on the optional matching facets.
+                    scores = (supplementaries[facets] == dataset).sum(axis=1)
+                    matches = supplementaries[scores == scores.max()]
+                    if "version" in facets:
+                        # Select the latest version if there are multiple matches
+                        matches = matches[matches["version"] == matches["version"].max()]
+                    # Select one match per dataset
+                    indices.add(matches.index[0])
+
             supplementary_group = supplementary_group.loc[list(indices)].drop_duplicates()
 
         return pd.concat([group, supplementary_group])

climate_ref_core/datasets.py

@@ -5,7 +5,7 @@ Dataset management and filtering
 import enum
 import functools
 import hashlib
-from collections.abc import Collection, Iterable
+from collections.abc import Collection, Iterable, Iterator
 from typing import Any, Self
 
 import pandas as pd
@@ -172,9 +172,24 @@ class ExecutionDatasetCollection:
     def __hash__(self) -> int:
         return hash(self.hash)
 
+    def __iter__(self) -> Iterator[SourceDatasetType]:
+        return iter(self._collection)
+
+    def keys(self) -> Iterable[SourceDatasetType]:
+        """
+        Iterate over the source types in the collection.
+        """
+        return self._collection.keys()
+
+    def values(self) -> Iterable[DatasetCollection]:
+        """
+        Iterate over the datasets in the collection.
+        """
+        return self._collection.values()
+
     def items(self) -> Iterable[tuple[SourceDatasetType, DatasetCollection]]:
         """
-        Iterate over the datasets in the collection
+        Iterate over the items in the collection.
         """
         return self._collection.items()
 

climate_ref_core/diagnostics.py

@@ -14,6 +14,7 @@ from attrs import field, frozen
 from climate_ref_core.constraints import GroupConstraint
 from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
 from climate_ref_core.metric_values import SeriesMetricValue
+from climate_ref_core.metric_values.typing import SeriesDefinition
 from climate_ref_core.pycmec.metric import CMECMetric
 from climate_ref_core.pycmec.output import CMECOutput
 
@@ -182,9 +183,11 @@ class ExecutionResult:
     Whether the diagnostic execution ran successfully.
     """
 
-    series: Sequence[SeriesMetricValue] = field(factory=tuple)
+    series_filename: pathlib.Path | None = None
     """
     A collection of series metric values that were extracted from the execution.
+
+    These are written to a CSV file in the output directory.
     """
 
     @staticmethod
@@ -193,6 +196,7 @@ class ExecutionResult:
         *,
         cmec_output_bundle: CMECOutput | dict[str, Any],
         cmec_metric_bundle: CMECMetric | dict[str, Any],
+        series: Sequence[SeriesMetricValue] = tuple(),
     ) -> ExecutionResult:
         """
         Build a ExecutionResult from a CMEC output bundle.
@@ -205,6 +209,8 @@ class ExecutionResult:
             An output bundle in the CMEC format.
         cmec_metric_bundle
             An diagnostic bundle in the CMEC format.
+        series
+            Series metric values extracted from the execution.
 
         Returns
         -------
@@ -223,17 +229,21 @@ class ExecutionResult:
             cmec_metric = cmec_metric_bundle
 
         definition.to_output_path(filename=None).mkdir(parents=True, exist_ok=True)
-        bundle_path = definition.to_output_path("output.json")
-        cmec_output.dump_to_json(bundle_path)
 
-        definition.to_output_path(filename=None).mkdir(parents=True, exist_ok=True)
-        bundle_path = definition.to_output_path("diagnostic.json")
-        cmec_metric.dump_to_json(bundle_path)
+        output_filename = "output.json"
+        metric_filename = "diagnostic.json"
+        series_filename = "series.json"
 
+        cmec_output.dump_to_json(definition.to_output_path(output_filename))
+        cmec_metric.dump_to_json(definition.to_output_path(metric_filename))
+        SeriesMetricValue.dump_to_json(definition.to_output_path(series_filename), series)
+
+        # We are using relative paths for the output files for portability of the results
         return ExecutionResult(
             definition=definition,
-            output_bundle_filename=pathlib.Path("output.json"),
-            metric_bundle_filename=pathlib.Path("diagnostic.json"),
+            output_bundle_filename=pathlib.Path(output_filename),
+            metric_bundle_filename=pathlib.Path(metric_filename),
+            series_filename=pathlib.Path(series_filename),
             successful=True,
         )
 
@@ -432,6 +442,11 @@ class AbstractDiagnostic(Protocol):
     is raised.
     """
 
+    series: Sequence[SeriesDefinition]
+    """
+    Definition of the series that are produced by the diagnostic.
+    """
+
     provider: DiagnosticProvider
     """
     The provider that provides the diagnostic.
@@ -493,6 +508,8 @@ class Diagnostic(AbstractDiagnostic):
     See (climate_ref_example.example.ExampleDiagnostic)[] for an example implementation.
     """
 
+    series: Sequence[SeriesDefinition] = tuple()
+
     def __init__(self) -> None:
         super().__init__()
         self._provider: DiagnosticProvider | None = None

climate_ref_core/executor.py

@@ -160,12 +160,15 @@ def import_executor_cls(fqn: str) -> type[Executor]:
         imp = importlib.import_module(module)
         executor: type[Executor] = getattr(imp, attribute_name)
 
+        if isinstance(executor, Exception):
+            raise executor
+
         # We can't really check if the executor is a subclass of Executor here
         # Protocols can't be used with issubclass if they have non-method members
         # We have to check this at class instantiation time
 
         return executor
-    except ModuleNotFoundError:
+    except (ModuleNotFoundError, ImportError):
         logger.error(f"Package '{fqn}' not found")
         raise InvalidExecutorException(fqn, f"Module '{module}' not found")
     except AttributeError:

climate_ref_core/metric_values/typing.py

@@ -1,11 +1,37 @@
+import json
 from collections.abc import Sequence
-from typing import Self
+from pathlib import Path
+from typing import Any, Self
 
 from pydantic import BaseModel, model_validator
 
 Value = float | int
 
 
+class SeriesDefinition(BaseModel):
+    """
+    A definition of a 1-d array with an associated index and additional dimensions.
+    """
+
+    file_pattern: str
+    """A glob pattern to match files that contain the series values."""
+
+    sel: dict[str, Any] | None = None
+    """A dictionary of selection criteria to apply with :meth:`xarray.Dataset.sel` after loading the file."""
+
+    dimensions: dict[str, str]
+    """Key, value pairs that identify the dimensions of the metric."""
+
+    values_name: str
+    """The name of the variable in the file that contains the values of the series."""
+
+    index_name: str
+    """The name of the variable in the file that contains the index of the series."""
+
+    attributes: Sequence[str]
+    """A list of attributes that should be extracted from the file and included in the series metadata."""
+
+
 class SeriesMetricValue(BaseModel):
     """
     A 1-d array with an associated index and additional dimensions
@@ -52,6 +78,45 @@ class SeriesMetricValue(BaseModel):
             )
         return self
 
+    @classmethod
+    def dump_to_json(cls, path: Path, series: Sequence["SeriesMetricValue"]) -> None:
+        """
+        Dump a sequence of SeriesMetricValue to a JSON file.
+
+        Parameters
+        ----------
+        path
+            The path to the JSON file.
+
+            The directory containing this file must already exist.
+            This file will be overwritten if it already exists.
+        series
+            The series values to dump.
+        """
+        with open(path, "w") as f:
+            json.dump([s.model_dump() for s in series], f, indent=2)
+
+    @classmethod
+    def load_from_json(
+        cls,
+        path: Path,
+    ) -> list["SeriesMetricValue"]:
+        """
+        Dump a sequence of SeriesMetricValue to a JSON file.
+
+        Parameters
+        ----------
+        path
+            The path to the JSON file.
+        """
+        with open(path) as f:
+            data = json.load(f)
+
+        if not isinstance(data, list):
+            raise ValueError(f"Expected a list of series values, got {type(data)}")
+
+        return [cls.model_validate(s) for s in data]
+
 
 class ScalarMetricValue(BaseModel):
     """

climate_ref_core/providers.py

@@ -232,6 +232,27 @@ def _get_micromamba_url() -> str:
 class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
     """
     A provider for diagnostics that can be run from the command line in a conda environment.
+
+    Parameters
+    ----------
+    name
+        The name of the provider.
+    version
+        The version of the provider.
+    slug
+        A slugified version of the name.
+    repo
+        URL of the git repository to install a development version of the package from.
+    tag_or_commit
+        Tag or commit to install from the `repo` repository.
+
+    Attributes
+    ----------
+    env_vars
+        Environment variables to set when running commands in the conda environment.
+    url
+        URL to install a development version of the package from.
+
     """
 
     def __init__(
@@ -246,6 +267,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
         self._conda_exe: Path | None = None
         self._prefix: Path | None = None
         self.url = f"git+{repo}@{tag_or_commit}" if repo and tag_or_commit else None
+        self.env_vars: dict[str, str] = {}
 
     @property
     def prefix(self) -> Path:
@@ -404,6 +426,8 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
             *cmd,
         ]
         logger.info(f"Running '{' '.join(cmd)}'")
+        env_vars = os.environ.copy()
+        env_vars.update(self.env_vars)
         try:
             # This captures the log output until the execution is complete
             # We could poll using `subprocess.Popen` if we want something more responsive
@@ -413,6 +437,7 @@ class CondaDiagnosticProvider(CommandLineDiagnosticProvider):
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 text=True,
+                env=env_vars,
             )
             logger.info("Command output: \n" + res.stdout)
             logger.info("Command execution successful")

climate_ref_core/pycmec/controlled_vocabulary.py

@@ -1,4 +1,5 @@
 import pathlib
+from collections.abc import Iterable, Sequence
 from typing import Any
 
 from attrs import field, frozen, validators
@@ -7,6 +8,7 @@ from loguru import logger
 from yaml import safe_load
 
 from climate_ref_core.exceptions import ResultValidationError
+from climate_ref_core.metric_values import ScalarMetricValue, SeriesMetricValue
 from climate_ref_core.pycmec.metric import CMECMetric
 
 RESERVED_DIMENSION_NAMES = {"attributes", "json_structure", "created_at", "updated_at", "value", "id"}
@@ -122,33 +124,49 @@ class CV:
                 return dim
         raise KeyError(f"Dimension {name} not found")
 
-    def validate_metrics(self, metric_bundle: CMECMetric) -> None:
+    def _validate_value(self, metric_value: ScalarMetricValue | SeriesMetricValue) -> None:
         """
-        Validate a diagnostic bundle against a CV
+        Validate a single metric value against the CV
+        """
+        for k, v in metric_value.dimensions.items():
+            try:
+                dimension = self.get_dimension_by_name(k)
+            except KeyError:
+                raise ResultValidationError(f"Unknown dimension: {k!r}")
+            if not dimension.allow_extra_values:
+                if v not in [dv.name for dv in dimension.values]:
+                    raise ResultValidationError(f"Unknown value {v!r} for dimension {k!r}")
+
+        if hasattr(metric_value, "value") and not isinstance(metric_value.value, float):  # pragma: no cover
+            # This may not be possible with the current CMECMetric implementation
+            raise ResultValidationError(f"Unexpected value: {metric_value.value!r}")
+
+    def validate_metrics(self, metric_value_collection: CMECMetric | Sequence[SeriesMetricValue]) -> None:
+        """
+        Validate a set of metric values (either scalar or series) against a CV
 
         The CV describes the accepted dimensions and values within a bundle
 
         Parameters
         ----------
-        metric_bundle
+        metric_value_collection
+            A collection of metric values to validate.
+
+            This can be a CMECMetric instance or a sequence of SeriesMetricValue instances.
 
         Raises
         ------
         ResultValidationError
             If the validation of the dimensions or values fails
         """
-        for result in metric_bundle.iter_results():
-            for k, v in result.dimensions.items():
-                try:
-                    dimension = self.get_dimension_by_name(k)
-                except KeyError:
-                    raise ResultValidationError(f"Unknown dimension: {k!r}")
-                if not dimension.allow_extra_values:
-                    if v not in [dv.name for dv in dimension.values]:
-                        raise ResultValidationError(f"Unknown value {v!r} for dimension {k!r}")
-            if not isinstance(result.value, float):  # pragma: no cover
-                # This may not be possible with the current CMECMetric implementation
-                raise ResultValidationError(f"Unexpected value: {result.value!r}")
+        generator: Iterable[SeriesMetricValue | ScalarMetricValue]
+        if isinstance(metric_value_collection, CMECMetric):
+            generator = metric_value_collection.iter_results()
+        else:
+            generator = iter(metric_value_collection)
+
+        for result in generator:
+            self._validate_value(result)
 
     @staticmethod
     def load_from_file(filename: pathlib.Path | str) -> "CV":

{climate_ref_core-0.6.4.dist-info → climate_ref_core-0.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref-core
-Version: 0.6.4
+Version: 0.6.6
 Summary: Core library for the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0
@@ -29,7 +29,7 @@ Requires-Dist: pydantic>=2.10.6
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: requests
 Requires-Dist: rich
-Requires-Dist: setuptools>=75.8.0
+Requires-Dist: setuptools<81
 Requires-Dist: typing-extensions
 Description-Content-Type: text/markdown
 

{climate_ref_core-0.6.4.dist-info → climate_ref_core-0.6.6.dist-info}/RECORD

@@ -1,24 +1,24 @@
 climate_ref_core/__init__.py,sha256=MtmPThF2F9_2UODEN6rt1x30LDxrHIZ0wyRN_wsHx5I,127
-climate_ref_core/constraints.py,sha256=QOqMh5jDBxdWTnQw2HNBizJQDF6Uu97rfJp9WudQWHc,11819
+climate_ref_core/constraints.py,sha256=9dZPlBAPHwv8A4ZCK7pKj3EnP6SB5baASjVNW7nNrOY,12503
 climate_ref_core/dataset_registry.py,sha256=sQp2VT9xSVAaWsf0tF4E_VQxuEsvIxU2MZm5uNX1ynw,7172
-climate_ref_core/datasets.py,sha256=TK50WQwTfbase26s8wPEGEN1BwcedrOd8nk6IlEf3Ww,6124
-climate_ref_core/diagnostics.py,sha256=5KCtHuhToSpATqjW4HBi56PsOxT5WX4VkqoZPUvYR60,18769
+climate_ref_core/datasets.py,sha256=-Rto5NNx0WCHnJL4OSqB0-DnTgUrjpoovBC86OsEfbw,6570
+climate_ref_core/diagnostics.py,sha256=hhWik9WBjTMbQ-XW6AYPfBBZblP0LeTpffXIeqZ_yJA,19401
 climate_ref_core/env.py,sha256=Ph2dejVxTELfP3bL0xES086WLGvV5H6KvsOwCkL6m-k,753
 climate_ref_core/exceptions.py,sha256=7Mkz22P-kbiL-ZevAhlOuQaaeTio6zpwE9YA45OTGvs,1909
-climate_ref_core/executor.py,sha256=QiVOca-d9JxKIktQIinQQYZGr3ecV5mL3nvUwCdMiJQ,5372
+climate_ref_core/executor.py,sha256=9mKVkm0S7ikub3_FP7CrgdC4Qj9ynOi0r_DIfzCDS-0,5459
 climate_ref_core/logging.py,sha256=cg6CK2DHGjyLaoRJm75p-Ja82hnVhBBQ4riOKk3l9XY,7063
-climate_ref_core/providers.py,sha256=HZTpz1VFa9yZLPr1Cwofag2TCuwn6P4VPOrBaXbhn0A,12746
+climate_ref_core/providers.py,sha256=z5oD7EErIWprH5zv8I5yuU0IavEiSsi6SFkYMtiqE7g,13448
 climate_ref_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 climate_ref_core/metric_values/__init__.py,sha256=aHfwRrqzLOmmaBKf1-4q97DnHb8KwmW0Dhwd79ZQiNQ,634
-climate_ref_core/metric_values/typing.py,sha256=2DpzmjqQ7tqOPAyjthZ_O14c0-MhiYt-A_n9p6-bOao,1903
+climate_ref_core/metric_values/typing.py,sha256=4Qmr1LJQxQd2qkndwOzKTHq-hihiIBfWDZ_vzRWNbsI,3880
 climate_ref_core/pycmec/README.md,sha256=PzkovlPpsXqFopsYzz5GRvCAipNRGO1Wo-0gc17qr2Y,36
 climate_ref_core/pycmec/__init__.py,sha256=hXvKGEJQWyAp1i-ndr3D4zuYxkRhcR2LfXgFXlhYOk4,28
-climate_ref_core/pycmec/controlled_vocabulary.py,sha256=_GZ2Y6KAYF12I3IS27g8YdFBAcAONyGAhpucQ4oEcOA,5128
+climate_ref_core/pycmec/controlled_vocabulary.py,sha256=kgMEvQ1P6EwXC7sFgdC77IQDo8I0DnnQ2CPXXQaavjE,5944
 climate_ref_core/pycmec/cv_cmip7_aft.yaml,sha256=gx5QyW88pZQVUfiYXmsJtJO6AJg6NbIZgdU4vDIa3fE,4390
 climate_ref_core/pycmec/metric.py,sha256=zymXoutnjbdcxvG_fMJugFLLcBrfSPG0XoV-2tA0ujA,18499
 climate_ref_core/pycmec/output.py,sha256=4-RQ439sfgNLeQZVDPB1pewF_kTwX7nCK0Z4U6bvbd0,5709
-climate_ref_core-0.6.4.dist-info/METADATA,sha256=oYXP9COnofum1dw4zCrilxEK0AAC33wr032prB-z2-U,2930
-climate_ref_core-0.6.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-climate_ref_core-0.6.4.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-climate_ref_core-0.6.4.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
-climate_ref_core-0.6.4.dist-info/RECORD,,
+climate_ref_core-0.6.6.dist-info/METADATA,sha256=9vrBrbYGYhrERHV7pE1MmwsEPeEqrX83C9oLtaXUcBo,2925
+climate_ref_core-0.6.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+climate_ref_core-0.6.6.dist-info/licenses/LICENCE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+climate_ref_core-0.6.6.dist-info/licenses/NOTICE,sha256=4qTlax9aX2-mswYJuVrLqJ9jK1IkN5kSBqfVvYLF3Ws,128
+climate_ref_core-0.6.6.dist-info/RECORD,,