climate-ref 0.5.0__py3-none-any.whl

climate_ref/datasets/obs4mips.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+import re
+import traceback
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import xarray as xr
+from ecgtools import Builder
+from loguru import logger
+
+from climate_ref.datasets.base import DatasetAdapter
+from climate_ref.datasets.cmip6 import _parse_datetime
+from climate_ref.models.dataset import Dataset, Obs4MIPsDataset
+
+
+def extract_attr_with_regex(
+    input_str: str, regex: str, strip_chars: str | None, ignore_case: bool
+) -> list[Any] | None:
+    """
+    Extract version information from attribute with regular expressions.
+    """
+    if ignore_case:
+        pattern = re.compile(regex, re.IGNORECASE)
+    else:
+        pattern = re.compile(regex)
+    match = re.findall(pattern, input_str)
+    if match:
+        matchstr = max(match, key=len)
+        match = matchstr.strip(strip_chars) if strip_chars else matchstr.strip()
+        return match
+    else:
+        return None
+
+
+def parse_obs4mips(file: str) -> dict[str, Any | None]:
+    """Parser for obs4mips"""
+    keys = sorted(
+        list(
+            {
+                "activity_id",
+                "frequency",
+                "grid",
+                "grid_label",
+                "institution_id",
+                "nominal_resolution",
+                "realm",
+                "product",
+                "source_id",
+                "source_type",
+                "variable_id",
+                "variant_label",
+            }
+        )
+    )
+
+    try:
+        time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+        with xr.open_dataset(file, chunks={}, decode_times=time_coder) as ds:
+            has_none_value = any(ds.attrs.get(key) is None for key in keys)
+            if has_none_value:
+                missing_fields = [key for key in keys if ds.attrs.get(key) is None]
+                traceback_message = str(missing_fields) + " are missing from the file metadata"
+                raise AttributeError(traceback_message)
+            info = {key: ds.attrs.get(key) for key in keys}
+
+            if info["activity_id"] != "obs4MIPs":
+                traceback_message = f"{file} is not an obs4MIPs dataset"
+                raise TypeError(traceback_message)
+
+            variable_id = info["variable_id"]
+
+            if variable_id:
+                attrs = ds[variable_id].attrs
+                for attr in ["long_name", "units"]:
+                    info[attr] = attrs.get(attr)
+
+            # Set the default of # of vertical levels to 1
+            vertical_levels = 1
+            start_time, end_time = None, None
+            try:
+                vertical_levels = ds[ds.cf["vertical"].name].size
+            except (KeyError, AttributeError, ValueError):
+                ...
+            try:
+                start_time, end_time = str(ds.cf["T"][0].data), str(ds.cf["T"][-1].data)
+            except (KeyError, AttributeError, ValueError):
+                ...
+
+            info["vertical_levels"] = vertical_levels
+            info["start_time"] = start_time
+            info["end_time"] = end_time
+            if not (start_time and end_time):
+                info["time_range"] = None
+            else:
+                info["time_range"] = f"{start_time}-{end_time}"
+        info["path"] = str(file)
+        info["source_version_number"] = (
+            extract_attr_with_regex(
+                str(file), regex=r"v\d{4}\d{2}\d{2}|v\d{1}", strip_chars=None, ignore_case=True
+            )
+            or "v0"
+        )
+        return info
+
+    except (TypeError, AttributeError) as err:
+        if (len(err.args)) == 1:
+            logger.warning(str(err.args[0]))
+        else:
+            logger.warning(str(err.args))
+        return {"INVALID_ASSET": file, "TRACEBACK": traceback_message}
+    except Exception:
+        logger.warning(traceback.format_exc())
+        return {"INVALID_ASSET": file, "TRACEBACK": traceback.format_exc()}
+
+
+class Obs4MIPsDatasetAdapter(DatasetAdapter):
+    """
+    Adapter for obs4MIPs datasets
+    """
+
+    dataset_cls: type[Dataset] = Obs4MIPsDataset
+    slug_column = "instance_id"
+
+    dataset_specific_metadata = (
+        "activity_id",
+        "frequency",
+        "grid",
+        "grid_label",
+        "institution_id",
+        "nominal_resolution",
+        "product",
+        "realm",
+        "source_id",
+        "source_type",
+        "variable_id",
+        "variant_label",
+        "long_name",
+        "units",
+        "vertical_levels",
+        "source_version_number",
+        slug_column,
+    )
+
+    file_specific_metadata = ("start_time", "end_time", "path")
+
+    def __init__(self, n_jobs: int = 1):
+        self.n_jobs = n_jobs
+
+    def pretty_subset(self, data_catalog: pd.DataFrame) -> pd.DataFrame:
+        """
+        Get a subset of the data_catalog to pretty print
+
+        This is particularly useful for obs4MIPs datasets, which have a lot of metadata columns.
+
+        Parameters
+        ----------
+        data_catalog
+            Data catalog to subset
+
+        Returns
+        -------
+        :
+            Subset of the data catalog to pretty print
+
+        """
+        return data_catalog[
+            [
+                "activity_id",
+                "institution_id",
+                "source_id",
+                "variable_id",
+                "grid_label",
+                "source_version_number",
+            ]
+        ]
+
+    def find_local_datasets(self, file_or_directory: Path) -> pd.DataFrame:
+        """
+        Generate a data catalog from the specified file or directory
+
+        Each dataset may contain multiple files, which are represented as rows in the data catalog.
+        Each dataset has a unique identifier, which is in `slug_column`.
+
+        Parameters
+        ----------
+        file_or_directory
+            File or directory containing the datasets
+
+        Returns
+        -------
+        :
+            Data catalog containing the metadata for the dataset
+        """
+        builder = Builder(
+            paths=[str(file_or_directory)],
+            depth=10,
+            include_patterns=["*.nc"],
+            joblib_parallel_kwargs={"n_jobs": self.n_jobs},
+        ).build(parsing_func=parse_obs4mips)  # type: ignore[arg-type]
+
+        datasets = builder.df
+        if datasets.empty:
+            logger.error("No datasets found")
+            raise ValueError("No obs4MIPs-compliant datasets found")
+
+        # Convert the start_time and end_time columns to datetime objects
+        # We don't know the calendar used in the dataset (TODO: Check what ecgtools does)
+        datasets["start_time"] = _parse_datetime(datasets["start_time"])
+        datasets["end_time"] = _parse_datetime(datasets["end_time"])
+
+        drs_items = [
+            "activity_id",
+            "institution_id",
+            "source_id",
+            "variable_id",
+            "grid_label",
+            "source_version_number",
+        ]
+        datasets["instance_id"] = datasets.apply(
+            lambda row: "obs4MIPs." + ".".join([row[item] for item in drs_items]), axis=1
+        )
+        return datasets

climate_ref/datasets/pmp_climatology.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from climate_ref.datasets.obs4mips import Obs4MIPsDatasetAdapter
+from climate_ref.models.dataset import PMPClimatologyDataset
+
+
+class PMPClimatologyDatasetAdapter(Obs4MIPsDatasetAdapter):
+    """
+    Adapter for climatology datasets post-processed from obs4MIPs datasets by PMP.
+
+    These data look like obs4MIPs datasets and are ingested in the same way, but
+    are treated separately as they may have the same metadata as the obs4MIPs datasets.
+    """
+
+    dataset_cls = PMPClimatologyDataset

climate_ref/datasets/utils.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+
+
+def validate_path(raw_path: str) -> Path:
+    """
+    Validate the prefix of a dataset against the data directory
+    """
+    prefix = Path(raw_path)
+
+    if not prefix.exists():
+        raise FileNotFoundError(prefix)
+
+    if not prefix.is_absolute():
+        raise ValueError(f"Path {prefix} must be absolute")
+
+    return prefix

climate_ref/executor/__init__.py

@@ -0,0 +1,274 @@
+"""
+Execute diagnostics in different environments
+
+We support running diagnostics in different environments, such as locally,
+in a separate process, or in a container.
+These environments are represented by `climate_ref.executor.Executor` classes.
+
+The simplest executor is the `LocalExecutor`, which runs the diagnostic in the same process.
+This is useful for local testing and debugging.
+"""
+
+import importlib
+import pathlib
+import shutil
+from typing import TYPE_CHECKING
+
+from loguru import logger
+from sqlalchemy import insert
+
+from climate_ref.database import Database
+from climate_ref.models.execution import Execution, ExecutionOutput, ResultOutputType
+from climate_ref.models.metric_value import MetricValue
+from climate_ref_core.diagnostics import ExecutionResult, ensure_relative_path
+from climate_ref_core.exceptions import InvalidExecutorException, ResultValidationError
+from climate_ref_core.executor import EXECUTION_LOG_FILENAME, Executor
+from climate_ref_core.pycmec.controlled_vocabulary import CV
+from climate_ref_core.pycmec.metric import CMECMetric
+from climate_ref_core.pycmec.output import CMECOutput, OutputDict
+
+if TYPE_CHECKING:
+    from climate_ref.config import Config
+
+
+def import_executor_cls(fqn: str) -> type[Executor]:
+    """
+    Import an executor using a fully qualified module path
+
+    Parameters
+    ----------
+    fqn
+        Full package and attribute name of the executor to import
+
+        For example: `climate_ref_example.executor` will use the `executor` attribute from the
+        `climate_ref_example` package.
+
+    Raises
+    ------
+    climate_ref_core.exceptions.InvalidExecutorException
+        If the executor cannot be imported
+
+        If the executor isn't a valid `DiagnosticProvider`.
+
+    Returns
+    -------
+    :
+        Executor instance
+    """
+    module, attribute_name = fqn.rsplit(".", 1)
+
+    try:
+        imp = importlib.import_module(module)
+        executor: type[Executor] = getattr(imp, attribute_name)
+
+        # We can't really check if the executor is a subclass of Executor here
+        # Protocols can't be used with issubclass if they have non-method members
+        # We have to check this at class instantiation time
+
+        return executor
+    except ModuleNotFoundError:
+        logger.error(f"Package '{fqn}' not found")
+        raise InvalidExecutorException(fqn, f"Module '{module}' not found")
+    except AttributeError:
+        logger.error(f"Provider '{fqn}' not found")
+        raise InvalidExecutorException(fqn, f"Executor '{attribute_name}' not found in {module}")
+
+
+def _copy_file_to_results(
+    scratch_directory: pathlib.Path,
+    results_directory: pathlib.Path,
+    fragment: pathlib.Path | str,
+    filename: pathlib.Path | str,
+) -> None:
+    """
+    Copy a file from the scratch directory to the executions directory
+
+    Parameters
+    ----------
+    scratch_directory
+        The directory where the file is currently located
+    results_directory
+        The directory where the file should be copied to
+    fragment
+        The fragment of the executions directory where the file should be copied
+    filename
+        The name of the file to be copied
+    """
+    assert results_directory != scratch_directory  # noqa
+    input_directory = scratch_directory / fragment
+    output_directory = results_directory / fragment
+
+    filename = ensure_relative_path(filename, input_directory)
+
+    if not (input_directory / filename).exists():
+        raise FileNotFoundError(f"Could not find {filename} in {input_directory}")
+
+    output_filename = output_directory / filename
+    output_filename.parent.mkdir(parents=True, exist_ok=True)
+
+    shutil.copy(input_directory / filename, output_filename)
+
+
+def handle_execution_result(
+    config: "Config",
+    database: Database,
+    execution: Execution,
+    result: "ExecutionResult",
+) -> None:
+    """
+    Handle the result of a diagnostic execution
+
+    This will update the diagnostic execution result with the output of the diagnostic execution.
+    The output will be copied from the scratch directory to the executions directory.
+
+    Parameters
+    ----------
+    config
+        The configuration to use
+    database
+        The active database session to use
+    execution
+        The diagnostic execution result DB object to update
+    result
+        The result of the diagnostic execution, either successful or failed
+    """
+    # Always copy log data
+    _copy_file_to_results(
+        config.paths.scratch,
+        config.paths.results,
+        execution.output_fragment,
+        EXECUTION_LOG_FILENAME,
+    )
+
+    if result.successful and result.metric_bundle_filename is not None:
+        logger.info(f"{execution} successful")
+
+        _copy_file_to_results(
+            config.paths.scratch,
+            config.paths.results,
+            execution.output_fragment,
+            result.metric_bundle_filename,
+        )
+        execution.mark_successful(result.as_relative_path(result.metric_bundle_filename))
+
+        if result.output_bundle_filename:
+            _copy_file_to_results(
+                config.paths.scratch,
+                config.paths.results,
+                execution.output_fragment,
+                result.output_bundle_filename,
+            )
+            _handle_output_bundle(
+                config,
+                database,
+                execution,
+                result.to_output_path(result.output_bundle_filename),
+            )
+
+        cmec_metric_bundle = CMECMetric.load_from_json(result.to_output_path(result.metric_bundle_filename))
+
+        # Check that the diagnostic values conform with the controlled vocabulary
+        try:
+            cv = CV.load_from_file(config.paths.dimensions_cv)
+            cv.validate_metrics(cmec_metric_bundle)
+        except (ResultValidationError, AssertionError):
+            logger.exception("Diagnostic values do not conform with the controlled vocabulary")
+            # TODO: Mark the diagnostic execution result as failed once the CV has stabilised
+            # execution.mark_failed()
+
+        # Perform a bulk insert of a diagnostic bundle
+        # TODO: The section below will likely fail until we have agreed on a controlled vocabulary
+        # The current implementation will swallow the exception, but display a log message
+        try:
+            # Perform this in a nested transaction to (hopefully) gracefully rollback if something
+            # goes wrong
+            with database.session.begin_nested():
+                database.session.execute(
+                    insert(MetricValue),
+                    [
+                        {
+                            "execution_id": execution.id,
+                            "value": result.value,
+                            "attributes": result.attributes,
+                            **result.dimensions,
+                        }
+                        for result in cmec_metric_bundle.iter_results()
+                    ],
+                )
+        except Exception:
+            # TODO: Remove once we have settled on a controlled vocabulary
+            logger.exception("Something went wrong when ingesting diagnostic values")
+
+        # TODO: This should check if the result is the most recent for the execution,
+        # if so then update the dirty fields
+        # i.e. if there are outstanding executions don't make as clean
+        execution.execution_group.dirty = False
+    else:
+        logger.error(f"{execution} failed")
+        execution.mark_failed()
+
+
+def _handle_output_bundle(
+    config: "Config",
+    database: Database,
+    execution: Execution,
+    cmec_output_bundle_filename: pathlib.Path,
+) -> None:
+    # Extract the registered outputs
+    # Copy the content to the output directory
+    # Track in the db
+    cmec_output_bundle = CMECOutput.load_from_json(cmec_output_bundle_filename)
+    _handle_outputs(
+        cmec_output_bundle.plots,
+        output_type=ResultOutputType.Plot,
+        config=config,
+        database=database,
+        execution=execution,
+    )
+    _handle_outputs(
+        cmec_output_bundle.data,
+        output_type=ResultOutputType.Data,
+        config=config,
+        database=database,
+        execution=execution,
+    )
+    _handle_outputs(
+        cmec_output_bundle.html,
+        output_type=ResultOutputType.HTML,
+        config=config,
+        database=database,
+        execution=execution,
+    )
+
+
+def _handle_outputs(
+    outputs: dict[str, OutputDict] | None,
+    output_type: ResultOutputType,
+    config: "Config",
+    database: Database,
+    execution: Execution,
+) -> None:
+    if outputs is None:
+        return
+
+    for key, output_info in outputs.items():
+        filename = ensure_relative_path(
+            output_info.filename, config.paths.scratch / execution.output_fragment
+        )
+
+        _copy_file_to_results(
+            config.paths.scratch,
+            config.paths.results,
+            execution.output_fragment,
+            filename,
+        )
+        database.session.add(
+            ExecutionOutput(
+                execution_id=execution.id,
+                output_type=output_type,
+                filename=str(filename),
+                description=output_info.description,
+                short_name=key,
+                long_name=output_info.long_name,
+            )
+        )

climate_ref/executor/local.py

@@ -0,0 +1,89 @@
+from typing import Any
+
+from loguru import logger
+
+from climate_ref.config import Config
+from climate_ref.database import Database
+from climate_ref.executor import handle_execution_result
+from climate_ref.models import Execution
+from climate_ref_core.diagnostics import Diagnostic, ExecutionDefinition, ExecutionResult
+from climate_ref_core.logging import redirect_logs
+from climate_ref_core.providers import DiagnosticProvider
+
+
+class LocalExecutor:
+    """
+    Run a diagnostic locally, in-process.
+
+    This is mainly useful for debugging and testing.
+    The production executor will run the diagnostic in a separate process or container,
+    the exact manner of which is yet to be determined.
+    """
+
+    name = "local"
+
+    def __init__(
+        self, *, database: Database | None = None, config: Config | None = None, **kwargs: Any
+    ) -> None:
+        if config is None:
+            config = Config.default()
+        if database is None:
+            database = Database.from_config(config, run_migrations=False)
+
+        self.database = database
+        self.config = config
+
+    def run(
+        self,
+        provider: DiagnosticProvider,
+        diagnostic: Diagnostic,
+        definition: ExecutionDefinition,
+        execution: Execution | None = None,
+    ) -> None:
+        """
+        Run a diagnostic in process
+
+        Parameters
+        ----------
+        provider
+            The provider of the diagnostic
+        diagnostic
+            Diagnostic to run
+        definition
+            A description of the information needed for this execution of the diagnostic
+        execution
+            A database model representing the execution of the diagnostic.
+            If provided, the result will be updated in the database when completed.
+        """
+        definition.output_directory.mkdir(parents=True, exist_ok=True)
+
+        try:
+            with redirect_logs(definition, self.config.log_level):
+                result = diagnostic.run(definition=definition)
+        except Exception:
+            if execution is not None:  # pragma: no branch
+                info_msg = (
+                    f"\nAdditional information about this execution can be viewed using: "
+                    f"ref executions inspect {execution.execution_group_id}"
+                )
+            else:
+                info_msg = ""
+
+            logger.exception(f"Error running diagnostic {diagnostic.slug}. {info_msg}")
+            result = ExecutionResult.build_from_failure(definition)
+
+        if execution:
+            handle_execution_result(self.config, self.database, execution, result)
+
+    def join(self, timeout: float) -> None:
+        """
+        Wait for all diagnostics to finish
+
+        This returns immediately because the local executor runs diagnostics synchronously.
+
+        Parameters
+        ----------
+        timeout
+            Timeout in seconds (Not used)
+        """
+        return

climate_ref/migrations/README

@@ -0,0 +1,22 @@
+# Alembic
+
+Alembic is a database migration tool.
+It interoperates with SqlAlchemy to determine how the currently declared models differ from what the database
+expects and generates a migration to apply the changes.
+
+The migrations are applied at run-time automatically (see [ref.database.Database]()).
+
+## Generating migrations
+
+To generate a migration,
+you can use the `uv run` command with the `alembic` package and the `revision` command.
+The `--rev-id` flag is used to specify the revision id.
+If it is omitted the revision id will be generated automatically.
+
+```
+uv run --package ref alembic revision --rev-id 0.1.0 --message "initial table" --autogenerate
+```
+
+How we name and manage these migrations is still a work in progress.
+It might be nice to have a way to automatically generate the revision id based on the version of the package.
+This would allow us to easily track which migrations have been applied to the database.