climate-ref 0.5.0__py3-none-any.whl

climate_ref/__init__.py

@@ -0,0 +1,30 @@
+"""
+Rapid evaluating CMIP data
+"""
+
+import importlib.metadata
+
+__version__ = importlib.metadata.version("climate-ref")
+
+
+from climate_ref.testing import SAMPLE_DATA_VERSION
+from climate_ref_core.dataset_registry import dataset_registry_manager
+
+# Register the obs4REF data registry
+dataset_registry_manager.register(
+    "obs4ref",
+    "https://pub-b093171261094c4ea9adffa01f94ee06.r2.dev/",
+    package="climate_ref.dataset_registry",
+    resource="obs4ref_reference.txt",
+)
+# Register the sample data registry -- used for testing
+dataset_registry_manager.register(
+    "sample-data",
+    "https://raw.githubusercontent.com/Climate-REF/ref-sample-data/refs/tags/{version}/data/",
+    package="climate_ref.dataset_registry",
+    resource="sample_data.txt",
+    version=SAMPLE_DATA_VERSION,
+)
+
+
+__all__ = ["__version__"]

climate_ref/_config_helpers.py

@@ -0,0 +1,214 @@
+import typing
+from collections.abc import Callable
+from typing import Any, TypeVar, overload
+
+import attr
+from attrs import fields
+from cattrs import ClassValidationError, ForbiddenExtraKeysError, IterableValidationError
+from cattrs.v import format_exception as default_format_exception
+from environs.exceptions import EnvError
+from loguru import logger
+
+from climate_ref_core.env import env
+
+T = TypeVar("T")
+C = TypeVar("C", bound=type)
+
+
+def _pop_empty(d: dict[str, Any]) -> None:
+    keys = list(d.keys())
+    for key in keys:
+        value = d[key]
+        if isinstance(value, dict):
+            _pop_empty(value)
+            if not value:
+                d.pop(key)
+
+
+def _format_key_exception(exc: BaseException, _: type | None) -> str | None:
+    """Format a n error exception."""
+    if isinstance(exc, ForbiddenExtraKeysError):
+        return f"extra fields found ({', '.join(exc.extra_fields)})"
+    else:
+        return None
+
+
+def _format_exception(exc: BaseException, type: type | None) -> str:  # noqa: A002
+    """Format an exception into a string description of the error.
+
+    Parameters
+    ----------
+    exc
+        The exception to format into an error message.
+    type
+        The type that the value was expected to be.
+    """
+    # Any custom handling of error goes here before falling back to the default
+    if isinstance(exc, EnvError):  # pragma: no cover
+        return str(exc)
+
+    return default_format_exception(exc, type)
+
+
+def transform_error(
+    exc: ClassValidationError | IterableValidationError | BaseException,
+    path: str = "$",
+    format_exception: Callable[[BaseException, type | None], str | None] = _format_exception,
+) -> list[str]:
+    """Transform an exception into a list of error messages.
+
+    This is based on [cattrs.transform_error][cattrs.transform_error],
+     but modified to be able to ignore errors
+
+    To get detailed error messages, the exception should be produced by a converter
+    with `detailed_validation` set.
+
+    By default, the error messages are in the form of `{description} @ {path}`.
+
+    While traversing the exception and subexceptions, the path is formed:
+
+    * by appending `.{field_name}` for fields in classes
+    * by appending `[{int}]` for indices in iterables, like lists
+    * by appending `[{str}]` for keys in mappings, like dictionaries
+
+    Parameters
+    ----------
+    exc
+        The exception to transform into error messages.
+    path
+        The root path to use.
+    format_exception
+        A callable to use to transform `Exceptions` into string descriptions of errors.
+    """
+    errors = []
+
+    def _maybe_append_error(exc: BaseException, _: type | None, path: str) -> str | None:
+        error_message = format_exception(exc, None)
+        if error_message:
+            errors.append(f"{error_message} @ {path}")
+        return None
+
+    if isinstance(exc, IterableValidationError):
+        iterable_validation_notes, without = exc.group_exceptions()
+        for inner_exc, iterable_note in iterable_validation_notes:
+            p = f"{path}[{iterable_note.index!r}]"
+            if isinstance(inner_exc, ClassValidationError | IterableValidationError):
+                errors.extend(transform_error(inner_exc, p, format_exception))
+            else:
+                _maybe_append_error(inner_exc, iterable_note.type, p)
+        for inner_exc in without:
+            _maybe_append_error(inner_exc, None, path)
+    elif isinstance(exc, ClassValidationError):
+        class_validation_notes, without = exc.group_exceptions()
+        for inner_exc, class_note in class_validation_notes:
+            p = f"{path}.{class_note.name}"
+            if isinstance(inner_exc, ClassValidationError | IterableValidationError):
+                errors.extend(transform_error(inner_exc, p, format_exception))
+            else:
+                _maybe_append_error(inner_exc, class_note.type, p)
+        for inner_exc in without:
+            _maybe_append_error(inner_exc, None, path)
+    else:
+        _maybe_append_error(exc, None, path)
+
+    return errors
+
+
+def _environment_override(value: T, env_name: str, convertor: Callable[[Any], T] | None = None) -> T:
+    try:
+        env_value = env.str(env_name)
+    except EnvError:
+        return value
+
+    logger.debug(f"Overriding {env_name} with {env_value}")
+    if convertor:
+        return convertor(env_value)
+
+    return typing.cast(T, env_value)
+
+
+def _environ_post_init(cls: Any) -> None:
+    for f in fields(cls.__class__):
+        if f.metadata.get("env"):
+            env_name = f"{cls._prefix}_{f.metadata['env']}"
+
+            # This is a bit of a hack to get around the fact that we can't update values on frozen classes
+            # https://www.attrs.org/en/stable/how-does-it-work.html#how-frozen
+            object.__setattr__(
+                cls, f.name, _environment_override(getattr(cls, f.name), env_name, f.converter)
+            )
+
+
+@overload
+def config(
+    *,
+    prefix: str,
+    frozen: bool = False,
+) -> Callable[[C], C]: ...
+
+
+@overload
+def config(
+    maybe_cls: C,
+    *,
+    prefix: str,
+    frozen: bool = False,
+) -> C: ...
+
+
+def config(
+    maybe_cls: C | None = None,
+    *,
+    prefix: str,
+    frozen: bool = False,
+) -> C | Callable[[C], C]:
+    """
+    Make a class a configuration class.
+
+    Parameters
+    ----------
+    prefix:
+        The prefix that is used for the env variables.
+
+        This is be prepended to all the fields that use an `env_field`.
+    frozen:
+        The configuration will be immutable after instantiation, if `True`.
+    """
+
+    def wrap(cls: C) -> C:
+        setattr(cls, "_prefix", prefix)
+        setattr(cls, "__attrs_post_init__", _environ_post_init)
+
+        return attr.s(cls, frozen=frozen, slots=True)
+
+    # maybe_cls's type depends on the usage of the decorator.  It's a class
+    # if it's used as `@attrs` but `None` if used as `@attrs()`.
+    if maybe_cls is None:
+        return wrap
+
+    return wrap(maybe_cls)
+
+
+def env_field(name: str, **kwargs: Any) -> Any:
+    """
+    Create a field with an environment variable override.
+
+    This field will use a value from the environment if it is set.
+
+    This field requires the class to be decorated with `config` to work.
+    The environment variable name is constructed by prefixing the field name with the class prefix.
+
+    Parameters
+    ----------
+    name
+        Name of the environment variable to use without a prefix
+    kwargs
+        Additional arguments to pass to the field
+
+    Returns
+    -------
+    field
+        The field with the environment variable override
+
+    """
+    return attr.attrib(**kwargs, metadata={"env": name})

climate_ref/alembic.ini

@@ -0,0 +1,114 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = %(here)s/migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+file_template = %%(year)d-%%(month).2d-%%(day).2dT%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = newline
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# lint with attempts to fix using "ruff"
+hooks = ruff-fix, ruff-format
+
+ruff-fix.type = exec
+ruff-fix.executable = ruff
+ruff-fix.options = check -q --fix REVISION_SCRIPT_FILENAME
+
+ruff-format.type = exec
+ruff-format.executable = ruff
+ruff-format.options = format -q REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

climate_ref/cli/__init__.py

@@ -0,0 +1,138 @@
+"""Entrypoint for the CLI"""
+
+import importlib
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+from attrs import define
+from loguru import logger
+
+from climate_ref import __version__
+from climate_ref.cli import config, datasets, executions, providers, solve
+from climate_ref.config import Config
+from climate_ref.constants import config_filename
+from climate_ref.database import Database
+from climate_ref_core import __version__ as __core_version__
+from climate_ref_core.logging import add_log_handler
+
+
+class LogLevel(str, Enum):
+    """
+    Log levels for the CLI
+    """
+
+    Normal = "WARNING"
+    Debug = "DEBUG"
+    Info = "INFO"
+
+
+@define
+class CLIContext:
+    """
+    Context object that can be passed to commands
+    """
+
+    config: Config
+    database: Database
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        print(f"climate_ref: {__version__}")
+        print(f"climate_ref-core: {__core_version__}")
+        raise typer.Exit()
+
+
+def _load_config(configuration_directory: Path | None = None) -> Config:
+    """
+    Load the configuration from the specified directory
+
+    Parameters
+    ----------
+    configuration_directory
+        The directory to load the configuration from
+
+        If the specified directory is not found, the process will exit with an exit code of 1
+
+        If None, the default configuration will be loaded
+
+    Returns
+    -------
+    :
+        The configuration loaded from the specified directory
+    """
+    try:
+        if configuration_directory:
+            config = Config.load(configuration_directory / config_filename, allow_missing=False)
+        else:
+            config = Config.default()
+    except FileNotFoundError:
+        typer.secho("Configuration file not found", fg=typer.colors.RED)
+        raise typer.Exit(1)
+    return config
+
+
+def build_app() -> typer.Typer:
+    """
+    Build the CLI app
+
+    This registers all the commands and subcommands of the CLI app.
+    Some commands may not be available if certain dependencies are not installed,
+    for example the Celery CLI is only available if the `climate-ref-celery` package is installed.
+
+    Returns
+    -------
+    :
+        The CLI app
+    """
+    app = typer.Typer(name="climate_ref", no_args_is_help=True)
+
+    app.command(name="solve")(solve.solve)
+    app.add_typer(config.app, name="config")
+    app.add_typer(datasets.app, name="datasets")
+    app.add_typer(executions.app, name="executions")
+    app.add_typer(providers.app, name="providers")
+
+    try:
+        celery_app = importlib.import_module("climate_ref_celery.cli").app
+
+        app.add_typer(celery_app, name="celery")
+    except ImportError:
+        logger.debug("Celery CLI not available")
+
+    return app
+
+
+app = build_app()
+
+
+@app.callback()
+def main(
+    ctx: typer.Context,
+    configuration_directory: Annotated[Path | None, typer.Option(help="Configuration directory")] = None,
+    verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False,
+    log_level: Annotated[LogLevel, typer.Option(case_sensitive=False)] = LogLevel.Normal,
+    version: Annotated[
+        Optional[bool],
+        typer.Option("--version", callback=_version_callback, is_eager=True),
+    ] = None,
+) -> None:
+    """
+    climate_ref: A CLI for the CMIP Rapid Evaluation Framework
+    """
+    if verbose:
+        log_level = LogLevel.Debug
+
+    logger.remove()
+    add_log_handler(level=log_level.value)
+
+    config = _load_config(configuration_directory)
+    config.log_level = log_level.value
+
+    ctx.obj = CLIContext(config=config, database=Database.from_config(config))
+
+
+if __name__ == "__main__":
+    app()

climate_ref/cli/_utils.py

@@ -0,0 +1,68 @@
+import pandas as pd
+from loguru import logger
+from rich import box
+from rich.console import Console
+from rich.table import Table
+
+
+def df_to_table(df: pd.DataFrame, max_col_count: int = -1) -> Table:
+    """
+    Convert a DataFrame to a rich Table instance
+
+    Parameters
+    ----------
+    df
+        DataFrame to convert
+    max_col_count
+        Maximum number of columns to display
+
+        If the DataFrame has more columns than this, the excess columns will be truncated
+        If set to -1, all columns will be displayed.
+        For very wide DataFrames, then this may result in no values at all being displayed
+        if the column-width ends up being less than 1 char.
+
+    Returns
+    -------
+        Rich Table instance representing the DataFrame
+    """
+    # Initiate a Table instance to be modified
+    if max_col_count > 0 and len(df.columns) > max_col_count:
+        logger.warning(f"Too many columns to display ({len(df.columns)}), truncating to {max_col_count}")
+        df = df.iloc[:, :max_col_count]
+
+    table = Table(*[str(column) for column in df.columns])
+
+    for index, value_list in enumerate(df.values.tolist()):
+        row = [str(x) for x in value_list]
+        table.add_row(*row)
+
+    # Update the style of the table
+    table.row_styles = ["none", "dim"]
+    table.box = box.SIMPLE_HEAD
+
+    return table
+
+
+def pretty_print_df(df: pd.DataFrame, console: Console | None = None) -> None:
+    """
+    Pretty print a DataFrame
+
+    Parameters
+    ----------
+    df
+        DataFrame to print
+    console
+        Console to use for printing
+
+        If not provided, a new Console instance will be created
+    """
+    # Drop duplicates as they are not informative to CLI users.
+    df = df.drop_duplicates()
+
+    if console is None:
+        console = Console()
+
+    max_col_count = console.width // 10
+    table = df_to_table(df, max_col_count=max_col_count)
+
+    console.print(table)

climate_ref/cli/config.py

@@ -0,0 +1,28 @@
+"""
+View and update the REF configuration
+"""
+
+import typer
+
+app = typer.Typer(help=__doc__)
+
+
+@app.command(name="list")
+def list_(ctx: typer.Context) -> None:
+    """
+    Print the current climate_ref configuration
+
+    If a configuration directory is provided,
+    the configuration will attempt to load from the specified directory.
+    """
+    config = ctx.obj.config
+
+    print(config.dumps(defaults=True))
+
+
+@app.command()
+def update() -> None:
+    """
+    Update a configuration value
+    """
+    print("config")