climate-ref 0.8.0__tar.gz → 0.9.0__tar.gz

{climate_ref-0.8.0 → climate_ref-0.9.0}/.gitignore

@@ -150,7 +150,7 @@ dmypy.json
 
 # Generated output
 out
-.ref
+.ref*
 
 # Ignore copied LICENCE/NOTICE files
 packages/*/LICENCE
@@ -158,3 +158,12 @@ packages/*/NOTICE
 
 # Local directory for data
 /data
+
+# Generated SDK
+/climate_ref_client
+
+# User-specific catalog paths (test data)
+*.paths.yaml
+
+# Helm dependencies
+helm/charts/*

{climate_ref-0.8.0 → climate_ref-0.9.0}/Dockerfile

@@ -2,7 +2,7 @@
 # This docker container packages up the REF cli tool with the default set of diagnostic providers
 # used as part of the CMIP7 FastTrack process.
 
-FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim AS build
+FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim AS base
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
@@ -16,6 +16,7 @@ ENV UV_LINK_MODE=copy
 
 WORKDIR /app
 
+FROM base AS build
 # Install the project's dependencies using the lockfile and settings
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
@@ -24,22 +25,18 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 
 # Then, add the rest of the project source code and install it
 # Installing separately from its dependencies allows optimal layer caching
-ADD . /app
+COPY . /app
 
 # Sync the project as non-editable installs
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-editable --no-dev
 
 # Runtime container
-FROM python:3.11-slim AS runtime
+FROM base AS runtime
 
 LABEL maintainer="Jared Lewis <jared.lewis@climate-resource.com>"
 LABEL description="Base Docker image for the REF compute engine"
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
 # Create a non-root user
 RUN useradd -m -u 1000 app
 
@@ -48,18 +45,30 @@ ENV PATH="/app/.venv/bin:${PATH}"
 ENV VIRTUAL_ENV=/app/.venv
 
 # Copy the installed packages from the build stage
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /app/.venv/bin/
 COPY --from=build --chown=app:app /app/.venv /app/.venv
 COPY --from=build --chown=app:app /app/scripts /app/scripts
 
 # Location of the REF configuration files
 ENV REF_CONFIGURATION=/ref
 
+# Set matplotlib config directory to a location with write permissions
+ENV MPLCONFIGDIR=$REF_CONFIGURATION/software/matplotlib
+
+# Set micromamba root to locations with write permissions
+ENV MAMBA_ROOT_PREFIX=$REF_CONFIGURATION/software/conda
+ENV CONDA_PKGS_DIRS=$REF_CONFIGURATION/software/conda/pkgs
+ENV XDG_CACHE_HOME=$REF_CONFIGURATION/cache
+
 # Create necessary directories with proper permissions
-RUN mkdir -p /ref /app/cache && chown -R app:app /ref /app/cache
+RUN install --owner=app --group=app -d /ref /app/cache && \
+    install --mode=555 -d /home/app/.conda && \
+    touch /home/app/.conda/environments.txt  # conda tries to write to ~/.conda/environments.txt despite its root being set elsewhere. See: https://github.com/conda/conda/issues/8804
+
+# Switch to non-root user -- use numeric ID for k8s systems that enforce runAsUser
+USER 1000
 
-# Switch to non-root user
-USER app
+# Pre-cache matplotlib fonts and ilamb3 config
+RUN /app/.venv/bin/python -c "import matplotlib; import ilamb3"
 
 # Run the REF CLI tool by default
 ENTRYPOINT ["/app/.venv/bin/ref"]

{climate_ref-0.8.0 → climate_ref-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: climate-ref
-Version: 0.8.0
+Version: 0.9.0
 Summary: Application which runs the CMIP Rapid Evaluation Framework
 Author-email: Jared Lewis <jared.lewis@climate-resource.com>, Mika Pflueger <mika.pflueger@climate-resource.com>, Bouwe Andela <b.andela@esciencecenter.nl>, Jiwoo Lee <lee1043@llnl.gov>, Min Xu <xum1@ornl.gov>, Nathan Collier <collierno@ornl.gov>, Dora Hegedus <dora.hegedus@stfc.ac.uk>
 License-Expression: Apache-2.0
@@ -25,6 +25,7 @@ Requires-Dist: cattrs>=24.1.2
 Requires-Dist: climate-ref-core
 Requires-Dist: ecgtools>=2024.7.31
 Requires-Dist: environs>=11.0.0
+Requires-Dist: gitpython>=3.1.0
 Requires-Dist: loguru>=0.7.2
 Requires-Dist: parsl>=2025.5.19; sys_platform != 'win32'
 Requires-Dist: platformdirs>=4.3.6

{climate_ref-0.8.0 → climate_ref-0.9.0}/conftest.py

@@ -1,5 +1,6 @@
 import importlib.resources
 import shutil
+from collections.abc import Generator
 from pathlib import Path
 from urllib import parse as urlparse
 
@@ -38,8 +39,10 @@ def _clone_db(target_db_url: str, template_db_path: Path) -> None:
 
 
 @pytest.fixture
-def db(config) -> Database:
-    return Database.from_config(config, run_migrations=True)
+def db(config) -> Generator[Database, None, None]:
+    database = Database.from_config(config, run_migrations=True)
+    yield database
+    database.close()
 
 
 @pytest.fixture(scope="session")
@@ -77,6 +80,8 @@ def db_seeded_template(tmp_path_session, cmip6_data_catalog, obs4mips_data_catal
     with database.session.begin():
         _register_provider(database, example_provider)
 
+    database.close()
+
     return template_db_path
 
 
@@ -85,4 +90,6 @@ def db_seeded(db_seeded_template, config) -> Database:
     # Copy the template database to the location in the config
     _clone_db(config.db.database_url, db_seeded_template)
 
-    return Database.from_config(config, run_migrations=True)
+    database = Database.from_config(config, run_migrations=True)
+    yield database
+    database.close()

{climate_ref-0.8.0 → climate_ref-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "climate-ref"
-version = "0.8.0"
+version = "0.9.0"
 description = "Application which runs the CMIP Rapid Evaluation Framework"
 readme = "README.md"
 authors = [
@@ -41,6 +41,7 @@ dependencies = [
     "ecgtools>=2024.7.31",
     "platformdirs>=4.3.6",
     "tqdm>=4.67.1",
+    "gitpython>=3.1.0",
     # parsl doesn't support Windows yet
     # We don't target Windows either, but this __might__ allow Windows users to install the package
     'parsl>=2025.5.19; sys_platform != "win32"'

{climate_ref-0.8.0 → climate_ref-0.9.0}/src/climate_ref/cli/__init__.py

@@ -11,7 +11,7 @@ from loguru import logger
 from rich.console import Console
 
 from climate_ref import __version__
-from climate_ref.cli import config, datasets, executions, providers, solve
+from climate_ref.cli import config, datasets, executions, providers, solve, test_cases
 from climate_ref.config import Config
 from climate_ref.constants import CONFIG_FILENAME
 from climate_ref.database import Database
@@ -104,6 +104,7 @@ def build_app() -> typer.Typer:
     app.add_typer(datasets.app, name="datasets")
     app.add_typer(executions.app, name="executions")
     app.add_typer(providers.app, name="providers")
+    app.add_typer(test_cases.app, name="test-cases")
 
     try:
         celery_app = importlib.import_module("climate_ref_celery.cli").app
@@ -164,7 +165,9 @@ def main(  # noqa: PLR0913
 
     logger.debug(f"Configuration loaded from: {config._config_file!s}")
 
-    ctx.obj = CLIContext(config=config, database=Database.from_config(config), console=_create_console())
+    # Use ctx.with_resource to ensure the database connection is closed when the CLI exits
+    database = ctx.with_resource(Database.from_config(config))
+    ctx.obj = CLIContext(config=config, database=database, console=_create_console())
 
 
 if __name__ == "__main__":

climate_ref-0.9.0/src/climate_ref/cli/_git_utils.py

@@ -0,0 +1,112 @@
+"""Git utilities for CLI commands."""
+
+from pathlib import Path
+from typing import Any
+
+from git import InvalidGitRepositoryError, Repo
+
+
+def get_repo_for_path(path: Path) -> Repo | None:
+    """
+    Get the git repository containing the given path.
+
+    Parameters
+    ----------
+    path
+        Path to a file or directory
+
+    Returns
+    -------
+    :
+        The Repo object if path is within a git repository, None otherwise
+    """
+    try:
+        return Repo(path, search_parent_directories=True)
+    except InvalidGitRepositoryError:
+        return None
+
+
+def get_git_status(file_path: Path, repo: Repo) -> str:
+    """
+    Get git status for a file using GitPython.
+
+    Parameters
+    ----------
+    file_path
+        Absolute path to the file
+    repo
+        GitPython Repo object
+
+    Returns
+    -------
+    :
+        Status string: "new", "staged", "modified", "tracked", "untracked", or "unknown"
+    """
+    try:
+        rel_path = str(file_path.relative_to(repo.working_dir))
+
+        # Check if untracked
+        if rel_path in repo.untracked_files:
+            return "new"
+
+        # Check staged changes (index vs HEAD)
+        staged_files = {item.a_path for item in repo.index.diff("HEAD")}
+        if rel_path in staged_files:
+            return "staged"
+
+        # Check unstaged changes (working tree vs index)
+        unstaged_files = {item.a_path for item in repo.index.diff(None)}
+        if rel_path in unstaged_files:
+            return "modified"
+
+        # Check if file is tracked
+        try:
+            repo.git.ls_files("--error-unmatch", rel_path)
+            return "tracked"
+        except Exception:
+            return "untracked"
+    except Exception:
+        return "unknown"
+
+
+def collect_regression_file_info(
+    regression_dir: Path,
+    repo: Repo | None,
+    size_threshold_bytes: int,
+) -> list[dict[str, Any]]:
+    """
+    Collect file information from a regression directory.
+
+    Parameters
+    ----------
+    regression_dir
+        Path to the regression data directory
+    repo
+        Git repository object, or None if not in a repo
+    size_threshold_bytes
+        Files larger than this will be flagged as large
+
+    Returns
+    -------
+    :
+        List of dicts with keys: rel_path, size, is_large, git_status
+    """
+    files = sorted(regression_dir.rglob("*"))
+    files = [f for f in files if f.is_file()]
+
+    file_info: list[dict[str, Any]] = []
+    for file_path in files:
+        size = file_path.stat().st_size
+        rel_path = str(file_path.relative_to(regression_dir))
+        git_status = get_git_status(file_path, repo) if repo else "unknown"
+
+        file_info.append(
+            {
+                "rel_path": rel_path,
+                "size": size,
+                "is_large": size > size_threshold_bytes,
+                "git_status": git_status,
+            }
+        )
+
+    return file_info

{climate_ref-0.8.0 → climate_ref-0.9.0}/src/climate_ref/cli/_utils.py

@@ -4,6 +4,30 @@ from rich import box
 from rich.console import Console
 from rich.table import Table
 
+_BYTES_PER_UNIT = 1024
+
+
+def format_size(size_bytes: int | float) -> str:
+    """
+    Format file size in human-readable form.
+
+    Parameters
+    ----------
+    size_bytes
+        Size in bytes
+
+    Returns
+    -------
+    :
+        Human-readable size string (e.g., "1.5 MB")
+    """
+    size = float(size_bytes)
+    for unit in ("B", "KB", "MB", "GB"):
+        if size < _BYTES_PER_UNIT:
+            return f"{size:.1f} {unit}"
+        size /= _BYTES_PER_UNIT
+    return f"{size:.1f} TB"
+
 
 def parse_facet_filters(filters: list[str] | None) -> dict[str, str]:
     """

{climate_ref-0.8.0 → climate_ref-0.9.0}/src/climate_ref/cli/datasets.py

@@ -217,6 +217,7 @@ def _fetch_sample_data(
     This operation may fail if the test data directory does not exist,
     as is the case for non-source-based installations.
     """
+    # TODO: Remove
     fetch_sample_data(force_cleanup=force_cleanup, symlink=symlink)
 
 

climate_ref-0.9.0/src/climate_ref/cli/providers.py

@@ -0,0 +1,183 @@
+"""
+Manage the REF providers.
+"""
+
+import warnings
+from typing import Annotated
+
+import pandas as pd
+import typer
+from loguru import logger
+
+from climate_ref.cli._utils import pretty_print_df
+from climate_ref.provider_registry import ProviderRegistry
+from climate_ref_core.providers import CondaDiagnosticProvider, DiagnosticProvider
+
+app = typer.Typer(help=__doc__)
+
+
+@app.command(name="list")
+def list_(ctx: typer.Context) -> None:
+    """
+    Print the available providers.
+    """
+    config = ctx.obj.config
+    db = ctx.obj.database
+    console = ctx.obj.console
+    provider_registry = ProviderRegistry.build_from_config(config, db)
+
+    def get_env(provider: DiagnosticProvider) -> str:
+        env = ""
+        if isinstance(provider, CondaDiagnosticProvider):
+            env = f"{provider.env_path}"
+            if not provider.env_path.exists():
+                env += " (not installed)"
+        return env
+
+    def get_data_path(provider: DiagnosticProvider) -> str:
+        """Get the data cache path for a provider."""
+        data_path = provider.get_data_path()
+        if data_path is None:
+            return ""
+        path_str = str(data_path)
+        if not data_path.exists():
+            path_str += " (not fetched)"
+        return path_str
+
+    results_df = pd.DataFrame(
+        [
+            {
+                "provider": provider.slug,
+                "version": provider.version,
+                "conda environment": get_env(provider),
+                "data path": get_data_path(provider),
+            }
+            for provider in provider_registry.providers
+        ]
+    )
+    pretty_print_df(results_df, console=console)
+
+
+@app.command(deprecated=True)
+def create_env(
+    ctx: typer.Context,
+    provider: Annotated[
+        str | None,
+        typer.Option(help="Only install the environment for the named provider."),
+    ] = None,
+) -> None:
+    """
+    Create a conda environment containing the provider software.
+
+    .. deprecated::
+        Use `ref providers setup` instead, which handles both environment creation
+        and data fetching in a single command.
+
+    If no provider is specified, all providers will be installed.
+    If the provider is up to date or does not use a conda environment, it will be skipped.
+    """
+    warnings.warn(
+        "create-env is deprecated. Use 'ref providers setup' instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    config = ctx.obj.config
+    db = ctx.obj.database
+    providers = ProviderRegistry.build_from_config(config, db).providers
+
+    if provider is not None:
+        available = ", ".join([f'"{p.slug}"' for p in providers])
+        providers = [p for p in providers if p.slug == provider]
+        if not providers:
+            msg = f'Provider "{provider}" not available. Choose from: {available}'
+            logger.error(msg)
+            raise typer.Exit(code=1)
+
+    for provider_ in providers:
+        txt = f"conda environment for provider {provider_.slug}"
+        if isinstance(provider_, CondaDiagnosticProvider):
+            logger.info(f"Creating {txt} in {provider_.env_path}")
+            provider_.create_env()
+            logger.info(f"Finished creating {txt}")
+        else:
+            logger.info(f"Skipping creating {txt} because it does not use conda environments.")
+
+    list_(ctx)
+
+
+@app.command()
+def setup(
+    ctx: typer.Context,
+    provider: Annotated[
+        str | None,
+        typer.Option(help="Only run setup for the named provider."),
+    ] = None,
+    skip_env: Annotated[
+        bool,
+        typer.Option(help="Skip environment setup (e.g., conda)."),
+    ] = False,
+    skip_data: Annotated[
+        bool,
+        typer.Option(help="Skip data fetching."),
+    ] = False,
+    validate_only: Annotated[
+        bool,
+        typer.Option(help="Only validate setup, don't run it."),
+    ] = False,
+) -> None:
+    """
+    Run provider setup for offline execution.
+
+    This command prepares all providers for offline execution by:
+
+    1. Creating conda environments (if applicable)
+
+    2. Fetching required reference datasets to pooch cache
+
+    All operations are idempotent and safe to run multiple times.
+    Run this on a login node with internet access before solving on compute nodes.
+    """
+    config = ctx.obj.config
+    db = ctx.obj.database
+    console = ctx.obj.console
+    providers = ProviderRegistry.build_from_config(config, db).providers
+
+    if provider is not None:
+        available = ", ".join([f'"{p.slug}"' for p in providers])
+        providers = [p for p in providers if p.slug == provider]
+        if not providers:
+            msg = f'Provider "{provider}" not available. Choose from: {available}'
+            logger.error(msg)
+            raise typer.Exit(code=1)
+
+    failed_providers: list[str] = []
+
+    for provider_ in providers:
+        if validate_only:
+            is_valid = provider_.validate_setup(config)
+            status = "[green]valid[/green]" if is_valid else "[red]invalid[/red]"
+            console.print(f"Provider {provider_.slug}: {status}")
+            if not is_valid:
+                failed_providers.append(provider_.slug)
+            continue
+
+        logger.info(f"Setting up provider {provider_.slug}")
+        try:
+            provider_.setup(config, skip_env=skip_env, skip_data=skip_data)
+            is_valid = provider_.validate_setup(config)
+            if not is_valid:
+                logger.error(f"Provider {provider_.slug} setup completed but validation failed")
+                failed_providers.append(provider_.slug)
+            else:
+                logger.info(f"Finished setting up provider {provider_.slug}")
+        except Exception as e:
+            logger.opt(exception=True).error(f"Failed to setup provider {provider_.slug}: {e}")
+            failed_providers.append(provider_.slug)
+
+    if failed_providers:
+        msg = f"Setup failed for providers: {', '.join(failed_providers)}"
+        logger.error(msg)
+        raise typer.Exit(code=1)
+
+    if not validate_only:
+        list_(ctx)