climate-ref 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

climate_ref/cli/__init__.py

@@ -11,7 +11,7 @@ from loguru import logger
 from rich.console import Console
 
 from climate_ref import __version__
-from climate_ref.cli import config, datasets, executions, providers, solve
+from climate_ref.cli import config, datasets, executions, providers, solve, test_cases
 from climate_ref.config import Config
 from climate_ref.constants import CONFIG_FILENAME
 from climate_ref.database import Database
@@ -104,6 +104,7 @@ def build_app() -> typer.Typer:
     app.add_typer(datasets.app, name="datasets")
     app.add_typer(executions.app, name="executions")
     app.add_typer(providers.app, name="providers")
+    app.add_typer(test_cases.app, name="test-cases")
 
     try:
         celery_app = importlib.import_module("climate_ref_celery.cli").app
@@ -164,7 +165,9 @@ def main(  # noqa: PLR0913
 
     logger.debug(f"Configuration loaded from: {config._config_file!s}")
 
-    ctx.obj = CLIContext(config=config, database=Database.from_config(config), console=_create_console())
+    # Use ctx.with_resource to ensure the database connection is closed when the CLI exits
+    database = ctx.with_resource(Database.from_config(config))
+    ctx.obj = CLIContext(config=config, database=database, console=_create_console())
 
 
 if __name__ == "__main__":

climate_ref/cli/_git_utils.py

@@ -0,0 +1,112 @@
+"""Git utilities for CLI commands."""
+
+from pathlib import Path
+from typing import Any
+
+from git import InvalidGitRepositoryError, Repo
+
+
+def get_repo_for_path(path: Path) -> Repo | None:
+    """
+    Get the git repository containing the given path.
+
+    Parameters
+    ----------
+    path
+        Path to a file or directory
+
+    Returns
+    -------
+    :
+        The Repo object if path is within a git repository, None otherwise
+    """
+    try:
+        return Repo(path, search_parent_directories=True)
+    except InvalidGitRepositoryError:
+        return None
+
+
+def get_git_status(file_path: Path, repo: Repo) -> str:
+    """
+    Get git status for a file using GitPython.
+
+    Parameters
+    ----------
+    file_path
+        Absolute path to the file
+    repo
+        GitPython Repo object
+
+    Returns
+    -------
+    :
+        Status string: "new", "staged", "modified", "tracked", "untracked", or "unknown"
+    """
+    try:
+        rel_path = str(file_path.relative_to(repo.working_dir))
+
+        # Check if untracked
+        if rel_path in repo.untracked_files:
+            return "new"
+
+        # Check staged changes (index vs HEAD)
+        staged_files = {item.a_path for item in repo.index.diff("HEAD")}
+        if rel_path in staged_files:
+            return "staged"
+
+        # Check unstaged changes (working tree vs index)
+        unstaged_files = {item.a_path for item in repo.index.diff(None)}
+        if rel_path in unstaged_files:
+            return "modified"
+
+        # Check if file is tracked
+        try:
+            repo.git.ls_files("--error-unmatch", rel_path)
+            return "tracked"
+        except Exception:
+            return "untracked"
+    except Exception:
+        return "unknown"
+
+
+def collect_regression_file_info(
+    regression_dir: Path,
+    repo: Repo | None,
+    size_threshold_bytes: int,
+) -> list[dict[str, Any]]:
+    """
+    Collect file information from a regression directory.
+
+    Parameters
+    ----------
+    regression_dir
+        Path to the regression data directory
+    repo
+        Git repository object, or None if not in a repo
+    size_threshold_bytes
+        Files larger than this will be flagged as large
+
+    Returns
+    -------
+    :
+        List of dicts with keys: rel_path, size, is_large, git_status
+    """
+    files = sorted(regression_dir.rglob("*"))
+    files = [f for f in files if f.is_file()]
+
+    file_info: list[dict[str, Any]] = []
+    for file_path in files:
+        size = file_path.stat().st_size
+        rel_path = str(file_path.relative_to(regression_dir))
+        git_status = get_git_status(file_path, repo) if repo else "unknown"
+
+        file_info.append(
+            {
+                "rel_path": rel_path,
+                "size": size,
+                "is_large": size > size_threshold_bytes,
+                "git_status": git_status,
+            }
+        )
+
+    return file_info

climate_ref/cli/_utils.py

@@ -4,6 +4,30 @@ from rich import box
 from rich.console import Console
 from rich.table import Table
 
+_BYTES_PER_UNIT = 1024
+
+
+def format_size(size_bytes: int | float) -> str:
+    """
+    Format file size in human-readable form.
+
+    Parameters
+    ----------
+    size_bytes
+        Size in bytes
+
+    Returns
+    -------
+    :
+        Human-readable size string (e.g., "1.5 MB")
+    """
+    size = float(size_bytes)
+    for unit in ("B", "KB", "MB", "GB"):
+        if size < _BYTES_PER_UNIT:
+            return f"{size:.1f} {unit}"
+        size /= _BYTES_PER_UNIT
+    return f"{size:.1f} TB"
+
 
 def parse_facet_filters(filters: list[str] | None) -> dict[str, str]:
     """

climate_ref/cli/datasets.py

@@ -217,6 +217,7 @@ def _fetch_sample_data(
     This operation may fail if the test data directory does not exist,
     as is the case for non-source-based installations.
     """
+    # TODO: Remove
     fetch_sample_data(force_cleanup=force_cleanup, symlink=symlink)
 
 

climate_ref/cli/providers.py

@@ -2,6 +2,7 @@
 Manage the REF providers.
 """
 
+import warnings
 from typing import Annotated
 
 import pandas as pd
@@ -33,12 +34,23 @@ def list_(ctx: typer.Context) -> None:
                 env += " (not installed)"
         return env
 
+    def get_data_path(provider: DiagnosticProvider) -> str:
+        """Get the data cache path for a provider."""
+        data_path = provider.get_data_path()
+        if data_path is None:
+            return ""
+        path_str = str(data_path)
+        if not data_path.exists():
+            path_str += " (not fetched)"
+        return path_str
+
     results_df = pd.DataFrame(
         [
             {
                 "provider": provider.slug,
                 "version": provider.version,
                 "conda environment": get_env(provider),
+                "data path": get_data_path(provider),
             }
             for provider in provider_registry.providers
         ]
@@ -46,7 +58,7 @@ def list_(ctx: typer.Context) -> None:
     pretty_print_df(results_df, console=console)
 
 
-@app.command()
+@app.command(deprecated=True)
 def create_env(
     ctx: typer.Context,
     provider: Annotated[
@@ -57,9 +69,18 @@ def create_env(
     """
     Create a conda environment containing the provider software.
 
+    .. deprecated::
+        Use `ref providers setup` instead, which handles both environment creation
+        and data fetching in a single command.
+
     If no provider is specified, all providers will be installed.
-    If the provider is up to date or does not use a virtual environment, it will be skipped.
+    If the provider is up to date or does not use a conda environment, it will be skipped.
     """
+    warnings.warn(
+        "create-env is deprecated. Use 'ref providers setup' instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     config = ctx.obj.config
     db = ctx.obj.database
     providers = ProviderRegistry.build_from_config(config, db).providers
@@ -73,12 +94,90 @@ def create_env(
             raise typer.Exit(code=1)
 
     for provider_ in providers:
-        txt = f"virtual environment for provider {provider_.slug}"
+        txt = f"conda environment for provider {provider_.slug}"
         if isinstance(provider_, CondaDiagnosticProvider):
             logger.info(f"Creating {txt} in {provider_.env_path}")
             provider_.create_env()
             logger.info(f"Finished creating {txt}")
         else:
-            logger.info(f"Skipping creating {txt} because it does use virtual environments.")
+            logger.info(f"Skipping creating {txt} because it does not use conda environments.")
 
     list_(ctx)
+
+
+@app.command()
+def setup(
+    ctx: typer.Context,
+    provider: Annotated[
+        str | None,
+        typer.Option(help="Only run setup for the named provider."),
+    ] = None,
+    skip_env: Annotated[
+        bool,
+        typer.Option(help="Skip environment setup (e.g., conda)."),
+    ] = False,
+    skip_data: Annotated[
+        bool,
+        typer.Option(help="Skip data fetching."),
+    ] = False,
+    validate_only: Annotated[
+        bool,
+        typer.Option(help="Only validate setup, don't run it."),
+    ] = False,
+) -> None:
+    """
+    Run provider setup for offline execution.
+
+    This command prepares all providers for offline execution by:
+
+    1. Creating conda environments (if applicable)
+
+    2. Fetching required reference datasets to pooch cache
+
+    All operations are idempotent and safe to run multiple times.
+    Run this on a login node with internet access before solving on compute nodes.
+    """
+    config = ctx.obj.config
+    db = ctx.obj.database
+    console = ctx.obj.console
+    providers = ProviderRegistry.build_from_config(config, db).providers
+
+    if provider is not None:
+        available = ", ".join([f'"{p.slug}"' for p in providers])
+        providers = [p for p in providers if p.slug == provider]
+        if not providers:
+            msg = f'Provider "{provider}" not available. Choose from: {available}'
+            logger.error(msg)
+            raise typer.Exit(code=1)
+
+    failed_providers: list[str] = []
+
+    for provider_ in providers:
+        if validate_only:
+            is_valid = provider_.validate_setup(config)
+            status = "[green]valid[/green]" if is_valid else "[red]invalid[/red]"
+            console.print(f"Provider {provider_.slug}: {status}")
+            if not is_valid:
+                failed_providers.append(provider_.slug)
+            continue
+
+        logger.info(f"Setting up provider {provider_.slug}")
+        try:
+            provider_.setup(config, skip_env=skip_env, skip_data=skip_data)
+            is_valid = provider_.validate_setup(config)
+            if not is_valid:
+                logger.error(f"Provider {provider_.slug} setup completed but validation failed")
+                failed_providers.append(provider_.slug)
+            else:
+                logger.info(f"Finished setting up provider {provider_.slug}")
+        except Exception as e:
+            logger.opt(exception=True).error(f"Failed to setup provider {provider_.slug}: {e}")
+            failed_providers.append(provider_.slug)
+
+    if failed_providers:
+        msg = f"Setup failed for providers: {', '.join(failed_providers)}"
+        logger.error(msg)
+        raise typer.Exit(code=1)
+
+    if not validate_only:
+        list_(ctx)