climate-ref 0.6.6__py3-none-any.whl → 0.8.0__py3-none-any.whl

climate_ref/cli/__init__.py

@@ -3,11 +3,12 @@
 import importlib
 from enum import Enum
 from pathlib import Path
-from typing import Annotated, Optional
+from typing import Annotated
 
 import typer
 from attrs import define
 from loguru import logger
+from rich.console import Console
 
 from climate_ref import __version__
 from climate_ref.cli import config, datasets, executions, providers, solve
@@ -37,6 +38,7 @@ class CLIContext:
 
     config: Config
     database: Database
+    console: Console
 
 
 def _version_callback(value: bool) -> None:
@@ -46,6 +48,13 @@ def _version_callback(value: bool) -> None:
         raise typer.Exit()
 
 
+def _create_console() -> Console:
+    # Hook for testing to disable color output
+
+    # Rich respects the NO_COLOR environment variabl
+    return Console()
+
+
 def _load_config(configuration_directory: Path | None = None) -> Config:
     """
     Load the configuration from the specified directory
@@ -129,7 +138,7 @@ def main(  # noqa: PLR0913
         typer.Option(case_sensitive=False, help="Set the level of logging information to display"),
     ] = LogLevel.Info,
     version: Annotated[
-        Optional[bool],
+        bool | None,
         typer.Option(
             "--version", callback=_version_callback, is_eager=True, help="Print the version and exit"
         ),
@@ -155,7 +164,7 @@ def main(  # noqa: PLR0913
 
     logger.debug(f"Configuration loaded from: {config._config_file!s}")
 
-    ctx.obj = CLIContext(config=config, database=Database.from_config(config))
+    ctx.obj = CLIContext(config=config, database=Database.from_config(config), console=_create_console())
 
 
 if __name__ == "__main__":

climate_ref/cli/_utils.py

@@ -5,6 +5,59 @@ from rich.console import Console
 from rich.table import Table
 
 
+def parse_facet_filters(filters: list[str] | None) -> dict[str, str]:
+    """
+    Parse facet filters from key=value format into a dictionary.
+
+    Parameters
+    ----------
+    filters
+        List of filter strings in 'key=value' format
+
+    Returns
+    -------
+    dict[str, str]
+        Dictionary mapping facet keys to values
+
+    Raises
+    ------
+    ValueError
+        If a filter string is not in valid 'key=value' format
+
+    Examples
+    --------
+    >>> parse_facet_filters(["source_id=GFDL-ESM4", "variable_id=tas"])
+    {'source_id': 'GFDL-ESM4', 'variable_id': 'tas'}
+    """
+    if not filters:
+        return {}
+
+    parsed: dict[str, str] = {}
+    for filter_str in filters:
+        if "=" not in filter_str:
+            raise ValueError(
+                f"Invalid filter format: '{filter_str}'. "
+                f"Expected format: 'key=value' or 'dataset_type.key=value' "
+                f"(e.g., 'source_id=GFDL-ESM4' or 'cmip6.source_id=GFDL-ESM4')"
+            )
+
+        key, value = filter_str.split("=", 1)
+        key = key.strip()
+        value = value.strip()
+
+        if not key:
+            raise ValueError(f"Empty key in filter: '{filter_str}'")
+        if not value:
+            raise ValueError(f"Empty value in filter: '{filter_str}'")
+
+        if key in parsed:
+            logger.warning(f"Filter key '{key}' specified multiple times. Using last value: '{value}'")
+
+        parsed[key] = value
+
+    return parsed
+
+
 def df_to_table(df: pd.DataFrame, max_col_count: int = -1) -> Table:
     """
     Convert a DataFrame to a rich Table instance
@@ -32,7 +85,7 @@ def df_to_table(df: pd.DataFrame, max_col_count: int = -1) -> Table:
 
     table = Table(*[str(column) for column in df.columns])
 
-    for index, value_list in enumerate(df.values.tolist()):
+    for value_list in df.values.tolist():
         row = [str(x) for x in value_list]
         table.add_row(*row)
 
@@ -59,7 +112,8 @@ def pretty_print_df(df: pd.DataFrame, console: Console | None = None) -> None:
     # Drop duplicates as they are not informative to CLI users.
     df = df.drop_duplicates()
 
-    if console is None:
+    if console is None:  # pragma: no branch
+        logger.debug("Creating new console for pretty printing")
         console = Console()
 
     max_col_count = console.width // 10

climate_ref/cli/datasets.py

@@ -6,8 +6,6 @@ which executions are required for a given diagnostic without having to re-parse
 
 """
 
-import errno
-import os
 import shutil
 from collections.abc import Iterable
 from pathlib import Path
@@ -15,9 +13,9 @@ from typing import Annotated
 
 import typer
 from loguru import logger
-from rich.console import Console
 
 from climate_ref.cli._utils import pretty_print_df
+from climate_ref.database import ModelState
 from climate_ref.datasets import get_dataset_adapter
 from climate_ref.models import Dataset
 from climate_ref.provider_registry import ProviderRegistry
@@ -27,7 +25,6 @@ from climate_ref_core.dataset_registry import dataset_registry_manager, fetch_al
 from climate_ref_core.datasets import SourceDatasetType
 
 app = typer.Typer(help=__doc__)
-console = Console()
 
 
 @app.command(name="list")
@@ -70,7 +67,7 @@ def list_(
             raise typer.Exit(code=1)
         data_catalog = data_catalog[column].sort_values(by=column)
 
-    pretty_print_df(data_catalog, console=console)
+    pretty_print_df(data_catalog, console=ctx.obj.console)
 
 
 @app.command()
@@ -97,7 +94,7 @@ def list_columns(
 
 
 @app.command()
-def ingest(  # noqa: PLR0913
+def ingest(  # noqa
     ctx: typer.Context,
     file_or_directory: list[Path],
     source_type: Annotated[SourceDatasetType, typer.Option(help="Type of source dataset")],
@@ -106,7 +103,7 @@ def ingest(  # noqa: PLR0913
     n_jobs: Annotated[int | None, typer.Option(help="Number of jobs to run in parallel")] = None,
     skip_invalid: Annotated[
         bool, typer.Option(help="Ignore (but log) any datasets that don't pass validation")
-    ] = False,
+    ] = True,
 ) -> None:
     """
     Ingest a directory of datasets into the database
@@ -118,6 +115,7 @@ def ingest(  # noqa: PLR0913
     """
     config = ctx.obj.config
     db = ctx.obj.database
+    console = ctx.obj.console
 
     kwargs = {}
 
@@ -133,16 +131,35 @@ def ingest(  # noqa: PLR0913
 
         if not _dir.exists():
             logger.error(f"File or directory {_dir} does not exist")
-            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), _dir)
+            continue
+
+        # TODO: This assumes that all datasets are nc files.
+        # THis is true for CMIP6 and obs4MIPs but may not be true for other dataset types in the future.
+        if not _dir.rglob("*.nc"):
+            logger.error(f"No .nc files found in {_dir}")
+            continue
 
-        data_catalog = adapter.find_local_datasets(_dir)
-        data_catalog = adapter.validate_data_catalog(data_catalog, skip_invalid=skip_invalid)
+        try:
+            data_catalog = adapter.find_local_datasets(_dir)
+            data_catalog = adapter.validate_data_catalog(data_catalog, skip_invalid=skip_invalid)
+        except Exception as e:
+            logger.error(f"Error ingesting datasets from {_dir}: {e}")
+            continue
 
         logger.info(
             f"Found {len(data_catalog)} files for {len(data_catalog[adapter.slug_column].unique())} datasets"
         )
         pretty_print_df(adapter.pretty_subset(data_catalog), console=console)
 
+        # track stats for a given directory
+        num_created_datasets = 0
+        num_updated_datasets = 0
+        num_unchanged_datasets = 0
+        num_created_files = 0
+        num_updated_files = 0
+        num_removed_files = 0
+        num_unchanged_files = 0
+
         for instance_id, data_catalog_dataset in data_catalog.groupby(adapter.slug_column):
             logger.debug(f"Processing dataset {instance_id}")
             with db.session.begin():
@@ -154,9 +171,29 @@ def ingest(  # noqa: PLR0913
                     )
                     if not dataset:
                         logger.info(f"Would save dataset {instance_id} to the database")
-                        continue
                 else:
-                    adapter.register_dataset(config, db, data_catalog_dataset)
+                    results = adapter.register_dataset(config, db, data_catalog_dataset)
+
+                    if results.dataset_state == ModelState.CREATED:
+                        num_created_datasets += 1
+                    elif results.dataset_state == ModelState.UPDATED:
+                        num_updated_datasets += 1
+                    else:
+                        num_unchanged_datasets += 1
+                    num_created_files += len(results.files_added)
+                    num_updated_files += len(results.files_updated)
+                    num_removed_files += len(results.files_removed)
+                    num_unchanged_files += len(results.files_unchanged)
+
+        if not dry_run:
+            ingestion_msg = (
+                f"Datasets: {num_created_datasets}/{num_updated_datasets}/{num_unchanged_datasets}"
+                " (created/updated/unchanged), "
+                f"Files: "
+                f"{num_created_files}/{num_updated_files}/{num_removed_files}/{num_unchanged_files}"
+                " (created/updated/removed/unchanged)"
+            )
+            logger.info(ingestion_msg)
 
     if solve:
         solve_required_executions(

climate_ref/cli/executions.py

@@ -2,38 +2,90 @@
 View execution groups and their results
 """
 
+import json
 import pathlib
+import shutil
+from dataclasses import dataclass
 from typing import Annotated
 from urllib.parse import quote
 
 import pandas as pd
 import typer
 from loguru import logger
-from rich.console import Console, Group
+from rich.console import Group
 from rich.filesize import decimal
 from rich.markup import escape
 from rich.panel import Panel
 from rich.text import Text
 from rich.tree import Tree
 
-from climate_ref.cli._utils import df_to_table, pretty_print_df
+from climate_ref.cli._utils import df_to_table, parse_facet_filters, pretty_print_df
 from climate_ref.config import Config
 from climate_ref.models import Execution, ExecutionGroup
-from climate_ref.models.execution import get_execution_group_and_latest
+from climate_ref.models.execution import execution_datasets, get_execution_group_and_latest_filtered
 from climate_ref_core.logging import EXECUTION_LOG_FILENAME
 
 app = typer.Typer(help=__doc__)
-console = Console()
+
+
+@dataclass
+class ListGroupsFilterOptions:
+    """Options to filter execution groups in list-groups command"""
+
+    diagnostic: list[str] | None = None
+    """Filter by diagnostic slug (substring, case-insensitive)"""
+
+    provider: list[str] | None = None
+    """Filter by provider slug (substring, case-insensitive)"""
+
+    facets: dict[str, str] | None = None
+    """Filter by facet key-value pairs (exact match)"""
 
 
 @app.command()
-def list_groups(
+def list_groups(  # noqa: PLR0913
     ctx: typer.Context,
     column: Annotated[
         list[str] | None,
         typer.Option(help="Only include specified columns in the output"),
     ] = None,
     limit: int = typer.Option(100, help="Limit the number of rows to display"),
+    diagnostic: Annotated[
+        list[str] | None,
+        typer.Option(
+            help="Filter by diagnostic slug (substring match, case-insensitive)."
+            "Multiple values can be provided."
+        ),
+    ] = None,
+    provider: Annotated[
+        list[str] | None,
+        typer.Option(
+            help="Filter by provider slug (substring match, case-insensitive)."
+            "Multiple values can be provided."
+        ),
+    ] = None,
+    filter: Annotated[  # noqa: A002
+        list[str] | None,
+        typer.Option(
+            "--filter",
+            help="Filter by facet key=value pairs (exact match). Multiple filters can be provided.",
+        ),
+    ] = None,
+    successful: Annotated[
+        bool | None,
+        typer.Option(
+            "--successful/--not-successful",
+            help="Filter by successful or unsuccessful executions.",
+        ),
+    ] = None,
+    dirty: Annotated[
+        bool | None,
+        typer.Option(
+            "--dirty/--not-dirty",
+            help="Filter to include only dirty or clean execution groups."
+            "These execution groups will be re-computed on the next run.",
+        ),
+    ] = None,
 ) -> None:
     """
     List the diagnostic execution groups that have been identified
@@ -41,42 +93,273 @@ def list_groups(
     The data catalog is sorted by the date that the execution group was created (first = newest).
     If the `--column` option is provided, only the specified columns will be displayed.
 
+    Filters can be combined using AND logic across filter types and OR logic within a filter type.
+
     The output will be in a tabular format.
     """
     session = ctx.obj.database.session
+    console = ctx.obj.console
 
-    execution_groups_results = get_execution_group_and_latest(session).limit(limit).all()
-    execution_count = session.query(ExecutionGroup).count()
+    # Parse facet filters
+    try:
+        facet_filters = parse_facet_filters(filter)
+    except ValueError as e:
+        logger.error(str(e))
+        raise typer.Exit(code=1)
 
-    results_df = pd.DataFrame(
-        [
-            {
-                "id": execution_groups.id,
-                "key": execution_groups.key,
-                "provider": execution_groups.diagnostic.provider.slug,
-                "diagnostic": execution_groups.diagnostic.slug,
-                "dirty": execution_groups.dirty,
-                "successful": result.successful if result else None,
-                "created_at": execution_groups.created_at,
-                "updated_at": execution_groups.updated_at,
-            }
-            for execution_groups, result in execution_groups_results
-        ]
+    # Build filter options
+    filters = ListGroupsFilterOptions(
+        diagnostic=diagnostic,
+        provider=provider,
+        facets=facet_filters if facet_filters else None,
     )
+    logger.debug(f"Applying filters: {filters}")
+
+    # Get total count before filtering for warning messages
+    total_count = session.query(ExecutionGroup).count()
+
+    # Apply filters to query
+    try:
+        all_filtered_results = get_execution_group_and_latest_filtered(
+            session,
+            diagnostic_filters=filters.diagnostic,
+            provider_filters=filters.provider,
+            facet_filters=filters.facets,
+            successful=successful,
+            dirty=dirty,
+        )
+        execution_groups_results = all_filtered_results[:limit]
+    except Exception as e:  # pragma: no cover
+        logger.error(f"Error applying filters: {e}")
+        raise typer.Exit(code=1)
+
+    # Check if any results found
+    if not execution_groups_results:
+        emit_no_results_warning(filters, total_count)
+        results_df = pd.DataFrame(
+            columns=[
+                "id",
+                "key",
+                "provider",
+                "diagnostic",
+                "dirty",
+                "successful",
+                "created_at",
+                "updated_at",
+                "selectors",
+            ]
+        )
+    else:
+        results_df = pd.DataFrame(
+            [
+                {
+                    "id": eg.id,
+                    "key": eg.key,
+                    "provider": eg.diagnostic.provider.slug,
+                    "diagnostic": eg.diagnostic.slug,
+                    "dirty": eg.dirty,
+                    "successful": result.successful if result else None,
+                    "created_at": eg.created_at,
+                    "updated_at": eg.updated_at,
+                    "selectors": json.dumps(eg.selectors),
+                }
+                for eg, result in execution_groups_results
+            ]
+        )
 
-    if column:
+    # Apply column filtering
+    if column and not results_df.empty:  # Only apply if df is not empty
         if not all(col in results_df.columns for col in column):
             logger.error(f"Column not found in data catalog: {column}")
             raise typer.Exit(code=1)
         results_df = results_df[column]
 
+    # Display results
     pretty_print_df(results_df, console=console)
-    if execution_count > limit:
+
+    # Show limit warning if applicable
+    filtered_count = len(all_filtered_results)
+    if filtered_count > limit:
         logger.warning(
-            f"Displaying {limit} of {execution_count} results. Use the `--limit` option to display more."
+            f"Displaying {limit} of {filtered_count} filtered results. "
+            f"Use the `--limit` option to display more."
         )
 
 
+@app.command()
+def delete_groups(  # noqa: PLR0912, PLR0913
+    ctx: typer.Context,
+    diagnostic: Annotated[
+        list[str] | None,
+        typer.Option(
+            help="Filter by diagnostic slug (substring match, case-insensitive)."
+            "Multiple values can be provided."
+        ),
+    ] = None,
+    provider: Annotated[
+        list[str] | None,
+        typer.Option(
+            help="Filter by provider slug (substring match, case-insensitive)."
+            "Multiple values can be provided."
+        ),
+    ] = None,
+    filter: Annotated[  # noqa: A002
+        list[str] | None,
+        typer.Option(
+            "--filter",
+            help="Filter by facet key=value pairs (exact match). Multiple filters can be provided.",
+        ),
+    ] = None,
+    successful: Annotated[
+        bool | None,
+        typer.Option(
+            "--successful/--not-successful",
+            help="Filter by successful or unsuccessful executions.",
+        ),
+    ] = None,
+    dirty: Annotated[
+        bool | None,
+        typer.Option(
+            "--dirty/--not-dirty",
+            help="Filter to include only dirty or clean execution groups."
+            "These execution groups will be re-computed on the next run.",
+        ),
+    ] = None,
+    remove_outputs: bool = typer.Option(
+        False, "--remove-outputs", help="Also remove output directories from the filesystem"
+    ),
+    force: bool = typer.Option(False, help="Skip confirmation prompt"),
+) -> None:
+    """
+    Delete execution groups matching the specified filters.
+
+    This command will delete execution groups and their associated executions.
+    Use filters to specify which groups to delete. At least one filter must be provided
+    to prevent accidental deletion of all groups.
+
+    Filters can be combined using AND logic across filter types and OR logic within a filter type.
+    """
+    session = ctx.obj.database.session
+    console = ctx.obj.console
+
+    # Parse facet filters
+    try:
+        facet_filters = parse_facet_filters(filter)
+    except ValueError as e:
+        logger.error(str(e))
+        raise typer.Exit(code=1)
+
+    if not any([diagnostic, provider, facet_filters, successful is not None, dirty is not None]):
+        logger.warning("THIS WILL DELETE ALL EXECUTION GROUPS IN THE DATABASE")
+        raise typer.Exit(code=1)
+
+    # Build filter options
+    filters = ListGroupsFilterOptions(
+        diagnostic=diagnostic,
+        provider=provider,
+        facets=facet_filters if facet_filters else None,
+    )
+    logger.debug(f"Applying filters: {filters}")
+
+    # Apply filters to query
+    try:
+        all_filtered_results = get_execution_group_and_latest_filtered(
+            session,
+            diagnostic_filters=filters.diagnostic,
+            provider_filters=filters.provider,
+            facet_filters=filters.facets,
+            successful=successful,
+            dirty=dirty,
+        )
+    except Exception as e:  # pragma: no cover
+        logger.error(f"Error applying filters: {e}")
+        raise typer.Exit(code=1)
+
+    # Check if any results found
+    if not all_filtered_results:
+        emit_no_results_warning(filters, session.query(ExecutionGroup).count())
+        return
+
+    # Convert to DataFrame for preview
+    results_df = pd.DataFrame(
+        [
+            {
+                "id": eg.id,
+                "key": eg.key,
+                "provider": eg.diagnostic.provider.slug,
+                "diagnostic": eg.diagnostic.slug,
+                "dirty": eg.dirty,
+                "successful": result.successful if result else None,
+                "created_at": eg.created_at,
+                "updated_at": eg.updated_at,
+                "selectors": json.dumps(eg.selectors),
+            }
+            for eg, result in all_filtered_results
+        ]
+    )
+
+    # Display preview
+    console.print("Execution groups to be deleted:")
+    pretty_print_df(results_df, console=console)
+
+    count = len(all_filtered_results)
+    console.print(f"\nWill delete {count} execution group(s).")
+
+    # Confirm unless force is set
+    if not force:
+        if not typer.confirm("Do you want to proceed with deletion?"):
+            console.print("Deletion cancelled.")
+            return
+
+    # Remove output directories if requested
+    if remove_outputs:
+        config = ctx.obj.config
+        for eg, _ in all_filtered_results:
+            for execution in eg.executions:
+                output_dir = config.paths.results / execution.output_fragment
+
+                # Safety check
+                if not output_dir.is_relative_to(config.paths.results):  # pragma: no cover
+                    logger.error(f"Skipping unsafe path: {output_dir}")
+                    continue
+
+                if output_dir.exists():
+                    try:
+                        logger.warning(f"Removing output directory: {output_dir}")
+                        shutil.rmtree(output_dir)
+                    except Exception as e:
+                        logger.error(f"Failed to remove {output_dir}: {e}")
+
+    # Delete execution groups and all related records
+    # TODO: Add cascade delete to FK relationships and simplify this code
+    with session.begin_nested() if session.in_transaction() else session.begin():
+        for eg, _ in all_filtered_results:
+            for execution in eg.executions:
+                # Delete MetricValues first
+                for metric_value in execution.values:
+                    session.delete(metric_value)
+
+                # Delete ExecutionOutputs
+                for output in execution.outputs:
+                    session.delete(output)
+
+                # Delete many-to-many associations with datasets
+                session.execute(
+                    execution_datasets.delete().where(execution_datasets.c.execution_id == execution.id)
+                )
+
+                # Now delete the execution
+                session.delete(execution)
+
+            # Finally delete the execution group
+            session.delete(eg)
+
+    if remove_outputs:
+        console.print(f"[green]Successfully deleted {count} execution group(s) and their output directories.")
+    else:
+        console.print(f"[green]Successfully deleted {count} execution group(s).")
+
+
 def walk_directory(directory: pathlib.Path, tree: Tree) -> None:
     """Recursively build a Tree with directory contents."""
     # Sort dirs first then by filename
@@ -182,6 +465,29 @@ def _log_panel(result_directory: pathlib.Path) -> Panel | None:
         )
 
 
+def emit_no_results_warning(
+    filters: ListGroupsFilterOptions,
+    total_count: int,
+) -> None:
+    """
+    Emit informative warning when filters produce no results.
+    """
+    filter_parts = []
+    if filters.diagnostic:
+        filter_parts.append(f"diagnostic filters: {filters.diagnostic}")
+    if filters.provider:
+        filter_parts.append(f"provider filters: {filters.provider}")
+    if filters.facets:
+        facet_strs = [f"{k}={v}" for k, v in filters.facets.items()]
+        filter_parts.append(f"facet filters: {facet_strs}")
+
+    logger.warning(
+        f"No execution groups match the specified filters. "
+        f"Total execution groups in database: {total_count}. "
+        f"Applied filters: {', '.join(filter_parts)}"
+    )
+
+
 @app.command()
 def inspect(ctx: typer.Context, execution_id: int) -> None:
     """
@@ -191,6 +497,8 @@ def inspect(ctx: typer.Context, execution_id: int) -> None:
     """
     config: Config = ctx.obj.config
     session = ctx.obj.database.session
+    console = ctx.obj.console
+
     execution_group = session.get(ExecutionGroup, execution_id)
 
     if not execution_group:
@@ -217,6 +525,7 @@ def flag_dirty(ctx: typer.Context, execution_id: int) -> None:
     Flag an execution group for recomputation
     """
     session = ctx.obj.database.session
+    console = ctx.obj.console
     with session.begin():
         execution_group = session.get(ExecutionGroup, execution_id)