cmem-cmemc 25.5.0rc1__py3-none-any.whl → 26.1.0rc1__py3-none-any.whl

cmem_cmemc/commands/config.py

@@ -1,34 +1,128 @@
 """configuration commands for cmem command line interface."""
 
 import click
-from click import ClickException
+from click import Context
 
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
-from cmem_cmemc.context import KNOWN_CONFIG_KEYS, ApplicationContext
+from cmem_cmemc.context import KNOWN_CONFIG_KEYS, ApplicationContext, build_caption
+from cmem_cmemc.exceptions import CmemcError
+from cmem_cmemc.object_list import (
+    DirectValuePropertyFilter,
+    ObjectList,
+    compare_regex,
+)
+
+
+def get_connections(ctx: Context) -> list[dict]:
+    """Get connections for object list"""
+    app = ctx.obj
+    if app is not None:
+        config = app.get_config()
+    else:
+        # when in completion mode, obj is not set :-(
+        # return empty list
+        return []
+
+    connections = []
+
+    for section_name in config.sections():
+        if section_name != "DEFAULT":
+            section = config[section_name]
+            connections.append(
+                {
+                    "name": section_name,
+                    "base_uri": section.get("CMEM_BASE_URI", "-"),
+                    "grant_type": section.get("OAUTH_GRANT_TYPE", "-"),
+                }
+            )
+
+    return connections
+
+
+config_list = ObjectList(
+    name="connections",
+    get_objects=get_connections,
+    filters=[
+        DirectValuePropertyFilter(
+            name="regex",
+            description="Filter by regex matching the connection name.",
+            property_key="name",
+            compare=compare_regex,
+            completion_method="none",
+        ),
+    ],
+)
 
 
 @click.command(cls=CmemcCommand, name="list")
-@click.pass_obj
-def list_command(app: ApplicationContext) -> None:
+@click.option(
+    "--filter",
+    "filter_",
+    type=(str, str),
+    multiple=True,
+    help=config_list.get_filter_help_text(),
+    shell_complete=config_list.complete_values,
+)
+@click.option(
+    "--id-only",
+    is_flag=True,
+    help="Lists only connection names. "
+    "This is useful for piping the names into other cmemc commands.",
+)
+@click.pass_context
+def list_command(ctx: Context, filter_: tuple[tuple[str, str]], id_only: bool) -> None:
     """List configured connections.
 
-    This command lists all configured
-    connections from the currently used config file.
+    This command lists all configured connections from the currently used config file.
+    Each connection is listed with its name, base URI, and grant type.
 
     The connection identifier can be used with the --connection option
     in order to use a specific Corporate Memory instance.
 
+    You can use the --filter option to filter connections by regex matching
+    the connection name.
+
     In order to apply commands on more than one instance, you need to use
     typical unix gear such as xargs or parallel.
 
-    Example: cmemc config list | xargs -I % sh -c 'cmemc -c % admin status'
+    Example: cmemc config list
+
+    Example: cmemc config list --id-only | xargs -I % sh -c 'cmemc -c % admin status'
 
-    Example: cmemc config list | parallel --jobs 5 cmemc -c {} admin status
+    Example: cmemc config list --id-only | parallel --jobs 5 cmemc -c {} admin status
     """
-    for section_string in sorted(app.get_config(), key=str.casefold):
-        if section_string != "DEFAULT":
-            app.echo_result(section_string)
+    app = ctx.obj
+    connections = config_list.apply_filters(ctx=ctx, filter_=filter_)
+
+    # Sort connections case-insensitively by name
+    connections = sorted(connections, key=lambda c: c["name"].casefold())
+
+    if id_only:
+        for connection in connections:
+            app.echo_result(connection["name"])
+        return
+
+    table = [
+        (
+            connection["name"],
+            connection["base_uri"],
+            connection["grant_type"],
+        )
+        for connection in connections
+    ]
+
+    filtered = len(filter_) > 0
+    app.echo_info_table(
+        table,
+        headers=["Connection", "Base URI", "Grant Type"],
+        caption=build_caption(
+            len(table), "connection", instance=str(app.config_file), filtered=filtered
+        ),
+        empty_table_message="No connections found for these filters."
+        if filtered
+        else "No connections found.",
+    )
 
 
 @click.command(cls=CmemcCommand, name="edit")
@@ -60,7 +154,7 @@ def get_command(app: ApplicationContext, key: str) -> None:
     value = KNOWN_CONFIG_KEYS[key]()
     app.echo_debug(f"Type of {key} value is {type(value)}")
     if value is None:
-        raise ClickException(f"Configuration key {key} is not used in this configuration.")
+        raise CmemcError(f"Configuration key {key} is not used in this configuration.")
     app.echo_info(str(value))
 
 

cmem_cmemc/commands/dataset.py

@@ -1,11 +1,10 @@
 """dataset commands for cmem command line interface."""
 
 import json
-import re
 
 import click
 import requests.exceptions
-from click import ClickException, UsageError
+from click import Context, UsageError
 from cmem.cmempy.config import get_cmem_base_uri
 from cmem.cmempy.workspace import get_task_plugin_description, get_task_plugins
 from cmem.cmempy.workspace.projects.datasets.dataset import (
@@ -16,79 +15,77 @@ from cmem.cmempy.workspace.projects.datasets.dataset import (
     update_dataset,
 )
 from cmem.cmempy.workspace.projects.resources.resource import (
-    create_resource,
     get_resource_response,
-    resource_exist,
 )
 from cmem.cmempy.workspace.search import list_items
 
 from cmem_cmemc import completion
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
-from cmem_cmemc.commands.resource import resource
+from cmem_cmemc.commands.file import _upload_file_resource, resource
 from cmem_cmemc.completion import get_dataset_file_mapping
-from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.context import ApplicationContext, build_caption
 from cmem_cmemc.exceptions import CmemcError
+from cmem_cmemc.object_list import (
+    DirectListPropertyFilter,
+    DirectMultiValuePropertyFilter,
+    DirectValuePropertyFilter,
+    ObjectList,
+    compare_regex,
+    transform_extract_labels,
+)
 from cmem_cmemc.parameter_types.path import ClickSmartPath
 from cmem_cmemc.smart_path import SmartPath as Path
+from cmem_cmemc.string_processor import DatasetLink, DatasetTypeLink
+from cmem_cmemc.title_helper import DatasetTypeTitleHelper, ProjectTitleHelper
 from cmem_cmemc.utils import check_or_select_project, struct_to_table
 
-DATASET_FILTER_TYPES = sorted(["project", "regex", "tag", "type"])
-DATASET_LIST_FILTER_HELP_TEXT = (
-    "Filter datasets based on metadata. First parameter"
-    f" can be one of the following values: {', '.join(DATASET_FILTER_TYPES)}."
-    " The options for the second parameter depend on the first parameter."
-)
-DATASET_DELETE_FILTER_HELP_TEXT = (
-    "Delete datasets based on metadata. First parameter --filter"
-    f" CHOICE can be one of {DATASET_FILTER_TYPES!s}."
-    " The second parameter is based on CHOICE."
-)
-
-
-def _get_dataset_tag_labels(dataset_: dict) -> list[str]:
-    """Output a list of tag labels from a single dataset."""
-    return [_["label"] for _ in dataset_["tags"]]
-
 
-def _get_datasets_filtered(
-    datasets: list[dict], filter_name: str, filter_value: str | int
-) -> list[dict]:
-    """Get dataset filtered according to filter name and value.
-
-    Args:
-    ----
-        datasets: list of datasets to filter
-        filter_name (str): one of DATASET_FILTER_TYPES
-        filter_value (str|int): value according to filter
-
-    Returns:
-    -------
-        list of filtered datasets from the get_query_status API call
-
-    Raises:
-    ------
-        UsageError
-
-    """
-    if filter_name not in DATASET_FILTER_TYPES:
-        raise UsageError(
-            f"{filter_name} is an unknown filter name. " f"Use one of {DATASET_FILTER_TYPES}."
-        )
-    # filter by project ID
-    if filter_name == "project":
-        return [_ for _ in datasets if _["projectId"] == filter_value]
-    # filter by regex on the label
-    if filter_name == "regex":
-        return [_ for _ in datasets if re.search(str(filter_value), _["label"])]
-    # filter by dataset type
-    if filter_name == "type":
-        return [_ for _ in datasets if re.search(str(filter_value), _["pluginId"])]
-    # filter by tag label
-    if filter_name == "tag":
-        return [_ for _ in datasets if filter_value in _get_dataset_tag_labels(_)]
-    # default is unfiltered
-    return datasets
+def get_datasets(ctx: Context) -> list[dict]:
+    """Get datasets for object list."""
+    _ = ctx
+    return list_items(item_type="dataset")["results"]  # type: ignore[no-any-return]
+
+
+dataset_list = ObjectList(
+    name="datasets",
+    get_objects=get_datasets,
+    filters=[
+        DirectValuePropertyFilter(
+            name="project",
+            description="Filter by project ID.",
+            property_key="projectId",
+            completion_method="values",
+            title_helper=ProjectTitleHelper(),
+        ),
+        DirectValuePropertyFilter(
+            name="regex",
+            description="Filter by regex matching the dataset label.",
+            property_key="label",
+            compare=compare_regex,
+            completion_method="none",
+        ),
+        DirectValuePropertyFilter(
+            name="type",
+            description="Filter by dataset type.",
+            property_key="pluginId",
+            compare=compare_regex,
+            completion_method="values",
+            title_helper=DatasetTypeTitleHelper(),
+        ),
+        DirectListPropertyFilter(
+            name="tag",
+            description="Filter by tag label.",
+            property_key="tags",
+            transform=transform_extract_labels,
+        ),
+        DirectMultiValuePropertyFilter(
+            name="ids",
+            description="Internal filter for multiple dataset IDs.",
+            property_key="id",
+        ),
+    ],
+)
 
 
 def _validate_and_split_dataset_id(dataset_id: str) -> tuple[str, str]:
@@ -107,13 +104,64 @@ def _validate_and_split_dataset_id(dataset_id: str) -> tuple[str, str]:
         project_part = dataset_id.split(":")[0]
         dataset_part = dataset_id.split(":")[1]
     except IndexError as error:
-        raise ClickException(
+        raise CmemcError(
             f"{dataset_id} is not a valid dataset ID. Use the "
             "'dataset list' command to get a list of existing datasets."
         ) from error
     return project_part, dataset_part
 
 
+def _validate_dataset_ids(dataset_ids: tuple[str, ...]) -> None:
+    """Validate that all provided dataset IDs exist."""
+    if not dataset_ids:
+        return
+    all_datasets = list_items(item_type="dataset")["results"]
+    all_dataset_ids = [_["projectId"] + ":" + _["id"] for _ in all_datasets]
+    for dataset_id in dataset_ids:
+        if dataset_id not in all_dataset_ids:
+            raise CmemcError(
+                f"Dataset {dataset_id} not available. Use the 'dataset list' "
+                "command to get a list of existing datasets."
+            )
+
+
+def _get_datasets_to_delete(
+    ctx: Context,
+    dataset_ids: tuple[str, ...],
+    all_: bool,
+    filter_: tuple[tuple[str, str], ...],
+) -> list[str]:
+    """Get the list of dataset IDs to delete based on selection method."""
+    if all_:
+        # Get all datasets
+        datasets = list_items(item_type="dataset")["results"]
+        return [_["projectId"] + ":" + _["id"] for _ in datasets]
+
+    # Validate provided IDs exist before proceeding
+    _validate_dataset_ids(dataset_ids)
+
+    # Build filter list
+    filter_to_apply = list(filter_) if filter_ else []
+
+    # Add IDs if provided (using internal multi-value filter)
+    if dataset_ids:
+        # Extract just the dataset ID part (after the colon) for filtering
+        dataset_id_parts = [_.split(":")[1] for _ in dataset_ids]
+        filter_to_apply.append(("ids", ",".join(dataset_id_parts)))
+
+    # Apply filters
+    datasets = dataset_list.apply_filters(ctx=ctx, filter_=filter_to_apply)
+
+    # Build full dataset IDs
+    result = [_["projectId"] + ":" + _["id"] for _ in datasets]
+
+    # Validation: ensure we found datasets
+    if not result and not dataset_ids:
+        raise CmemcError("No datasets found matching the provided filters.")
+
+    return result
+
+
 def _post_file_resource(
     app: ApplicationContext,
     project_id: str,
@@ -147,57 +195,6 @@ def _post_file_resource(
     app.echo_success("done")
 
 
-def _upload_file_resource(
-    app: ApplicationContext,
-    project_id: str,
-    local_file_name: str,
-    remote_file_name: str,
-    replace: bool,
-) -> None:
-    """Upload a local file as a dataset resource to a project.
-
-    Args:
-    ----
-        app: the click cli app context.
-        project_id: The project ID in the workspace.
-        local_file_name: The path to the local file name
-        remote_file_name: The remote file name
-        replace: Replace resource if needed.
-
-    Raises:
-    ------
-        ValueError: if resource exists and no replace
-
-    """
-    exist = resource_exist(project_name=project_id, resource_name=remote_file_name)
-    if exist and not replace:
-        raise ClickException(
-            f"A file resource with the name '{remote_file_name}' already "
-            "exists in this project. \n"
-            "Please rename the file or use the '--replace' "
-            "parameter in order to overwrite the remote file."
-        )
-    if exist:
-        app.echo_info(
-            f"Replace content of {remote_file_name} with content from "
-            f"{local_file_name} in project {project_id} ... ",
-            nl=False,
-        )
-    else:
-        app.echo_info(
-            f"Upload {local_file_name} as a file resource "
-            f"{remote_file_name} to project {project_id} ... ",
-            nl=False,
-        )
-    create_resource(
-        project_name=project_id,
-        resource_name=remote_file_name,
-        file_resource=ClickSmartPath.open(local_file_name),
-        replace=replace,
-    )
-    app.echo_success("done")
-
-
 def _get_metadata_out_of_parameter(parameter_dict: dict) -> dict:
     """Extract metadata keys out of the parameter dict.
 
@@ -235,7 +232,7 @@ def _get_read_only_out_of_parameter(parameter_dict: dict) -> bool:
         return True
     if read_only in ("false", False, "False"):
         return False
-    raise ClickException(f"readOnly parameter should be 'true' or 'false' - was {read_only!r}")
+    raise CmemcError(f"readOnly parameter should be 'true' or 'false' - was {read_only!r}")
 
 
 def _extend_parameter_with_metadata(
@@ -338,7 +335,12 @@ def _show_parameter_list(app: ApplicationContext, dataset_type: str) -> None:
     # metadata always on top, then sorted by key
     table = sorted(table, key=lambda k: k[0].lower())
     table = completion.add_metadata_parameter(table)
-    app.echo_info_table(table, headers=["Parameter", "Description"])
+    app.echo_info_table(
+        table,
+        headers=["Parameter", "Description"],
+        caption=build_caption(len(table), f"{dataset_type} dataset parameter"),
+        empty_table_message="No parameters found for this dataset type.",
+    )
 
 
 def _show_type_list(app: ApplicationContext) -> None:
@@ -362,7 +364,13 @@ def _show_type_list(app: ApplicationContext) -> None:
                 f"{title}: {description}",
             ]
             table.append(row)
-    app.echo_info_table(table, headers=["Dataset Type", "Description"], sort_column=1)
+    app.echo_info_table(
+        table,
+        headers=["Dataset Type", "Description"],
+        sort_column=1,
+        caption=build_caption(len(table), "dataset type"),
+        empty_table_message="No dataset types found.",
+    )
 
 
 def _check_or_select_dataset_type(app: ApplicationContext, dataset_type: str) -> tuple[str, dict]:
@@ -386,7 +394,7 @@ def _check_or_select_dataset_type(app: ApplicationContext, dataset_type: str) ->
         app.echo_debug(f"check type {dataset_type}")
         plugin = get_task_plugin_description(dataset_type)
     except requests.exceptions.HTTPError as error:
-        raise CmemcError(app, f"Unknown dataset type: {dataset_type}.") from error
+        raise CmemcError(f"Unknown dataset type: {dataset_type}.") from error
     else:
         return dataset_type, plugin
 
@@ -397,8 +405,8 @@ def _check_or_select_dataset_type(app: ApplicationContext, dataset_type: str) ->
     "filter_",
     type=(str, str),
     multiple=True,
-    shell_complete=completion.dataset_list_filter,
-    help=DATASET_LIST_FILTER_HELP_TEXT,
+    shell_complete=dataset_list.complete_values,
+    help=dataset_list.get_filter_help_text(),
 )
 @click.option(
     "--raw", is_flag=True, help="Outputs raw JSON objects of the dataset search API response."
@@ -409,19 +417,15 @@ def _check_or_select_dataset_type(app: ApplicationContext, dataset_type: str) ->
     help="Lists only dataset IDs and no labels or other metadata. "
     "This is useful for piping the IDs into other cmemc commands.",
 )
-@click.pass_obj
-def list_command(
-    app: ApplicationContext, filter_: tuple[tuple[str, str]], raw: bool, id_only: bool
-) -> None:
+@click.pass_context
+def list_command(ctx: Context, filter_: tuple[tuple[str, str]], raw: bool, id_only: bool) -> None:
     """List available datasets.
 
     Output and filter a list of available datasets. Each dataset is listed
     with its ID, type and label.
     """
-    datasets = list_items(item_type="dataset")["results"]
-    for _ in filter_:
-        filter_type, filter_name = _
-        datasets = _get_datasets_filtered(datasets, filter_type, filter_name)
+    app = ctx.obj
+    datasets = dataset_list.apply_filters(ctx=ctx, filter_=filter_)
 
     if raw:
         app.echo_info_json(datasets)
@@ -431,18 +435,21 @@ def list_command(
     else:
         table = []
         for _ in datasets:
-            row = [
-                _["projectId"] + ":" + _["id"],
-                _["pluginId"],
-                _["label"],
-            ]
+            # Build row with dataset ID; the Label column will be transformed by DatasetLink
+            dataset_id = _["projectId"] + ":" + _["id"]
+            row = [dataset_id, _["pluginId"], dataset_id]
+
             table.append(row)
+        filtered = len(filter_) > 0
         app.echo_info_table(
             table,
             headers=["Dataset ID", "Type", "Label"],
             sort_column=2,
-            empty_table_message="No datasets found. "
-            "Use the `dataset create` command to create a new dataset.",
+            cell_processing={2: DatasetLink(), 1: DatasetTypeLink()},
+            caption=build_caption(len(table), "dataset", filtered=filtered),
+            empty_table_message="No datasets found for these filters."
+            if filtered
+            else "No datasets found. Use the `dataset create` command to create a new dataset.",
         )
 
 
@@ -454,28 +461,18 @@ def list_command(
     is_flag=True,
     help="Delete all datasets. " "This is a dangerous option, so use it with care.",
 )
-@click.option(
-    "--project",
-    "project_id",
-    type=click.STRING,
-    shell_complete=completion.project_ids,
-    help="In combination with the '--all' flag, this option allows for "
-    "deletion of all datasets of a certain project. The behaviour is "
-    "similar to the 'dataset list --project' command.",
-)
 @click.option(
     "--filter",
     "filter_",
     type=(str, str),
     multiple=True,
-    shell_complete=completion.dataset_list_filter,
-    help=DATASET_DELETE_FILTER_HELP_TEXT,
+    shell_complete=dataset_list.complete_values,
+    help=dataset_list.get_filter_help_text(),
 )
 @click.argument("dataset_ids", nargs=-1, type=click.STRING, shell_complete=completion.dataset_ids)
-@click.pass_obj
+@click.pass_context
 def delete_command(
-    app: ApplicationContext,
-    project_id: str,
+    ctx: Context,
     all_: bool,
     filter_: tuple[tuple[str, str]],
     dataset_ids: tuple[str],
@@ -489,12 +486,10 @@ def delete_command(
 
     Note: Datasets can be listed by using the `dataset list` command.
     """
-    if project_id:
-        app.echo_warning(
-            "Option '--project' is deprecated and will be removed. "
-            "Please use '--filter project XXX' instead."
-        )
-    if dataset_ids == () and not all_ and not filter_:
+    app = ctx.obj
+
+    # Validation: require at least one selection method
+    if not dataset_ids and not all_ and not filter_:
         raise UsageError(
             "Either specify at least one dataset ID"
             " or use a --filter option,"
@@ -504,25 +499,21 @@ def delete_command(
     if dataset_ids and (all_ or filter_):
         raise UsageError("Either specify a dataset ID OR" " use a --filter or the --all option.")
 
-    if all_ or filter_:
-        # in case --all or --filter is given, a list of datasets is fetched
-        dataset_ids = []
-        datasets = list_items(item_type="dataset", project=project_id)["results"]
-        for _ in filter_:
-            filter_type, filter_name = _
-            datasets = _get_datasets_filtered(datasets, filter_type, filter_name)
-        for _ in datasets:
-            dataset_ids.append(_["projectId"] + ":" + _["id"])
+    # Get datasets to delete based on selection method
+    datasets_to_delete = _get_datasets_to_delete(ctx, dataset_ids, all_, filter_)
 
-    count = len(dataset_ids)
-    current = 1
-    for _ in dataset_ids:
-        app.echo_info(f"Delete dataset {current}/{count}: {_} ... ", nl=False)
-        project_part, dataset_part = _validate_and_split_dataset_id(_)
+    # Avoid double removal as well as sort IDs
+    processed_ids = sorted(set(datasets_to_delete), key=lambda v: v.lower())
+    count = len(processed_ids)
+
+    # Delete each dataset
+    for current, dataset_id in enumerate(processed_ids, start=1):
+        current_string = str(current).zfill(len(str(count)))
+        app.echo_info(f"Delete dataset {current_string}/{count}: {dataset_id} ... ", nl=False)
+        project_part, dataset_part = _validate_and_split_dataset_id(dataset_id)
         app.echo_debug(f"Project ID is {project_part}, dataset ID is {dataset_part}")
         delete_dataset(project_part, dataset_part)
-        app.echo_success("done")
-        current = current + 1
+        app.echo_success("deleted")
 
 
 @click.command(cls=CmemcCommand, name="download")
@@ -564,7 +555,7 @@ def download_command(
         file = project["data"]["parameters"]["file"]
     except KeyError as no_file_resource:
         raise CmemcError(
-            app, f"The dataset {dataset_id} has no associated file resource."
+            f"The dataset {dataset_id} has no associated file resource."
         ) from no_file_resource
     if Path(output_path).exists() and replace is not True:
         raise UsageError(
@@ -897,7 +888,7 @@ def open_command(app: ApplicationContext, dataset_ids: tuple[str]) -> None:
             app.echo_debug(f"Open {_}: {full_url}")
             click.launch(full_url)
         else:
-            raise ClickException(f"Dataset '{_}' not found.")
+            raise CmemcError(f"Dataset '{_}' not found.")
 
 
 @click.group(cls=CmemcGroup)