cmem-cmemc 25.6.0__py3-none-any.whl → 26.1.0rc1__py3-none-any.whl

cmem_cmemc/commands/variable.py

@@ -1,59 +1,209 @@
 """Build (DataIntegration) variable commands for cmemc."""
 
-import re
+from collections import defaultdict
 
 import click
-from click import UsageError
+from click import Context, UsageError
+from click.shell_completion import CompletionItem
 from cmem.cmempy.workspace.projects.variables import (
     create_or_update_variable,
     delete_variable,
     get_all_variables,
     get_variable,
 )
+from jinja2 import Environment, TemplateSyntaxError, nodes
 
 from cmem_cmemc import completion
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
-from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.context import ApplicationContext, build_caption
+from cmem_cmemc.exceptions import CmemcError
+from cmem_cmemc.object_list import (
+    DirectMultiValuePropertyFilter,
+    DirectValuePropertyFilter,
+    MultiFieldPropertyFilter,
+    ObjectList,
+    compare_regex,
+)
 from cmem_cmemc.utils import check_or_select_project, split_task_id
 
-VARIABLES_FILTER_TYPES = ["project", "regex"]
-VARIABLES_FILTER_TYPES_HIDDEN = ["ids"]
-VARIABLES_FILTER_TEXT = (
-    "Filter variables based on metadata. "
-    f"First parameter CHOICE can be one of {VARIABLES_FILTER_TYPES!s}"
-    ". The second parameter is based on CHOICE, e.g. a project "
-    "ID or a regular expression string."
+
+def get_variables(ctx: Context) -> list[dict]:  # noqa: ARG001
+    """Get variables for object list."""
+    _: list[dict] = get_all_variables()
+    return _
+
+
+variable_list_obj = ObjectList(
+    name="project variables",
+    get_objects=get_variables,
+    filters=[
+        DirectValuePropertyFilter(
+            name="project",
+            description="Filter variables by project ID.",
+            property_key="project",
+            completion_method="values",
+        ),
+        MultiFieldPropertyFilter(
+            name="regex",
+            description="Filter by regex matching variable id, value, description, or template.",
+            property_keys=["id", "value", "description", "template"],
+            compare=compare_regex,
+            fixed_completion=[
+                CompletionItem("^ending$", help="Variables name ends with 'ending'."),
+                CompletionItem("^starting", help="Variables name starts with 'starting'."),
+            ],
+            fixed_completion_only=False,
+        ),
+        DirectMultiValuePropertyFilter(
+            name="ids",
+            description="Internal filter for multiple variable IDs.",
+            property_key="id",
+        ),
+    ],
 )
 
 
-def _get_variables_filtered(
-    variables: list[dict], filter_name: str, filter_value: str
+def _extract_dependencies(variable: dict) -> list[str]:
+    """Extract variable dependencies from template field using Jinja parser.
+
+    Parses template strings like '{{project.var1}}+{{project.var2}}' to extract
+    dependencies. Returns a list of variable IDs that this variable depends on.
+
+    Args:
+        variable: Variable dict containing 'template', 'project', and 'id' fields
+
+    Returns:
+        List of variable IDs (e.g., ['project:var1', 'project:var2']) that
+        this variable depends on
+
+    """
+    dependencies: list[str] = []
+    template = variable.get("template", "")
+
+    if not template:
+        return dependencies
+
+    try:
+        # Parse the template using Jinja2's AST
+        env = Environment(autoescape=True)
+        ast = env.parse(template)
+
+        # Walk through the AST to find all Getattr nodes (attribute access like project.var1)
+        for node in ast.find_all(nodes.Getattr):
+            # Look for patterns like project.var1 or global.var2
+            if isinstance(node.node, nodes.Name) and isinstance(node.attr, str):
+                scope = node.node.name  # "project" or "global"
+                var_name = node.attr  # "var1", "var2", etc.
+
+                if scope == "project":
+                    # Build the full variable ID: projectname:varname
+                    project_name = variable["project"]
+                    dep_id = f"{project_name}:{var_name}"
+                    dependencies.append(dep_id)
+                # Note: We skip global variables as they're not in the same deletion scope
+
+    except TemplateSyntaxError:
+        # If template parsing fails, return empty dependencies
+        # This is safer than failing the entire deletion operation
+        pass
+
+    return dependencies
+
+
+def _sort_variables_by_dependency(variables: list[dict]) -> list[str]:
+    """Sort variables in reverse topological order for deletion.
+
+    Variables that depend on others should be deleted first (dependents before
+    dependencies). This ensures we don't try to delete a variable that is still
+    referenced by another variable's template.
+
+    Args:
+        variables: List of variable dicts
+
+    Returns:
+        List of variable IDs sorted for safe deletion (dependents first)
+
+    """
+    # Build dependency graph: variable_id -> list of variables it depends on
+    dependencies: dict[str, list[str]] = {}
+    # Build reverse graph: variable_id -> list of variables that depend on it
+    dependents: dict[str, list[str]] = defaultdict(list)
+    # Track all variable IDs in our deletion set
+    variable_ids = {v["id"] for v in variables}
+
+    for variable in variables:
+        var_id = variable["id"]
+        deps = _extract_dependencies(variable)
+        # Only track dependencies within our deletion set
+        deps_in_set = [d for d in deps if d in variable_ids]
+        dependencies[var_id] = deps_in_set
+
+        for dep_id in deps_in_set:
+            dependents[dep_id].append(var_id)
+
+    # Topological sort using Kahn's algorithm (modified for reverse order)
+    # We want to delete dependents first, so we start with variables that
+    # have no dependents (leaf nodes in the dependency tree)
+
+    # Count how many other variables depend on each variable
+    dependent_count = {var_id: len(dependents[var_id]) for var_id in variable_ids}
+
+    # Start with variables that nothing depends on (can be deleted first)
+    queue = [var_id for var_id in variable_ids if dependent_count[var_id] == 0]
+    # Sort queue to ensure deterministic output
+    queue.sort()
+
+    result = []
+
+    while queue:
+        # Take the next variable with no dependents
+        var_id = queue.pop(0)
+        result.append(var_id)
+
+        # For each variable this one depends on, decrease the dependent count
+        for dep_id in dependencies[var_id]:
+            dependent_count[dep_id] -= 1
+            # If no more dependents, can be deleted next
+            if dependent_count[dep_id] == 0:
+                queue.append(dep_id)
+                queue.sort()
+
+    return result
+
+
+def _validate_variable_ids(variable_ids: tuple[str, ...]) -> None:
+    """Validate that provided variable IDs exist."""
+    all_variables = get_all_variables()
+    all_variable_ids = [_["id"] for _ in all_variables]
+    for variable_id in variable_ids:
+        if variable_id not in all_variable_ids:
+            raise CmemcError(f"Variable {variable_id} not available.")
+
+
+def _get_variables_to_delete(
+    ctx: Context,
+    variable_ids: tuple[str, ...],
+    all_: bool,
+    filter_: tuple[tuple[str, str], ...],
 ) -> list[dict]:
-    """Get variables but filtered according to name and value."""
-    filter_types = VARIABLES_FILTER_TYPES + VARIABLES_FILTER_TYPES_HIDDEN
-    # check for correct filter names (filter ids is used internally only)
-    if filter_name not in filter_types:
-        raise UsageError(
-            f"{filter_name} is an unknown filter name. " f"Use one of {VARIABLES_FILTER_TYPES}."
-        )
-    # filter by ID list
-    if filter_name == "ids":
-        return [_ for _ in variables if _["id"] in filter_value]
-    # filter by project
-    if filter_name == "project":
-        return [_ for _ in variables if _["project"] == filter_value]
-    # filter by regex
-    if filter_name == "regex":
-        return [
-            _
-            for _ in variables
-            if re.search(filter_value, _["id"])
-            or re.search(filter_value, _["value"])
-            or re.search(filter_value, _.get("description", ""))
-            or re.search(filter_value, _.get("template", ""))
-        ]
-    # return unfiltered list
+    """Get the list of variables to delete based on selection method."""
+    if all_:
+        _: list[dict] = get_all_variables()
+        return _
+
+    _validate_variable_ids(variable_ids)
+
+    filter_to_apply = list(filter_) if filter_ else []
+
+    if variable_ids:
+        filter_to_apply.append(("ids", ",".join(variable_ids)))
+
+    variables = variable_list_obj.apply_filters(ctx=ctx, filter_=filter_to_apply)
+
+    if not variables and not variable_ids:
+        raise CmemcError("No variables found matching the provided filters.")
+
     return variables
 
 
@@ -67,25 +217,21 @@ def _get_variables_filtered(
 )
 @click.option(
     "--filter",
-    "filters_",
+    "filter_",
     multiple=True,
     type=(str, str),
-    shell_complete=completion.variable_list_filter,
-    help=VARIABLES_FILTER_TEXT,
+    shell_complete=variable_list_obj.complete_values,
+    help=variable_list_obj.get_filter_help_text(),
 )
-@click.pass_obj
-def list_command(
-    app: ApplicationContext, raw: bool, id_only: bool, filters_: tuple[tuple[str, str]]
-) -> None:
+@click.pass_context
+def list_command(ctx: Context, raw: bool, id_only: bool, filter_: tuple[tuple[str, str]]) -> None:
     """List available project variables.
 
     Outputs a table or a list of project variables.
     """
-    variables = get_all_variables()
+    app: ApplicationContext = ctx.obj
+    variables = variable_list_obj.apply_filters(ctx=ctx, filter_=filter_)
 
-    for _ in filters_:
-        filter_name, filter_value = _
-        variables = _get_variables_filtered(variables, filter_name, filter_value)
     if raw:
         app.echo_info_json(variables)
         return
@@ -104,11 +250,15 @@ def list_command(
             _.get("description", ""),
         ]
         table.append(row)
+    filtered = len(filter_) > 0
     app.echo_info_table(
         table,
         headers=headers,
         sort_column=0,
-        empty_table_message="No project variables found. "
+        caption=build_caption(len(table), "project variable", filtered=filtered),
+        empty_table_message="No project variables found for these filters."
+        if filtered
+        else "No project variables found. "
         "Use the `project variable create` command to create a new project variable.",
     )
 
@@ -147,24 +297,77 @@ def get_command(app: ApplicationContext, variable_id: str, key: str, raw: bool)
 
 
 @click.command(cls=CmemcCommand, name="delete")
-@click.argument(
-    "variable_id", required=True, type=click.STRING, shell_complete=completion.variable_ids
+@click.argument("variable_ids", nargs=-1, type=click.STRING, shell_complete=completion.variable_ids)
+@click.option(
+    "-a",
+    "--all",
+    "all_",
+    is_flag=True,
+    help="Delete all variables. This is a dangerous option, so use it with care.",
 )
-@click.pass_obj
-def delete_command(app: ApplicationContext, variable_id: str) -> None:
-    """Delete a project variable.
+@click.option(
+    "--filter",
+    "filter_",
+    multiple=True,
+    type=(str, str),
+    shell_complete=variable_list_obj.complete_values,
+    help=variable_list_obj.get_filter_help_text(),
+)
+@click.pass_context
+def delete_command(
+    ctx: Context,
+    variable_ids: tuple[str, ...],
+    all_: bool,
+    filter_: tuple[tuple[str, str], ...],
+) -> None:
+    """Delete project variables.
 
-    Note: You can not delete a variable which is used by another
-    (template based) variable. In order to do so, delete the template based
-    variable first.
+    There are three selection mechanisms: with specific IDs - only those
+    specified variables will be deleted; by using --filter - variables based
+    on the filter type and value will be deleted; by using --all, which will
+    delete all variables.
+
+    Variables are automatically sorted by their dependencies and deleted in the
+    correct order (template-based variables that depend on others are deleted
+    first, then their dependencies).
     """
-    project_name, variable_name = split_task_id(variable_id)
-    app.echo_info(f"Delete variable {variable_name} from project {project_name} ... ", nl=False)
-    delete_variable(variable_name=variable_name, project_name=project_name)
-    app.echo_success("done")
+    app: ApplicationContext = ctx.obj
+
+    # Validation: require at least one selection method
+    if not variable_ids and not all_ and not filter_:
+        raise UsageError(
+            "Either specify at least one variable ID or use the --all or "
+            "--filter options to specify variables for deletion."
+        )
+
+    # Get variables to delete based on selection method
+    variables_to_delete = _get_variables_to_delete(ctx, variable_ids, all_, filter_)
+
+    # Remove duplicates while preserving variable objects for dependency analysis
+    unique_variables = list({v["id"]: v for v in variables_to_delete}.values())
+
+    # Sort by dependency order (dependents first, then dependencies)
+    processed_ids = _sort_variables_by_dependency(unique_variables)
+    count = len(processed_ids)
+
+    # Delete each variable
+    for current, variable_id in enumerate(processed_ids, start=1):
+        project_name, variable_name = split_task_id(variable_id)
+        if count > 1:
+            current_string = str(current).zfill(len(str(count)))
+            app.echo_info(
+                f"Delete variable {current_string}/{count}: {variable_name} "
+                f"from project {project_name} ... ",
+                nl=False,
+            )
+        else:
+            app.echo_info(
+                f"Delete variable {variable_name} from project {project_name} ... ", nl=False
+            )
+        delete_variable(variable_name=variable_name, project_name=project_name)
+        app.echo_success("done")
 
 
-# pylint: disable=too-many-arguments
 @click.command(cls=CmemcCommand, name="create")
 @click.argument(
     "variable_name",

cmem_cmemc/commands/vocabulary.py

@@ -6,7 +6,6 @@ from re import match
 from urllib.parse import urlparse
 
 import click
-from click import ClickException
 from cmem.cmempy.config import get_cmem_base_uri
 from cmem.cmempy.dp.proxy import graph as graph_api
 from cmem.cmempy.dp.titles import resolve
@@ -25,7 +24,8 @@ from six.moves.urllib.parse import quote
 from cmem_cmemc import completion
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
-from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.context import ApplicationContext, build_caption
+from cmem_cmemc.exceptions import CmemcError
 from cmem_cmemc.parameter_types.path import ClickSmartPath
 
 GET_ONTOLOGY_IRI_QUERY = """
@@ -109,7 +109,7 @@ def _validate_namespace(app: ApplicationContext, namespace: tuple[str | None, st
     """User input validation for the namespace."""
     prefix, uri = namespace
     if prefix is None or uri is None:
-        raise ClickException("No namespace given.")
+        raise CmemcError("No namespace given.")
 
     if uri[-1] not in ("/", "#"):
         app.echo_warning(
@@ -119,10 +119,10 @@ def _validate_namespace(app: ApplicationContext, namespace: tuple[str | None, st
     parsed_url = urlparse(uri)
     app.echo_debug(str(parsed_url))
     if parsed_url.scheme not in ("http", "https", "urn"):
-        raise ClickException(f"Namespace IRI '{uri}' is not a https(s) URL or an URN.")
+        raise CmemcError(f"Namespace IRI '{uri}' is not a https(s) URL or an URN.")
     prefix_expression = r"^[a-z][a-z0-9]*$"
     if not match(prefix_expression, prefix):
-        raise ClickException(
+        raise CmemcError(
             "Prefix string does not match this regular" f" expression: {prefix_expression}"
         )
 
@@ -172,14 +172,14 @@ def _get_vocabulary_metadata_from_file(
     try:
         graph = Graph().parse(file, format="ttl")
     except BadSyntax as error:
-        raise ClickException("File {file} could not be parsed as turtle.") from error
+        raise CmemcError("File {file} could not be parsed as turtle.") from error
 
     ontology_iris = graph.query(GET_ONTOLOGY_IRI_QUERY)
     if len(ontology_iris) == 0:
-        raise ClickException("There is no owl:Ontology resource described " "in the turtle file.")
+        raise CmemcError("There is no owl:Ontology resource described " "in the turtle file.")
     if len(ontology_iris) > 1:
         ontology_iris_str = [str(iri[0]) for iri in ontology_iris]  # type: ignore[index]
-        raise ClickException(
+        raise CmemcError(
             "There are more than one owl:Ontology resources described "
             f"in the turtle file: {ontology_iris_str}"
         )
@@ -187,7 +187,7 @@ def _get_vocabulary_metadata_from_file(
     metadata["iri"] = iri
     vann_data = graph.query(GET_PREFIX_DECLARATION.format(iri))
     if not vann_data and not namespace_given:
-        raise ClickException(
+        raise CmemcError(
             "There is no namespace defined "
             f"for the ontology '{iri}'.\n"
             "Please add a prefix and namespace to the sources"
@@ -196,13 +196,13 @@ def _get_vocabulary_metadata_from_file(
             "https://vocab.org/vann/ for more information."
         )
     if vann_data and namespace_given:
-        raise ClickException(
+        raise CmemcError(
             "There is already a namespace defined "
             f"in the file for the ontology '{iri}'.\n"
             "You can not use the --namespace option with this file."
         )
     if len(vann_data) > 1:
-        raise ClickException(
+        raise CmemcError(
             "There is more than one vann namespace defined " f"for the ontology: {iri}"
         )
     if not namespace_given:
@@ -278,11 +278,15 @@ def list_command(app: ApplicationContext, id_only: bool, filter_: str, raw: bool
             except (KeyError, TypeError):
                 label = _["vocabularyLabel"] if _["vocabularyLabel"] else "[no label given]"
             table.append((iri, label))
+        filtered = filter_ != "installed"
         app.echo_info_table(
             table,
             headers=["Vocabulary Graph IRI", "Label"],
             sort_column=1,
-            empty_table_message="No installed vocabularies found. "
+            caption=build_caption(len(table), "vocabulary", filtered=filtered),
+            empty_table_message="No vocabularies found for this filter."
+            if filtered
+            else "No installed vocabularies found. "
             "Use the `vocabulary install` command to install vocabulary from the catalog.",
         )
 
@@ -388,7 +392,7 @@ def import_command(
         if replace:
             success_message = "replaced"
         else:
-            raise ClickException(f"Proposed graph {iri} does already exist.")
+            raise CmemcError(f"Proposed graph {iri} does already exist.")
     app.echo_info(f"Import {file} as vocabulary to {iri} ... ", nl=False)
     # upload graph
     _buffer.seek(0)
@@ -465,6 +469,7 @@ def cache_list_command(app: ApplicationContext, id_only: bool, raw: bool) -> Non
             table,
             headers=["IRI", "Type", "Label"],
             sort_column=0,
+            caption=build_caption(len(table), "vocabulary cache entry"),
             empty_table_message="No cache entries found. "
             "Use the `vocabulary install` command to install a vocabulary.",
         )

cmem_cmemc/commands/workflow.py

@@ -6,8 +6,7 @@ import time
 from datetime import datetime, timezone
 
 import click
-import timeago
-from click import ClickException, UsageError
+from click import UsageError
 from cmem.cmempy.workflow import get_workflows
 from cmem.cmempy.workflow.workflow import (
     execute_workflow_io,
@@ -22,6 +21,7 @@ from cmem.cmempy.workspace.activities.taskactivities import get_activities_statu
 from cmem.cmempy.workspace.activities.taskactivity import get_activity_status, start_task_activity
 from cmem.cmempy.workspace.projects.project import get_projects
 from cmem.cmempy.workspace.search import list_items
+from humanize import naturaltime
 from requests import Response
 from requests.status_codes import codes
 
@@ -29,9 +29,11 @@ from cmem_cmemc import completion
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.commands.scheduler import scheduler
-from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.context import ApplicationContext, build_caption
+from cmem_cmemc.exceptions import CmemcError
 from cmem_cmemc.parameter_types.path import ClickSmartPath
 from cmem_cmemc.smart_path import SmartPath as Path
+from cmem_cmemc.string_processor import WorkflowLink
 
 WORKFLOW_FILTER_TYPES = sorted(["project", "regex", "tag", "io"])
 WORKFLOW_LIST_FILTER_HELP_TEXT = (
@@ -215,7 +217,7 @@ def _io_get_info(project_id: str, workflow_id: str) -> dict[str, str]:
         info: dict[str, str] = _
         if info["id"] == workflow_id and info["projectId"] == project_id:
             return info
-    raise ClickException(
+    raise CmemcError(
         "The given workflow does not exist or is not suitable to be executed "
         "with this command.\n"
         "An io workflow needs exactly one variable input and/or one variable "
@@ -332,7 +334,7 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
     # prepare human friendly relative time
     now = datetime.now(tz=timezone.utc)
     stamp = datetime.fromtimestamp(status["lastUpdateTime"] / 1000, tz=timezone.utc)
-    time_ago = timeago.format(stamp, now, "en")
+    time_ago = naturaltime(stamp, when=now)
     status_name = status["statusName"]
     status_message = status["message"]
     # prepare message
@@ -345,7 +347,7 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
         if status_name in ("Running", "Canceling", "Waiting"):
             app.echo_warning(message)
             return
-        raise ClickException(
+        raise CmemcError(
             f"statusName is {status_name}, expecting one of: " "Running, Canceling or Waiting."
         )
     # not running can be Idle or Finished
@@ -360,7 +362,6 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
         app.echo_success(message)
 
 
-# pylint: disable=too-many-arguments
 @click.command(cls=CmemcCommand, name="execute")
 @click.option("-a", "--all", "all_", is_flag=True, help="Execute all available workflows.")
 @click.option("--wait", is_flag=True, help="Wait until workflows are completed.")
@@ -445,7 +446,6 @@ def execute_command(  # noqa: PLR0913
                 sys.exit(1)
 
 
-# pylint: disable=too-many-arguments
 @click.command(cls=CmemcCommand, name="io")
 @click.option(
     "--input",
@@ -621,19 +621,29 @@ def list_command(
             app.echo_info(_["projectId"] + ":" + _["id"])
         return
     # output a user table
+    # Create a dict mapping workflow IDs to workflow data for the WorkflowLink processor
+    workflows_dict = {
+        workflow["projectId"] + ":" + workflow["id"]: workflow for workflow in workflows
+    }
     table = []
     for _ in workflows:
+        workflow_id = _["projectId"] + ":" + _["id"]
         row = [
-            _["projectId"] + ":" + _["id"],
-            _["label"],
+            workflow_id,
+            workflow_id,  # Pass workflow ID to be processed by WorkflowLink
         ]
         table.append(row)
+    filtered = len(filter_) > 0
     app.echo_info_table(
         table,
         headers=["Workflow ID", "Label"],
         sort_column=1,
-        empty_table_message="No workflows found. "
+        caption=build_caption(len(table), "workflow", filtered=filtered),
+        empty_table_message="No workflows found for these filters."
+        if filtered
+        else "No workflows found. "
         "Open a project in the web interface and create a new workflow there.",
+        cell_processing={1: WorkflowLink(workflows=workflows_dict)},
     )