cmem-cmemc 23.2__py3-none-any.whl → 23.3.0__py3-none-any.whl

cmem/cmemc/cli/commands/variable.py

@@ -0,0 +1,364 @@
+"""DataIntegration variable commands for cmemc."""
+import re
+
+import click
+from click import UsageError
+
+from cmem.cmemc.cli import completion
+from cmem.cmemc.cli.commands import CmemcCommand, CmemcGroup
+from cmem.cmemc.cli.utils import split_task_id, check_or_select_project
+from cmem.cmempy.workspace.projects.variables import (
+    create_or_update_variable,
+    delete_variable,
+    get_all_variables,
+    get_variable
+)
+from cmem.cmemc.cli.context import ApplicationContext
+
+VARIABLES_FILTER_TYPES = ["project", 'regex']
+VARIABLES_FILTER_TYPES_HIDDEN = ["ids"]
+VARIABLES_FILTER_TEXT = (
+    "Filter variables based on metadata. "
+    f"First parameter CHOICE can be one of {str(VARIABLES_FILTER_TYPES)}"
+    ". The second parameter is based on CHOICE, e.g. a project "
+    "ID or a regular expression string."
+)
+
+
+def _get_variables_filtered(variables, filter_name, filter_value):
+    """Get variables but filtered according to name and value."""
+    filter_types = VARIABLES_FILTER_TYPES + VARIABLES_FILTER_TYPES_HIDDEN
+    # check for correct filter names (filter ids is used internally only)
+    if filter_name not in filter_types:
+        raise ValueError(
+            f"{filter_name} is an unknown filter name. "
+            f"Use one of {VARIABLES_FILTER_TYPES}."
+        )
+    # filter by ID list
+    if filter_name == "ids":
+        return [_ for _ in variables if _["id"] in filter_value]
+    # filter by project
+    if filter_name == "project":
+        return [_ for _ in variables if _["project"] == filter_value]
+    # filter by regex
+    if filter_name == "regex":
+        return [
+            _ for _ in variables
+            if re.search(filter_value, _["id"])
+            or re.search(filter_value, _["value"])
+            or re.search(filter_value, _.get("description", ""))
+            or re.search(filter_value, _.get("template", ""))
+        ]
+    # return unfiltered list
+    return variables
+
+
+@click.command(cls=CmemcCommand, name="list")
+@click.option(
+    "--raw",
+    is_flag=True,
+    help="Outputs raw JSON."
+)
+@click.option(
+    "--id-only",
+    is_flag=True,
+    help="Lists only variables names and no other metadata. "
+         "This is useful for piping the IDs into other commands."
+)
+@click.option(
+    "--filter", "filters_",
+    multiple=True,
+    type=(str, str),
+    shell_complete=completion.variable_list_filter,
+    help=VARIABLES_FILTER_TEXT
+)
+@click.pass_obj
+def list_command(app: ApplicationContext, raw, id_only, filters_):
+    """List available project variables.
+
+    Outputs a table or a list of project variables.
+    """
+    variables = get_all_variables()
+
+    for _ in filters_:
+        filter_name, filter_value = _
+        variables = _get_variables_filtered(
+            variables, filter_name, filter_value
+        )
+    if raw:
+        app.echo_info_json(variables)
+        return
+    if id_only:
+        for _ in sorted(_["id"] for _ in variables):
+            app.echo_result(_)
+        return
+    # output a user table
+    table = []
+    headers = ["ID", "Value", "Template", "Description"]
+    for _ in variables:
+        row = [
+            _["id"],
+            _["value"],
+            _.get("template", ""),
+            _.get("description", ""),
+        ]
+        table.append(row)
+    app.echo_info_table(table, headers=headers, sort_column=0)
+
+
+@click.command(cls=CmemcCommand, name="get")
+@click.argument(
+    "variable_id",
+    required=True,
+    type=click.STRING,
+    shell_complete=completion.variable_ids
+)
+@click.option(
+    "--key",
+    type=click.Choice(
+        ["value", "template", "description"],
+        case_sensitive=False
+    ),
+    default="value",
+    show_default=True,
+    help="Specify the name of the value you want to get."
+)
+@click.option(
+    "--raw",
+    is_flag=True,
+    help="Outputs raw json."
+)
+@click.pass_obj
+def get_command(app: ApplicationContext, variable_id, key, raw):
+    """Get the value or other data of a project variable.
+
+    Use the `--key` option to specify which information you want to get.
+
+    Note: Only the `value` key is always available on a project variable.
+    Static value variables have no `template` key, and the `description` key
+    is optional for both types of variables.
+    """
+    project_name, variable_name = split_task_id(variable_id)
+    _ = get_variable(variable_name=variable_name, project_name=project_name)
+    if raw:
+        app.echo_info_json(_)
+        return
+    try:
+        app.echo_info(_[key], nl=False)
+    except KeyError as error:
+        raise UsageError(
+            f"Variable {variable_name} has no value of '{key}'."
+        ) from error
+
+
+@click.command(cls=CmemcCommand, name="delete")
+@click.argument(
+    "variable_id",
+    required=True,
+    type=click.STRING,
+    shell_complete=completion.variable_ids
+)
+@click.pass_obj
+def delete_command(app: ApplicationContext, variable_id):
+    """Delete a project variable.
+
+    Note: You can not delete a variable which is used by another
+    (template based) variable. In order to do so, delete the template based
+    variable first.
+    """
+    project_name, variable_name = split_task_id(variable_id)
+    app.echo_info(
+        f"Delete variable {variable_name} from project {project_name} ... ",
+        nl=False
+    )
+    delete_variable(variable_name=variable_name, project_name=project_name)
+    app.echo_success("done")
+
+
+# pylint: disable=too-many-arguments
+@click.command(cls=CmemcCommand, name="create")
+@click.argument(
+    "variable_name",
+    required=True,
+    type=click.STRING,
+)
+@click.option(
+    "--value",
+    type=click.STRING,
+    help="The value of the new project variable."
+)
+@click.option(
+    "--template",
+    type=click.STRING,
+    help="The template of the new project variable. You can use Jinja template "
+         "syntax, e.g. use '{{global.myVar}}' for accessing global variables, or "
+         "'{{project.myVar}}' for accessing variables from the same project."
+)
+@click.option(
+    "--description",
+    type=click.STRING,
+    help="The optional description of the new project variable."
+)
+@click.option(
+    "--project", "project_id",
+    type=click.STRING,
+    shell_complete=completion.project_ids,
+    help="The project, where you want to create the variable in. If there is "
+         "only one project in the workspace, this option can be omitted."
+)
+@click.pass_obj
+def create_command(
+        app: ApplicationContext,
+        variable_name, value, template, description,
+        project_id
+):
+    """Create a new project variable.
+
+    Variables need to be created with a value or a template (not both).
+    In addition to that, a project ID and a name are mandatory.
+
+    Example: cmemc project variable create my_var --project my_project --value abc
+
+    Note: cmemc is currently not able to manage the order of the variables in a
+    project. This means you have to create plain value variables in advance,
+    before you can create template based variables, which access these values.
+    """
+    if value and template:
+        raise UsageError(
+            "Either use '--value' or '--template' but not both."
+        )
+    if not value and not template:
+        raise UsageError(
+            "Use '--value' or '--template' to create a new variable."
+        )
+    project_id = check_or_select_project(app, project_id)
+    data = get_variable(project_name=project_id, variable_name=variable_name)
+    if data:
+        raise UsageError(
+            f"Variable '{variable_name}' already exist in project '{project_id}'."
+            )
+    data = {
+        "name": variable_name,
+        "isSensitive": False,
+        "scope": "project"
+    }
+    if value:
+        data["value"] = value
+    if template:
+        data["template"] = template
+    if description:
+        data["description"] = description
+    app.echo_info(
+        f"Create variable {variable_name} in project {project_id} ... ",
+        nl=False
+    )
+    create_or_update_variable(
+        project_name=project_id,
+        variable_name=variable_name,
+        data=data
+    )
+    app.echo_success("done")
+
+
+@click.command(cls=CmemcCommand, name="update")
+@click.argument(
+    "variable_id",
+    required=True,
+    type=click.STRING,
+    shell_complete=completion.variable_ids
+)
+@click.option(
+    "--value",
+    type=click.STRING,
+    help="The new value of the project variable."
+)
+@click.option(
+    "--template",
+    type=click.STRING,
+    help="The new template of the project variable. You can use Jinja template "
+         "syntax, e.g. use '{{global.myVar}}' for accessing global variables, or "
+         "'{{project.myVar}}' for accessing variables from the same project."
+)
+@click.option(
+    "--description",
+    type=click.STRING,
+    help="The new description of the project variable."
+)
+@click.pass_obj
+def update_command(
+        app: ApplicationContext,
+        variable_id, value, template, description,
+):
+    """Update data of an existing project variable.
+
+    With this command you can update the value or the template, as well as the
+    description of a project variable.
+
+    Note: If you update the template of a static variable, it will be transformed
+    to a template based variable. If you want to change the value of a template
+    based variable, an error will be shown.
+    """
+    project_id, variable_name = split_task_id(variable_id)
+    data = get_variable(project_name=project_id, variable_name=variable_name)
+    if not data:
+        raise UsageError(
+            f"Variable '{variable_name}' does not exist in project '{project_id}'."
+            )
+    if value and template:
+        raise UsageError(
+            "Project variables are based on a static value or on a template, but not "
+            "both."
+        )
+    if not value and not template and not description:
+        raise UsageError(
+            "Please specify what you want to update. "
+            "Use at least one of the following options: "
+            "'--value', '--template', '--description'."
+        )
+    if value:
+        if data.get("template", None):
+            raise UsageError(
+                "You can not change the value of a template based variable."
+            )
+        data["value"] = value
+    if template:
+        if not data.get("template", None):
+            app.echo_warning(f"Variable '{variable_id}' will be converted from a "
+                             f"simple to a template based variable.")
+        data["template"] = template
+    if description:
+        data["description"] = description
+    app.echo_info(
+        f"Update variable {variable_name} in project {project_id} ... ",
+        nl=False
+    )
+    create_or_update_variable(
+        project_name=project_id,
+        variable_name=variable_name,
+        data=data
+    )
+    app.echo_success("done")
+
+
+@click.group(cls=CmemcGroup)
+def variable():
+    """List, create, delete or get data from project variables.
+
+    Project variables can be used in dataset and task parameters, and in the template
+    transform operator.
+    Variables are either based on a static value or based on a template.
+    They may use templates that access globally configured
+    variables or other preceding variables from the same project.
+
+    Variables are identified by a VARIABLE_ID. To get a list of existing
+    variables, execute the list command or use tab-completion.
+    The VARIABLE_ID is a concatenation of a PROJECT_ID and a VARIABLE_NAME,
+    such as `my-project:my-variable`.
+    """
+
+
+variable.add_command(list_command)
+variable.add_command(get_command)
+variable.add_command(delete_command)
+variable.add_command(create_command)
+variable.add_command(update_command)

cmem/cmemc/cli/commands/vocabulary.py

@@ -202,7 +202,7 @@ def _transform_cache_to_table(cache_category, table):
 @click.argument(
     "iri",
     type=click.STRING,
-    autocompletion=completion.installed_vocabularies
+    shell_complete=completion.installed_vocabularies
 )
 @click.pass_obj
 def open_command(app, iri):
@@ -276,7 +276,7 @@ def list_command(app, id_only, filter_, raw):
     "iris",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.installable_vocabularies
+    shell_complete=completion.installable_vocabularies
 )
 @click.option(
     "-a", "--all", "all_",
@@ -316,7 +316,7 @@ def install_command(app: ApplicationContext, iris: tuple[str, ...], all_):
     "iris",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.installed_vocabularies
+    shell_complete=completion.installed_vocabularies
 )
 @click.option(
     "-a", "--all", "all_",
@@ -355,7 +355,7 @@ def uninstall_command(app: ApplicationContext, iris: tuple[str, ...], all_):
 @click.argument(
     "FILE",
     required=True,
-    autocompletion=completion.triple_files,
+    shell_complete=completion.triple_files,
     type=click.Path(
         allow_dash=False,
         readable=True
@@ -437,7 +437,7 @@ def import_command(app: ApplicationContext, file, namespace, replace):
     "iris",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.installed_vocabularies
+    shell_complete=completion.installed_vocabularies
 )
 @click.option(
     "-a", "--all", "all_",

cmem/cmemc/cli/commands/workflow.py

@@ -8,6 +8,7 @@ import time
 import click
 
 import timeago
+from click import UsageError
 
 from cmem.cmemc.cli import completion
 from cmem.cmemc.cli.commands import CmemcCommand, CmemcGroup
@@ -31,6 +32,7 @@ from cmem.cmempy.workspace.activities.taskactivity import (
 )
 from cmem.cmempy.workspace.projects.project import get_projects
 from cmem.cmempy.workspace.search import list_items
+from cmem.cmemc.cli.context import ApplicationContext
 
 WORKFLOW_FILTER_TYPES = sorted(['project', 'regex', 'tag', 'io'])
 WORKFLOW_LIST_FILTER_HELP_TEXT = (
@@ -276,19 +278,54 @@ def _workflows_get_ids():
     return ids
 
 
-def _workflow_wait_until_finished(project_id, task_id, polling_interval):
-    """Poll workflow status until workflow is finished and return status.
-
-    :type project_id: string
-    :type task_id: string
-    :type polling_interval: int
-    """
-    while True:
-        status = get_activity_status(project_id, task_id)
-        # wait until isRunning is false
-        if not status['isRunning']:
-            break
-        time.sleep(polling_interval)
+def _get_progress_bar_info(status):
+    """get the workflow message from status response with colors."""
+    if status is None:
+        return ""
+    if status["isRunning"]:
+        return click.style(status["message"], fg="yellow")
+    if status["concreteStatus"] == "Failed":
+        return click.style(status["message"], fg="red")
+    return click.style(status["message"], fg="green")
+
+
+def _workflow_wait_until_finished(
+        app: ApplicationContext,
+        project_id: str,
+        task_id: str,
+        polling_interval: int,
+        log_progress: bool
+):
+    """Poll workflow status until workflow is finished and return status."""
+    progress = 0
+    if log_progress:
+        app.echo_success(message="", nl=True)
+        with click.progressbar(
+            show_eta=False,
+            length=100,
+            item_show_func=_get_progress_bar_info,
+        ) as progress_bar:
+            while True:
+                status = get_activity_status(project_id, task_id)
+                if progress is not status['progress']:
+                    progress_bar.update(
+                        n_steps=status['progress'] - progress,
+                        current_item=status
+                    )
+                    progress = status['progress']
+                app.echo_debug(f"{status['statusName']}({status['message']})")
+                # wait until isRunning is false
+                if not status['isRunning']:
+                    break
+                time.sleep(polling_interval)
+    else:
+        while True:
+            status = get_activity_status(project_id, task_id)
+            app.echo_debug(f"{status['statusName']}({status['message']})")
+            # wait until isRunning is false
+            if not status['isRunning']:
+                break
+            time.sleep(polling_interval)
     return status
 
 
@@ -336,6 +373,7 @@ def _workflow_echo_status(app, status):
             app.echo_success(message)
 
 
+# pylint: disable=too-many-arguments
 @click.command(cls=CmemcCommand, name="execute")
 @click.option(
     "-a", "--all", "all_",
@@ -345,7 +383,12 @@ def _workflow_echo_status(app, status):
 @click.option(
     "--wait",
     is_flag=True,
-    help="Wait until all executed workflows are completed."
+    help="Wait until workflows are completed."
+)
+@click.option(
+    "--progress",
+    is_flag=True,
+    help="Wait until workflows are completed and show a progress bar."
 )
 @click.option(
     "--polling-interval",
@@ -359,10 +402,10 @@ def _workflow_echo_status(app, status):
     "workflow_ids",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.workflow_ids
+    shell_complete=completion.workflow_ids
 )
 @click.pass_obj
-def execute_command(app, all_, wait, polling_interval, workflow_ids):
+def execute_command(app, all_, wait, polling_interval, workflow_ids, progress):
     """Execute workflow(s).
 
     With this command, you can start one or more workflows at the same time or
@@ -377,16 +420,16 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
     polls the status of a workflow until it is finished. In case of an error of
     a workflow, the next workflow is not started.
     """
-    all_workflow_ids = _workflows_get_ids()
     if workflow_ids == () and not all_:
         raise ValueError("Either specify at least one workflow or use the"
                          + " --all option to execute all workflows.")
+    all_workflow_ids = _workflows_get_ids()
     if all_:
         workflow_ids = all_workflow_ids
 
     for workflow_id in workflow_ids:
         if workflow_id not in all_workflow_ids:
-            raise ValueError(
+            raise UsageError(
                 f"Workflow '{workflow_id}' is not available."
             )
         project_id, task_id = workflow_id.split(":")
@@ -394,7 +437,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
 
         # before we start, we fetch the status
         status = get_activity_status(project_id, task_id)
-        if not wait:
+        if not wait and not progress:
             if status['isRunning']:
                 # in case of a running workflow, we only output status
                 app.echo_info('Already Running')
@@ -403,7 +446,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
                 start_task_activity(project_id, task_id)
                 app.echo_info('Started')
         else:
-            # in case of --wait, we poll the status until finished
+            # in case of --wait or --progress, we poll the status until finished
             if status['isRunning']:
                 # in case of a running workflow, we only output status
                 app.echo_info('Already Running ... ', nl=False)
@@ -412,10 +455,11 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
                 app.echo_info('Started ... ', nl=False)
 
             status = _workflow_wait_until_finished(
-                project_id, task_id, polling_interval
+                app, project_id, task_id, polling_interval, progress
             )
             # when we have a Finished status, we output it
-            _workflow_echo_status(app, status)
+            if not progress:
+                _workflow_echo_status(app, status)
             # in case of failure, the following workflows are not executed
             if status['failed']:
                 sys.exit(1)
@@ -430,7 +474,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
         dir_okay=False,
         readable=True
     ),
-    autocompletion=completion.workflow_io_input_files,
+    shell_complete=completion.workflow_io_input_files,
     help="From which file the input is taken. If the workflow "
          "has no defined variable input dataset, this option is not allowed."
 )
@@ -441,7 +485,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
         dir_okay=False,
         writable=True,
     ),
-    autocompletion=completion.workflow_io_output_files,
+    shell_complete=completion.workflow_io_output_files,
     help="To which file the result is written to. Use '-' in order to output "
          "the result to stdout. If the workflow has no defined variable "
          "output dataset, this option is not allowed. Please note that the io "
@@ -475,7 +519,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
 @click.argument(
     "workflow_id",
     type=click.STRING,
-    autocompletion=completion.workflow_io_ids
+    shell_complete=completion.workflow_io_ids
 )
 @click.pass_obj
 def io_command(
@@ -548,7 +592,7 @@ def io_command(
     "--filter", "filter_",
     type=(str, str),
     multiple=True,
-    autocompletion=completion.workflow_list_filter,
+    shell_complete=completion.workflow_list_filter,
     help=WORKFLOW_LIST_FILTER_HELP_TEXT,
 )
 @click.option(
@@ -601,7 +645,7 @@ def list_command(app, filter_, raw, id_only):
 @click.option(
     "--project", "project_id",
     type=click.STRING,
-    autocompletion=completion.project_ids,
+    shell_complete=completion.project_ids,
     help="The project, from which you want to list the workflows. "
          "Project IDs can be listed with the 'project list' command."
 )
@@ -619,7 +663,7 @@ def list_command(app, filter_, raw, id_only):
     "workflow_ids",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.workflow_ids
+    shell_complete=completion.workflow_ids
 )
 @click.pass_obj
 def status_command(app, project_id, raw, _filter, workflow_ids):
@@ -644,7 +688,7 @@ def status_command(app, project_id, raw, _filter, workflow_ids):
 @click.argument(
     "workflow_id",
     type=click.STRING,
-    autocompletion=completion.workflow_ids
+    shell_complete=completion.workflow_ids
 )
 @click.pass_obj
 def open_command(app, workflow_id):

cmem/cmemc/cli/commands/workspace.py

@@ -26,7 +26,7 @@ from cmem.cmempy.workspace.import_ import import_workspace
     default="xmlZip",
     show_default=True,
     type=click.STRING,
-    autocompletion=completion.marshalling_plugins,
+    shell_complete=completion.marshalling_plugins,
     help="Type of the exported workspace file."
 )
 @click.option(
@@ -34,7 +34,7 @@ from cmem.cmempy.workspace.import_ import import_workspace
     default="{{date}}-{{connection}}.workspace",
     show_default=True,
     type=click.STRING,
-    autocompletion=completion.workspace_export_templates,
+    shell_complete=completion.workspace_export_templates,
     help="Template for the export file name. "
          "Possible placeholders are (Jinja2): "
          "{{connection}} (from the --connection option) and "
@@ -44,7 +44,7 @@ from cmem.cmempy.workspace.import_ import import_workspace
 )
 @click.argument(
     "file",
-    autocompletion=completion.workspace_files,
+    shell_complete=completion.workspace_files,
     required=False,
     type=click.Path(
         writable=True,
@@ -95,12 +95,12 @@ def export_command(app, overwrite, marshalling_plugin, template, file):
     default="xmlZip",
     show_default=True,
     type=click.STRING,
-    autocompletion=completion.marshalling_plugins,
+    shell_complete=completion.marshalling_plugins,
     help="Type of the exported workspace file."
 )
 @click.argument(
     "file",
-    autocompletion=completion.workspace_files,
+    shell_complete=completion.workspace_files,
     type=click.Path(
         readable=True,
         allow_dash=False,