cmem-cmemc 23.1.3__py3-none-any.whl → 23.3.0__py3-none-any.whl

cmem/cmemc/cli/commands/workflow.py

@@ -8,15 +8,16 @@ import time
 import click
 
 import timeago
+from click import UsageError
 
 from cmem.cmemc.cli import completion
 from cmem.cmemc.cli.commands import CmemcCommand, CmemcGroup
 from cmem.cmemc.cli.commands.scheduler import scheduler
-from cmem.cmempy.workflow import get_workflows
+from cmem.cmempy.workflow import get_workflows, get_resource_based_dataset_types
 from cmem.cmempy.workflow.workflow import (
     execute_workflow_io,
     get_workflow_editor_uri,
-    get_workflows_io
+    get_workflows_io,
 )
 from cmem.cmempy.workspace.activities import (
     ACTIVITY_TYPE_EXECUTE_DEFAULTWORKFLOW,
@@ -31,6 +32,7 @@ from cmem.cmempy.workspace.activities.taskactivity import (
 )
 from cmem.cmempy.workspace.projects.project import get_projects
 from cmem.cmempy.workspace.search import list_items
+from cmem.cmemc.cli.context import ApplicationContext
 
 WORKFLOW_FILTER_TYPES = sorted(['project', 'regex', 'tag', 'io'])
 WORKFLOW_LIST_FILTER_HELP_TEXT = (
@@ -43,11 +45,15 @@ IO_WARNING_NO_RESULT = "The workflow was executed but produced no result."
 IO_WARNING_NO_OUTPUT_DEFINED = "The workflow was executed, a result was "\
                                "received but dropped."
 
-MIME_CSV = "text/csv"
-MIME_XLS = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+MIME_CSV = "application/x-plugin-csv"
+MIME_XLS = "application/x-plugin-excel"
 MIME_NT = "application/n-triples"
-MIME_JSON = "application/json"
+MIME_JSON = "application/x-plugin-json"
 MIME_XML = "application/xml"
+MIME_FILE = "application/octet-stream"
+MIME_ZIP = "application/x-plugin-multiCsv"
+MIME_ALIGNMENT = "text/alignment"
+MIME_TEXT = "text/plain"
 
 VALID_OUTPUT_EXTENSIONS = {
     ".csv": MIME_CSV,
@@ -61,10 +67,23 @@ VALID_OUTPUT_EXTENSIONS = {
 VALID_INPUT_EXTENSIONS = {
     ".csv": MIME_CSV,
     ".json": MIME_JSON,
-    ".xml": MIME_XML
+    ".xml": MIME_XML,
+    ".xlsx": MIME_XLS,
+    ".file": MIME_FILE,
+    ".zip": MIME_ZIP,
+    ".txt": MIME_TEXT,
 }
 
 
+def is_supported_mime_type(mime_type: str) -> bool:
+    """Boolean to determine if a request is multipart or not"""
+    supported_mime_types = get_resource_based_dataset_types()
+    for supported_mime_type in supported_mime_types:
+        if supported_mime_type in mime_type:
+            return True
+    return False
+
+
 def _get_workflow_tag_labels(workflow_) -> list:
     """Output a list of tag labels from a single workflow."""
     tag_labels = []
@@ -226,7 +245,8 @@ def _io_guess_output(output_file):
         return None
     _, file_extension = os.path.splitext(output_file)
     if file_extension in VALID_OUTPUT_EXTENSIONS:
-        return VALID_OUTPUT_EXTENSIONS[file_extension]
+        if is_supported_mime_type(VALID_OUTPUT_EXTENSIONS[file_extension]):
+            return VALID_OUTPUT_EXTENSIONS[file_extension]
     valid_extensions = ', '.join(VALID_OUTPUT_EXTENSIONS.keys())
     raise ValueError(
         f"Files with the extension {file_extension} can not be generated. "
@@ -240,7 +260,8 @@ def _io_guess_input(input_file):
         return None
     _, file_extension = os.path.splitext(input_file)
     if file_extension in VALID_INPUT_EXTENSIONS:
-        return VALID_INPUT_EXTENSIONS[file_extension]
+        if is_supported_mime_type(VALID_INPUT_EXTENSIONS[file_extension]):
+            return VALID_INPUT_EXTENSIONS[file_extension]
     valid_extensions = ', '.join(VALID_INPUT_EXTENSIONS.keys())
     raise ValueError(
         f"Files with the extension {file_extension} can not be processed. "
@@ -257,19 +278,54 @@ def _workflows_get_ids():
     return ids
 
 
-def _workflow_wait_until_finished(project_id, task_id, polling_interval):
-    """Poll workflow status until workflow is finished and return status.
-
-    :type project_id: string
-    :type task_id: string
-    :type polling_interval: int
-    """
-    while True:
-        status = get_activity_status(project_id, task_id)
-        # wait until isRunning is false
-        if not status['isRunning']:
-            break
-        time.sleep(polling_interval)
+def _get_progress_bar_info(status):
+    """get the workflow message from status response with colors."""
+    if status is None:
+        return ""
+    if status["isRunning"]:
+        return click.style(status["message"], fg="yellow")
+    if status["concreteStatus"] == "Failed":
+        return click.style(status["message"], fg="red")
+    return click.style(status["message"], fg="green")
+
+
+def _workflow_wait_until_finished(
+        app: ApplicationContext,
+        project_id: str,
+        task_id: str,
+        polling_interval: int,
+        log_progress: bool
+):
+    """Poll workflow status until workflow is finished and return status."""
+    progress = 0
+    if log_progress:
+        app.echo_success(message="", nl=True)
+        with click.progressbar(
+            show_eta=False,
+            length=100,
+            item_show_func=_get_progress_bar_info,
+        ) as progress_bar:
+            while True:
+                status = get_activity_status(project_id, task_id)
+                if progress is not status['progress']:
+                    progress_bar.update(
+                        n_steps=status['progress'] - progress,
+                        current_item=status
+                    )
+                    progress = status['progress']
+                app.echo_debug(f"{status['statusName']}({status['message']})")
+                # wait until isRunning is false
+                if not status['isRunning']:
+                    break
+                time.sleep(polling_interval)
+    else:
+        while True:
+            status = get_activity_status(project_id, task_id)
+            app.echo_debug(f"{status['statusName']}({status['message']})")
+            # wait until isRunning is false
+            if not status['isRunning']:
+                break
+            time.sleep(polling_interval)
     return status
 
 
@@ -317,6 +373,7 @@ def _workflow_echo_status(app, status):
             app.echo_success(message)
 
 
+# pylint: disable=too-many-arguments
 @click.command(cls=CmemcCommand, name="execute")
 @click.option(
     "-a", "--all", "all_",
@@ -326,7 +383,12 @@ def _workflow_echo_status(app, status):
 @click.option(
     "--wait",
     is_flag=True,
-    help="Wait until all executed workflows are completed."
+    help="Wait until workflows are completed."
+)
+@click.option(
+    "--progress",
+    is_flag=True,
+    help="Wait until workflows are completed and show a progress bar."
 )
 @click.option(
     "--polling-interval",
@@ -340,10 +402,10 @@ def _workflow_echo_status(app, status):
     "workflow_ids",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.workflow_ids
+    shell_complete=completion.workflow_ids
 )
 @click.pass_obj
-def execute_command(app, all_, wait, polling_interval, workflow_ids):
+def execute_command(app, all_, wait, polling_interval, workflow_ids, progress):
     """Execute workflow(s).
 
     With this command, you can start one or more workflows at the same time or
@@ -358,16 +420,16 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
     polls the status of a workflow until it is finished. In case of an error of
     a workflow, the next workflow is not started.
     """
-    all_workflow_ids = _workflows_get_ids()
     if workflow_ids == () and not all_:
         raise ValueError("Either specify at least one workflow or use the"
                          + " --all option to execute all workflows.")
+    all_workflow_ids = _workflows_get_ids()
     if all_:
         workflow_ids = all_workflow_ids
 
     for workflow_id in workflow_ids:
         if workflow_id not in all_workflow_ids:
-            raise ValueError(
+            raise UsageError(
                 f"Workflow '{workflow_id}' is not available."
             )
         project_id, task_id = workflow_id.split(":")
@@ -375,7 +437,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
 
         # before we start, we fetch the status
         status = get_activity_status(project_id, task_id)
-        if not wait:
+        if not wait and not progress:
             if status['isRunning']:
                 # in case of a running workflow, we only output status
                 app.echo_info('Already Running')
@@ -384,7 +446,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
                 start_task_activity(project_id, task_id)
                 app.echo_info('Started')
         else:
-            # in case of --wait, we poll the status until finished
+            # in case of --wait or --progress, we poll the status until finished
             if status['isRunning']:
                 # in case of a running workflow, we only output status
                 app.echo_info('Already Running ... ', nl=False)
@@ -393,10 +455,11 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
                 app.echo_info('Started ... ', nl=False)
 
             status = _workflow_wait_until_finished(
-                project_id, task_id, polling_interval
+                app, project_id, task_id, polling_interval, progress
             )
             # when we have a Finished status, we output it
-            _workflow_echo_status(app, status)
+            if not progress:
+                _workflow_echo_status(app, status)
             # in case of failure, the following workflows are not executed
             if status['failed']:
                 sys.exit(1)
@@ -411,10 +474,9 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
         dir_okay=False,
         readable=True
     ),
-    autocompletion=completion.workflow_io_input_files,
-    help="From which file the input is taken: note that the maximum file size "
-         "to upload is limited to a server configured value. If the workflow "
-         "has no defined variable input dataset, this can be ignored."
+    shell_complete=completion.workflow_io_input_files,
+    help="From which file the input is taken. If the workflow "
+         "has no defined variable input dataset, this option is not allowed."
 )
 @click.option(
     "--output", "-o", "output_file",
@@ -423,25 +485,18 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
         dir_okay=False,
         writable=True,
     ),
-    autocompletion=completion.workflow_io_output_files,
-    help="To which file the result is written to: use '-' in order to output "
+    shell_complete=completion.workflow_io_output_files,
+    help="To which file the result is written to. Use '-' in order to output "
          "the result to stdout. If the workflow has no defined variable "
-         "output dataset, this can be ignored. Please note that the io "
+         "output dataset, this option is not allowed. Please note that the io "
          "command will not warn you on overwriting existing output files."
 )
 @click.option(
     "--input-mimetype",
     help="Which input format should be processed: If not given, cmemc will "
          "try to guess the mime type based on the file extension or will "
-         "fail",
-    type=click.Choice(
-        [
-            "guess",
-            "application/xml",
-            "application/json",
-            "text/csv"
-        ]
-    ),
+         "fail.",
+    type=click.Choice(list(VALID_INPUT_EXTENSIONS.values()) + ["guess"]),
     default="guess"
 )
 @click.option(
@@ -449,10 +504,8 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
     help="Which output format should be requested: If not given, cmemc will "
          "try to guess the mime type based on the file extension or will "
          "fail. In case of an output to stdout, a default mime type "
-         "will be used (currently xml).",
-    type=click.Choice(
-        ["guess", MIME_XML, MIME_JSON, MIME_NT, MIME_XLS, MIME_CSV]
-    ),
+         "will be used (JSON).",
+    type=click.Choice(list(VALID_OUTPUT_EXTENSIONS.values()) + ["guess"]),
     default="guess"
 )
 @click.option(
@@ -466,7 +519,7 @@ def execute_command(app, all_, wait, polling_interval, workflow_ids):
 @click.argument(
     "workflow_id",
     type=click.STRING,
-    autocompletion=completion.workflow_io_ids
+    shell_complete=completion.workflow_io_ids
 )
 @click.pass_obj
 def io_command(
@@ -496,6 +549,16 @@ def io_command(
         output_mimetype=output_mimetype
     )
 
+    app.echo_debug(
+        f"On workflow io execution:"
+        f"project_name={project_id}, "
+        f"task_name={task_id}, "
+        f"input_file={input_file}, "
+        f"input_mime_type={input_mimetype}, "
+        f"output_mime_type={output_mimetype}, "
+        f"auto_config={autoconfig}"
+    )
+
     response = execute_workflow_io(
         project_name=project_id,
         task_name=task_id,
@@ -529,7 +592,7 @@ def io_command(
     "--filter", "filter_",
     type=(str, str),
     multiple=True,
-    autocompletion=completion.workflow_list_filter,
+    shell_complete=completion.workflow_list_filter,
     help=WORKFLOW_LIST_FILTER_HELP_TEXT,
 )
 @click.option(
@@ -582,7 +645,7 @@ def list_command(app, filter_, raw, id_only):
 @click.option(
     "--project", "project_id",
     type=click.STRING,
-    autocompletion=completion.project_ids,
+    shell_complete=completion.project_ids,
     help="The project, from which you want to list the workflows. "
          "Project IDs can be listed with the 'project list' command."
 )
@@ -600,7 +663,7 @@ def list_command(app, filter_, raw, id_only):
     "workflow_ids",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.workflow_ids
+    shell_complete=completion.workflow_ids
 )
 @click.pass_obj
 def status_command(app, project_id, raw, _filter, workflow_ids):
@@ -625,7 +688,7 @@ def status_command(app, project_id, raw, _filter, workflow_ids):
 @click.argument(
     "workflow_id",
     type=click.STRING,
-    autocompletion=completion.workflow_ids
+    shell_complete=completion.workflow_ids
 )
 @click.pass_obj
 def open_command(app, workflow_id):

cmem/cmemc/cli/commands/workspace.py

@@ -26,7 +26,7 @@ from cmem.cmempy.workspace.import_ import import_workspace
     default="xmlZip",
     show_default=True,
     type=click.STRING,
-    autocompletion=completion.marshalling_plugins,
+    shell_complete=completion.marshalling_plugins,
     help="Type of the exported workspace file."
 )
 @click.option(
@@ -34,7 +34,7 @@ from cmem.cmempy.workspace.import_ import import_workspace
     default="{{date}}-{{connection}}.workspace",
     show_default=True,
     type=click.STRING,
-    autocompletion=completion.workspace_export_templates,
+    shell_complete=completion.workspace_export_templates,
     help="Template for the export file name. "
          "Possible placeholders are (Jinja2): "
          "{{connection}} (from the --connection option) and "
@@ -44,7 +44,7 @@ from cmem.cmempy.workspace.import_ import import_workspace
 )
 @click.argument(
     "file",
-    autocompletion=completion.workspace_files,
+    shell_complete=completion.workspace_files,
     required=False,
     type=click.Path(
         writable=True,
@@ -95,12 +95,12 @@ def export_command(app, overwrite, marshalling_plugin, template, file):
     default="xmlZip",
     show_default=True,
     type=click.STRING,
-    autocompletion=completion.marshalling_plugins,
+    shell_complete=completion.marshalling_plugins,
     help="Type of the exported workspace file."
 )
 @click.argument(
     "file",
-    autocompletion=completion.workspace_files,
+    shell_complete=completion.workspace_files,
     type=click.Path(
         readable=True,
         allow_dash=False,