cmem-cmemc 23.1.3__py3-none-any.whl → 23.3.0__py3-none-any.whl

cmem/cmemc/cli/completion.py

@@ -4,9 +4,9 @@ import os
 from collections import Counter
 from contextlib import suppress
 
-import requests
-from bs4 import BeautifulSoup
 from click import Context
+from click.parser import split_arg_string
+from click.shell_completion import CompletionItem
 from natsort import natsorted, ns
 from prometheus_client.parser import text_string_to_metric_families
 
@@ -14,10 +14,11 @@ from cmem.cmemc.cli.context import CONTEXT
 from cmem.cmemc.cli.utils import (
     get_graphs,
     metric_get_labels,
-    metrics_get_dict, struct_to_table
+    metrics_get_dict, struct_to_table, get_published_packages
 )
 from cmem.cmempy.dp.admin import get_prometheus_data
 from cmem.cmempy.health import get_complete_status_info
+from cmem.cmempy.keycloak.client import list_open_id_clients
 from cmem.cmempy.keycloak.group import list_groups
 from cmem.cmempy.keycloak.user import list_users, get_user_by_username, user_groups
 from cmem.cmempy.plugins.marshalling import get_marshalling_plugins
@@ -30,9 +31,12 @@ from cmem.cmempy.workspace import (
 )
 from cmem.cmempy.workspace.projects.project import get_projects
 from cmem.cmempy.workspace.projects.resources import get_all_resources
+from cmem.cmempy.workspace.projects.variables import get_all_variables
 from cmem.cmempy.workspace.python import list_packages
 from cmem.cmempy.workspace.search import list_items
 
+from cmem.cmempy.workspace.projects.datasets.dataset import get_dataset
+
 SORT_BY_KEY = 0
 SORT_BY_DESC = 1
 
@@ -68,7 +72,6 @@ def _finalize_completion(
         return candidates
     # remove duplicates
     candidates = list(set(candidates))
-
     if isinstance(candidates[0], str):
         # list of strings filtering and sorting
         filtered_candidates = [
@@ -96,22 +99,56 @@ def _finalize_completion(
             or str(element[1]).lower().find(incomplete) != -1
         ]
         if nat_sort:
-            return natsorted(
+            sorted_list = natsorted(
                 seq=filtered_candidates,
                 key=lambda k: k[sort_by],   # type: ignore
                 alg=ns.IGNORECASE,
                 reverse=reverse
             )
-        return sorted(
-            filtered_candidates,
-            key=lambda x: (str(x[sort_by]).casefold(), str(x[sort_by])),
-            reverse=reverse
-        )
+        else:
+            sorted_list = sorted(
+                filtered_candidates,
+                key=lambda x: (str(x[sort_by]).casefold(), str(x[sort_by])),
+                reverse=reverse
+            )
+        return [
+            CompletionItem(
+                value=element[0].replace(":", r"\:"),
+                help=element[1]
+            ) for element in sorted_list
+        ]
     raise ValueError(
         "candidates should be a list of strings or a list of tuples."
     )
 
 
+def _get_completion_args(incomplete):
+    """get completion args
+
+    This is a workaround to get partial tuple options in a completion function
+    see https://github.com/pallets/click/issues/2597
+    """
+    args = split_arg_string(os.environ["COMP_WORDS"])
+    if incomplete and len(args) > 0 and args[len(args)-1] == incomplete:
+        args.pop()
+    return args
+
+
+def _ignore_option(option, params):
+    """
+    Check if the given 'option' is present in the 'params' dictionary
+    or any of its values.
+    """
+    ignore_project_id = False
+    for _ in params:
+        if hasattr(params[_], '__iter__') and option in params[_]:
+            ignore_project_id = True
+        elif option == params[_]:
+            ignore_project_id = True
+
+    return ignore_project_id
+
+
 def add_metadata_parameter(list_=None):
     """Extend a list with metadata keys and key descriptions."""
     if list_ is None:
@@ -127,19 +164,56 @@ def add_metadata_parameter(list_=None):
     return list_
 
 
-def dataset_parameter(ctx, args, incomplete):
+def add_read_only_and_uri_property_parameters(list_=None):
+    """Extend a list with readonly/uriProperty keys and key descriptions."""
+    if list_ is None:
+        list_ = []
+    list_.append(
+        (
+            "readOnly",
+            "Read-only: If enabled, all write operations using this dataset object "
+            "will fail, e.g. when used as output in workflows or transform/linking "
+            "executions. This will NOT protect the underlying resource in general, "
+            "e.g. files, databases or knowledge graphs could still be changed "
+            "externally."
+         )
+    )
+    list_.append(
+        (
+            "uriProperty",
+            "URI attribute: When reading data from the dataset, the specified "
+            "attribute will be used to get the URIs of the entities. "
+            "When writing to a dataset, the specified attribute will be automatically "
+            "added to the schema as well as the generated entity URIs will be added as "
+            "values for each entity. If the entered value is not a valid URI, "
+            "it will be converted to a valid URI."
+        )
+    )
+    return list_
+
+
+def dataset_parameter(ctx, param, incomplete):
     """Prepare a list of dataset parameters for a dataset type."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    args = _get_completion_args(incomplete)
     incomplete = incomplete.lower()
     # look if cursor is in value position of the -p option and
     # return nothing in case it is (values are not completed atm)
     if args[len(args) - 2] in ("-p", "--parameter"):
         return []
     # try to determine the dataset type
-    dataset_type = None
-    for num, arg in enumerate(args):
-        if arg == "--type":
-            dataset_type = args[num + 1]
+    dataset_type = ctx.params.get('dataset_type')
+    if dataset_type is None:
+        try:
+            dataset_id = ctx.args[0]
+            project = get_dataset(
+                project_name=dataset_id.split(":")[0],
+                dataset_name=dataset_id.split(":")[1]
+            )
+            dataset_type = project["data"]["type"]
+        except IndexError:
+            pass
+
     # without type, we know nothing
     if dataset_type is None:
         return []
@@ -151,6 +225,8 @@ def dataset_parameter(ctx, args, incomplete):
         description = properties[key]["description"]
         option = f"{title}: {description}"
         options.append((key, option))
+
+    options = add_read_only_and_uri_property_parameters(options)
     # sorting: metadata on top, then parameter per key
     options = sorted(options, key=lambda k: k[0].lower())
     options = add_metadata_parameter(options)
@@ -161,12 +237,13 @@ def dataset_parameter(ctx, args, incomplete):
             or key[1].lower().find(incomplete.lower()) != -1
             )
     ]
-    return options
+
+    return [CompletionItem(value=option[0], help=option[1]) for option in options]
 
 
-def dataset_types(ctx, args, incomplete):
+def dataset_types(ctx, param, incomplete):
     """Prepare a list of dataset types."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     incomplete = incomplete.lower()
     options = []
     plugins = get_task_plugins()
@@ -184,31 +261,33 @@ def dataset_types(ctx, args, incomplete):
                     option
                 )
             )
-    options = sorted(options, key=lambda k: k[1].lower())
-    return options
+    return _finalize_completion(
+        candidates=options,
+        incomplete=incomplete,
+        sort_by=SORT_BY_DESC
+    )
 
 
-def dataset_ids(ctx, args, incomplete):
+def dataset_ids(ctx, param, incomplete):
     """Prepare a list of projectid:datasetid dataset identifier."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     results = list_items(item_type="dataset")
     datasets = results["results"]
     for _ in datasets:
         options.append(
             (
-                _["projectId"] + r"\:" + _["id"],
+                _['projectId'] + ":" + _['id'],
                 _["label"]
             )
         )
     return _finalize_completion(
         candidates=options,
-        incomplete=incomplete,
-        sort_by=SORT_BY_DESC
+        incomplete=incomplete
     )
 
 
-def dataset_list_filter(ctx, args, incomplete):
+def dataset_list_filter(ctx, param, incomplete):
     """Prepare a list of filter names and values for dataset list filter."""
     filter_names = [
         (
@@ -239,17 +318,18 @@ def dataset_list_filter(ctx, args, incomplete):
         )
     ]
     options = []
+    args = _get_completion_args(incomplete)
     if args[len(args) - 1] == "--filter":
         options = _finalize_completion(
             candidates=filter_names,
             incomplete=incomplete
         )
     if args[len(args) - 1] == "type":
-        options = dataset_types(ctx, args, incomplete)
+        options = dataset_types(ctx, param, incomplete)
     if args[len(args) - 1] == "project":
-        options = project_ids(ctx, args, incomplete)
+        options = project_ids(ctx, param, incomplete)
     if args[len(args) - 1] == "tag":
-        options = tag_labels(ctx, args, incomplete, "dataset")
+        options = tag_labels(ctx, param, incomplete, "dataset")
     if args[len(args) - 1] == "regex":
         options = _finalize_completion(
             candidates=filter_regex,
@@ -258,9 +338,9 @@ def dataset_list_filter(ctx, args, incomplete):
     return options
 
 
-def resource_ids(ctx, args, incomplete):
+def resource_ids(ctx, param, incomplete):
     """Prepare a list of projectid:resourceid resource identifier."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     for _ in get_all_resources():
         options.append(
@@ -275,9 +355,9 @@ def resource_ids(ctx, args, incomplete):
     )
 
 
-def scheduler_ids(ctx, args, incomplete):
+def scheduler_ids(ctx, param, incomplete):
     """Prepare a list of projectid:schedulerid scheduler identifier."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     schedulers = list_items(
         item_type="task",
@@ -288,11 +368,11 @@ def scheduler_ids(ctx, args, incomplete):
         }]
     )["results"]
     for _ in schedulers:
-        if _["projectId"] + ":" + _["id"] in args:
+        if _ignore_option(_["projectId"] + ":" + _["id"], ctx.params):
             continue
         options.append(
             (
-                _["projectId"] + r"\:" + _["id"],
+                _["projectId"] + ":" + _["id"],
                 _["label"]
             )
         )
@@ -303,9 +383,9 @@ def scheduler_ids(ctx, args, incomplete):
     )
 
 
-def metric_ids(ctx, args, incomplete):
+def metric_ids(ctx, param, incomplete):
     """Prepare a list of metric identifier."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
 
     data = get_prometheus_data().text
@@ -323,13 +403,13 @@ def metric_ids(ctx, args, incomplete):
     )
 
 
-def metric_label_filter(ctx, args, incomplete):
+def metric_label_filter(ctx, param, incomplete):
     """Prepare a list of label name or values."""
-    # TODO: this completion does not take care of the --job option
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    args = _get_completion_args(incomplete)
     incomplete = incomplete.lower()
     options = []
-    metric_id = args[3]
+    metric_id = ctx.args[0]
     labels = metric_get_labels(metrics_get_dict()[metric_id])
     if args[len(args) - 1] in "--filter":
         # we are in the name position
@@ -345,9 +425,9 @@ def metric_label_filter(ctx, args, incomplete):
     )
 
 
-def vocabularies(ctx, args, incomplete, filter_="all"):
+def vocabularies(ctx, param, incomplete, filter_="all"):
     """Prepare a list of vocabulary graphs for auto-completion."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     try:
         vocabs = get_vocabularies(filter_=filter_)
@@ -356,9 +436,9 @@ def vocabularies(ctx, args, incomplete, filter_="all"):
         return []
     for _ in vocabs:
         url = _["iri"]
-        if url in args:
+        if _ignore_option(url, ctx.params):
             continue
-        url = _["iri"].replace(":", r"\:")
+        url = _["iri"]
         try:
             label = _["label"]["title"]
         except (KeyError, TypeError):
@@ -371,14 +451,14 @@ def vocabularies(ctx, args, incomplete, filter_="all"):
     )
 
 
-def installed_vocabularies(ctx, args, incomplete):
+def installed_vocabularies(ctx, param, incomplete):
     """Prepare a list of installed vocabulary graphs."""
-    return vocabularies(ctx, args, incomplete, filter_="installed")
+    return vocabularies(ctx, param, incomplete, filter_="installed")
 
 
-def installable_vocabularies(ctx, args, incomplete):
+def installable_vocabularies(ctx, param, incomplete):
     """Prepare a list of installable vocabulary graphs."""
-    return vocabularies(ctx, args, incomplete, filter_="installable")
+    return vocabularies(ctx, param, incomplete, filter_="installable")
 
 
 def file_list(incomplete="", suffix="", description="", prefix=""):
@@ -397,12 +477,12 @@ def file_list(incomplete="", suffix="", description="", prefix=""):
     )
 
 
-def workflow_io_ids(ctx, args, incomplete):
+def workflow_io_ids(ctx, param, incomplete):
     """Prepare a list of io workflows."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     for _ in get_workflows_io():
-        workflow_id = _["projectId"] + r"\:" + _["id"]
+        workflow_id = _["projectId"] + ":" + _["id"]
         label = _["label"]
         options.append((workflow_id, label))
     return _finalize_completion(
@@ -412,7 +492,7 @@ def workflow_io_ids(ctx, args, incomplete):
     )
 
 
-def replay_files(ctx, args, incomplete):
+def replay_files(ctx, param, incomplete):
     """Prepare a list of JSON replay files."""
     return file_list(
         incomplete=incomplete,
@@ -421,9 +501,9 @@ def replay_files(ctx, args, incomplete):
     )
 
 
-def installed_package_names(ctx, args, incomplete):
+def installed_package_names(ctx, param, incomplete):
     """Prepare a list of installed packages."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_args(ctx.find_root().params)
     options = []
     packages = list_packages()
     for _ in packages:
@@ -440,31 +520,14 @@ def installed_package_names(ctx, args, incomplete):
     )
 
 
-def published_package_names(ctx, args, incomplete):
+def published_package_names(ctx, param, incomplete):
     """List of plugin packages scraped from pypi.org."""
     options = []
-    url = "https://pypi.org/search/?q=%22cmem-plugin-%22"
-    soup = BeautifulSoup(
-        requests.get(url, timeout=5).content,
-        "html.parser"
-    )
-    packages = soup.find_all("a", class_="package-snippet")
-    for package in packages:
-        name = package.findChildren(
-            class_="package-snippet__name"
-        )[0].getText()
-        if name == "cmem-plugin-base":
-            continue
-        description = package.findChildren(
-            class_="package-snippet__description"
-        )[0].getText()
-        version = package.findChildren(
-            class_="package-snippet__version"
-        )[0].getText()
+    for _ in get_published_packages():
         options.append(
             (
-                name,
-                f"{version}: {description}"
+                _.name,
+                f"{_.version}: {_.description}"
             )
         )
 
@@ -475,7 +538,7 @@ def published_package_names(ctx, args, incomplete):
     )
 
 
-def python_package_files(ctx, args, incomplete):
+def python_package_files(ctx, param, incomplete):
     """Prepare a list of acceptable python package files."""
     return file_list(
         incomplete=incomplete,
@@ -485,14 +548,14 @@ def python_package_files(ctx, args, incomplete):
     )
 
 
-def installable_packages(ctx, args, incomplete):
+def installable_packages(ctx, param, incomplete):
     """Installable packages from files and pypi.org."""
     return python_package_files(
-        ctx, args, incomplete) + published_package_names(
-        ctx, args, incomplete)
+        ctx, param, incomplete) + published_package_names(
+        ctx, param, incomplete)
 
 
-def workflow_io_input_files(ctx, args, incomplete):
+def workflow_io_input_files(ctx, param, incomplete):
     """Prepare a list of acceptable workflow io input files."""
     return file_list(
         incomplete=incomplete,
@@ -506,10 +569,51 @@ def workflow_io_input_files(ctx, args, incomplete):
         incomplete=incomplete,
         suffix=".json",
         description="JSON Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".xlsx",
+        description="Excel Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".txt",
+        description="Text Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".zip",
+        description="Multi CSV Dataset resource"
     )
 
 
-def workflow_io_output_files(ctx, args, incomplete):
+def workflow_io_input_mimetypes(ctx, args, incomplete):
+    """Prepare a list of acceptable workflow io input mimetypes."""
+    return file_list(
+        incomplete=incomplete,
+        suffix=".csv",
+        description="CSV Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".xml",
+        description="XML Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".json",
+        description="JSON Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".xlsx",
+        description="Excel Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".txt",
+        description="Text Dataset resource"
+    ) + file_list(
+        incomplete=incomplete,
+        suffix=".zip",
+        description="Multi CSV Dataset resource"
+    )
+
+
+def workflow_io_output_files(ctx, param, incomplete):
     """Prepare a list of acceptable workflow io output files."""
     return file_list(
         incomplete=incomplete,
@@ -538,7 +642,7 @@ def workflow_io_output_files(ctx, args, incomplete):
     )
 
 
-def dataset_files(ctx, args, incomplete):
+def dataset_files(ctx, param, incomplete):
     """Prepare a list of SPARQL files."""
     return file_list(
         incomplete=incomplete,
@@ -571,7 +675,7 @@ def dataset_files(ctx, args, incomplete):
     )
 
 
-def graph_backup_files(ctx, args, incomplete):
+def graph_backup_files(ctx, param, incomplete):
     """Prepare a list of workspace files."""
     return file_list(
         incomplete=incomplete,
@@ -580,7 +684,7 @@ def graph_backup_files(ctx, args, incomplete):
     )
 
 
-def project_files(ctx, args, incomplete):
+def project_files(ctx, param, incomplete):
     """Prepare a list of workspace files."""
     return file_list(
         incomplete=incomplete,
@@ -589,7 +693,7 @@ def project_files(ctx, args, incomplete):
     )
 
 
-def ini_files(ctx, args, incomplete):
+def ini_files(ctx, param, incomplete):
     """Prepare a list of workspace files."""
     return file_list(
         incomplete=incomplete,
@@ -598,7 +702,7 @@ def ini_files(ctx, args, incomplete):
     )
 
 
-def workspace_files(ctx, args, incomplete):
+def workspace_files(ctx, param, incomplete):
     """Prepare a list of workspace files."""
     return file_list(
         incomplete=incomplete,
@@ -607,7 +711,7 @@ def workspace_files(ctx, args, incomplete):
     )
 
 
-def sparql_files(ctx, args, incomplete):
+def sparql_files(ctx, param, incomplete):
     """Prepare a list of SPARQL files."""
     return file_list(
         incomplete=incomplete,
@@ -620,7 +724,7 @@ def sparql_files(ctx, args, incomplete):
     )
 
 
-def triple_files(ctx, args, incomplete):
+def triple_files(ctx, param, incomplete):
     """Prepare a list of triple files."""
     return file_list(
         incomplete=incomplete,
@@ -633,14 +737,15 @@ def triple_files(ctx, args, incomplete):
     )
 
 
-def placeholder(ctx, args, incomplete):
+def placeholder(ctx, param, incomplete):
     """Prepare a list of placeholder from the to-be executed queries."""
     # look if cursor is in value position of the -p option and
     # return nothing in case it is (values are not completed atm)
+    args = _get_completion_args(incomplete)
     if args[len(args) - 2] in ("-p", "--parameter"):
         return []
     # setup configuration
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     # extract placeholder from given queries in the command line
     options = []
     for num, arg in enumerate(args):
@@ -660,12 +765,12 @@ def placeholder(ctx, args, incomplete):
     )
 
 
-def remote_queries(ctx, args, incomplete):
+def remote_queries(ctx, param, incomplete):
     """Prepare a list of query URIs."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     for _, query in QUERY_CATALOG.get_queries().items():
-        url = query.short_url.replace(":", r"\:")
+        url = query.short_url
         label = query.label
         options.append((url, label))
     return _finalize_completion(
@@ -675,24 +780,24 @@ def remote_queries(ctx, args, incomplete):
     )
 
 
-def remote_queries_and_sparql_files(ctx, args, incomplete):
+def remote_queries_and_sparql_files(ctx, param, incomplete):
     """Prepare a list of named queries, query files and directories."""
-    remote = remote_queries(ctx, args, incomplete)
-    files = sparql_files(ctx, args, incomplete)
+    remote = remote_queries(ctx, param, incomplete)
+    files = sparql_files(ctx, param, incomplete)
     return remote + files
 
 
-def workflow_ids(ctx, args, incomplete):
+def workflow_ids(ctx, param, incomplete):
     """Prepare a list of projectid:taskid workflow identifier."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     workflows = list_items(item_type="workflow")["results"]
     options = []
     for _ in workflows:
         workflow = _["projectId"] + ":" + _["id"]
         label = _["label"]
-        if workflow in args:
+        if _ignore_option(workflow, ctx.params):
             continue
-        options.append((workflow.replace(":", r"\:"), label))
+        options.append((workflow, label))
     return _finalize_completion(
         candidates=options,
         incomplete=incomplete,
@@ -700,19 +805,26 @@ def workflow_ids(ctx, args, incomplete):
     )
 
 
-def marshalling_plugins(ctx, args, incomplete):
+def marshalling_plugins(ctx, param, incomplete):
     """Prepare a list of supported workspace/project import/export plugins."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = get_marshalling_plugins()
     if "description" in options[0].keys():
-        return [(_["id"], _["description"]) for _ in options]
-    # in case, no descriptions are available, labels are fine as well
-    return [(_["id"], _["label"]) for _ in options]
+        final_options = [(_["id"], _["description"]) for _ in options]
+    else:
+        # in case, no descriptions are available, labels are fine as well
+        final_options = [(_["id"], _["label"]) for _ in options]
+
+    return _finalize_completion(
+        candidates=final_options,
+        incomplete=incomplete,
+        sort_by=SORT_BY_DESC
+    )
 
 
-def project_ids(ctx, args, incomplete):
+def project_ids(ctx, param, incomplete):
     """Prepare a list of project IDs for auto-completion."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     try:
         projects = get_projects()
     except Exception:
@@ -723,7 +835,7 @@ def project_ids(ctx, args, incomplete):
         project_id = _["name"]
         label = _["metaData"]["label"]
         # do not add project if already in the command line
-        if project_id in args:
+        if _ignore_option(project_id, ctx.params):
             continue
         options.append((project_id, label))
     return _finalize_completion(
@@ -733,9 +845,9 @@ def project_ids(ctx, args, incomplete):
     )
 
 
-def graph_uris(ctx, args, incomplete, writeable=True, readonly=True):
+def graph_uris(ctx, param, incomplete, writeable=True, readonly=True):
     """Prepare a list of graphs for auto-completion."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     try:
         graphs = get_graphs()
     except Exception:
@@ -746,9 +858,9 @@ def graph_uris(ctx, args, incomplete, writeable=True, readonly=True):
         iri = _["iri"]
         label = _["label"]["title"]
         # do not add graph if already in the command line
-        if iri in args:
+        if _ignore_option(iri, ctx.params):
             continue
-        options.append((iri.replace(":", r"\:"), label))
+        options.append((iri, label))
     return _finalize_completion(
         candidates=options,
         incomplete=incomplete,
@@ -756,15 +868,15 @@ def graph_uris(ctx, args, incomplete, writeable=True, readonly=True):
     )
 
 
-def writable_graph_uris(ctx, args, incomplete):
+def writable_graph_uris(ctx, param, incomplete):
     """Prepare a list of writable graphs for auto-completion."""
-    return graph_uris(ctx, args, incomplete, writeable=True, readonly=False)
+    return graph_uris(ctx, param, incomplete, writeable=True, readonly=False)
 
 
-def connections(ctx, args, incomplete):
+def connections(ctx, param, incomplete):
     """Prepare a list of config connections for auto-completion."""
     # since ctx does not have an obj here, we re-create the object
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     for section in CONTEXT.config.sections():
         options.append(section)
@@ -774,7 +886,7 @@ def connections(ctx, args, incomplete):
     )
 
 
-def graph_export_templates(ctx, args, incomplete):
+def graph_export_templates(ctx, param, incomplete):
     """Prepare a list of example templates for the graph export command."""
     examples = [
         (
@@ -800,7 +912,7 @@ def graph_export_templates(ctx, args, incomplete):
     )
 
 
-def project_export_templates(ctx, args, incomplete):
+def project_export_templates(ctx, param, incomplete):
     """Prepare a list of example templates for the project export command."""
     examples = [
         (
@@ -821,7 +933,7 @@ def project_export_templates(ctx, args, incomplete):
     )
 
 
-def workspace_export_templates(ctx, args, incomplete):
+def workspace_export_templates(ctx, param, incomplete):
     """Prepare a list of example templates for the workspace export command."""
     examples = [
         (
@@ -842,7 +954,7 @@ def workspace_export_templates(ctx, args, incomplete):
     )
 
 
-def query_status_filter(ctx, args, incomplete):
+def query_status_filter(ctx, param, incomplete):
     """Prepare a list of filter names and values for query status filter."""
     filter_names = [
         (
@@ -963,7 +1075,7 @@ def query_status_filter(ctx, args, incomplete):
             "Queries of unknown type."
         )
     ]
-
+    args = _get_completion_args(incomplete)
     last_argument = args[len(args) - 1]
     options = None
     if last_argument == "--filter":
@@ -993,11 +1105,11 @@ def query_status_filter(ctx, args, incomplete):
             incomplete=incomplete
         )
     if last_argument == "user":
-        options = query_account_iris(ctx, args, incomplete)
+        options = query_account_iris(ctx, param, incomplete)
     if last_argument == "trace-id":
-        options = query_trace_ids(ctx, args, incomplete)
+        options = query_trace_ids(ctx, param, incomplete)
     if last_argument == "graph":
-        options = query_graphs(ctx, args, incomplete)
+        options = query_graphs(ctx, param, incomplete)
 
     if not options:
         raise ValueError(
@@ -1006,9 +1118,9 @@ def query_status_filter(ctx, args, incomplete):
     return options
 
 
-def query_account_iris(ctx, args, incomplete):
+def query_account_iris(ctx, param, incomplete):
     """Prepare a list account IRIs from the query status."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     accounts = {}
     for _ in get_query_status():
         if _["user"] in accounts:
@@ -1016,7 +1128,7 @@ def query_account_iris(ctx, args, incomplete):
         else:
             accounts[_["user"]] = 1
     options = [
-        (account.replace(":", r"\:"), f"{count} queries")
+        (account, f"{count} queries")
         for account, count in accounts.items()
     ]
     return _finalize_completion(
@@ -1025,9 +1137,9 @@ def query_account_iris(ctx, args, incomplete):
     )
 
 
-def query_trace_ids(ctx, args, incomplete):
+def query_trace_ids(ctx, param, incomplete):
     """Prepare a list trace IDs from the query status."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = Counter(
         [query["traceId"] for query in get_query_status()]
     ).most_common()
@@ -1038,9 +1150,9 @@ def query_trace_ids(ctx, args, incomplete):
     )
 
 
-def query_graphs(ctx, args, incomplete):
+def query_graphs(ctx, param, incomplete):
     """Prepare a list graph URLs from the query status."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = Counter()
     for query in get_query_status():
         for graph in query.get("affectedGraphs", []):
@@ -1052,7 +1164,7 @@ def query_graphs(ctx, args, incomplete):
     )
 
 
-def graph_list_filter(ctx, args, incomplete):
+def graph_list_filter(ctx, param, incomplete):
     """Prepare a list of filter names and values for graph list filter."""
     filter_names = [
         (
@@ -1076,7 +1188,7 @@ def graph_list_filter(ctx, args, incomplete):
             "List only graphs which ARE writeable for the current user."
         )
     ]
-
+    args = _get_completion_args(incomplete)
     options = []
     if args[len(args) - 1] == "--filter":
         options = _finalize_completion(
@@ -1089,11 +1201,72 @@ def graph_list_filter(ctx, args, incomplete):
             incomplete=incomplete
         )
     if args[len(args) - 1] == "imported-by":
-        options = graph_uris(ctx, args, incomplete)
+        options = graph_uris(ctx, param, incomplete)
     return options
 
 
-def resource_list_filter(ctx, args, incomplete):
+def variable_ids(ctx, param, incomplete):
+    """Prepare a list of variables IDs for auto-completion."""
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    try:
+        variables = get_all_variables()
+    except Exception:
+        # if something went wrong, die silently
+        return []
+    options = []
+    for _ in variables:
+        variable_id = _["id"]
+        label = _.get("description", "").partition('\n')[0]
+        if label == "":
+            label = f"Current value: {_['value']}"
+        # do not add project if already in the command line
+        if _ignore_option(variable_id, ctx.params):
+            continue
+        options.append((variable_id, label))
+    return _finalize_completion(
+        candidates=options,
+        incomplete=incomplete,
+        sort_by=SORT_BY_KEY
+    )
+
+
+def variable_list_filter(ctx, param, incomplete):
+    """Prepare a list of filter names and values for variable list filter."""
+    filter_names = [
+        (
+            "project",
+            "Filter for variables from a specific project."
+        ),
+        (
+            "regex",
+            "Filter for variables with a regular expression search over "
+            "id, value and description."
+        )
+    ]
+    filter_values_regex = [
+        (
+            "ending$",
+            "Variables name ends with 'ending'."
+        ),
+        (
+            "^starting",
+            "Variables name starts with 'starting'."
+        )
+    ]
+    args = _get_completion_args(incomplete)
+    if args[len(args) - 1] == "--filter":
+        return [CompletionItem(value=f[0], help=f[1]) for f in filter_names]
+    if args[len(args) - 1] == "regex":
+        return _finalize_completion(
+            candidates=filter_values_regex,
+            incomplete=incomplete
+        )
+    if args[len(args) - 1] == "project":
+        return project_ids(ctx, param, incomplete)
+    return []
+
+
+def resource_list_filter(ctx, param, incomplete):
     """Prepare a list of filter names and values for resource list filter."""
     filter_names = [
         (
@@ -1116,12 +1289,12 @@ def resource_list_filter(ctx, args, incomplete):
             "File resources which name has a date from 2021-10 in it"
         )
     ]
-
+    args = _get_completion_args(incomplete)
     if args[len(args) - 1] == "--filter":
-        return filter_names
+        return [CompletionItem(value=f[0], help=f[1]) for f in filter_names]
     if args[len(args) - 1] == "project":
         return _finalize_completion(
-            candidates=project_ids(ctx, args, incomplete),
+            candidates=project_ids(ctx, param, incomplete),
             incomplete=incomplete
         )
     if args[len(args) - 1] == "regex":
@@ -1132,7 +1305,7 @@ def resource_list_filter(ctx, args, incomplete):
     return []
 
 
-def workflow_list_filter(ctx, args, incomplete):
+def workflow_list_filter(ctx, param, incomplete):
     """Prepare a list of filter names and values for workflow list filter."""
     filter_names = [
         (
@@ -1181,23 +1354,28 @@ def workflow_list_filter(ctx, args, incomplete):
         )
     ]
     options = []
+    args = _get_completion_args(incomplete)
     if args[len(args) - 1] == "--filter":
         options = filter_names
     if args[len(args) - 1] == "io":
         options = filter_values_io
     if args[len(args) - 1] == "project":
-        options = project_ids(ctx, args, incomplete)
+        options = project_ids(ctx, param, incomplete)
     if args[len(args) - 1] == "tag":
-        options = tag_labels(ctx, args, incomplete, "workflow")
+        options = tag_labels(ctx, param, incomplete, "workflow")
     if args[len(args) - 1] == "regex":
         options = filter_regex
+
+    if len(options) > 0 and isinstance(options[0], CompletionItem):
+        return options
+
     return _finalize_completion(
         candidates=options,
         incomplete=incomplete
     )
 
 
-def tag_labels(ctx, args, incomplete, item_type):
+def tag_labels(ctx, param, incomplete, item_type):
     """Prepare a list of tag labels for a item_type."""
     datasets = list_items(item_type=item_type)
     options = []
@@ -1218,7 +1396,7 @@ def tag_labels(ctx, args, incomplete, item_type):
     )
 
 
-def status_keys(ctx, args, incomplete):
+def status_keys(ctx, param, incomplete):
     """Prepare a list of status keys for the admin status command."""
     options = ["all"]
     os.environ["CMEMPY_IS_CHATTY"] = "false"
@@ -1231,9 +1409,9 @@ def status_keys(ctx, args, incomplete):
     )
 
 
-def user_ids(ctx, args, incomplete):
+def user_ids(ctx, param, incomplete):
     """Prepare a list of username for admin update/delete/password command."""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     options = []
     for _ in list_users():
         options.append(
@@ -1248,17 +1426,17 @@ def user_ids(ctx, args, incomplete):
     )
 
 
-def user_group_ids(ctx: Context, args, incomplete):
+def user_group_ids(ctx: Context, param, incomplete):
     """Prepare a list of group name for admin user update
      --unassign-group/--assign-group parameter"""
-    CONTEXT.set_connection_from_args(args)
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
     if not ctx.args:
         return []
     users = get_user_by_username(username=str(ctx.args[0]))
     if not users:
         return []
 
-    if args[len(args) - 1] == "--unassign-group":
+    if param.name == "unassign_group":
         groups = user_groups(user_id=users[0]["id"])
     else:
         user_group_names = (
@@ -1274,10 +1452,71 @@ def user_group_ids(ctx: Context, args, incomplete):
                 _["name"]
             )
         )
-    for num, arg in enumerate(args):
-        if num - 1 > 0 and args[num - 1] == args[len(args) - 1]:
-            with suppress(ValueError):
-                options.remove(arg)
+
+    for arg in ctx.params["assign_group"]:
+        with suppress(ValueError):
+            options.remove(arg)
+    for arg in ctx.params["unassign_group"]:
+        with suppress(ValueError):
+            options.remove(arg)
+
+    return _finalize_completion(
+        candidates=options,
+        incomplete=incomplete,
+        sort_by=SORT_BY_DESC
+    )
+
+
+def client_ids(ctx, param, incomplete):
+    """Prepare a list of client ids for admin secret and update command."""
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    options = []
+    for _ in list_open_id_clients():
+        options.append(
+            (
+                _["clientId"]
+            )
+        )
+    return _finalize_completion(
+        candidates=options,
+        incomplete=incomplete,
+        sort_by=SORT_BY_DESC
+    )
+
+
+def transformation_task_ids(ctx, param, incomplete):
+    """Prepare a list of projectId:transformation task identifier."""
+    CONTEXT.set_connection_from_params(ctx.find_root().params)
+    options = []
+    results = list_items(item_type="transform")
+    datasets = results["results"]
+    for _ in datasets:
+        options.append(
+            (
+                _["projectId"] + ":" + _["id"],
+                _["label"]
+            )
+        )
+    return _finalize_completion(
+        candidates=options,
+        incomplete=incomplete,
+        sort_by=SORT_BY_DESC
+    )
+
+
+def linking_task_ids(ctx, args, incomplete):
+    """Prepare a list of projectId:linking task identifier."""
+    CONTEXT.set_connection_from_args(args)
+    options = []
+    results = list_items(item_type="linking")
+    datasets = results["results"]
+    for _ in datasets:
+        options.append(
+            (
+                _["projectId"] + r"\:" + _["id"],
+                _["label"]
+            )
+        )
     return _finalize_completion(
         candidates=options,
         incomplete=incomplete,