cmem-cmemc 23.1.3__py3-none-any.whl → 23.3.0__py3-none-any.whl

cmem/cmemc/cli/__init__.py

@@ -1,5 +1,5 @@
 """The main command line interface."""
-
+from importlib.resources import open_text
 import os
 from subprocess import CalledProcessError  # nosec
 import sys
@@ -38,6 +38,12 @@ from cmem.cmemc.cli.commands import CmemcGroup
 
 CMEMC_VERSION = get_version()
 
+# this will output a custom zsh completion function
+if os.environ.get("_CMEMC_COMPLETE", "") == "zsh_source":
+    with open_text("cmem.cmemc.cli", "_cmemc.zsh") as zsh_output:
+        print(zsh_output.read())
+    sys.exit(0)
+
 version = sys.version_info
 PYTHON_VERSION = f"{version.major}.{version.minor}.{version.micro}"
 PYTHON_EXPECTED = "3.11"
@@ -65,12 +71,12 @@ CONTEXT_SETTINGS = {
 @click.option(
     '-c', '--connection',
     type=click.STRING,
-    autocompletion=completion.connections,
+    shell_complete=completion.connections,
     help='Use a specific connection from the config file.'
 )
 @click.option(
     '--config-file',
-    autocompletion=completion.ini_files,
+    shell_complete=completion.ini_files,
     type=click.Path(
         readable=True,
         allow_dash=False,

cmem/cmemc/cli/_cmemc.zsh

@@ -0,0 +1,44 @@
+#compdef cmemc
+
+_cmemc_completion() {
+    local -a completions
+    local -a completions_with_descriptions
+    local -a response
+    (( ! $+commands[cmemc] )) && return 1
+
+    response=("${(@f)$(env COMP_WORDS="${words[*]}" COMP_CWORD=$((CURRENT-1)) _CMEMC_COMPLETE=zsh_complete cmemc)}")
+
+    for type key descr in ${response}; do
+        if [[ "$type" == "plain" ]]; then
+            if [[ "$descr" == "_" ]]; then
+                completions+=("$key")
+            else
+                completions_with_descriptions+=("$key":"$descr")
+            fi
+        elif [[ "$type" == "dir" ]]; then
+            _path_files -/
+        elif [[ "$type" == "file" ]]; then
+            _path_files -f
+        fi
+    done
+
+    if [ -n "$completions_with_descriptions" ]; then
+        _describe -V unsorted completions_with_descriptions -U
+    fi
+
+    if [ -n "$completions" ]; then
+        compadd -U -V unsorted -a completions
+    fi
+    # Other than the standard click completion function, we re-added this line
+    # in order to have working completions for task IDs
+    compstate[insert]="automenu"
+}
+
+if [[ $zsh_eval_context[-1] == loadautofunc ]]; then
+    # autoload from fpath, call function directly
+    _cmemc_completion "$@"
+else
+    # eval/source/. command, register function for later
+    compdef _cmemc_completion cmemc
+fi
+

cmem/cmemc/cli/commands/__init__.py

@@ -29,6 +29,7 @@ class CmemcGroup(HelpColorsGroup, DYMGroup):
                 "showcase": COLOR_FOR_WRITING_COMMANDS,
                 "delete": COLOR_FOR_WRITING_COMMANDS,
                 "password": COLOR_FOR_WRITING_COMMANDS,
+                "secret": COLOR_FOR_WRITING_COMMANDS,
                 "upload": COLOR_FOR_WRITING_COMMANDS,
                 "import": COLOR_FOR_WRITING_COMMANDS,
                 "create": COLOR_FOR_WRITING_COMMANDS,
@@ -59,6 +60,8 @@ class CmemcGroup(HelpColorsGroup, DYMGroup):
                 "python": COLOR_FOR_COMMAND_GROUPS,
                 "cache": COLOR_FOR_COMMAND_GROUPS,
                 "resource": COLOR_FOR_COMMAND_GROUPS,
+                "client": COLOR_FOR_COMMAND_GROUPS,
+                "variable": COLOR_FOR_COMMAND_GROUPS,
             }
         )
         super().__init__(*args, **kwargs)

cmem/cmemc/cli/commands/admin.py

@@ -4,6 +4,7 @@ import click
 import jwt
 
 from cmem.cmemc.cli import completion
+from cmem.cmemc.cli.commands.client import client
 from cmem.cmemc.cli.commands.user import user
 from cmem.cmemc.cli.context import ApplicationContext
 from cmem.cmemc.cli.utils import struct_to_table
@@ -20,7 +21,7 @@ from cmem.cmempy.health import get_complete_status_info
 @click.command(cls=CmemcCommand, name="status")
 @click.option(
     "--key", "key",
-    autocompletion=completion.status_keys,
+    shell_complete=completion.status_keys,
     help="Get only specific key(s) from the status / info output. There are "
          "two special keys available: 'all' will list all available keys in "
          "the table, 'overall.healthy' with result in  UP in case all "
@@ -192,3 +193,4 @@ admin.add_command(metrics)
 admin.add_command(workspace)
 admin.add_command(store)
 admin.add_command(user)
+admin.add_command(client)

cmem/cmemc/cli/commands/client.py

@@ -0,0 +1,173 @@
+"""Keycloak client management commands"""
+
+import click
+
+from cmem.cmemc.cli import completion
+from cmem.cmemc.cli.commands import CmemcCommand, CmemcGroup
+from cmem.cmemc.cli.context import ApplicationContext
+from cmem.cmempy.config import get_keycloak_base_uri, get_keycloak_realm_id
+from cmem.cmempy.keycloak.client import (
+    get_client_by_client_id,
+    generate_client_secret,
+    get_client_secret, list_open_id_clients
+)
+
+NO_CLIENT_ERROR = "{} is not a valid client account. " \
+                "Use the 'admin client list' command " \
+                "to get a list of existing client accounts."
+
+
+@click.command(cls=CmemcCommand, name="list")
+@click.option(
+    "--raw",
+    is_flag=True,
+    help="Outputs raw JSON."
+)
+@click.option(
+    "--id-only",
+    is_flag=True,
+    help="Lists only Client ID. "
+         "This is useful for piping the IDs into other commands."
+)
+@click.pass_obj
+def list_command(app: ApplicationContext, raw, id_only):
+    """
+    List client accounts.
+
+    Outputs a list of client accounts, which can be used to get an overview as well
+    as a reference for the other commands of the `admin client` command group.
+
+    Note: The list command only outputs clients which have a client secret.
+    Use the `--raw` option to get a JSON description of all clients.
+    """
+    clients = list_open_id_clients()
+    if raw:
+        app.echo_info_json(clients)
+        return
+    if id_only:
+        for cnt in clients:
+            app.echo_info(cnt["clientId"])
+        return
+    table = []
+    for cnt in clients:
+        table.append((
+            cnt["clientId"],
+            cnt.get("description", "-")
+        ))
+    app.echo_info_table(
+        table,
+        headers=["Client ID", "Description"],
+        sort_column=0
+    )
+
+
+@click.command(cls=CmemcCommand, name="secret")
+@click.argument(
+    "client-id",
+    shell_complete=completion.client_ids
+)
+@click.option(
+    "--generate",
+    is_flag=True,
+    help="Generate a new secret"
+)
+@click.option(
+    "--output",
+    is_flag=True,
+    help="Display client secret"
+)
+@click.pass_obj
+def secret_command(app: ApplicationContext, client_id, generate, output):
+    """
+    Get or generate a new secret for a client account.
+
+    This command retrieves or generates a new secret for a client account from a realm.
+    """
+    if not output and not generate:
+        app.echo_info(click.get_current_context().get_help())
+        raise ValueError(
+            "You need to use '--output' or '--generate' as an option."
+        )
+
+    clients = get_client_by_client_id(client_id)
+    if not clients:
+        raise ValueError(NO_CLIENT_ERROR.format(client_id))
+
+    if generate:
+        if not output:
+            app.echo_info(
+                f"Generating a new secret for {client_id} ... ",
+                nl=False
+            )
+        credential = generate_client_secret(client_id=clients[0]["id"])
+        if not output:
+            app.echo_success("done")
+    else:
+        credential = get_client_secret(client_id=clients[0]["id"])
+
+    if output:
+        app.echo_result(credential["value"])
+
+
+@click.command(cls=CmemcCommand, name="open")
+@click.argument(
+    "client-ids",
+    nargs=-1,
+    required=False,
+    type=click.STRING,
+    shell_complete=completion.client_ids
+)
+@click.pass_obj
+def open_command(app: ApplicationContext, client_ids):
+    """Open clients in the browser.
+
+    With this command, you can open a client in the keycloak web
+    interface in your browser.
+
+    The command accepts multiple client IDs which results in
+    opening multiple browser tabs.
+    """
+    open_client_base_uri = (
+        f"{get_keycloak_base_uri()}/admin/master/console/#/"
+        f"{get_keycloak_realm_id()}/clients"
+    )
+    if not client_ids:
+        app.echo_debug(f"Open users list: {open_client_base_uri}")
+        click.launch(open_client_base_uri)
+    else:
+        clients = list_open_id_clients()
+        client_id_map = {c["clientId"]: c["id"] for c in clients}
+        for _ in client_ids:
+            if _ not in client_id_map.keys():
+                raise ValueError(NO_CLIENT_ERROR.format(_))
+            client_id = client_id_map[_]
+            open_user_uri = f"{open_client_base_uri}/{client_id}/settings"
+
+            app.echo_debug(f"Open {_}: {open_user_uri}")
+            click.launch(open_user_uri)
+
+
+@click.group(cls=CmemcGroup)
+def client():
+    """
+    List client accounts, get or generate client account secrets.
+
+    This command group is an opinionated interface to the Keycloak realm of your
+    Corporate Memory instance. In order to be able to use the commands in this group,
+    the configured cmemc connection account needs to be equipped with the
+    `manage-clients` role in the used realm.
+
+    Client accounts are identified by a client ID which is unique in the scope of
+    the used realm.
+
+    In case your Corporate Memory deployment does not use the default deployment
+    layout, the following additional config variables can be used in your
+    connection configuration: `KEYCLOAK_BASE_URI` defaults to
+    `{CMEM_BASE_URI}/auth` and locates your Keycloak deployment;
+    `KEYCLOAK_REALM_ID` defaults to `cmem` and identifies the used realm.
+    """
+
+
+client.add_command(list_command)
+client.add_command(secret_command)
+client.add_command(open_command)

cmem/cmemc/cli/commands/config.py

@@ -41,7 +41,7 @@ def edit_command(app):
     "KEY",
     nargs=1,
     type=click.Choice(
-        KNOWN_CONFIG_KEYS.keys(),
+        list(KNOWN_CONFIG_KEYS.keys()),
         case_sensitive=False
     )
 )

cmem/cmemc/cli/commands/dataset.py

@@ -6,21 +6,21 @@ import re
 import click
 
 import requests.exceptions
+from click import UsageError
 
 from cmem.cmemc.cli import completion
 from cmem.cmemc.cli.commands import CmemcCommand, CmemcGroup
 from cmem.cmemc.cli.commands.resource import resource
 from cmem.cmemc.cli.context import ApplicationContext
-from cmem.cmemc.cli.utils import struct_to_table
+from cmem.cmemc.cli.utils import struct_to_table, check_or_select_project
 from cmem.cmempy.config import get_cmem_base_uri
 from cmem.cmempy.workspace.search import list_items
-from cmem.cmempy.workspace.projects.project import (
-    get_projects
-)
 from cmem.cmempy.workspace.projects.datasets.dataset import (
     create_dataset,
     delete_dataset,
-    get_dataset
+    get_dataset,
+    post_resource,
+    update_dataset
 )
 from cmem.cmempy.workspace.projects.resources.resource import (
     create_resource,
@@ -118,6 +118,37 @@ def _validate_and_split_dataset_id(dataset_id):
     return project_part, dataset_part
 
 
+def _post_file_resource(
+    app=None,
+    project_id=None,
+    dataset_id=None,
+    local_file_name=None
+
+):
+    """Upload a local file as a dataset resource to a project.
+
+    Args:
+        app (ApplicationContext): the click cli app context.
+        project_id (str): The project ID in the workspace.
+        dataset_id (str): The dataset ID in the workspace.
+        local_file_name (str): The path to the local file name
+
+    Raises:
+        ValueError: if resource exists and no replace
+    """
+    app.echo_info(
+        f"Upload {local_file_name} as a file resource of dataset "
+        f"{dataset_id} to project {project_id} ... ",
+        nl=False
+    )
+    post_resource(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        file_resource=click.open_file(local_file_name, "rb"),
+    )
+    app.echo_success("done")
+
+
 def _upload_file_resource(
         app=None,
         project_id=None,
@@ -185,6 +216,23 @@ def _get_metadata_out_of_parameter(parameter_dict):
     return metadata_dict
 
 
+def _get_read_only_out_of_parameter(parameter_dict):
+    """Extract readonly key value out of the parameter dict.
+
+    Args:
+        parameter_dict (dict): the dictionary of given parameters.
+
+    Returns:
+        The value of read only field.
+    """
+    read_only = parameter_dict.get("readOnly", False)
+    if read_only == "true":
+        return True
+    if read_only == "false":
+        return False
+    return read_only
+
+
 def _extend_parameter_with_metadata(
         app,
         parameter_dict=None,
@@ -263,7 +311,7 @@ def _check_or_set_dataset_type(
     return dataset_type
 
 
-def _show_parameter_list(app, dataset_type=None):
+def _show_parameter_list(app, dataset_type):
     """Output the parameter list for a given dataset type.
 
     Args:
@@ -286,6 +334,9 @@ def _show_parameter_list(app, dataset_type=None):
             description,
         ]
         table.append(row)
+
+    table = completion.add_read_only_and_uri_property_parameters(table)
+
     # metadata always on top, then sorted by key
     table = sorted(table, key=lambda k: k[0].lower())
     table = completion.add_metadata_parameter(table)
@@ -321,45 +372,6 @@ def _show_type_list(app):
     )
 
 
-def _check_or_select_project(app, project_id=None):
-    """Check for given project, select the first one if there is only one.
-
-    Args:
-        app (ApplicationContext): the click cli app context.
-        project_id (str): The project ID.
-
-    Raises:
-        ValueError: if no projects available.
-        ValueError: if more than one project is.
-
-    Returns:
-        Maybe project_id if there was no project_id before.
-    """
-    if project_id is not None:
-        return project_id
-
-    projects = get_projects()
-    if len(projects) == 1:
-        project_name = projects[0]["name"]
-        app.echo_warning(
-            "Missing project (--project) - since there is only one project, "
-            f"this is selected: {project_name}"
-        )
-        return project_name
-
-    if len(projects) == 0:
-        raise ValueError(
-            "There are no projects available. "
-            "Please create a project with 'cmemc project create'."
-        )
-
-    # more than one project
-    raise ValueError(
-        "There is more than one project available so you need to "
-        "specify the project with '--project'."
-    )
-
-
 def _check_or_select_dataset_type(app, dataset_type):
     """Test type and return plugin.
 
@@ -388,7 +400,7 @@ def _check_or_select_dataset_type(app, dataset_type):
     "--filter", "filter_",
     type=(str, str),
     multiple=True,
-    autocompletion=completion.dataset_list_filter,
+    shell_complete=completion.dataset_list_filter,
     help=DATASET_LIST_FILTER_HELP_TEXT,
 )
 @click.option(
@@ -445,7 +457,7 @@ def list_command(app: ApplicationContext, filter_, raw, id_only):
 @click.option(
     "--project", "project_id",
     type=click.STRING,
-    autocompletion=completion.project_ids,
+    shell_complete=completion.project_ids,
     help="In combination with the '--all' flag, this option allows for "
          "deletion of all datasets of a certain project. The behaviour is "
          "similar to the 'dataset list --project' command."
@@ -454,14 +466,14 @@ def list_command(app: ApplicationContext, filter_, raw, id_only):
     "--filter", "filter_",
     type=(str, str),
     multiple=True,
-    autocompletion=completion.dataset_list_filter,
+    shell_complete=completion.dataset_list_filter,
     help=DATASET_DELETE_FILTER_HELP_TEXT,
 )
 @click.argument(
     "dataset_ids",
     nargs=-1,
     type=click.STRING,
-    autocompletion=completion.dataset_ids
+    shell_complete=completion.dataset_ids
 )
 @click.pass_obj
 def delete_command(app, project_id, all_, filter_, dataset_ids):
@@ -525,7 +537,7 @@ def delete_command(app, project_id, all_, filter_, dataset_ids):
 @click.argument(
     "dataset_id",
     type=click.STRING,
-    autocompletion=completion.dataset_ids
+    shell_complete=completion.dataset_ids
 )
 @click.argument(
     "output_path",
@@ -594,12 +606,12 @@ def download_command(app, dataset_id, output_path, replace):
 @click.argument(
     "dataset_id",
     type=click.STRING,
-    autocompletion=completion.dataset_ids
+    shell_complete=completion.dataset_ids
 )
 @click.argument(
     "input_path",
     required=True,
-    autocompletion=completion.dataset_files,
+    shell_complete=completion.dataset_files,
     type=click.Path(
         allow_dash=True,
         dir_okay=False,
@@ -625,20 +637,12 @@ def upload_command(app, dataset_id, input_path):
     Example: cmemc dataset upload cmem:my-dataset new-file.csv
     """
     project_part, dataset_part = _validate_and_split_dataset_id(dataset_id)
-    project = get_dataset(project_part, dataset_part)
-    try:
-        remote_file_name = project["data"]["parameters"]["file"]
-    except KeyError as error:
-        raise ValueError(
-            f"Dataset {dataset_id} has no attached file resource to replace."
-        ) from error
 
-    _upload_file_resource(
+    _post_file_resource(
         app=app,
         project_id=project_part,
+        dataset_id=dataset_part,
         local_file_name=input_path,
-        remote_file_name=remote_file_name,
-        replace=True
     )
 
 
@@ -646,7 +650,7 @@ def upload_command(app, dataset_id, input_path):
 @click.argument(
     "dataset_id",
     type=click.STRING,
-    autocompletion=completion.dataset_ids
+    shell_complete=completion.dataset_ids
 )
 @click.option(
     "--raw",
@@ -678,7 +682,7 @@ def inspect_command(app, dataset_id, raw):
 @click.argument(
     "DATASET_FILE",
     required=False,
-    autocompletion=completion.dataset_files,
+    shell_complete=completion.dataset_files,
     type=click.Path(
         allow_dash=False,
         readable=True,
@@ -690,21 +694,21 @@ def inspect_command(app, dataset_id, raw):
     "--type", "-t", "dataset_type",
     multiple=False,
     type=click.STRING,
-    autocompletion=completion.dataset_types,
+    shell_complete=completion.dataset_types,
     help="The dataset type of the dataset to create. Example types are 'csv',"
          "'json' and 'eccencaDataPlatform' (-> Knowledge Graph)."
 )
 @click.option(
     "--project", "project_id",
     type=click.STRING,
-    autocompletion=completion.project_ids,
+    shell_complete=completion.project_ids,
     help="The project, where you want to create the dataset in. If there is "
          "only one project in the workspace, this option can be omitted."
 )
 @click.option(
     "--parameter", "-p",
     type=(str, str),
-    autocompletion=completion.dataset_parameter,
+    shell_complete=completion.dataset_parameter,
     multiple=True,
     help="A set of key/value pairs. Each dataset type has different "
          "parameters (such as charset, arraySeparator, ignoreBadLines, ...). "
@@ -782,7 +786,7 @@ def create_command(
         dataset_file=dataset_file
     )
 
-    project_id = _check_or_select_project(app, project_id)
+    project_id = check_or_select_project(app, project_id)
 
     # file required but not given
     if "file" in plugin["required"] \
@@ -816,20 +820,106 @@ def create_command(
     created_dataset = create_dataset(
         dataset_id=dataset_id, project_id=project_id,
         dataset_type=dataset_type, parameter=parameter_dict,
-        metadata=_get_metadata_out_of_parameter(parameter_dict)
+        metadata=_get_metadata_out_of_parameter(parameter_dict),
+        read_only=_get_read_only_out_of_parameter(parameter_dict),
+        uri_property=parameter_dict.get("uriProperty", "")
     )
     returned_id = json.loads(created_dataset)["id"]
     app.echo_info(f"{returned_id} ... ", nl=False)
     app.echo_success("done")
 
 
+@click.command(cls=CmemcCommand, name="update")
+@click.argument(
+    "dataset_id",
+    type=click.STRING,
+    required=True,
+    shell_complete=completion.dataset_ids,
+)
+@click.option(
+    "--parameter", "-p",
+    type=(str, str),
+    shell_complete=completion.dataset_parameter,
+    multiple=True,
+    help="A configuration parameter key/value pair. Each dataset type has different "
+         "parameters (such as charset, arraySeparator, ignoreBadLines, ...). "
+         "In order to get a list of possible parameter, use the"
+         "'--help-parameter' option."
+)
+@click.option(
+    "--help-parameter",
+    is_flag=True,
+    help="Lists all possible (optional and mandatory) configuration parameter for"
+         " a given dataset. Note that this option already needs access to the instance."
+)
+@click.pass_obj
+def update_command(
+        app, dataset_id, parameter,
+        help_parameter
+):
+    """Update a dataset.
+
+    With this command, you can update the configuration of an existing dataset.
+    Similar to the `dataset create` command, you need to use configuration key/value
+    pairs on the `--parameter` option.
+
+    To get more information about the available configuration parameters on a dataset,
+    use the `--help-parameter` option.
+
+    Example: cmemc dataset update my-project:my-csv -p separator ";"
+    """
+    project_part, dataset_part = _validate_and_split_dataset_id(dataset_id)
+    try:
+        project = get_dataset(project_part, dataset_part)
+    except requests.exceptions.HTTPError as http_exception:
+        if http_exception.response.status_code == 404:
+            raise UsageError(
+                f"Dataset {dataset_part} does not exist in project {project_part}."
+            ) from http_exception
+    dataset_type = project["data"]["type"]
+
+    if help_parameter:
+        _show_parameter_list(app, dataset_type=dataset_type)
+        return
+
+    if not parameter:
+        raise UsageError("You need to use the `--parameter/-p` option at least once,"
+                         " in order to execute this command.")
+
+    desc = get_task_plugin_description(dataset_type)
+    possible_keys = ["label", "description", "readOnly"]
+    possible_keys.extend(desc["properties"])
+    possible_keys.extend(desc["required"])
+
+    # transform the parameter list of tuple to a dictionary
+    parameter_dict = {}
+    for key, value in parameter:
+        if key not in possible_keys:
+            raise UsageError(
+                f"Configuration key '{key}' is not valid for"
+                f" the dataset type '{dataset_type}'."
+            )
+        parameter_dict[key] = value
+
+    app.echo_info(f"Updating dataset {dataset_id} ... ", nl=False)
+    update_dataset(
+        dataset_id=dataset_part,
+        project_id=project_part,
+        parameters=parameter_dict,
+        metadata=_get_metadata_out_of_parameter(parameter_dict),
+        read_only=_get_read_only_out_of_parameter(parameter_dict),
+        uri_property=parameter_dict.get("uriProperty", "")
+    )
+    app.echo_success("done")
+
+
 @click.command(cls=CmemcCommand, name="open")
 @click.argument(
     "dataset_ids",
     nargs=-1,
     required=True,
     type=click.STRING,
-    autocompletion=completion.dataset_ids
+    shell_complete=completion.dataset_ids
 )
 @click.pass_obj
 def open_command(app, dataset_ids):
@@ -879,4 +969,5 @@ dataset.add_command(upload_command)
 dataset.add_command(inspect_command)
 dataset.add_command(create_command)
 dataset.add_command(open_command)
+dataset.add_command(update_command)
 dataset.add_command(resource)