cmem-cmemc 25.5.0__py3-none-any.whl → 25.6.0__py3-none-any.whl

cmem_cmemc/command_group.py

@@ -38,6 +38,7 @@ class CmemcGroup(HelpColorsGroup, DYMGroup):
                 "enable": self.color_for_writing_commands,
                 "eval": self.color_for_writing_commands,
                 "execute": self.color_for_writing_commands,
+                "file": self.color_for_command_groups,
                 "graph": self.color_for_command_groups,
                 "import": self.color_for_writing_commands,
                 "imports": self.color_for_command_groups,

cmem_cmemc/commands/dataset.py

@@ -16,16 +16,14 @@ from cmem.cmempy.workspace.projects.datasets.dataset import (
     update_dataset,
 )
 from cmem.cmempy.workspace.projects.resources.resource import (
-    create_resource,
     get_resource_response,
-    resource_exist,
 )
 from cmem.cmempy.workspace.search import list_items
 
 from cmem_cmemc import completion
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
-from cmem_cmemc.commands.resource import resource
+from cmem_cmemc.commands.file import _upload_file_resource, resource
 from cmem_cmemc.completion import get_dataset_file_mapping
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.exceptions import CmemcError
@@ -147,57 +145,6 @@ def _post_file_resource(
     app.echo_success("done")
 
 
-def _upload_file_resource(
-    app: ApplicationContext,
-    project_id: str,
-    local_file_name: str,
-    remote_file_name: str,
-    replace: bool,
-) -> None:
-    """Upload a local file as a dataset resource to a project.
-
-    Args:
-    ----
-        app: the click cli app context.
-        project_id: The project ID in the workspace.
-        local_file_name: The path to the local file name
-        remote_file_name: The remote file name
-        replace: Replace resource if needed.
-
-    Raises:
-    ------
-        ValueError: if resource exists and no replace
-
-    """
-    exist = resource_exist(project_name=project_id, resource_name=remote_file_name)
-    if exist and not replace:
-        raise ClickException(
-            f"A file resource with the name '{remote_file_name}' already "
-            "exists in this project. \n"
-            "Please rename the file or use the '--replace' "
-            "parameter in order to overwrite the remote file."
-        )
-    if exist:
-        app.echo_info(
-            f"Replace content of {remote_file_name} with content from "
-            f"{local_file_name} in project {project_id} ... ",
-            nl=False,
-        )
-    else:
-        app.echo_info(
-            f"Upload {local_file_name} as a file resource "
-            f"{remote_file_name} to project {project_id} ... ",
-            nl=False,
-        )
-    create_resource(
-        project_name=project_id,
-        resource_name=remote_file_name,
-        file_resource=ClickSmartPath.open(local_file_name),
-        replace=replace,
-    )
-    app.echo_success("done")
-
-
 def _get_metadata_out_of_parameter(parameter_dict: dict) -> dict:
     """Extract metadata keys out of the parameter dict.
 

cmem_cmemc/commands/file.py

@@ -0,0 +1,465 @@
+"""Build project file commands for cmemc."""
+
+import re
+
+import click
+from click import ClickException, Context, UsageError
+from cmem.cmempy.config import get_cmem_base_uri
+from cmem.cmempy.workspace.projects.resources import get_all_resources
+from cmem.cmempy.workspace.projects.resources.resource import (
+    create_resource,
+    delete_resource,
+    get_resource_metadata,
+    get_resource_response,
+    get_resource_usage_data,
+    resource_exist,
+)
+
+from cmem_cmemc import completion
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
+from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.exceptions import CmemcError
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
+from cmem_cmemc.string_processor import FileSize, TimeAgo
+from cmem_cmemc.utils import check_or_select_project, split_task_id, struct_to_table
+
+RESOURCE_FILTER_TYPES = ["project", "regex"]
+RESOURCE_FILTER_TYPES_HIDDEN = ["ids"]
+RESOURCE_FILTER_TEXT = (
+    "Filter file resources based on metadata. "
+    f"First parameter CHOICE can be one of {RESOURCE_FILTER_TYPES!s}"
+    ". The second parameter is based on CHOICE, e.g. a project "
+    "ID or a regular expression string."
+)
+
+
+def _upload_file_resource(
+    app: ApplicationContext,
+    project_id: str,
+    local_file_name: str,
+    remote_file_name: str,
+    replace: bool,
+) -> None:
+    """Upload a local file as a dataset resource to a project.
+
+    Args:
+    ----
+        app: the click cli app context.
+        project_id: The project ID in the workspace.
+        local_file_name: The path to the local file name
+        remote_file_name: The remote file name
+        replace: Replace resource if needed.
+
+    Raises:
+    ------
+        ValueError: if resource exists and no replace
+
+    """
+    exist = resource_exist(project_name=project_id, resource_name=remote_file_name)
+    if exist and not replace:
+        raise CmemcError(
+            app,
+            f"A file resource with the name '{remote_file_name}' already "
+            "exists in this project. \n"
+            "Please rename the file or use the '--replace' "
+            "parameter in order to overwrite the remote file.",
+        )
+    if exist:
+        app.echo_info(
+            f"Replace content of {remote_file_name} with content from "
+            f"{local_file_name} in project {project_id} ... ",
+            nl=False,
+        )
+    else:
+        app.echo_info(
+            f"Upload {local_file_name} as a file resource "
+            f"{remote_file_name} to project {project_id} ... ",
+            nl=False,
+        )
+    create_resource(
+        project_name=project_id,
+        resource_name=remote_file_name,
+        file_resource=ClickSmartPath.open(local_file_name),
+        replace=replace,
+    )
+    app.echo_success("done")
+
+
+def _get_resources_filtered(
+    resources: list[dict], filter_name: str, filter_value: str | tuple[str, ...]
+) -> list[dict]:
+    """Get file resources but filtered according to name and value."""
+    # check for correct filter names (filter ids is used internally only)
+    if filter_name not in RESOURCE_FILTER_TYPES + RESOURCE_FILTER_TYPES_HIDDEN:
+        raise UsageError(
+            f"{filter_name} is an unknown filter name. " f"Use one of {RESOURCE_FILTER_TYPES}."
+        )
+    # filter by ID list
+    if filter_name == "ids":
+        return [_ for _ in resources if _["id"] in filter_value]
+    # filter by project
+    if filter_name == "project":
+        return [_ for _ in resources if _["project"] == str(filter_value)]
+    # filter by regex
+    if filter_name == "regex":
+        return [_ for _ in resources if re.search(str(filter_value), _["name"])]
+    # return unfiltered list
+    return resources
+
+
+@click.command(cls=CmemcCommand, name="list")
+@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
+@click.option(
+    "--id-only",
+    is_flag=True,
+    help="Lists only resource IDs and no other metadata. "
+    "This is useful for piping the IDs into other commands.",
+)
+@click.option(
+    "--filter",
+    "filters_",
+    multiple=True,
+    type=(str, str),
+    shell_complete=completion.resource_list_filter,
+    help=RESOURCE_FILTER_TEXT,
+)
+@click.pass_obj
+def list_command(
+    app: ApplicationContext, raw: bool, id_only: bool, filters_: tuple[tuple[str, str], ...]
+) -> None:
+    """List available file resources.
+
+    Outputs a table or a list of file resources.
+    """
+    resources = get_all_resources()
+    for _ in filters_:
+        filter_name, filter_value = _
+        resources = _get_resources_filtered(resources, filter_name, filter_value)
+    if raw:
+        app.echo_info_json(resources)
+        return
+    if id_only:
+        for _ in sorted(_["id"] for _ in resources):
+            app.echo_result(_)
+        return
+    # output a user table
+    table = []
+    headers = ["ID", "Modified", "Size"]
+    for _ in resources:
+        row = [
+            _["id"],
+            _["modified"],
+            _["size"],
+        ]
+        table.append(row)
+
+    caption = f"{len(table)} files of {get_cmem_base_uri()}"
+    empty_note = "No resources found."
+    if len(filters_) > 0:
+        caption += " (filtered)"
+        empty_note = "No resources found for these filters."
+
+    app.echo_info_table(
+        table,
+        headers=headers,
+        sort_column=0,
+        cell_processing={1: TimeAgo(), 2: FileSize()},
+        caption=caption,
+        empty_table_message=f"{empty_note} "
+        "Use the `dataset create` command to create a new file-based dataset, or "
+        "the `project file upload` command to create only a file resource.",
+    )
+
+
+@click.command(cls=CmemcCommand, name="delete")
+@click.argument("resource_ids", nargs=-1, type=click.STRING, shell_complete=completion.resource_ids)
+@click.option("--force", is_flag=True, help="Delete resource even if in use by a task.")
+@click.option(
+    "-a",
+    "--all",
+    "all_",
+    is_flag=True,
+    help="Delete all resources. " "This is a dangerous option, so use it with care.",
+)
+@click.option(
+    "--filter",
+    "filters_",
+    multiple=True,
+    type=(str, str),
+    shell_complete=completion.resource_list_filter,
+    help=RESOURCE_FILTER_TEXT,
+)
+@click.pass_obj
+def delete_command(
+    app: ApplicationContext,
+    resource_ids: tuple[str, ...],
+    force: bool,
+    all_: bool,
+    filters_: tuple[tuple[str, str], ...],
+) -> None:
+    """Delete file resources.
+
+    There are three selection mechanisms: with specific IDs - only those
+    specified resources will be deleted; by using --filter - resources based
+    on the filter type and value will be deleted; by using --all, which will
+    delete all resources.
+    """
+    if resource_ids == () and not all_ and filters_ == ():
+        raise UsageError(
+            "Either specify at least one resource ID or use the --all or "
+            "--filter options to specify resources for deletion."
+        )
+
+    resources = get_all_resources()
+    if len(resource_ids) > 0:
+        for resource_id in resource_ids:
+            if resource_id not in [_["id"] for _ in resources]:
+                raise ClickException(f"Resource {resource_id} not available.")
+        # "filter" by id
+        resources = _get_resources_filtered(resources, "ids", resource_ids)
+    for _ in filters_:
+        resources = _get_resources_filtered(resources, _[0], _[1])
+
+    # avoid double removal as well as sort IDs
+    processed_ids = sorted({_["id"] for _ in resources}, key=lambda v: v.lower())
+    count = len(processed_ids)
+    for current, resource_id in enumerate(processed_ids, start=1):
+        current_string = str(current).zfill(len(str(count)))
+        app.echo_info(f"Delete resource {current_string}/{count}: {resource_id} ... ", nl=False)
+        project_id, resource_local_id = split_task_id(resource_id)
+        usage = get_resource_usage_data(project_id, resource_local_id)
+        if len(usage) > 0:
+            app.echo_error(f"in use by {len(usage)} task(s)", nl=False)
+            if force:
+                app.echo_info(" ... ", nl=False)
+            else:
+                app.echo_info("")
+                continue
+        delete_resource(project_name=project_id, resource_name=resource_local_id)
+        app.echo_success("deleted")
+
+
+@click.command(cls=CmemcCommand, name="download")
+@click.argument("resource_ids", nargs=-1, type=click.STRING, shell_complete=completion.resource_ids)
+@click.option(
+    "--output-dir",
+    default=".",
+    show_default=True,
+    type=ClickSmartPath(writable=True, file_okay=False),
+    help="The directory where the downloaded files will be saved. "
+    "If this directory does not exist, it will be created.",
+)
+@click.option(
+    "--replace",
+    is_flag=True,
+    help="Replace existing files. This is a dangerous option, " "so use it with care!",
+)
+@click.pass_obj
+def download_command(
+    app: ApplicationContext, resource_ids: tuple[str], output_dir: str, replace: bool
+) -> None:
+    """Download file resources to the local file system.
+
+    This command downloads one or more file resources from projects to your local
+    file system. Files are saved with their resource names in the output directory.
+
+    Resources are identified by their IDs in the format PROJECT_ID:RESOURCE_NAME.
+
+    Example: cmemc project file download my-proj:my-file.csv
+
+    Example: cmemc project file download my-proj:file1.csv my-proj:file2.csv --output-dir /tmp
+    """
+    import os
+
+    if not resource_ids:
+        raise UsageError(
+            "At least one resource ID must be specified. "
+            "Use 'project file list' to see available resources."
+        )
+
+    count = len(resource_ids)
+    for current, resource_id in enumerate(resource_ids, start=1):
+        try:
+            project_id, resource_name = split_task_id(resource_id)
+        except ValueError:
+            app.echo_error(f"Invalid resource ID format: {resource_id}")
+            continue
+
+        # Build output path
+        output_path = os.path.normpath(str(Path(output_dir) / resource_name))
+
+        app.echo_info(
+            f"Download resource {current}/{count}: {resource_id} to {output_path} ... ",
+            nl=False,
+        )
+
+        if Path(output_path).exists() and replace is not True:
+            app.echo_error("target file exists")
+            continue
+
+        # Create parent directory if it doesn't exist
+        Path(output_path).parent.mkdir(exist_ok=True, parents=True)
+
+        try:
+            with (
+                get_resource_response(project_id, resource_name) as response,
+                click.open_file(output_path, "wb") as resource_file,
+            ):
+                for chunk in response.iter_content(chunk_size=8192):
+                    resource_file.write(chunk)
+            app.echo_success("done")
+        except (OSError, ClickException) as error:
+            app.echo_error(f"failed: {error!s}")
+            continue
+
+
+@click.command(cls=CmemcCommand, name="upload")
+@click.argument(
+    "input_path",
+    required=True,
+    type=ClickSmartPath(
+        allow_dash=False, dir_okay=False, readable=True, exists=True, remote_okay=True
+    ),
+)
+@click.option(
+    "--project",
+    "project_id",
+    type=click.STRING,
+    shell_complete=completion.project_ids,
+    help="The project where you want to upload the file. If there is "
+    "only one project in the workspace, this option can be omitted.",
+)
+@click.option(
+    "--path",
+    "remote_name",
+    type=click.STRING,
+    shell_complete=completion.resource_paths,
+    help="The path/name of the file resource in the project (e.g., 'data/file.csv'). "
+    "If not specified, the local file name will be used.",
+)
+@click.option(
+    "--replace",
+    is_flag=True,
+    help="Replace existing file resource. This is a dangerous option, " "so use it with care!",
+)
+@click.pass_obj
+def upload_command(
+    app: ApplicationContext, input_path: str, project_id: str, remote_name: str, replace: bool
+) -> None:
+    """Upload a file to a project.
+
+    This command uploads a file to a project as a file resource.
+
+    Note: If you want to create a dataset from your file, the `dataset create`
+    command is maybe the better option.
+
+    Example: cmemc project file upload my-file.csv --project my-project
+    """
+    project_id = check_or_select_project(app, project_id)
+    local_file_name = Path(input_path).name
+
+    if remote_name and remote_name.endswith("/"):
+        app.echo_warning(
+            f"Remote path ends with a slash, so the local file name is appended: {local_file_name}."
+        )
+        remote_name = remote_name + local_file_name
+
+    # Use local filename if remote name not specified
+    if not remote_name:
+        remote_name = local_file_name
+
+    _upload_file_resource(
+        app=app,
+        remote_file_name=remote_name,
+        project_id=project_id,
+        local_file_name=input_path,
+        replace=replace,
+    )
+
+
+@click.command(cls=CmemcCommand, name="inspect")
+@click.argument("resource_id", type=click.STRING, shell_complete=completion.resource_ids)
+@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
+@click.pass_obj
+def inspect_command(app: ApplicationContext, resource_id: str, raw: bool) -> None:
+    """Display all metadata of a file resource."""
+    project_id, resource_id = split_task_id(resource_id)
+    resource_data = get_resource_metadata(project_id, resource_id)
+    if raw:
+        app.echo_info_json(resource_data)
+    else:
+        table = struct_to_table(resource_data)
+        app.echo_info_table(table, headers=["Key", "Value"], sort_column=0)
+
+
+@click.command(cls=CmemcCommand, name="usage")
+@click.argument("resource_id", type=click.STRING, shell_complete=completion.resource_ids)
+@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
+@click.pass_obj
+def usage_command(app: ApplicationContext, resource_id: str, raw: bool) -> None:
+    """Display all usage data of a file resource."""
+    project_id, resource_id = split_task_id(resource_id)
+    usage = get_resource_usage_data(project_id, resource_id)
+    if raw:
+        app.echo_info_json(usage)
+        return
+    # output a user table
+    table = []
+    headers = ["Task ID", "Type", "Label"]
+    for _ in usage:
+        row = [project_id + ":" + _["id"], _["taskType"], _["label"]]
+        table.append(row)
+    app.echo_info_table(
+        table,
+        empty_table_message=f"The file resource '{resource_id}' is not used in "
+        f"any task in project '{project_id}'.",
+        headers=headers,
+        sort_column=2,
+    )
+
+
+@click.group(
+    cls=CmemcGroup,
+    hidden=True,
+)
+@click.pass_context
+def resource(ctx: Context) -> None:
+    """List, inspect or delete dataset file resources.
+
+    File resources are identified by their paths and project IDs.
+
+    Warning: This command group is deprecated and will be removed with the next major release.
+    Please use the `project file` command group instead.
+    """
+    app: ApplicationContext = ctx.obj
+    app.echo_warning(
+        "The 'dataset resource' command group is deprecated and will be removed with the next"
+        " major release. Please use the 'project file' command group instead.",
+    )
+
+
+@click.group(cls=CmemcGroup)
+def file() -> CmemcGroup:  # type: ignore[empty-body]
+    """List, inspect, up-/download or delete project file resources.
+
+    File resources are identified with a RESOURCE_ID which is a concatenation
+    of its project ID and its relative path, e.g. `my-project:path-to/table.csv`.
+
+    Note: To get a list of existing file resources, execute the `project file list` command
+    or use tab-completion.
+    """
+
+
+resource.add_command(list_command)
+resource.add_command(delete_command)
+resource.add_command(inspect_command)
+resource.add_command(usage_command)
+
+file.add_command(list_command)
+file.add_command(delete_command)
+file.add_command(download_command)
+file.add_command(upload_command)
+file.add_command(inspect_command)
+file.add_command(usage_command)

cmem_cmemc/commands/project.py

@@ -32,6 +32,7 @@ from jinja2 import Template
 from cmem_cmemc import completion
 from cmem_cmemc.command import CmemcCommand
 from cmem_cmemc.command_group import CmemcGroup
+from cmem_cmemc.commands.file import file
 from cmem_cmemc.commands.variable import variable
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.parameter_types.path import ClickSmartPath
@@ -584,3 +585,4 @@ project.add_command(delete_command)
 project.add_command(create_command)
 project.add_command(reload_command)
 project.add_command(variable)
+project.add_command(file)

cmem_cmemc/commands/python.py

@@ -267,7 +267,7 @@ def list_plugins_command(
         table,
         headers=["ID", "Package ID", "Type", "Label"],
         sort_column=0,
-        empty_table_message="No plugin plugins found. "
+        empty_table_message="No plugins found. "
         "Use the `admin workspace python install` command to install python packages with plugins.",
     )
     if "error" in raw_output:

cmem_cmemc/completion.py

@@ -31,7 +31,7 @@ from cmem.cmempy.workspace import (
 )
 from cmem.cmempy.workspace.projects.datasets.dataset import get_dataset
 from cmem.cmempy.workspace.projects.project import get_projects
-from cmem.cmempy.workspace.projects.resources import get_all_resources
+from cmem.cmempy.workspace.projects.resources import get_all_resources, get_resources
 from cmem.cmempy.workspace.projects.variables import get_all_variables
 from cmem.cmempy.workspace.python import list_packages
 from cmem.cmempy.workspace.search import list_items
@@ -1053,3 +1053,22 @@ def transformation_task_ids(ctx: Context, param: Argument, incomplete: str) -> l
     datasets = results["results"]
     options = [(f"{_['projectId']}:{_['id']}", _["label"]) for _ in datasets]
     return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
+
+
+def resource_paths(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
+    """Prepare a list of file resource paths within a project.
+
+    Returns the full path of file resources (not including the project ID prefix).
+    If a project_id is available in context, lists resources from that project.
+    If only one project exists, automatically uses that project.
+    """
+    ApplicationContext.set_connection_from_params(ctx.find_root().params)
+    project_id = ctx.params.get("project_id")
+    if not project_id:
+        projects = get_projects()
+        if len(projects) == 1:
+            project_id = projects[0]["name"]
+    if project_id is None:
+        return []
+    options = [_["fullPath"] for _ in list(get_resources(project_id))]
+    return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)

cmem_cmemc/manual_helper/graph.py

@@ -56,7 +56,7 @@ def print_group_manual_graph_recursive(
             ctx.obj.echo_info(f"{iri} a cli:CommandGroup .")
             ctx.obj.echo_info(f"{iri} rdfs:label '{prefix}{key} Command Group' .")
             ctx.obj.echo_info(f"{iri} cli:subGroupOf {sub_group_iri} .")
-            ctx.obj.echo_info(f"{iri} rdfs:comment '{comment}' .")
+            ctx.obj.echo_info(f'{iri} rdfs:comment """{comment}""" .')
             print_group_manual_graph_recursive(item, ctx=ctx, prefix=f"{prefix}{key}-")
         elif isinstance(item, click.Command):
             comment = item.get_short_help_str(limit=200)
@@ -64,7 +64,7 @@ def print_group_manual_graph_recursive(
             ctx.obj.echo_info(f"{iri} a cli:Command .")
             ctx.obj.echo_info(f"{iri} rdfs:label '{prefix}{key} Command' .")
             ctx.obj.echo_info(f"{iri} cli:group {group_iri} .")
-            ctx.obj.echo_info(f"{iri} rdfs:comment '{comment}' .")
+            ctx.obj.echo_info(f'{iri} rdfs:comment """{comment}""" .')
             for parameter in item.params:
                 print_parameter_manual_graph(parameter, ctx=ctx, prefix=f"{prefix}{key}-")
         else:

cmem_cmemc/manual_helper/multi_page.py

@@ -26,8 +26,8 @@ def get_icon_for_command_group(full_name: str) -> str:
         "admin workspace python": "material/language-python",
         "config": "material/cog-outline",
         "dataset": "eccenca/artefact-dataset",
-        "dataset resource": "eccenca/artefact-file",
         "project": "eccenca/artefact-project",
+        "project file": "eccenca/artefact-file",
         "project variable": "material/variable-box",
         "query": "eccenca/application-queries",
         "graph": "eccenca/artefact-dataset-eccencadataplatform",
@@ -55,8 +55,8 @@ def get_tags_for_command_group(full_name: str) -> str:
         "admin workspace python": ["Python", "cmemc"],
         "config": ["Configuration", "cmemc"],
         "dataset": ["cmemc"],
-        "dataset resource": ["cmemc"],
         "project": ["Project", "cmemc"],
+        "project file": ["Files", "cmemc"],
         "project variable": ["Variables", "cmemc"],
         "query": ["SPARQL", "cmemc"],
         "graph": ["KnowledgeGraph", "cmemc"],

cmem_cmemc/placeholder.py

@@ -36,7 +36,7 @@ class QueryPlaceholder:
         """Prepare a list of placeholder values"""
         result = self.value_query.get_json_results()
         projection_vars = result["head"]["vars"]
-        bindings = result["results"]["bindings"]
+        bindings: list[dict] = result["results"]["bindings"]
         if "value" not in projection_vars:
             return []
 
@@ -50,7 +50,7 @@ class QueryPlaceholder:
         values_with_description = []
         for _ in bindings:
             value = str(_["value"]["value"])
-            description = str(_["description"]["value"])
+            description = _.get("description", {}).get("value", "")
             values_with_description.append((value, description))
         return values_with_description
 

cmem_cmemc/string_processor.py

@@ -6,6 +6,7 @@ from urllib.parse import quote
 
 import timeago
 from cmem.cmempy.config import get_cmem_base_uri
+from humanize import naturalsize
 
 from cmem_cmemc.title_helper import TitleHelper
 from cmem_cmemc.utils import get_graphs_as_dict
@@ -19,6 +20,17 @@ class StringProcessor(ABC):
         """Process a single string content and output the processed string."""
 
 
+class FileSize(StringProcessor):
+    """Create a human-readable file size string."""
+
+    def process(self, text: str) -> str:
+        """Process a single string content and output the processed string."""
+        try:
+            return "" if text is None else naturalsize(value=text, gnu=True)
+        except ValueError:
+            return text
+
+
 class TimeAgo(StringProcessor):
     """Create a string similar to 'x minutes ago' from a timestamp or iso-formated string."""
 

{cmem_cmemc-25.5.0.dist-info → cmem_cmemc-25.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cmem-cmemc
-Version: 25.5.0
+Version: 25.6.0
 Summary: Command line client for eccenca Corporate Memory
 License: Apache-2.0
 License-File: LICENSE
@@ -34,6 +34,7 @@ Requires-Dist: click-didyoumean (>=0.3.1,<0.4.0)
 Requires-Dist: click-help-colors (>=0.9.4,<0.10.0)
 Requires-Dist: cmem-cmempy (==25.4.0)
 Requires-Dist: configparser (>=7.2.0,<8.0.0)
+Requires-Dist: humanize (>=4.14.0,<5.0.0)
 Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
 Requires-Dist: junit-xml (>=1.9,<2.0)
 Requires-Dist: natsort (>=8.4.0,<9.0.0)

{cmem_cmemc-25.5.0.dist-info → cmem_cmemc-25.6.0.dist-info}/RECORD

@@ -2,23 +2,23 @@ cmem_cmemc/__init__.py,sha256=-RPEVweA-fcmEAynszDDMKwArJgxZpGW61UBiV7O4Og,24
 cmem_cmemc/_cmemc.zsh,sha256=fmkrBHIQxus8cp2AgO1tzZ5mNZdGL_83cYz3a9uAdsg,1326
 cmem_cmemc/cli.py,sha256=vDdSHFmXUstC3T7OlbPSd0hXxyigJE4VVgRMcsNz5cc,4538
 cmem_cmemc/command.py,sha256=nBtrwPKFJLRpD3IPk5hKyn2LOMl-1ae7SV9iRhgky8k,1958
-cmem_cmemc/command_group.py,sha256=LSCQfvRQm8R-HsBZfUs-5lzEGEf_LMaleFwZ1AKpZRE,3488
+cmem_cmemc/command_group.py,sha256=72_7yiBl04JumnhpHWIZ-cggdCUjj71Rm5zT-SPdv78,3543
 cmem_cmemc/commands/__init__.py,sha256=NaGM5jOzf0S_-4UIAwlVDOf2AZ3mliGPoRLXQJfTyZs,22
 cmem_cmemc/commands/acl.py,sha256=vJ3H5eeWiVCtGk1ZpEF2OqXeHurZgPwakD8YT09lnVM,19030
 cmem_cmemc/commands/admin.py,sha256=F-393oXTVYV7HxK5NxuhONlBIEg7wffxE7DAKDuasG4,10192
 cmem_cmemc/commands/client.py,sha256=nBs7MoF2wF45AteTCeIQrXcOwKmHHCd8_lG_SM2mQSA,5127
 cmem_cmemc/commands/config.py,sha256=VHiVkW6NFuz-tpKXRPl7dO1gIXQLOuEhlGVxb422qwA,5803
-cmem_cmemc/commands/dataset.py,sha256=zylVsfBD92LzHgQ96edNKaQNsf0zKoPro0VQdX--Mu8,30488
+cmem_cmemc/commands/dataset.py,sha256=vz4vqm6mSCY2ELCdEjCDUwM5YbszXjvRW2NR2ILP0aI,28891
+cmem_cmemc/commands/file.py,sha256=CGut4VpSB-7A7wGMl4uP0m4xv6G--pnYO2faVI72PbM,15935
 cmem_cmemc/commands/graph.py,sha256=uAI7dEDRjwLSGl6QsJ0PsAycJf2-EZNNn_lIFliDE9Q,32662
 cmem_cmemc/commands/graph_imports.py,sha256=CYgTUSj8giHoWzz0Qjtup0H1V5GKZEI3xXdA_9w_btI,14046
 cmem_cmemc/commands/graph_insights.py,sha256=ARgaYeKBH1b2OIKVjgOPwgAa8mVzo3CLCstcw4Z53z4,13193
 cmem_cmemc/commands/manual.py,sha256=-sZWeFL92Kj8gL3VYsbpKh2ZaVTyM3LgKaUcpNn9u3A,2179
 cmem_cmemc/commands/metrics.py,sha256=pIBRTq90f7MEI99HLdFLN3D1xQ2Z2u6VKUeTIz0X7DY,12205
 cmem_cmemc/commands/migration.py,sha256=y9v4Be7WELGjDGDBZrfLBeqU_G_JH1fnP5UVG9qjB3g,9638
-cmem_cmemc/commands/project.py,sha256=0cnThtaRZOc04EXR31JFEUSONKbh3ANQG4vZANGwfWA,20601
-cmem_cmemc/commands/python.py,sha256=7ExdKr7nTED5ZZGjeBk5UngNI3F_KNcavwUgwaxApV0,11965
+cmem_cmemc/commands/project.py,sha256=H4UBlqJXjGzkL_afpn4dbFO3tpQbQa5L4XDsCyKS5RM,20669
+cmem_cmemc/commands/python.py,sha256=dXFRV3pD4nRC7iTuy3dFFbdpilip-zZVuxkjyJX67Ls,11958
 cmem_cmemc/commands/query.py,sha256=Dra4BHshfec1nvVJbDvLT9VTjfesWJGSQCbyNmnnMWI,28788
-cmem_cmemc/commands/resource.py,sha256=74cn_yqMv3a6xOQAPpNCuluTWEH-_2PGENJnl7y8qz4,7778
 cmem_cmemc/commands/scheduler.py,sha256=zYeO1-Hlxh9D-I9JIweQ-SEA0la0wv0EicY_UY7rNCg,8751
 cmem_cmemc/commands/store.py,sha256=W_6LXq98If50-X-XYZUQsYodVwUjOSm3_jyMp62pFuA,10599
 cmem_cmemc/commands/user.py,sha256=ANZpeOBA46xiqOcNPrueComsCV0gEBbav-vOL9VgyX4,12535
@@ -27,14 +27,14 @@ cmem_cmemc/commands/variable.py,sha256=aLRH_rFe0h7JBpKIqzcevbk26vczgUGokIDY8g6LP
 cmem_cmemc/commands/vocabulary.py,sha256=erf3zqSRqCVrN_OlCZj5Z5w4L6MRwSaUmC6V11v2vHc,18095
 cmem_cmemc/commands/workflow.py,sha256=BINC-P5RsDvKTkHUbKZpzkfV5M12Cl7EPD4RLmygDOQ,25798
 cmem_cmemc/commands/workspace.py,sha256=IcZgBsvtulLRFofS70qpln6oKQIZunrVLfSAUeiFhCA,4579
-cmem_cmemc/completion.py,sha256=JchfZKbgpOIxN8rPyu7ZBPerIzNKXLo0tOgz8xJa5uw,45237
+cmem_cmemc/completion.py,sha256=mLNikJ2lHojj0El4PlhUivTOu4gPlWEdo49VoHe3cnI,46111
 cmem_cmemc/config_parser.py,sha256=NduwOT-BB_uAk3pz1Y-ex18RQJW-jjHzkQKCEUUK6Hc,1276
 cmem_cmemc/constants.py,sha256=pzZYbSaTDUiWmE-VOAHB20oivHew5_FP9UTejySsVK4,550
 cmem_cmemc/context.py,sha256=Axk5zTbhAMPBSMPnk9jIaOQr4GvS3Ih5Jb6ZZc-c5O0,22247
 cmem_cmemc/exceptions.py,sha256=0lsGOfXhciNGJloJGERMbbPuBbs5IwIIJ_5YnY9qQ-8,546
 cmem_cmemc/manual_helper/__init__.py,sha256=G3Lqw2aPxo8x63Tg7L0aa5VD9BMaRzZDmhrog7IuEPg,43
-cmem_cmemc/manual_helper/graph.py,sha256=HU04NYWeJ6LmW4UC7qHr1v1qsm2Md61pJ-pgWUHFmHY,3647
-cmem_cmemc/manual_helper/multi_page.py,sha256=v24CZpirD-ThVYJI34axQhC-IB9APQ_VyjMrl6WNQQk,12320
+cmem_cmemc/manual_helper/graph.py,sha256=dTkFXgU9fgySn54rE93t79v1MjWjQkprKRIfJhc7Jps,3655
+cmem_cmemc/manual_helper/multi_page.py,sha256=asJ8QdAxRBU5EXZxP7UWaWnK7JtQ3cN7f3JY4ArjX4c,12321
 cmem_cmemc/manual_helper/single_page.py,sha256=0mMn_IJwFCe-WPKAmxGEStb8IINLpQRxAx_F1pIxg1E,1526
 cmem_cmemc/migrations/__init__.py,sha256=i6Ri7qN58ou_MwOzm2KibPkXOD7u-1ELky-nUE5LjAA,24
 cmem_cmemc/migrations/abc.py,sha256=UGJzrvMzUFdp2-sosp49ObRI-SrUSzLJqLEhvB4QTzg,3564
@@ -47,15 +47,15 @@ cmem_cmemc/migrations/workspace_configurations.py,sha256=tFmCdfEL10ICjqMXQEIf-9f
 cmem_cmemc/object_list.py,sha256=PKDZ-p61ep2MzygRXAe4uzP9CB9D8esoiC3lwra84P4,14734
 cmem_cmemc/parameter_types/__init__.py,sha256=Jqhwnw5a2oPNMClzUyovWiieK60RCl3rvSNr-t3wP84,36
 cmem_cmemc/parameter_types/path.py,sha256=M56PGdjploN2pEYaNAk6_qomAX54crLW8E9XZsFvRuI,2270
-cmem_cmemc/placeholder.py,sha256=PzZWpVBa7WPP5_5f-HlSoaZaOnto-1Dr9XuCTSpPnNM,2386
+cmem_cmemc/placeholder.py,sha256=Rf20OqwDjISnVPJsYlvuSgzeUbfJ2sklE2PWnZ5TSYg,2409
 cmem_cmemc/smart_path/__init__.py,sha256=zDgm1kDrzLyCuIcNb8VXSdnb_CcVNjGkjgiIDVlsh74,3023
 cmem_cmemc/smart_path/clients/__init__.py,sha256=YFOm69BfTCRvAcJjN_CoUmCv3kzEciyYOPUG337p_pA,1696
 cmem_cmemc/smart_path/clients/http.py,sha256=3clZu2v4uuOvPY4MY_8SVSy7hIXJDNooahFRBRpy0ok,2347
-cmem_cmemc/string_processor.py,sha256=0EZVl3UeVgV5EEYGLPvCGHrJDMd0Ezekkwg6mDQVBZI,3112
+cmem_cmemc/string_processor.py,sha256=bwFs6BYoxX3DrUaxdqkonTC4AO99doBSmwqI05qvYyY,3494
 cmem_cmemc/title_helper.py,sha256=7frjAR54_Xc1gszOWXfzSmKFTawNJQ7kkXhZcHmQLyw,1250
 cmem_cmemc/utils.py,sha256=LlvAMHxciY9ge-REdwHQhRetJGrYghRqBZADxqE0yL4,14657
-cmem_cmemc-25.5.0.dist-info/METADATA,sha256=F3j2Ueql34apDjUfI0IH2TgvOpRzq-Ki-QGvd37Kqq8,5715
-cmem_cmemc-25.5.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-cmem_cmemc-25.5.0.dist-info/entry_points.txt,sha256=2G0AWAyz501EHpFTjIxccdlCTsHt80NT0pdUGP1QkPA,45
-cmem_cmemc-25.5.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-cmem_cmemc-25.5.0.dist-info/RECORD,,
+cmem_cmemc-25.6.0.dist-info/METADATA,sha256=ZUPmXLqT9akk7zF8ErvMXTr7zTgBr1h-UBSBm0V_dZQ,5757
+cmem_cmemc-25.6.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+cmem_cmemc-25.6.0.dist-info/entry_points.txt,sha256=2G0AWAyz501EHpFTjIxccdlCTsHt80NT0pdUGP1QkPA,45
+cmem_cmemc-25.6.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+cmem_cmemc-25.6.0.dist-info/RECORD,,

cmem_cmemc/commands/resource.py

@@ -1,220 +0,0 @@
-"""Build dataset resource commands for cmemc."""
-
-import re
-
-import click
-from click import ClickException, UsageError
-from cmem.cmempy.workspace.projects.resources import get_all_resources
-from cmem.cmempy.workspace.projects.resources.resource import (
-    delete_resource,
-    get_resource_metadata,
-    get_resource_usage_data,
-)
-
-from cmem_cmemc import completion
-from cmem_cmemc.command import CmemcCommand
-from cmem_cmemc.command_group import CmemcGroup
-from cmem_cmemc.context import ApplicationContext
-from cmem_cmemc.utils import split_task_id, struct_to_table
-
-RESOURCE_FILTER_TYPES = ["project", "regex"]
-RESOURCE_FILTER_TYPES_HIDDEN = ["ids"]
-RESOURCE_FILTER_TEXT = (
-    "Filter file resources based on metadata. "
-    f"First parameter CHOICE can be one of {RESOURCE_FILTER_TYPES!s}"
-    ". The second parameter is based on CHOICE, e.g. a project "
-    "ID or a regular expression string."
-)
-
-
-def _get_resources_filtered(
-    resources: list[dict], filter_name: str, filter_value: str | tuple[str, ...]
-) -> list[dict]:
-    """Get file resources but filtered according to name and value."""
-    # check for correct filter names (filter ids is used internally only)
-    if filter_name not in RESOURCE_FILTER_TYPES + RESOURCE_FILTER_TYPES_HIDDEN:
-        raise UsageError(
-            f"{filter_name} is an unknown filter name. " f"Use one of {RESOURCE_FILTER_TYPES}."
-        )
-    # filter by ID list
-    if filter_name == "ids":
-        return [_ for _ in resources if _["id"] in filter_value]
-    # filter by project
-    if filter_name == "project":
-        return [_ for _ in resources if _["project"] == str(filter_value)]
-    # filter by regex
-    if filter_name == "regex":
-        return [_ for _ in resources if re.search(str(filter_value), _["name"])]
-    # return unfiltered list
-    return resources
-
-
-@click.command(cls=CmemcCommand, name="list")
-@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
-@click.option(
-    "--id-only",
-    is_flag=True,
-    help="Lists only resource names and no other metadata. "
-    "This is useful for piping the IDs into other commands.",
-)
-@click.option(
-    "--filter",
-    "filters_",
-    multiple=True,
-    type=(str, str),
-    shell_complete=completion.resource_list_filter,
-    help=RESOURCE_FILTER_TEXT,
-)
-@click.pass_obj
-def list_command(
-    app: ApplicationContext, raw: bool, id_only: bool, filters_: tuple[tuple[str, str], ...]
-) -> None:
-    """List available file resources.
-
-    Outputs a table or a list of dataset resources (files).
-    """
-    resources = get_all_resources()
-    for _ in filters_:
-        filter_name, filter_value = _
-        resources = _get_resources_filtered(resources, filter_name, filter_value)
-    if raw:
-        app.echo_info_json(resources)
-        return
-    if id_only:
-        for _ in sorted(_["id"] for _ in resources):
-            app.echo_result(_)
-        return
-    # output a user table
-    table = []
-    headers = ["ID", "Modified", "Size"]
-    for _ in resources:
-        row = [
-            _["id"],
-            _["modified"],
-            _["size"],
-        ]
-        table.append(row)
-    app.echo_info_table(
-        table,
-        headers=headers,
-        sort_column=0,
-        empty_table_message="No dataset resources found. "
-        "Use the `dataset create` command to create a new file based dataset.",
-    )
-
-
-@click.command(cls=CmemcCommand, name="delete")
-@click.argument("resource_ids", nargs=-1, type=click.STRING, shell_complete=completion.resource_ids)
-@click.option("--force", is_flag=True, help="Delete resource even if in use by a task.")
-@click.option(
-    "-a",
-    "--all",
-    "all_",
-    is_flag=True,
-    help="Delete all resources. " "This is a dangerous option, so use it with care.",
-)
-@click.option(
-    "--filter",
-    "filters_",
-    multiple=True,
-    type=(str, str),
-    shell_complete=completion.resource_list_filter,
-    help=RESOURCE_FILTER_TEXT,
-)
-@click.pass_obj
-def delete_command(
-    app: ApplicationContext,
-    resource_ids: tuple[str, ...],
-    force: bool,
-    all_: bool,
-    filters_: tuple[tuple[str, str], ...],
-) -> None:
-    """Delete file resources.
-
-    There are three selection mechanisms: with specific IDs, only those
-    specified resources will be deleted; by using --filter, resources based
-    on the filter type and value will be deleted; using --all will delete
-    all resources.
-    """
-    if resource_ids == () and not all_ and filters_ == ():
-        raise UsageError(
-            "Either specify at least one resource ID or use the --all or "
-            "--filter options to specify resources for deletion."
-        )
-
-    resources = get_all_resources()
-    if len(resource_ids) > 0:
-        for resource_id in resource_ids:
-            if resource_id not in [_["id"] for _ in resources]:
-                raise ClickException(f"Resource {resource_id} not available.")
-        # "filter" by id
-        resources = _get_resources_filtered(resources, "ids", resource_ids)
-    for _ in filters_:
-        resources = _get_resources_filtered(resources, _[0], _[1])
-
-    # avoid double removal as well as sort IDs
-    processed_ids = sorted({_["id"] for _ in resources}, key=lambda v: v.lower())
-    count = len(processed_ids)
-    for current, resource_id in enumerate(processed_ids, start=1):
-        current_string = str(current).zfill(len(str(count)))
-        app.echo_info(f"Delete resource {current_string}/{count}: {resource_id} ... ", nl=False)
-        project_id, resource_local_id = split_task_id(resource_id)
-        usage = get_resource_usage_data(project_id, resource_local_id)
-        if len(usage) > 0:
-            app.echo_error(f"in use by {len(usage)} task(s)", nl=False)
-            if force:
-                app.echo_info(" ... ", nl=False)
-            else:
-                app.echo_info("")
-                continue
-        delete_resource(project_name=project_id, resource_name=resource_local_id)
-        app.echo_success("deleted")
-
-
-@click.command(cls=CmemcCommand, name="inspect")
-@click.argument("resource_id", type=click.STRING, shell_complete=completion.resource_ids)
-@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
-@click.pass_obj
-def inspect_command(app: ApplicationContext, resource_id: str, raw: bool) -> None:
-    """Display all metadata of a file resource."""
-    project_id, resource_id = split_task_id(resource_id)
-    resource_data = get_resource_metadata(project_id, resource_id)
-    if raw:
-        app.echo_info_json(resource_data)
-    else:
-        table = struct_to_table(resource_data)
-        app.echo_info_table(table, headers=["Key", "Value"], sort_column=0)
-
-
-@click.command(cls=CmemcCommand, name="usage")
-@click.argument("resource_id", type=click.STRING, shell_complete=completion.resource_ids)
-@click.option("--raw", is_flag=True, help="Outputs raw JSON.")
-@click.pass_obj
-def usage_command(app: ApplicationContext, resource_id: str, raw: bool) -> None:
-    """Display all usage data of a file resource."""
-    project_id, resource_id = split_task_id(resource_id)
-    usage = get_resource_usage_data(project_id, resource_id)
-    if raw:
-        app.echo_info_json(usage)
-        return
-    # output a user table
-    table = []
-    headers = ["Task ID", "Type", "Label"]
-    for _ in usage:
-        row = [project_id + ":" + _["id"], _["taskType"], _["label"]]
-        table.append(row)
-    app.echo_info_table(table, headers=headers, sort_column=2)
-
-
-@click.group(cls=CmemcGroup)
-def resource() -> CmemcGroup:  # type: ignore[empty-body]
-    """List, inspect or delete dataset file resources.
-
-    File resources are identified by their paths and project IDs.
-    """
-
-
-resource.add_command(list_command)
-resource.add_command(delete_command)
-resource.add_command(inspect_command)
-resource.add_command(usage_command)