cmem-cmemc 24.2.0rc2__py3-none-any.whl → 24.3.0rc2__py3-none-any.whl

cmem_cmemc/commands/project.py

@@ -1,8 +1,9 @@
 """DataIntegration project commands for the cmem command line interface."""
+
 import os
+import pathlib
 import shutil
 import tempfile
-from pathlib import Path
 from zipfile import ZipFile
 
 import click
@@ -29,9 +30,12 @@ from cmem.cmempy.workspace.projects.project import (
 from jinja2 import Template
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.commands.variable import variable
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
 
 
 def _validate_projects_to_process(project_ids: tuple[str], all_flag: bool) -> list[str]:
@@ -137,7 +141,13 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool) -> None:
             _["metaData"]["label"],
         ]
         table.append(row)
-    app.echo_info_table(table, headers=["Project ID", "Label"], sort_column=1)
+    app.echo_info_table(
+        table,
+        headers=["Project ID", "Label"],
+        sort_column=1,
+        empty_table_message="No projects found. "
+        "Use the `project create` command to create a new project.",
+    )
 
 
 @click.command(cls=CmemcCommand, name="delete")
@@ -283,13 +293,18 @@ def create_command(
     "-o",
     "--overwrite",
     is_flag=True,
-    help="Overwrite existing files. This is a dangerous option, " "so use it with care.",
+    hidden=True,
+)
+@click.option(
+    "--replace",
+    is_flag=True,
+    help="Replace existing files. This is a dangerous option, so use it with care.",
 )
 @click.option(
     "--output-dir",
     default=".",
     show_default=True,
-    type=click.Path(writable=True, file_okay=False),
+    type=ClickSmartPath(writable=True, file_okay=False),
     help="The base directory, where the project files will be created. "
     "If this directory does not exist, it will be silently created.",
 )
@@ -334,6 +349,7 @@ def export_command(  # noqa: PLR0913
     all_: bool,
     project_ids: tuple[str],
     overwrite: bool,
+    replace: bool,
     marshalling_plugin: str,
     template: str,
     output_dir: str,
@@ -356,6 +372,13 @@ def export_command(  # noqa: PLR0913
     Example: cmemc config list | parallel -I% cmemc -c % project export --all
     -t "dump/{{connection}}/{{date}}-{{id}}.project"
     """
+    if overwrite:
+        replace = overwrite
+        app.echo_warning(
+            "The option --overwrite is deprecated and will be removed with the next major release."
+            " Please use the --replace option instead."
+        )
+
     if help_types:
         _show_type_list(app)
         return
@@ -384,12 +407,12 @@ def export_command(  # noqa: PLR0913
                 + get_extension_by_plugin(marshalling_plugin)
             )
         # join with given output directory and normalize full path
-        export_path = os.path.normpath(Path(output_dir) / local_name)
+        export_path = os.path.normpath(str(Path(output_dir) / local_name))
 
         app.echo_info(
             f"Export project {current}/{count}: " f"{project_id} to {export_path} ... ", nl=False
         )
-        if Path(export_path).exists() and overwrite is not True:
+        if Path(export_path).exists() and replace is not True:
             app.echo_error("target file or directory exists")
             continue
 
@@ -418,7 +441,9 @@ def export_command(  # noqa: PLR0913
 @click.argument(
     "path",
     shell_complete=completion.project_files,
-    type=click.Path(allow_dash=False, dir_okay=True, readable=True, exists=True),
+    type=ClickSmartPath(
+        allow_dash=False, dir_okay=True, readable=True, exists=True, remote_okay=True
+    ),
 )
 @click.argument(
     "project_id",
@@ -431,20 +456,34 @@ def export_command(  # noqa: PLR0913
     "-o",
     "--overwrite",
     is_flag=True,
-    help="Overwrite an existing project. This is a dangerous option, " "so use it with care.",
+    hidden=True,
+)
+@click.option(
+    "--replace",
+    is_flag=True,
+    help="Replace an existing project. This is a dangerous option, so use it with care.",
 )
 @click.pass_obj
-def import_command(app: ApplicationContext, path: str, project_id: str, overwrite: bool) -> None:
+def import_command(
+    app: ApplicationContext, path: str, project_id: str, overwrite: bool, replace: bool
+) -> None:
     """Import a project from a file or directory.
 
     Example: cmemc project import my_project.zip my_project
     """
+    if overwrite:
+        replace = overwrite
+        app.echo_warning(
+            "The option --overwrite is deprecated and will be removed with the next major release."
+            " Please use the --replace option instead."
+        )
+
     all_projects = get_projects()
-    if project_id and not overwrite and project_id in ([_["name"] for _ in all_projects]):
+    if project_id and not replace and project_id in ([_["name"] for _ in all_projects]):
         raise ValueError(f"Project {project_id} is already there.")
 
     if Path(path).is_dir():
-        if not Path(Path(path) / "config.xml").is_file():
+        if not (Path(path) / "config.xml").is_file():
             # fail early if directory is not an export
             raise ValueError(f"Directory {path} seems not to be a export directory.")
 
@@ -453,17 +492,25 @@ def import_command(app: ApplicationContext, path: str, project_id: str, overwrit
         app.echo_info("zipping ... ", nl=False)
         with tempfile.NamedTemporaryFile() as _:
             shutil.make_archive(
-                _.name, "zip", base_dir=Path(path).name, root_dir=Path(path).parent.absolute()
+                _.name,
+                "zip",
+                base_dir=pathlib.Path(path).name,
+                root_dir=str(Path(path).parent.absolute()),
             )
             # make_archive adds a .zip automatically ...
             uploaded_file = _.name + ".zip"
             app.echo_debug(f"Uploaded file is {uploaded_file}")
     else:
         app.echo_info(f"Import file {path} to project {project_id} ... ", nl=False)
-        uploaded_file = path
+        with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as _:
+            with ClickSmartPath.open(path) as _buffer:
+                _.write(_buffer.read())
+            uploaded_file = _.name
 
     # upload file and get validation report
     validation_response = upload_project(uploaded_file)
+    # Remove the temporary file
+    pathlib.Path.unlink(pathlib.Path(uploaded_file))
     if "errorMessage" in validation_response:
         raise ValueError(validation_response["errorMessage"])
     import_id = validation_response["projectImportId"]
@@ -476,9 +523,7 @@ def import_command(app: ApplicationContext, path: str, project_id: str, overwrit
 
     # start import of project from upload using import ID as a reference
     # this fails if project_id already exists
-    import_from_upload_start(
-        import_id=import_id, project_id=project_id, overwrite_existing=overwrite
-    )
+    import_from_upload_start(import_id=import_id, project_id=project_id, overwrite_existing=replace)
 
     # loop until "success" boolean is in status response
     status = import_from_upload_status(import_id)

cmem_cmemc/commands/python.py

@@ -1,4 +1,5 @@
 """DataIntegration python management commands."""
+
 import sys
 from dataclasses import asdict
 from re import match
@@ -15,8 +16,10 @@ from cmem.cmempy.workspace.python import (
 )
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
 from cmem_cmemc.utils import get_published_packages
 
 
@@ -27,16 +30,14 @@ def _get_package_id(module_name: str) -> str:
 
 def _looks_like_a_package(package: str) -> bool:
     """Check if a string looks like a package requirement string."""
-    if match(r"^[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*((==|<=|>=|>|<).*)?$", package):
-        return True
-    return False
+    return bool(match("^[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*((==|<=|>=|>|<).*)?$", package))
 
 
 @click.command(cls=CmemcCommand, name="install")
 @click.argument(
     "PACKAGE",
     shell_complete=completion.installable_packages,
-    type=click.Path(readable=True, allow_dash=False, dir_okay=False),
+    type=ClickSmartPath(readable=True, allow_dash=False, dir_okay=False),
 )
 @click.pass_obj
 def install_command(app: ApplicationContext, package: str) -> None:
@@ -49,33 +50,48 @@ def install_command(app: ApplicationContext, package: str) -> None:
     .tar.gz file, by uploading a build distribution .whl file, or by
     specifying a package name, i.e., a pip requirement specifier with a
     package name available on pypi.org (e.g. `requests==2.27.1`).
+
+    Note: The tab-completion of this command lists only public packages from
+    pypi.org and not from additional or changed python package repositories you
+    may have configured on the server.
     """
     app.echo_info(f"Install package {package} ... ", nl=False)
     try:
-        response = install_package_by_file(package_file=package)
-    except FileNotFoundError as error:
+        install_response = install_package_by_file(package_file=package)
+    except FileNotFoundError as not_found_error:
         if not _looks_like_a_package(package):
             raise ValueError(
                 f"{package} does not look like a package name or requirement "
                 "string, and a file with this name also does not exists."
-            ) from error
-        response = install_package_by_name(package_name=package)
+            ) from not_found_error
+        install_response = install_package_by_name(package_name=package)
 
     # DI >= 24.1 has a combine 'output' key, before 24.1 'standardOutput' and 'errorOutput' existed
-    output: list[str] = []
-    output.extend(response.get("output", "").splitlines())
-    output.extend(response.get("standardOutput", "").splitlines())
-    output.extend(response.get("errorOutput", "").splitlines())
-    for output_line in output:
-        app.echo_debug(output_line)
-
-    if response["success"]:
+    install_output: list[str] = []
+    install_output.extend(install_response.get("output", "").splitlines())
+    install_output.extend(install_response.get("standardOutput", "").splitlines())
+    install_output.extend(install_response.get("errorOutput", "").splitlines())
+    app.echo_debug(install_output)
+
+    update_plugin_response = update_plugins()
+    app.echo_debug(f"Updated Plugins: {update_plugin_response!s}")
+    update_errors = update_plugin_response.get("errors", [])
+    if install_response["success"] is True and len(update_errors) == 0:
         app.echo_success("done")
-    else:
-        app.echo_error("error")
-        for output_line in output:
-            app.echo_error(output_line)
-    app.echo_debug("Updated Plugins: " + str(update_plugins()))
+        return
+
+    # something went wrong
+    app.echo_error("error")
+    app.echo_error(install_output, prepend_line=True)
+    for update_error in update_errors:
+        app.echo_error(
+            f"Error while updating the plugins of "
+            f"{update_error.get('packageName')}: {update_error.get('errorMessage')} "
+            f"({update_error.get('errorType')})",
+            prepend_line=True,
+        )
+        app.echo_error(update_error.get("stackTrace"))
+    sys.exit(1)
 
 
 @click.command(cls=CmemcCommand, name="uninstall")
@@ -168,7 +184,10 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool, available: b
             (_.name, _.version, str(_.published)[:10], _.description) for _ in published_packages
         ]
         app.echo_info_table(
-            table_published, headers=["Name", "Version", "Published", "Description"], sort_column=0
+            table_published,
+            headers=["Name", "Version", "Published", "Description"],
+            sort_column=0,
+            empty_table_message="No available python packages found.",
         )
         return
 
@@ -183,7 +202,13 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool, available: b
         return
 
     table_installed = [(_["name"], _["version"]) for _ in installed_packages]
-    app.echo_info_table(table_installed, headers=["Name", "Version"], sort_column=0)
+    app.echo_info_table(
+        table_installed,
+        headers=["Name", "Version"],
+        sort_column=0,
+        empty_table_message="No installed python packages found."
+        "Most likely, this is due to a wrong deployment.",
+    )
 
 
 @click.command(cls=CmemcCommand, name="list-plugins")
@@ -232,7 +257,13 @@ def list_plugins_command(
         )
         for _ in sorted(plugins, key=lambda k: k["id"].lower())
     ]
-    app.echo_info_table(table, headers=["ID", "Package ID", "Type", "Label"], sort_column=0)
+    app.echo_info_table(
+        table,
+        headers=["ID", "Package ID", "Type", "Label"],
+        sort_column=0,
+        empty_table_message="No plugin plugins found. "
+        "Use the `admin workspace python install` command to install python packages with plugins.",
+    )
     if "error" in raw_output:
         app.echo_error(raw_output["error"])
 

cmem_cmemc/commands/query.py

@@ -1,9 +1,9 @@
 """query commands for cmem command line interface."""
+
 import json
 import sys
 from hashlib import sha1
 from json import JSONDecodeError, load
-from pathlib import Path
 from shutil import get_terminal_size
 from time import sleep, time
 from uuid import uuid4
@@ -11,11 +11,17 @@ from uuid import uuid4
 import click
 from click import UsageError
 from click.shell_completion import CompletionItem
-from cmem.cmempy.queries import QUERY_CATALOG, SparqlQuery, cancel_query, get_query_status
+from cmem.cmempy.queries import (
+    QueryCatalog,
+    SparqlQuery,
+    cancel_query,
+    get_query_status,
+)
 from requests import HTTPError
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.object_list import (
     DirectListPropertyFilter,
@@ -25,6 +31,8 @@ from cmem_cmemc.object_list import (
     compare_int_greater_than,
     compare_regex,
 )
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
 from cmem_cmemc.utils import extract_error_message, struct_to_table
 
 QUERY_FILTER_TYPES = sorted(["graph", "status", "slower-than", "type", "regex", "trace-id", "user"])
@@ -51,7 +59,7 @@ class ReplayStatistics:
     query_count: int = 0
     error_count: int = 0
     query_average: float
-    catalog = QUERY_CATALOG
+    catalog = QueryCatalog()
     app: ApplicationContext
 
     def __init__(self, app: ApplicationContext, label: str | None = None):
@@ -161,11 +169,12 @@ class ReplayStatistics:
 
     def create_output(self) -> dict:
         """Create the structure for the output commands."""
-        output = {}
-        for key, value in vars(self).items():
-            if key not in ("current_loop_key", "app"):
-                # ignore some object vars on output
-                output[key] = value
+        # create dict from object but ignore some internal vars on output
+        output = {
+            key: value
+            for (key, value) in dict(vars(self)).items()
+            if key not in ("current_loop_key", "app")
+        }
         loop_average = 0
         loop_minimum = None
         loop_maximum = None
@@ -331,7 +340,7 @@ def list_command(app: ApplicationContext, id_only: bool) -> None:
     Outputs a list of query URIs which can be used as reference for
     the query execute command.
     """
-    queries = QUERY_CATALOG.get_queries().items()
+    queries = QueryCatalog().get_queries().items()
     if id_only:
         # sort dict by short_url - https://docs.python.org/3/howto/sorting.html
         for _, sparql_query in sorted(queries, key=lambda k: k[1].short_url.lower()):
@@ -443,7 +452,7 @@ def execute_command(  # noqa: PLR0913
     app.echo_debug("Parameter: " + str(placeholder))
     for file_or_uri in queries:
         app.echo_debug(f"Start of execution: {file_or_uri} with " f"placeholder {placeholder}")
-        executed_query = QUERY_CATALOG.get_query(file_or_uri, placeholder=placeholder)
+        executed_query = QueryCatalog().get_query(file_or_uri, placeholder=placeholder)
         if executed_query is None:
             raise ValueError(f"{file_or_uri} is neither a (readable) file nor a query URI.")
         app.echo_debug(
@@ -486,7 +495,7 @@ def open_command(app: ApplicationContext, queries: tuple[str, ...]) -> None:
     opening multiple browser tabs.
     """
     for file_or_uri in queries:
-        opened_query = QUERY_CATALOG.get_query(file_or_uri)
+        opened_query = QueryCatalog().get_query(file_or_uri)
         if opened_query is None:
             raise ValueError(f"{file_or_uri} is neither a (readable) file nor a query URI.")
         open_query_uri = opened_query.get_editor_url()
@@ -575,7 +584,7 @@ def status_command(
     "REPLAY_FILE",
     required=True,
     shell_complete=completion.replay_files,
-    type=click.Path(exists=True, allow_dash=False, readable=True, dir_okay=False),
+    type=ClickSmartPath(exists=True, allow_dash=False, readable=True, dir_okay=False),
 )
 @click.option("--raw", is_flag=True, help="Output the execution statistic as raw JSON.")
 @click.option(
@@ -603,7 +612,7 @@ def status_command(
     "of a successful command execution. The output can be stdout "
     "('-') - in this case, the execution statistic output is "
     "oppressed.",
-    type=click.Path(exists=False, allow_dash=True, writable=True, dir_okay=False),
+    type=ClickSmartPath(exists=False, allow_dash=True, writable=True, dir_okay=False),
 )
 @click.option("--run-label", type=click.STRING, help="Optional label of this replay run.")
 @click.pass_obj

cmem_cmemc/commands/resource.py

@@ -1,4 +1,5 @@
 """DataIntegration dataset resource commands for cmemc."""
+
 import re
 
 import click
@@ -10,7 +11,8 @@ from cmem.cmempy.workspace.projects.resources.resource import (
 )
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.utils import split_task_id, struct_to_table
 
@@ -91,7 +93,13 @@ def list_command(
             _["size"],
         ]
         table.append(row)
-    app.echo_info_table(table, headers=headers, sort_column=0)
+    app.echo_info_table(
+        table,
+        headers=headers,
+        sort_column=0,
+        empty_table_message="No dataset resources found. "
+        "Use the `dataset create` command to create a new file based dataset.",
+    )
 
 
 @click.command(cls=CmemcCommand, name="delete")

cmem_cmemc/commands/scheduler.py

@@ -1,4 +1,5 @@
 """DataIntegration scheduler commands for the cmem command line interface."""
+
 from typing import Any
 
 import click
@@ -9,7 +10,8 @@ from cmem.cmempy.workspace.search import list_items
 from cmem.cmempy.workspace.tasks import get_task, patch_parameter
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.utils import split_task_id, struct_to_table
 
@@ -114,7 +116,13 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool) -> None:
         ]
         table.append(row)
     # sort output by label - https://docs.python.org/3/howto/sorting.html
-    app.echo_info_table(table, headers=headers, sort_column=1)
+    app.echo_info_table(
+        table,
+        headers=headers,
+        sort_column=1,
+        empty_table_message="Now workflow scheduler found. "
+        "Open a project in the web interface and create a new workflow scheduler there.",
+    )
 
 
 @click.command(cls=CmemcCommand, name="inspect")