cmem-cmemc 24.2.0rc2__py3-none-any.whl → 24.3.0rc1__py3-none-any.whl

cmem_cmemc/commands/vocabulary.py

@@ -1,6 +1,7 @@
 """vocabularies commands for cmem command line interface."""
+
 import io
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from re import match
 from urllib.parse import urlparse
 
@@ -21,8 +22,10 @@ from rdflib.plugins.parsers.notation3 import BadSyntax
 from six.moves.urllib.parse import quote
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
 
 GET_ONTOLOGY_IRI_QUERY = """
 PREFIX owl: <http://www.w3.org/2002/07/owl#>
@@ -71,22 +74,29 @@ def _validate_vocabs_to_process(iris: tuple[str], filter_: str, all_flag: bool)
     list is without duplicates, and validated if they exist
     """
     if iris == () and not all_flag:
-        raise ValueError(
+        raise click.UsageError(
             "Either specify at least one vocabulary IRI "
             "or use the --all option to process over all vocabularies."
         )
-    vocabs_to_process = list(iris)
-    all_vocabs = [_["iri"] for _ in get_vocabularies(filter_=filter_)]
+    all_vocabs = {_["iri"]: _ for _ in get_vocabularies()}
     if all_flag:
-        # in case --all is given, all vocabs are processed
-        vocabs_to_process = all_vocabs
-    # avoid double removal
-    vocabs_to_process = list(set(vocabs_to_process))
-
-    # test if one of the vocabs does not exist
+        # in case --all is given, all installable / installed vocabs are processed
+        if filter_ == "installed":  # uninstall command
+            return [_ for _ in all_vocabs if all_vocabs[_]["installed"]]
+        # install command
+        return [_ for _ in all_vocabs if not all_vocabs[_]["installed"]]
+
+    vocabs_to_process = list(set(iris))  # avoid double removal / installation
+    # test if one of the vocabs does not exist or is already installed / not installed
     for _ in vocabs_to_process:
-        if _ not in all_vocabs:
-            raise ValueError(f"Vocabulary {_} does not exist.")
+        # uninstall command
+        if filter_ == "installed" and (_ not in all_vocabs or not all_vocabs[_]["installed"]):
+            raise click.UsageError(f"Vocabulary {_} not installed.")
+        if filter_ == "installable":  # install command
+            if _ not in all_vocabs:
+                raise click.UsageError(f"Vocabulary {_} does not exist.")
+            if all_vocabs[_]["installed"]:
+                raise click.UsageError(f"Vocabulary {_} already installed.")
     return vocabs_to_process
 
 
@@ -140,7 +150,7 @@ def _insert_catalog_entry(iri: str, prefix: str, namespace: str, label: str, lan
         iri=iri,
         prefix=prefix,
         namespace=namespace,
-        date=datetime.now(tz=UTC).date(),
+        date=datetime.now(tz=timezone.utc).date(),
         label=label,
         language=language,
         description="vocabulary imported with cmemc",
@@ -150,7 +160,7 @@ def _insert_catalog_entry(iri: str, prefix: str, namespace: str, label: str, lan
 
 
 def _get_vocabulary_metadata_from_file(
-    file: io.StringIO, namespace_given: bool = False
+    file: io.BytesIO, namespace_given: bool = False
 ) -> dict[str, str]:
     """Get potential graph iri and prefix/namespace from a turtle file."""
     metadata = {"iri": "", "prefix": "", "namespace": ""}
@@ -263,7 +273,13 @@ def list_command(app: ApplicationContext, id_only: bool, filter_: str, raw: bool
             except (KeyError, TypeError):
                 label = _["vocabularyLabel"] if _["vocabularyLabel"] else "[no label given]"
             table.append((iri, label))
-        app.echo_info_table(table, headers=["Vocabulary Graph IRI", "Label"], sort_column=1)
+        app.echo_info_table(
+            table,
+            headers=["Vocabulary Graph IRI", "Label"],
+            sort_column=1,
+            empty_table_message="No installed vocabularies found. "
+            "Use the `vocabulary install` command to install vocabulary from the catalog.",
+        )
 
 
 @click.command(cls=CmemcCommand, name="install")
@@ -315,10 +331,7 @@ def uninstall_command(app: ApplicationContext, iris: tuple[str], all_: bool) ->
     "FILE",
     required=True,
     shell_complete=completion.triple_files,
-    type=click.Path(
-        allow_dash=True,
-        readable=True,
-    ),
+    type=ClickSmartPath(allow_dash=True, readable=True, remote_okay=True),
 )
 @click.option(
     "--namespace",
@@ -345,19 +358,19 @@ def import_command(
     description which correctly uses the `vann:preferredNamespacePrefix` and
     `vann:preferredNamespaceUri` properties.
     """
-    string_buffer = io.StringIO()
-    with click.open_file(file, "rb") as file_handle:
-        string_buffer.write(file_handle.read().decode("utf-8"))
-    string_buffer.seek(0)
+    _buffer = io.BytesIO()
+    with ClickSmartPath.open(file) as file_handle:
+        _buffer.write(file_handle.read())
+    _buffer.seek(0)
 
     # fetch metadata
     if namespace != (None, None):
         _validate_namespace(app, namespace)
-        meta_data = _get_vocabulary_metadata_from_file(string_buffer, True)
+        meta_data = _get_vocabulary_metadata_from_file(_buffer, True)
         meta_data["prefix"] = namespace[0]  # type: ignore[assignment]
         meta_data["namespace"] = namespace[1]  # type: ignore[assignment]
     else:
-        meta_data = _get_vocabulary_metadata_from_file(string_buffer, False)
+        meta_data = _get_vocabulary_metadata_from_file(_buffer, False)
     iri = meta_data["iri"]
 
     success_message = "done"
@@ -368,8 +381,8 @@ def import_command(
             raise ValueError(f"Proposed graph {iri} does already exist.")
     app.echo_info(f"Import {file} as vocabulary to {iri} ... ", nl=False)
     # upload graph
-    string_buffer.seek(0)
-    graph_api.post_streamed(iri, string_buffer, replace=True)
+    _buffer.seek(0)
+    graph_api.post_streamed(iri, _buffer, replace=True)
 
     # resolve label
     resolved_label_object: dict = resolve([iri], graph=iri)[iri]
@@ -438,7 +451,13 @@ def cache_list_command(app: ApplicationContext, id_only: bool, raw: bool) -> Non
         for vocab in cache_["vocabularies"]:
             table = _transform_cache_to_table(vocab["classes"], table)
             table = _transform_cache_to_table(vocab["properties"], table)
-        app.echo_info_table(table, headers=["IRI", "Type", "Label"], sort_column=0)
+        app.echo_info_table(
+            table,
+            headers=["IRI", "Type", "Label"],
+            sort_column=0,
+            empty_table_message="No cache entries found. "
+            "Use the `vocabulary install` command to install a vocabulary.",
+        )
 
 
 @click.group(cls=CmemcGroup)

cmem_cmemc/commands/workflow.py

@@ -1,14 +1,14 @@
 """workflow commands for cmem command line interface."""
+
 import re
 import sys
 import time
-from datetime import UTC, datetime
-from pathlib import Path
+from datetime import datetime, timezone
 
 import click
 import timeago
 from click import UsageError
-from cmem.cmempy.workflow import get_resource_based_dataset_types, get_workflows
+from cmem.cmempy.workflow import get_workflows
 from cmem.cmempy.workflow.workflow import (
     execute_workflow_io,
     get_workflow_editor_uri,
@@ -26,9 +26,12 @@ from requests import Response
 from requests.status_codes import codes
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.commands.scheduler import scheduler
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
 
 WORKFLOW_FILTER_TYPES = sorted(["project", "regex", "tag", "io"])
 WORKFLOW_LIST_FILTER_HELP_TEXT = (
@@ -40,40 +43,41 @@ WORKFLOW_LIST_FILTER_HELP_TEXT = (
 IO_WARNING_NO_RESULT = "The workflow was executed but produced no result."
 IO_WARNING_NO_OUTPUT_DEFINED = "The workflow was executed, a result was " "received but dropped."
 
-MIME_CSV = "application/x-plugin-csv"
-MIME_XLS = "application/x-plugin-excel"
-MIME_NT = "application/n-triples"
-MIME_JSON = "application/x-plugin-json"
-MIME_XML = "application/xml"
-MIME_FILE = "application/octet-stream"
-MIME_ZIP = "application/x-plugin-multiCsv"
-MIME_ALIGNMENT = "text/alignment"
-MIME_TEXT = "text/plain"
-
-VALID_OUTPUT_EXTENSIONS = {
-    ".csv": MIME_CSV,
-    ".xlsx": MIME_XLS,
-    ".nt": MIME_NT,
-    ".ttl": MIME_NT,
-    ".json": MIME_JSON,
-    ".xml": MIME_XML,
-}
 
-VALID_INPUT_EXTENSIONS = {
-    ".csv": MIME_CSV,
-    ".json": MIME_JSON,
-    ".xml": MIME_XML,
-    ".xlsx": MIME_XLS,
-    ".file": MIME_FILE,
-    ".zip": MIME_ZIP,
-    ".txt": MIME_TEXT,
+FILE_EXTENSIONS_TO_PLUGIN_ID = {
+    ".nt": "file",
+    ".ttl": "file",
+    ".csv": "csv",
+    ".json": "json",
+    ".xml": "xml",
+    ".txt": "text",
+    ".xlsx": "excel",
+    ".zip": "multiCsv",
 }
 
-
-def is_supported_mime_type(mime_type: str) -> bool:
-    """Boolean to determine if a request is multipart or not"""
-    supported_mime_types = get_resource_based_dataset_types()
-    return any(supported_mime_type in mime_type for supported_mime_type in supported_mime_types)
+# Derive valid extensions from FILE_EXTENSIONS_TO_PLUGIN_ID keys
+VALID_EXTENSIONS = list(FILE_EXTENSIONS_TO_PLUGIN_ID.keys())
+PLUGIN_MIME_TYPES = [f"application/x-plugin-{_}" for _ in FILE_EXTENSIONS_TO_PLUGIN_ID.values()]
+# Define additional mime types for input and output
+EXTRA_INPUT_MIME_TYPES = [
+    "application/json",
+    "application/xml",
+    "text/csv",
+]
+
+EXTRA_OUTPUT_MIME_TYPES = [
+    "application/json",
+    "application/xml",
+    "application/n-triples",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "text/csv",
+]
+
+STDOUT_UNSUPPORTED_MIME_TYPES = {
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "excel",
+    "application/x-plugin-excel": "excel",
+    "application/x-plugin-multiCsv": "ZIP",
+}
 
 
 def _get_workflow_tag_labels(workflow_: dict) -> list:
@@ -185,9 +189,10 @@ def _io_check_request(info: dict, input_file: str, output_file: str, output_mime
             "This workflow has a defined output so you need to use the '-o' "
             "parameter to retrieve data from it."
         )
-    if output_mimetype == MIME_XLS and output_file == "-":
+    if output_mimetype in STDOUT_UNSUPPORTED_MIME_TYPES and output_file == "-":
         raise ValueError(
-            "Trying to output an excel document to stdout will fail.\n"
+            f"Trying to output an {STDOUT_UNSUPPORTED_MIME_TYPES[output_mimetype]} "
+            "file to stdout will fail.\n"
             "Please output to a regular file instead "
             "(workflow was not executed)."
         )
@@ -226,11 +231,9 @@ def _io_guess_output(output_file: str) -> str:
     if output_file == "-":
         raise ValueError("Output mime-type not guessable, please use the --output-mimetype option.")
     file_extension = Path(output_file).suffix
-    if file_extension in VALID_OUTPUT_EXTENSIONS and is_supported_mime_type(
-        VALID_OUTPUT_EXTENSIONS[file_extension]
-    ):
-        return VALID_OUTPUT_EXTENSIONS[file_extension]
-    valid_extensions = ", ".join(VALID_OUTPUT_EXTENSIONS.keys())
+    if file_extension in VALID_EXTENSIONS:
+        return f"application/x-plugin-{FILE_EXTENSIONS_TO_PLUGIN_ID[file_extension]}"
+    valid_extensions = ", ".join(VALID_EXTENSIONS)
     raise ValueError(
         f"Files with the extension {file_extension} can not be generated. "
         f"Try one of {valid_extensions}"
@@ -242,11 +245,9 @@ def _io_guess_input(input_file: str) -> str:
     if input_file == "-":
         raise ValueError("Input mime-type not guessable, please use the --output-mimetype option.")
     file_extension = Path(input_file).suffix
-    if file_extension in VALID_INPUT_EXTENSIONS and is_supported_mime_type(
-        VALID_INPUT_EXTENSIONS[file_extension]
-    ):
-        return VALID_INPUT_EXTENSIONS[file_extension]
-    valid_extensions = ", ".join(VALID_INPUT_EXTENSIONS.keys())
+    if file_extension in VALID_EXTENSIONS:
+        return f"application/x-plugin-{FILE_EXTENSIONS_TO_PLUGIN_ID[file_extension]}"
+    valid_extensions = ", ".join(VALID_EXTENSIONS)
     raise ValueError(
         f"Files with the extension {file_extension} can not be processed. "
         f"Try one of {valid_extensions}"
@@ -319,8 +320,8 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
     canceled only exists sometimes
     """
     # prepare human friendly relative time
-    now = datetime.now(tz=UTC)
-    stamp = datetime.fromtimestamp(status["lastUpdateTime"] / 1000, tz=UTC)
+    now = datetime.now(tz=timezone.utc)
+    stamp = datetime.fromtimestamp(status["lastUpdateTime"] / 1000, tz=timezone.utc)
     time_ago = timeago.format(stamp, now, "en")
     status_name = status["statusName"]
     status_message = status["message"]
@@ -338,9 +339,9 @@ def _workflow_echo_status(app: ApplicationContext, status: dict) -> None:
             f"statusName is {status_name}, expecting one of: " "Running, Canceling or Waiting."
         )
     # not running can be Idle or Finished
-    if "failed" in status and status["failed"]:
+    if status.get("failed"):
         app.echo_error(message, nl=True, err=False)
-    elif "cancelled" in status and status["cancelled"]:
+    elif status.get("cancelled"):
         app.echo_warning(message)
     elif status["statusName"] == "Idle":
         app.echo_info(message)
@@ -440,7 +441,7 @@ def execute_command(  # noqa: PLR0913
     "--input",
     "-i",
     "input_file",
-    type=click.Path(allow_dash=False, dir_okay=False, readable=True),
+    type=ClickSmartPath(allow_dash=False, dir_okay=False, readable=True),
     shell_complete=completion.workflow_io_input_files,
     help="From which file the input is taken. If the workflow "
     "has no defined variable input dataset, this option is not allowed.",
@@ -449,7 +450,7 @@ def execute_command(  # noqa: PLR0913
     "--output",
     "-o",
     "output_file",
-    type=click.Path(
+    type=ClickSmartPath(
         allow_dash=False,
         dir_okay=False,
         writable=True,
@@ -465,7 +466,13 @@ def execute_command(  # noqa: PLR0913
     help="Which input format should be processed: If not given, cmemc will "
     "try to guess the mime type based on the file extension or will "
     "fail.",
-    type=click.Choice([*list(VALID_INPUT_EXTENSIONS.values()), "guess"]),
+    type=click.Choice(
+        [
+            *PLUGIN_MIME_TYPES,
+            *EXTRA_INPUT_MIME_TYPES,
+            "guess",
+        ]
+    ),
     default="guess",
 )
 @click.option(
@@ -474,7 +481,13 @@ def execute_command(  # noqa: PLR0913
     "try to guess the mime type based on the file extension or will "
     "fail. In case of an output to stdout, a default mime type "
     "will be used (JSON).",
-    type=click.Choice([*list(VALID_OUTPUT_EXTENSIONS.values()), "guess"]),
+    type=click.Choice(
+        [
+            *PLUGIN_MIME_TYPES,
+            *EXTRA_OUTPUT_MIME_TYPES,
+            "guess",
+        ]
+    ),
     default="guess",
 )
 @click.option(
@@ -498,10 +511,19 @@ def io_command(  # noqa: PLR0913
 ) -> None:
     """Execute a workflow with file input/output.
 
-    With this command, you can execute a workflow that uses variable datasets
+    With this command, you can execute a workflow that uses replaceable datasets
     as input, output or for configuration. Use the input parameter to feed
     data into the workflow. Likewise, use output for retrieval of the workflow
-    result. Workflows without a variable dataset will throw an error.
+    result. Workflows without a replaceable dataset will throw an error.
+
+    Note: Regarding the input dataset configuration - the following rules apply:
+    If autoconfig is enabled ('--autoconfig', the default), the dataset
+    configuration is guessed.
+    If autoconfig is disabled ('--no-autoconfig') and the type of the dataset
+    file is the same as the replaceable dataset in the workflow, the configuration
+    from this dataset is copied.
+    If autoconfig is disabled and the type of the dataset file is different from the
+    replaceable dataset in the workflow, the default config is used.
     """
     project_id, task_id = workflow_id.split(":")
     if output_file and output_mimetype == "guess":
@@ -525,7 +547,6 @@ def io_command(  # noqa: PLR0913
         f"output_mime_type={output_mimetype}, "
         f"auto_config={autoconfig}"
     )
-
     response = execute_workflow_io(
         project_name=project_id,
         task_name=task_id,
@@ -597,7 +618,13 @@ def list_command(
             _["label"],
         ]
         table.append(row)
-    app.echo_info_table(table, headers=["Workflow ID", "Label"], sort_column=1)
+    app.echo_info_table(
+        table,
+        headers=["Workflow ID", "Label"],
+        sort_column=1,
+        empty_table_message="No workflows found. "
+        "Open a project in the web interface and create a new workflow there.",
+    )
 
 
 @click.command(cls=CmemcCommand, name="status")

cmem_cmemc/commands/workspace.py

@@ -1,6 +1,6 @@
 """workspace commands for cmem command line interface."""
+
 import os
-from pathlib import Path
 
 import click
 from cmem.cmempy.workspace import reload_workspace
@@ -9,9 +9,12 @@ from cmem.cmempy.workspace.import_ import import_workspace
 from jinja2 import Template
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.commands.python import python
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
 
 
 @click.command(cls=CmemcCommand, name="export")
@@ -19,7 +22,12 @@ from cmem_cmemc.context import ApplicationContext
     "-o",
     "--overwrite",
     is_flag=True,
-    help="Overwrite existing files. " "This is a dangerous option, so use it with care.",
+    hidden=True,
+)
+@click.option(
+    "--replace",
+    is_flag=True,
+    help="Replace existing files. This is a dangerous option, so use it with care.",
 )
 @click.option(
     "--type",
@@ -49,11 +57,16 @@ from cmem_cmemc.context import ApplicationContext
     "file",
     shell_complete=completion.workspace_files,
     required=False,
-    type=click.Path(writable=True, allow_dash=False, dir_okay=False),
+    type=ClickSmartPath(writable=True, allow_dash=False, dir_okay=False),
 )
 @click.pass_obj
-def export_command(
-    app: ApplicationContext, overwrite: bool, marshalling_plugin: str, template: str, file: str
+def export_command(  # noqa: PLR0913
+    app: ApplicationContext,
+    overwrite: bool,
+    replace: bool,
+    marshalling_plugin: str,
+    template: str,
+    file: str,
 ) -> None:
     """Export the complete workspace (all projects) to a ZIP file.
 
@@ -64,13 +77,19 @@ def export_command(
     The file name is optional and will be generated with by
     the template if absent.
     """
+    if overwrite:
+        replace = overwrite
+        app.echo_warning(
+            "The option --overwrite is deprecated and will be removed with the next major release."
+            " Please use the --replace option instead."
+        )
     if file is None:
         # prepare the template data and create the actual file incl. suffix
         template_data = app.get_template_data()
         file = Template(template).render(template_data) + ".zip"
     file = os.path.normpath(file)
     app.echo_info(f"Export workspace to {file} ... ", nl=False)
-    if Path(file).exists() and overwrite is not True:
+    if Path(file).exists() and replace is not True:
         app.echo_error("file exists")
     else:
         # output directory is created lazy
@@ -95,7 +114,7 @@ def export_command(
 @click.argument(
     "file",
     shell_complete=completion.workspace_files,
-    type=click.Path(readable=True, allow_dash=False, dir_okay=False),
+    type=ClickSmartPath(readable=True, allow_dash=False, dir_okay=False),
 )
 @click.pass_obj
 def import_command(app: ApplicationContext, file: str, marshalling_plugin: str) -> None: