cmem-cmemc 24.1.5__py3-none-any.whl → 24.2.0__py3-none-any.whl

cmem_cmemc/commands/query.py

@@ -4,7 +4,6 @@ import json
 import sys
 from hashlib import sha1
 from json import JSONDecodeError, load
-from pathlib import Path
 from shutil import get_terminal_size
 from time import sleep, time
 from uuid import uuid4
@@ -12,7 +11,12 @@ from uuid import uuid4
 import click
 from click import UsageError
 from click.shell_completion import CompletionItem
-from cmem.cmempy.queries import QUERY_CATALOG, SparqlQuery, cancel_query, get_query_status
+from cmem.cmempy.queries import (
+    QueryCatalog,
+    SparqlQuery,
+    cancel_query,
+    get_query_status,
+)
 from requests import HTTPError
 
 from cmem_cmemc import completion
@@ -26,6 +30,8 @@ from cmem_cmemc.object_list import (
     compare_int_greater_than,
     compare_regex,
 )
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
 from cmem_cmemc.utils import extract_error_message, struct_to_table
 
 QUERY_FILTER_TYPES = sorted(["graph", "status", "slower-than", "type", "regex", "trace-id", "user"])
@@ -52,7 +58,7 @@ class ReplayStatistics:
     query_count: int = 0
     error_count: int = 0
     query_average: float
-    catalog = QUERY_CATALOG
+    catalog = QueryCatalog()
     app: ApplicationContext
 
     def __init__(self, app: ApplicationContext, label: str | None = None):
@@ -162,11 +168,12 @@ class ReplayStatistics:
 
     def create_output(self) -> dict:
         """Create the structure for the output commands."""
-        output = {}
-        for key, value in vars(self).items():
-            if key not in ("current_loop_key", "app"):
-                # ignore some object vars on output
-                output[key] = value
+        # create dict from object but ignore some internal vars on output
+        output = {
+            key: value
+            for (key, value) in dict(vars(self)).items()
+            if key not in ("current_loop_key", "app")
+        }
         loop_average = 0
         loop_minimum = None
         loop_maximum = None
@@ -332,7 +339,7 @@ def list_command(app: ApplicationContext, id_only: bool) -> None:
     Outputs a list of query URIs which can be used as reference for
     the query execute command.
     """
-    queries = QUERY_CATALOG.get_queries().items()
+    queries = QueryCatalog().get_queries().items()
     if id_only:
         # sort dict by short_url - https://docs.python.org/3/howto/sorting.html
         for _, sparql_query in sorted(queries, key=lambda k: k[1].short_url.lower()):
@@ -444,7 +451,7 @@ def execute_command(  # noqa: PLR0913
     app.echo_debug("Parameter: " + str(placeholder))
     for file_or_uri in queries:
         app.echo_debug(f"Start of execution: {file_or_uri} with " f"placeholder {placeholder}")
-        executed_query = QUERY_CATALOG.get_query(file_or_uri, placeholder=placeholder)
+        executed_query = QueryCatalog().get_query(file_or_uri, placeholder=placeholder)
         if executed_query is None:
             raise ValueError(f"{file_or_uri} is neither a (readable) file nor a query URI.")
         app.echo_debug(
@@ -487,7 +494,7 @@ def open_command(app: ApplicationContext, queries: tuple[str, ...]) -> None:
     opening multiple browser tabs.
     """
     for file_or_uri in queries:
-        opened_query = QUERY_CATALOG.get_query(file_or_uri)
+        opened_query = QueryCatalog().get_query(file_or_uri)
         if opened_query is None:
             raise ValueError(f"{file_or_uri} is neither a (readable) file nor a query URI.")
         open_query_uri = opened_query.get_editor_url()
@@ -576,7 +583,7 @@ def status_command(
     "REPLAY_FILE",
     required=True,
     shell_complete=completion.replay_files,
-    type=click.Path(exists=True, allow_dash=False, readable=True, dir_okay=False),
+    type=ClickSmartPath(exists=True, allow_dash=False, readable=True, dir_okay=False),
 )
 @click.option("--raw", is_flag=True, help="Output the execution statistic as raw JSON.")
 @click.option(
@@ -604,7 +611,7 @@ def status_command(
     "of a successful command execution. The output can be stdout "
     "('-') - in this case, the execution statistic output is "
     "oppressed.",
-    type=click.Path(exists=False, allow_dash=True, writable=True, dir_okay=False),
+    type=ClickSmartPath(exists=False, allow_dash=True, writable=True, dir_okay=False),
 )
 @click.option("--run-label", type=click.STRING, help="Optional label of this replay run.")
 @click.pass_obj

cmem_cmemc/commands/store.py

@@ -1,16 +1,21 @@
 """DataPlatform store commands for the cmem command line interface."""
 
 import os
-from pathlib import Path
+from dataclasses import dataclass
 
 import click
 from click import UsageError
 from cmem.cmempy.dp.admin import create_showcase_data, delete_bootstrap_data, import_bootstrap_data
 from cmem.cmempy.dp.admin.backup import get_zip, post_zip
+from cmem.cmempy.dp.workspace import migrate_workspaces
+from cmem.cmempy.health import get_dp_info
 
 from cmem_cmemc import completion
 from cmem_cmemc.commands import CmemcCommand, CmemcGroup
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
+from cmem_cmemc.smart_path import SmartPath as Path
+from cmem_cmemc.utils import validate_zipfile
 
 
 @click.command(cls=CmemcCommand, name="bootstrap")
@@ -93,7 +98,7 @@ def showcase_command(app: ApplicationContext, scale: int, create: bool, delete:
     "BACKUP_FILE",
     shell_complete=completion.graph_backup_files,
     required=True,
-    type=click.Path(writable=True, allow_dash=False, dir_okay=False),
+    type=ClickSmartPath(writable=True, allow_dash=False, dir_okay=False),
 )
 @click.option(
     "--overwrite",
@@ -133,7 +138,11 @@ def export_command(app: ApplicationContext, backup_file: str, overwrite: bool) -
                     app.echo_info(".", nl=False)
                     byte_counter = 0
             app.echo_debug(f"Wrote {overall_byte_counter} bytes to {backup_file}.")
-            app.echo_success(" done")
+            if validate_zipfile(zipfile=backup_file):
+                app.echo_debug(f"{backup_file} successfully validated")
+                app.echo_success(" done")
+            else:
+                app.echo_error(" error (file corrupt)")
 
 
 @click.command(cls=CmemcCommand, name="import")
@@ -141,7 +150,7 @@ def export_command(app: ApplicationContext, backup_file: str, overwrite: bool) -
     "BACKUP_FILE",
     shell_complete=completion.graph_backup_files,
     required=True,
-    type=click.Path(readable=True, exists=True, allow_dash=False, dir_okay=False),
+    type=ClickSmartPath(readable=True, exists=True, allow_dash=False, dir_okay=False),
 )
 @click.pass_obj
 def import_command(app: ApplicationContext, backup_file: str) -> None:
@@ -167,6 +176,66 @@ def import_command(app: ApplicationContext, backup_file: str) -> None:
     app.echo_success(" done")
 
 
+@dataclass
+class CommandResult:
+    """Represents the result of a command execution"""
+
+    data: list
+    headers: list[str]
+    caption: str
+    empty_state_message: str
+
+
+def _migrate_workspaces() -> CommandResult:
+    """Migrate workspace configurations to the current CMEM version."""
+    request = migrate_workspaces()
+    return CommandResult(
+        data=[(iri,) for iri in request],
+        headers=["IRI"],
+        caption="Migrated workspace configurations",
+        empty_state_message="No migrateable workspace configurations found.",
+    )
+
+
+def _get_migrate_workspaces() -> CommandResult:
+    """Retrieve workspace configurations that have been migrated to the current CMEM version."""
+    dp_info = get_dp_info()
+    migratable_workspace_configurations = dp_info["workspaceConfiguration"]["workspacesToMigrate"]
+    return CommandResult(
+        data=[(_["iri"], _["label"]) for _ in migratable_workspace_configurations],
+        headers=["IRI", "LABEL"],
+        caption="Migrateable configurations workspaces",
+        empty_state_message="No migrateable workspace configurations found.",
+    )
+
+
+@click.command("migrate")
+@click.option(
+    "--workspaces",
+    is_flag=True,
+    help="Migrate workspace configurations to the current version.",
+)
+@click.pass_obj
+def migrate_command(app: ApplicationContext, workspaces: bool) -> None:
+    """Migrate configuration resources to the current version.
+
+    This command serves two purposes: (1) When invoked without an option, it lists
+    all migrateable configuration resources. (2) When invoked with the `--workspaces`
+    option, it migrates the workspace configurations to the current version.
+    """
+    result = _migrate_workspaces() if workspaces else _get_migrate_workspaces()
+
+    if result.data:
+        app.echo_info_table(
+            result.data,
+            headers=result.headers,
+            sort_column=0,
+            caption=result.caption,
+        )
+    else:
+        app.echo_success(result.empty_state_message)
+
+
 @click.group(cls=CmemcGroup)
 def store() -> CmemcGroup:  # type: ignore[empty-body]
     """Import, export and bootstrap the knowledge graph store.
@@ -180,3 +249,4 @@ store.add_command(showcase_command)
 store.add_command(bootstrap_command)
 store.add_command(export_command)
 store.add_command(import_command)
+store.add_command(migrate_command)

cmem_cmemc/commands/validation.py

@@ -1,7 +1,11 @@
 """graph validation command group"""
 
+import json
+import sys
 import time
+from collections import Counter
 from datetime import datetime, timezone
+from pathlib import Path
 
 import click
 import requests
@@ -9,6 +13,7 @@ import timeago
 from click import Context, UsageError
 from click.shell_completion import CompletionItem
 from cmem.cmempy.dp.shacl import validation
+from junit_xml import TestCase, TestSuite, to_xml_report_string
 from requests import HTTPError
 from rich.progress import Progress, SpinnerColumn, TaskID, TimeElapsedColumn
 
@@ -24,7 +29,62 @@ from cmem_cmemc.object_list import (
     compare_int_greater_than,
     transform_lower,
 )
-from cmem_cmemc.utils import struct_to_table
+from cmem_cmemc.utils import get_query_text, struct_to_table
+
+
+def _reports_to_junit(reports: list[dict]) -> str:
+    """Create a jUnit XML document from a list of report dictionaries"""
+    test_suites: list[TestSuite] = []
+
+    for report in reports:
+        test_cases: list[TestCase] = []
+        context_graph = report["contextGraphIri"]
+        shape_graph = report["shapeGraphIri"]
+        time_elapsed = (report["executionFinished"] - report["executionStarted"]) / 1000
+        violations: dict[str, list[dict]] = {}
+        for resource in sorted(report["resources"]):
+            # get a list of all tested resources
+            violations[resource] = []
+        average_elapsed_sec = time_elapsed / len(violations)
+        for result in report["results"]:
+            # collection violations per resource
+            resource_iri = result["resourceIri"]
+            violations[resource_iri] = result["violations"]
+        for resource in violations:
+            # create on test case per resource
+            resource_violations = violations[resource]
+            violations_count = len(violations[resource])
+            constraints = Counter(
+                _["reportEntryConstraintMessageTemplate"]["constraintName"]
+                for _ in resource_violations
+            )
+            test_case_name = f"{resource}"
+            if violations_count == 0:
+                test_case_name += " has no violations"
+            if violations_count == 1:
+                test_case_name += f" has 1 violation ({next(iter(constraints.keys()))})"
+            if violations_count > 1:
+                constrains_str = ""
+                for constraint, constraint_count in constraints.items():
+                    constrains_str += f", {constraint_count}x{constraint}"
+                test_case_name += f" has {violations_count} violations ({constrains_str[2:]})"
+
+            test_case = TestCase(
+                name=test_case_name,
+                classname=f"{context_graph} tested with {shape_graph}",
+                elapsed_sec=average_elapsed_sec,
+            )
+            if violations_count > 0:
+                test_case.add_failure_info(output=json.dumps(resource_violations, indent=2))
+            test_cases.append(test_case)
+        test_suite = TestSuite(
+            name=f"Testing {context_graph} with shapes from {shape_graph}.",
+            test_cases=test_cases,
+            id=report["id"],
+            timestamp=report["executionFinished"],
+        )
+        test_suites.append(test_suite)
+    return str(to_xml_report_string(test_suites, encoding="utf-8"))
 
 
 def get_sorted_validations_list(ctx: Context) -> list[dict]:  # noqa: ARG001
@@ -179,6 +239,20 @@ def _complete_running_batch_validations(
     return _finalize_completion(candidates=options, incomplete=incomplete)
 
 
+def _complete_finished_batch_validations(
+    ctx: click.Context,  # noqa: ARG001
+    param: click.Argument,  # noqa: ARG001
+    incomplete: str,
+) -> list[CompletionItem]:
+    """Provide completion for finished batch validations"""
+    options = [
+        _get_batch_validation_option(_)
+        for _ in validation.get_all_aggregations()
+        if _["state"] == validation.STATUS_FINISHED
+    ]
+    return _finalize_completion(candidates=options, incomplete=incomplete)
+
+
 def show_process_summary(app: ApplicationContext, process_id: str) -> None:
     """Show summary of the validation process"""
     app.echo_info_table(
@@ -307,6 +381,12 @@ def _get_violation_count(process_data: dict) -> str:
 
 @click.command(cls=CmemcCommand, name="execute")
 @click.argument("iri", type=click.STRING, shell_complete=completion.graph_uris)
+@click.option(
+    "--wait",
+    is_flag=True,
+    help="Wait until the process is finished. When using this option without the "
+    "`--id-only` flag, it will enable a progress bar and a summary view.",
+)
 @click.option(
     "--shape-graph",
     shell_complete=completion.graph_uris,
@@ -315,16 +395,39 @@ def _get_violation_count(process_data: dict) -> str:
     help="The shape catalog used for validation.",
 )
 @click.option(
-    "--id-only",
+    "--query",
+    shell_complete=completion.remote_queries_and_sparql_files,
+    help="SPARQL query to select the resources which you want to validate from "
+    "the data graph. "
+    "Can be provided as a local file or as a query catalog IRI. "
+    "[default: all typed resources]",
+)
+@click.option(
+    "--result-graph",
+    shell_complete=completion.writable_graph_uris,
+    help="(Optionally) write the validation results to a Knowledge Graph. " "[default: None]",
+)
+@click.option(
+    "--replace",
     is_flag=True,
-    help="Return the validation process identifier only. "
-    "This is useful for piping the ID into other commands.",
+    default=False,
+    help="Replace the result graph instead of just adding the new results. "
+    "This is a dangerous option, so use it with care!",
 )
 @click.option(
-    "--wait",
+    "--ignore-graph",
+    shell_complete=completion.ignore_graph_uris,
+    type=click.STRING,
+    multiple=True,
+    help="A set of data graph IRIs which are not queried in the resource selection. "
+    "This option is useful for validating only parts of an integration graph "
+    "which imports other graphs.",
+)
+@click.option(
+    "--id-only",
     is_flag=True,
-    help="Wait until the process is finished. When using this option without the "
-    "`--id-only` flag, it will enable a progress bar and a summary view.",
+    help="Return the validation process identifier only. "
+    "This is useful for piping the ID into other commands.",
 )
 @click.option(
     "--polling-interval",
@@ -339,17 +442,31 @@ def execute_command(  # noqa: PLR0913
     app: ApplicationContext,
     iri: str,
     shape_graph: str,
+    query: str,
+    result_graph: str,
+    replace: bool,
+    ignore_graph: list[str],
     id_only: bool,
     wait: bool,
     polling_interval: int,
 ) -> None:
     """Start a new validation process.
 
-    Validation is performed on all typed resources of a data / context graph (IRI).
-    Each resource is validated against all applicable node shapes from a
-    selected shape catalog graph (and its sub-graphs).
+    Validation is performed on all typed resources of the data / context graph
+    (and its sub-graphs). Each resource is validated against all applicable node
+    shapes from the shape catalog.
     """
-    process_id = validation.start(context_graph=iri, shape_graph=shape_graph)
+    query_str = None
+    if query:
+        query_str = get_query_text(query, {"resource"})
+    process_id = validation.start(
+        context_graph=iri,
+        shape_graph=shape_graph,
+        query=query_str,
+        result_graph=result_graph,
+        replace=replace,
+        ignore_graph=ignore_graph,
+    )
     if wait:
         _wait_for_process_completion(
             app=app, process_id=process_id, use_rich=not id_only, polling_interval=polling_interval
@@ -532,6 +649,94 @@ def cancel_command(app: ApplicationContext, process_id: str) -> None:
     app.echo_success("cancelled")
 
 
+@click.command(cls=CmemcCommand, name="export")
+@click.argument(
+    "process_ids", nargs=-1, type=click.STRING, shell_complete=_complete_finished_batch_validations
+)
+@click.option(
+    "--output-file",
+    type=click.Path(writable=True, allow_dash=False, dir_okay=False),
+    default="report.xml",
+    show_default=True,
+    help="Export the report to this file. Existing files will be overwritten.",
+)
+@click.option(
+    "--exit-1",
+    type=click.Choice(["never", "error"]),
+    default="error",
+    show_default=True,
+    help="Specify, when this command returns with exit code 1. Available options are "
+    "'never' (exit 0, even if there are violations in the reports), "
+    "'error' (exit 1 if there is at least one violation in a report).), ",
+)
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["JSON", "XML"], case_sensitive=True),
+    default="XML",
+    help="Export either the plain JSON report or a distilled jUnit XML report.",
+    show_default=True,
+)
+@click.pass_context
+def export_command(
+    ctx: Context, process_ids: tuple[str], output_file: str, exit_1: str, format_: str
+) -> None:
+    """Export a report of finished validations.
+
+    This command exports a jUnit XML or JSON report in order to process
+    them somewhere else (e.g. a CI pipeline).
+
+    You can export a single report of multiple validation processes.
+
+    For jUnit XML: Each validation process result will be transformed to
+    a single test suite. All violations of one resource in a result will be
+    collected and attached to a single test case in that test suite.
+
+    Note: Validation processes IDs can be listed with the `graph validation list`
+    command, or by utilizing the tab completion of this command.
+    """
+    if len(process_ids) == 0:
+        raise UsageError("This command needs at least one validation process ID.")
+    app: ApplicationContext = ctx.obj
+    process_ids_to_test = {_: True for _ in process_ids}
+    overall_violations = 0
+    overall_resources = 0
+    for _ in validation.get_all_aggregations():
+        if _["id"] in process_ids_to_test:
+            if _["state"] != "FINISHED":
+                raise UsageError(f"Validation process with ID '{_['id']}' is still running.")
+            del process_ids_to_test[_["id"]]
+            overall_violations += int(_["violationsCount"])
+            overall_resources += int(_["resourceProcessedCount"])
+    if len(process_ids_to_test) > 0:
+        raise UsageError(
+            "Validation processes with the following IDs not known (anymore): "
+            + ", ".join(process_ids_to_test)
+        )
+    reports = []
+    for process_id in process_ids:
+        report = validation.get(batch_id=process_id)
+        reports.append(report)
+    app.echo_info(
+        f"Export of {len(reports)} validation report(s) with"
+        f" {overall_violations} violations in {overall_resources} resources"
+        f" to {output_file} ... ",
+        nl=False,
+    )
+    with Path(output_file, mode="w", encoding="utf-8") as file:
+        if format_ == "XML":
+            file.write_text(_reports_to_junit(reports))
+        if format_ == "JSON":
+            file.write_text(json.dumps(reports, indent=2))
+    app.echo_success("done")
+    if exit_1 == "error" and overall_violations > 0:
+        app.echo_error(
+            "Exit 1 since violations where found in the reports "
+            "(can be suppressed with '--exit-1 never')."
+        )
+        sys.exit(1)
+
+
 @click.group(cls=CmemcGroup, name="validation")
 def validation_group() -> CmemcGroup:  # type: ignore[empty-body]
     """Validate resources in a graph.
@@ -552,3 +757,4 @@ validation_group.add_command(execute_command)
 validation_group.add_command(list_command)
 validation_group.add_command(inspect_command)
 validation_group.add_command(cancel_command)
+validation_group.add_command(export_command)

cmem_cmemc/commands/vocabulary.py

@@ -24,6 +24,7 @@ from six.moves.urllib.parse import quote
 from cmem_cmemc import completion
 from cmem_cmemc.commands import CmemcCommand, CmemcGroup
 from cmem_cmemc.context import ApplicationContext
+from cmem_cmemc.parameter_types.path import ClickSmartPath
 
 GET_ONTOLOGY_IRI_QUERY = """
 PREFIX owl: <http://www.w3.org/2002/07/owl#>
@@ -72,22 +73,29 @@ def _validate_vocabs_to_process(iris: tuple[str], filter_: str, all_flag: bool)
     list is without duplicates, and validated if they exist
     """
     if iris == () and not all_flag:
-        raise ValueError(
+        raise click.UsageError(
             "Either specify at least one vocabulary IRI "
             "or use the --all option to process over all vocabularies."
         )
-    vocabs_to_process = list(iris)
-    all_vocabs = [_["iri"] for _ in get_vocabularies(filter_=filter_)]
+    all_vocabs = {_["iri"]: _ for _ in get_vocabularies()}
     if all_flag:
-        # in case --all is given, all vocabs are processed
-        vocabs_to_process = all_vocabs
-    # avoid double removal
-    vocabs_to_process = list(set(vocabs_to_process))
-
-    # test if one of the vocabs does not exist
+        # in case --all is given, all installable / installed vocabs are processed
+        if filter_ == "installed":  # uninstall command
+            return [_ for _ in all_vocabs if all_vocabs[_]["installed"]]
+        # install command
+        return [_ for _ in all_vocabs if not all_vocabs[_]["installed"]]
+
+    vocabs_to_process = list(set(iris))  # avoid double removal / installation
+    # test if one of the vocabs does not exist or is already installed / not installed
     for _ in vocabs_to_process:
-        if _ not in all_vocabs:
-            raise ValueError(f"Vocabulary {_} does not exist.")
+        # uninstall command
+        if filter_ == "installed" and (_ not in all_vocabs or not all_vocabs[_]["installed"]):
+            raise click.UsageError(f"Vocabulary {_} not installed.")
+        if filter_ == "installable":  # install command
+            if _ not in all_vocabs:
+                raise click.UsageError(f"Vocabulary {_} does not exist.")
+            if all_vocabs[_]["installed"]:
+                raise click.UsageError(f"Vocabulary {_} already installed.")
     return vocabs_to_process
 
 
@@ -316,10 +324,7 @@ def uninstall_command(app: ApplicationContext, iris: tuple[str], all_: bool) ->
     "FILE",
     required=True,
     shell_complete=completion.triple_files,
-    type=click.Path(
-        allow_dash=True,
-        readable=True,
-    ),
+    type=ClickSmartPath(allow_dash=True, readable=True, remote_okay=True),
 )
 @click.option(
     "--namespace",
@@ -347,7 +352,7 @@ def import_command(
     `vann:preferredNamespaceUri` properties.
     """
     _buffer = io.BytesIO()
-    with click.open_file(file, "rb") as file_handle:
+    with ClickSmartPath.open(file) as file_handle:
         _buffer.write(file_handle.read())
     _buffer.seek(0)