cmem-cmemc 24.2.0rc1__py3-none-any.whl → 24.3.0rc1__py3-none-any.whl

cmem_cmemc/commands/validation.py

@@ -1,20 +1,25 @@
 """graph validation command group"""
+
+import json
+import sys
 import time
-from datetime import UTC, datetime
+from collections import Counter
+from datetime import datetime, timezone
+from pathlib import Path
 
 import click
-import requests
 import timeago
 from click import Context, UsageError
 from click.shell_completion import CompletionItem
+from cmem.cmempy.config import get_cmem_base_uri
 from cmem.cmempy.dp.shacl import validation
 from junit_xml import TestCase, TestSuite, to_xml_report_string
-from requests import HTTPError
 from rich.progress import Progress, SpinnerColumn, TaskID, TimeElapsedColumn
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
-from cmem_cmemc.completion import _finalize_completion
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
+from cmem_cmemc.completion import finalize_completion
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.exceptions import ServerError
 from cmem_cmemc.object_list import (
@@ -24,27 +29,64 @@ from cmem_cmemc.object_list import (
     compare_int_greater_than,
     transform_lower,
 )
-from cmem_cmemc.utils import struct_to_table
-
-
-def _report_to_junit(report: dict) -> str:
-    """Create a jUnit XML document from a report dictionary"""
-    test_cases: list[TestCase] = []
-
-    for result in report["results"]:
-        resource_iri = result["resourceIri"]
-        for _ in result["violations"]:
-            constraint_name = _["reportEntryConstraintMessageTemplate"]["constraintName"]
-            message = _["messages"][0]["value"]
-            new_case = TestCase(
-                name=f"{constraint_name} @ {resource_iri}",
-                url=resource_iri,
-                stdout=message,
-                category=_["severity"],
+from cmem_cmemc.string_processor import GraphLink, ResourceLink, TimeAgo
+from cmem_cmemc.title_helper import TitleHelper
+from cmem_cmemc.utils import get_query_text, struct_to_table
+
+
+def _reports_to_junit(reports: list[dict]) -> str:
+    """Create a jUnit XML document from a list of report dictionaries"""
+    test_suites: list[TestSuite] = []
+
+    for report in reports:
+        test_cases: list[TestCase] = []
+        context_graph = report["contextGraphIri"]
+        shape_graph = report["shapeGraphIri"]
+        time_elapsed = (report["executionFinished"] - report["executionStarted"]) / 1000
+        violations: dict[str, list[dict]] = {}
+        for resource in sorted(report["resources"]):
+            # get a list of all tested resources
+            violations[resource] = []
+        average_elapsed_sec = time_elapsed / len(violations)
+        for result in report["results"]:
+            # collection violations per resource
+            resource_iri = result["resourceIri"]
+            violations[resource_iri] = result["violations"]
+        for resource in violations:
+            # create on test case per resource
+            resource_violations = violations[resource]
+            violations_count = len(violations[resource])
+            constraints = Counter(
+                _["reportEntryConstraintMessageTemplate"]["constraintName"]
+                for _ in resource_violations
             )
-            test_cases.append(new_case)
-    test_suite = TestSuite("eccenca Corporate Memory Graph Validation", test_cases)
-    return str(to_xml_report_string([test_suite], encoding="utf-8"))
+            test_case_name = f"{resource}"
+            if violations_count == 0:
+                test_case_name += " has no violations"
+            if violations_count == 1:
+                test_case_name += f" has 1 violation ({next(iter(constraints.keys()))})"
+            if violations_count > 1:
+                constrains_str = ""
+                for constraint, constraint_count in constraints.items():
+                    constrains_str += f", {constraint_count}x{constraint}"
+                test_case_name += f" has {violations_count} violations ({constrains_str[2:]})"
+
+            test_case = TestCase(
+                name=test_case_name,
+                classname=f"{context_graph} tested with {shape_graph}",
+                elapsed_sec=average_elapsed_sec,
+            )
+            if violations_count > 0:
+                test_case.add_failure_info(output=json.dumps(resource_violations, indent=2))
+            test_cases.append(test_case)
+        test_suite = TestSuite(
+            name=f"Testing {context_graph} with shapes from {shape_graph}.",
+            test_cases=test_cases,
+            id=report["id"],
+            timestamp=report["executionFinished"],
+        )
+        test_suites.append(test_suite)
+    return str(to_xml_report_string(test_suites, encoding="utf-8"))
 
 
 def get_sorted_validations_list(ctx: Context) -> list[dict]:  # noqa: ARG001
@@ -144,17 +186,22 @@ violations_list = ObjectList(
             name="severity", description="Filter list by severity.", property_key="severity"
         ),
         DirectValuePropertyFilter(
-            name="resource", description="Filter list by resource IRI.", property_key="resourceIri"
+            name="resource",
+            description="Filter list by resource IRI.",
+            property_key="resourceIri",
+            title_helper=TitleHelper(),
         ),
         DirectListPropertyFilter(
             name="node-shape",
             description="Filter list by node shape IRI.",
             property_key="nodeShapes",
+            title_helper=TitleHelper(),
         ),
         DirectValuePropertyFilter(
-            name="property-shape",
-            description="Filter list by property shape IRI.",
+            name="source",
+            description="Filter list by constraint source.",
             property_key="source",
+            title_helper=TitleHelper(),
         ),
     ],
 )
@@ -165,8 +212,8 @@ def _get_batch_validation_option(validation_: dict) -> tuple[str, str]:
     id_ = validation_["id"]
     state = validation_["state"]
     graph = validation_["contextGraphIri"]
-    stamp = datetime.fromtimestamp(validation_["executionStarted"] / 1000, tz=UTC)
-    time_ago = timeago.format(stamp, datetime.now(tz=UTC))
+    stamp = datetime.fromtimestamp(validation_["executionStarted"] / 1000, tz=timezone.utc)
+    time_ago = timeago.format(stamp, datetime.now(tz=timezone.utc))
     resources = _get_resource_count(validation_)
     violations = _get_violation_count(validation_)
     return (
@@ -182,7 +229,7 @@ def _complete_all_batch_validations(
 ) -> list[CompletionItem]:
     """Provide completion for batch validation"""
     options = [_get_batch_validation_option(_) for _ in validation.get_all_aggregations()]
-    return _finalize_completion(candidates=options, incomplete=incomplete)
+    return finalize_completion(candidates=options, incomplete=incomplete)
 
 
 def _complete_running_batch_validations(
@@ -196,10 +243,24 @@ def _complete_running_batch_validations(
         for _ in validation.get_all_aggregations()
         if _["state"] == validation.STATUS_RUNNING
     ]
-    return _finalize_completion(candidates=options, incomplete=incomplete)
+    return finalize_completion(candidates=options, incomplete=incomplete)
+
+
+def _complete_finished_batch_validations(
+    ctx: click.Context,  # noqa: ARG001
+    param: click.Argument,  # noqa: ARG001
+    incomplete: str,
+) -> list[CompletionItem]:
+    """Provide completion for finished batch validations"""
+    options = [
+        _get_batch_validation_option(_)
+        for _ in validation.get_all_aggregations()
+        if _["state"] == validation.STATUS_FINISHED
+    ]
+    return finalize_completion(candidates=options, incomplete=incomplete)
 
 
-def show_process_summary(app: ApplicationContext, process_id: str) -> None:
+def _print_process_summary(app: ApplicationContext, process_id: str) -> None:
     """Show summary of the validation process"""
     app.echo_info_table(
         struct_to_table(validation.get_aggregation(batch_id=process_id)),
@@ -276,33 +337,64 @@ def _wait_for_process_completion(
     return state.status
 
 
-def _get_violation_table(violations: list[dict]) -> tuple[list, list]:
-    """Get violation table from batch validation result"""
+def _print_violation_table(
+    app: ApplicationContext, data_graph: str, shape_graph: str, violations: list[dict]
+) -> None:
+    """Print violation table from batch validation result"""
+    # fetch titles
+    resources = []
+    for violation in violations:
+        resources.append(str(violation.get("resourceIri")))
+        resources.extend(violation.get("nodeShapes", []))
+    title_helper = TitleHelper()
+    title_helper.get(resources)
+
+    # prepare link helper
+    resource_link = ResourceLink(graph_iri=data_graph, title_helper=title_helper)
+    shape_link = ResourceLink(graph_iri=shape_graph, title_helper=title_helper)
+
     table = []
     for violation in violations:
-        resource_iri: str = str(violation.get("resourceIri"))
+        combined_cell = ""
+
         path = violation.get("path", None)
-        constraint_name = violation.get("constraintName", "UNKNOWN")
+        if path is not None:
+            combined_cell = f"Path: {path}\n"
+
+        source = violation.get("source", None)
+        if source is not None:
+            combined_cell = f"{combined_cell}Source: {shape_link.process(text=source)}"
+
         node_shapes = violation.get("nodeShapes", [])
+        if len(node_shapes) == 1:
+            combined_cell = f"{combined_cell}\nNodeShape: {shape_link.process(text=node_shapes[0])}"
+        if len(node_shapes) > 1:
+            combined_cell = f"{combined_cell}\nNodeShapes:"
+            for node_shape in node_shapes:
+                combined_cell = f"{combined_cell}\n - {shape_link.process(text=node_shape)}"
+
         text = violation["messages"][0]["value"]  # default: use the text of the first message
         for message in violation["messages"]:
             # look for en non non-lang messages to use
             if message["lang"] == "" or message["lang"] == "en":
                 text = str(message["value"])
                 break
-        cell = ""
-        if path is not None:
-            cell = f"Path: {path}"
-        if len(node_shapes) == 1:
-            cell = f"{cell}\nNodeShape: {node_shapes[0]}"
-        if len(node_shapes) > 1:
-            cell = f"{cell}\nNodeShapes:"
-            for node_shape in node_shapes:
-                cell = f"{cell}\n - {node_shape}"
-        cell = f"{cell}\nMessage: {text}"
-        row = [resource_iri, constraint_name, cell]
+        combined_cell = f"{combined_cell}\nMessage: {text}"
+
+        row = [
+            resource_link.process(text=str(violation.get("resourceIri"))),
+            violation.get("constraintName", "UNKNOWN"),
+            combined_cell,
+        ]
         table.append(row)
-    return table, ["Resource IRI", "Constraint", "Details"]
+
+    app.echo_info_table(
+        table,
+        headers=["Resource", "Constraint", "Details"],
+        sort_column=0,
+        caption="Violation List",
+        empty_table_message="No violations found.",
+    )
 
 
 def _get_resource_count(batch_validation: dict) -> str:
@@ -316,7 +408,7 @@ def _get_resource_count(batch_validation: dict) -> str:
 
 def _get_violation_count(process_data: dict) -> str:
     """Get violation count from validation report"""
-    if process_data.get("executionStarted", None) is None:
+    if process_data.get("executionStarted") is None:
         return "-"
     resources = str(process_data.get("resourcesWithViolationsCount", "0"))
     violations = str(process_data.get("violationsCount", "0"))
@@ -327,13 +419,48 @@ def _get_violation_count(process_data: dict) -> str:
 
 @click.command(cls=CmemcCommand, name="execute")
 @click.argument("iri", type=click.STRING, shell_complete=completion.graph_uris)
+@click.option(
+    "--wait",
+    is_flag=True,
+    help="Wait until the process is finished. When using this option without the "
+    "`--id-only` flag, it will enable a progress bar and a summary view.",
+)
 @click.option(
     "--shape-graph",
-    shell_complete=completion.graph_uris,
+    shell_complete=completion.graph_uris_skip_check,
     default="https://vocab.eccenca.com/shacl/",
     show_default=True,
     help="The shape catalog used for validation.",
 )
+@click.option(
+    "--query",
+    shell_complete=completion.remote_queries_and_sparql_files,
+    help="SPARQL query to select the resources which you want to validate from "
+    "the data graph. "
+    "Can be provided as a local file or as a query catalog IRI. "
+    "[default: all typed resources]",
+)
+@click.option(
+    "--result-graph",
+    shell_complete=completion.writable_graph_uris,
+    help="(Optionally) write the validation results to a Knowledge Graph. " "[default: None]",
+)
+@click.option(
+    "--replace",
+    is_flag=True,
+    default=False,
+    help="Replace the result graph instead of just adding the new results. "
+    "This is a dangerous option, so use it with care!",
+)
+@click.option(
+    "--ignore-graph",
+    shell_complete=completion.ignore_graph_uris,
+    type=click.STRING,
+    multiple=True,
+    help="A set of data graph IRIs which are not queried in the resource selection. "
+    "This option is useful for validating only parts of an integration graph "
+    "which imports other graphs.",
+)
 @click.option(
     "--id-only",
     is_flag=True,
@@ -341,10 +468,9 @@ def _get_violation_count(process_data: dict) -> str:
     "This is useful for piping the ID into other commands.",
 )
 @click.option(
-    "--wait",
+    "--inspect",
     is_flag=True,
-    help="Wait until the process is finished. When using this option without the "
-    "`--id-only` flag, it will enable a progress bar and a summary view.",
+    help="Return the list of violations instead of the summary (includes --wait).",
 )
 @click.option(
     "--polling-interval",
@@ -354,30 +480,53 @@ def _get_violation_count(process_data: dict) -> str:
     help="How many seconds to wait between status polls. Status polls are"
     " cheap, so a higher polling interval is most likely not needed.",
 )
-@click.pass_obj
+@click.pass_context
 def execute_command(  # noqa: PLR0913
-    app: ApplicationContext,
+    ctx: Context,
     iri: str,
     shape_graph: str,
+    query: str,
+    result_graph: str,
+    replace: bool,
+    ignore_graph: list[str],
     id_only: bool,
     wait: bool,
+    inspect: bool,
     polling_interval: int,
 ) -> None:
     """Start a new validation process.
 
-    Validation is performed on all typed resources of a data / context graph (IRI).
-    Each resource is validated against all applicable node shapes from a
-    selected shape catalog graph (and its sub-graphs).
+    Validation is performed on all typed resources of the data / context graph
+    (and its sub-graphs). Each resource is validated against all applicable node
+    shapes from the shape catalog.
     """
-    process_id = validation.start(context_graph=iri, shape_graph=shape_graph)
-    if wait:
+    app: ApplicationContext = ctx.obj
+    if id_only and inspect:
+        raise UsageError(
+            "Output can be the summary (default), the process ID (--id-only) "
+            "or the violation list (--inspect)."
+        )
+    process_id = validation.start(
+        context_graph=iri,
+        shape_graph=shape_graph,
+        query=get_query_text(query, {"resource"}) if query else None,
+        result_graph=result_graph,
+        replace=replace,
+        ignore_graph=ignore_graph,
+    )
+    if wait or inspect:
         _wait_for_process_completion(
             app=app, process_id=process_id, use_rich=not id_only, polling_interval=polling_interval
         )
     if id_only:
         app.echo_info(process_id)
         return
-    show_process_summary(process_id=process_id, app=app)
+    if inspect:
+        ctx.params["process_id"] = process_id
+        data = violations_list.apply_filters(ctx=ctx, filter_=[])
+        _print_violation_table(app=app, data_graph=iri, shape_graph=shape_graph, violations=data)
+        return
+    _print_process_summary(process_id=process_id, app=app)
 
 
 @click.command(cls=CmemcCommand, name="list")
@@ -418,32 +567,25 @@ def list_command(ctx: Context, filter_: tuple[tuple[str, str]], id_only: bool, r
             app.echo_info(_["id"])
         return
 
-    if len(validations) == 0:
-        app.echo_warning(
-            "No validation processes found. "
-            "Use `graph validation execute` to start a new validation process."
-        )
-        return
-
     # output a user table
     table = []
     for _ in validations:
-        if "executionStarted" in _ and _["executionStarted"] is not None:
-            stamp = datetime.fromtimestamp(_["executionStarted"] / 1000, tz=UTC)
-            time_ago = timeago.format(stamp, datetime.now(tz=UTC))
-        else:
-            time_ago = f"{_['state']}"
         row = [
             _["id"],
             _["state"],
-            time_ago,
+            _.get("executionStarted", None),
             _["contextGraphIri"],
             _get_resource_count(_),
             _get_violation_count(_),
         ]
         table.append(row)
     app.echo_info_table(
-        table, headers=["ID", "Status", "Started", "Graph", "Resources", "Violations"]
+        table,
+        headers=["ID", "Status", "Started", "Graph", "Resources", "Violations"],
+        caption=f"Validation Processes of {get_cmem_base_uri()}",
+        cell_processing={2: TimeAgo(), 3: GraphLink()},
+        empty_table_message="No validation processes found. "
+        "Use `graph validation execute` to start a new validation process.",
     )
 
 
@@ -499,7 +641,7 @@ def inspect_command(  # noqa: PLR0913
         if raw:
             app.echo_info_json(validation.get_aggregation(batch_id=process_id))
         else:
-            show_process_summary(app=app, process_id=process_id)
+            _print_process_summary(app=app, process_id=process_id)
         return
 
     data = violations_list.apply_filters(ctx=ctx, filter_=filter_)
@@ -517,12 +659,15 @@ def inspect_command(  # noqa: PLR0913
             "The given validation process does not have any violations - "
             "I will show the summary instead."
         )
-        show_process_summary(app=app, process_id=process_id)
+        _print_process_summary(app=app, process_id=process_id)
     else:
-        messages_table, messages_header = _get_violation_table(violations=data)
-        if len(messages_table) > 0:
-            app.echo_info("")
-            app.echo_info_table(messages_table, headers=messages_header, sort_column=0)
+        process_data = validation.get_aggregation(batch_id=process_id)
+        _print_violation_table(
+            app=app,
+            violations=data,
+            shape_graph=process_data["shapeGraphIri"],
+            data_graph=process_data["contextGraphIri"],
+        )
 
 
 @click.command(cls=CmemcCommand, name="cancel")
@@ -535,58 +680,104 @@ def cancel_command(app: ApplicationContext, process_id: str) -> None:
     processes, use the `graph validation list` command with the option
     `--filter status running`, or utilize the tab completion of this command.
     """
-    try:
-        graph_validation = validation.get_aggregation(process_id)
-    except HTTPError as error:
-        if error.response is not None and error.response.status_code == requests.codes.not_found:
-            raise click.UsageError(
-                f"Graph validation process with ID {process_id} does not exist."
-            ) from error
-    else:
-        if graph_validation["state"] != validation.STATUS_RUNNING:
-            raise click.UsageError(
-                f"Graph validation process with ID {process_id} is not a running anymore."
-            )
-    app.echo_info(f"Graph validation process with ID {process_id} ... ", nl=False)
+    all_status = {_["id"]: _["state"] for _ in validation.get_all_aggregations()}
+    if process_id not in all_status:
+        raise UsageError(f"Validation process with ID '{process_id}' is not known (anymore).")
+    if all_status[process_id] != validation.STATUS_RUNNING:
+        raise click.UsageError(
+            f"Validation process with ID '{process_id}' is not a running anymore."
+        )
+    app.echo_info(f"Validation process with ID '{process_id}' ... ", nl=False)
     validation.cancel(batch_id=process_id)
     app.echo_success("cancelled")
 
 
 @click.command(cls=CmemcCommand, name="export")
-@click.argument("process_id", type=click.STRING, shell_complete=_complete_all_batch_validations)
+@click.argument(
+    "process_ids", nargs=-1, type=click.STRING, shell_complete=_complete_finished_batch_validations
+)
+@click.option(
+    "--output-file",
+    type=click.Path(writable=True, allow_dash=False, dir_okay=False),
+    default="report.xml",
+    show_default=True,
+    help="Export the report to this file. Existing files will be overwritten.",
+)
+@click.option(
+    "--exit-1",
+    type=click.Choice(["never", "error"]),
+    default="error",
+    show_default=True,
+    help="Specify, when this command returns with exit code 1. Available options are "
+    "'never' (exit 0, even if there are violations in the reports), "
+    "'error' (exit 1 if there is at least one violation in a report).), ",
+)
 @click.option(
     "--format",
     "format_",
     type=click.Choice(["JSON", "XML"], case_sensitive=True),
-    default="JSON",
+    default="XML",
     help="Export either the plain JSON report or a distilled jUnit XML report.",
     show_default=True,
 )
 @click.pass_context
-def export_command(ctx: Context, process_id: str, format_: str) -> None:
-    """Export the report of a finished validation process to a file.
+def export_command(
+    ctx: Context, process_ids: tuple[str], output_file: str, exit_1: str, format_: str
+) -> None:
+    """Export a report of finished validations.
 
-    This command exports JSON documents or jUnit XML reports in order to process
+    This command exports a jUnit XML or JSON report in order to process
     them somewhere else (e.g. a CI pipeline).
 
+    You can export a single report of multiple validation processes.
+
+    For jUnit XML: Each validation process result will be transformed to
+    a single test suite. All violations of one resource in a result will be
+    collected and attached to a single test case in that test suite.
+
     Note: Validation processes IDs can be listed with the `graph validation list`
     command, or by utilizing the tab completion of this command.
     """
+    if len(process_ids) == 0:
+        raise UsageError("This command needs at least one validation process ID.")
     app: ApplicationContext = ctx.obj
-    process = None
+    process_ids_to_test = {_: True for _ in process_ids}
+    overall_violations = 0
+    overall_resources = 0
     for _ in validation.get_all_aggregations():
-        if _["id"] == process_id:
-            process = _
-    if process is None:
-        raise UsageError(f"Validation process with ID '{process_id}' is not known (anymore).")
-    if process["state"] != "FINISHED":
-        raise UsageError(f"Validation process with ID '{process_id}' is still running.")
-
-    report = validation.get(batch_id=process_id)
-    if format_ == "JSON":
-        app.echo_info_json(report)
-    if format_ == "XML":
-        app.echo_info_xml(_report_to_junit(report))
+        if _["id"] in process_ids_to_test:
+            if _["state"] != "FINISHED":
+                raise UsageError(f"Validation process with ID '{_['id']}' is still running.")
+            del process_ids_to_test[_["id"]]
+            overall_violations += int(_["violationsCount"])
+            overall_resources += int(_["resourceProcessedCount"])
+    if len(process_ids_to_test) > 0:
+        raise UsageError(
+            "Validation processes with the following IDs not known (anymore): "
+            + ", ".join(process_ids_to_test)
+        )
+    reports = []
+    for process_id in process_ids:
+        report = validation.get(batch_id=process_id)
+        reports.append(report)
+    app.echo_info(
+        f"Export of {len(reports)} validation report(s) with"
+        f" {overall_violations} violations in {overall_resources} resources"
+        f" to {output_file} ... ",
+        nl=False,
+    )
+    with Path(output_file, mode="w", encoding="utf-8") as file:
+        if format_ == "XML":
+            file.write_text(_reports_to_junit(reports))
+        if format_ == "JSON":
+            file.write_text(json.dumps(reports, indent=2))
+    app.echo_success("done")
+    if exit_1 == "error" and overall_violations > 0:
+        app.echo_error(
+            "Exit 1 since violations where found in the reports "
+            "(can be suppressed with '--exit-1 never')."
+        )
+        sys.exit(1)
 
 
 @click.group(cls=CmemcGroup, name="validation")

cmem_cmemc/commands/variable.py

@@ -1,4 +1,5 @@
 """DataIntegration variable commands for cmemc."""
+
 import re
 
 import click
@@ -11,7 +12,8 @@ from cmem.cmempy.workspace.projects.variables import (
 )
 
 from cmem_cmemc import completion
-from cmem_cmemc.commands import CmemcCommand, CmemcGroup
+from cmem_cmemc.command import CmemcCommand
+from cmem_cmemc.command_group import CmemcGroup
 from cmem_cmemc.context import ApplicationContext
 from cmem_cmemc.utils import check_or_select_project, split_task_id
 
@@ -102,7 +104,13 @@ def list_command(
             _.get("description", ""),
         ]
         table.append(row)
-    app.echo_info_table(table, headers=headers, sort_column=0)
+    app.echo_info_table(
+        table,
+        headers=headers,
+        sort_column=0,
+        empty_table_message="No project variables found. "
+        "Use the `project variable create` command to create a new project variable.",
+    )
 
 
 @click.command(cls=CmemcCommand, name="get")