dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/validate_data/shacl_cli_validator.py

@@ -0,0 +1,70 @@
+import importlib.resources
+import subprocess
+from pathlib import Path
+
+import yaml
+from loguru import logger
+from rdflib import SH
+from rdflib import Graph
+
+from dsp_tools.commands.validate_data.models.api_responses import SHACLValidationReport
+from dsp_tools.commands.validate_data.models.validation import ValidationFilePaths
+from dsp_tools.config.logger_config import LOGGER_SAVEPATH
+from dsp_tools.error.exceptions import InternalError
+from dsp_tools.error.exceptions import ShaclValidationCliError
+
+
+class ShaclCliValidator:
+    def validate(self, file_paths: ValidationFilePaths) -> SHACLValidationReport:
+        try:
+            self._run_validate_cli(file_paths)
+        except subprocess.CalledProcessError as e:
+            logger.error(e)
+            msg = (
+                "Data validation requires Docker. Is your Docker Desktop Application open? "
+                "If it is, please contact the DSP-TOOLS development team (at support@dasch.swiss) "
+                f"with the log file at {LOGGER_SAVEPATH}."
+            )
+            logger.error(msg)
+            raise ShaclValidationCliError(msg) from None
+        return self._parse_validation_result(file_paths.directory / file_paths.report_file)
+
+    def _run_validate_cli(self, file_paths: ValidationFilePaths) -> None:
+        if not (file_paths.directory / file_paths.shacl_file).exists():
+            raise InternalError(f"SHACL file not found: {file_paths.shacl_file}")
+        if not (file_paths.directory / file_paths.data_file).exists():
+            raise InternalError(f"Data file not found: {file_paths.data_file}")
+
+        # Get relative paths within the container
+        shacl_path = f"/data/{file_paths.shacl_file}"
+        data_path = f"/data/{file_paths.data_file}"
+        report_path = f"/data/{file_paths.report_file}"
+
+        docker_file = importlib.resources.files("dsp_tools").joinpath("resources/validate_data/shacl-cli-image.yml")
+        docker_spec = yaml.safe_load(docker_file.read_bytes())
+        docker_image = docker_spec["image"]
+
+        d_cmd = (
+            f"docker run --rm -v {file_paths.directory.absolute()}:/data {docker_image} "
+            f"validate --shacl {shacl_path} --data {data_path} --report {report_path}"
+        )
+        logger.debug(f"Running SHACL validation: {d_cmd}")
+        result = subprocess.run(
+            d_cmd.split(),
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        if result.stdout:
+            logger.debug(f"Validation output: {result.stdout}")
+        if result.stderr:
+            logger.error(f"Validation output error: {result.stderr}")
+
+    def _parse_validation_result(self, filepath: Path) -> SHACLValidationReport:
+        if not filepath.exists():
+            raise InternalError(f"SHACL file not found: {filepath}")
+        logger.debug(f"Parse validation response: {filepath}.")
+        graph = Graph()
+        graph.parse(filepath)
+        conforms = bool(next(graph.objects(None, SH.conforms)))
+        return SHACLValidationReport(conforms=conforms, validation_graph=graph)

dsp_tools/commands/validate_data/sparql/cardinality_shacl.py

@@ -0,0 +1,209 @@
+from loguru import logger
+from rdflib import Graph
+
+
+def construct_cardinality_node_shapes(onto: Graph) -> Graph:
+    """
+    Returns the sh:NodeShapes for the resource classes in the ontology.
+
+    Args:
+        onto: ontology graph
+
+    Returns:
+        Graph with the resource class node shapes
+    """
+    logger.debug("Constructing cardinality shapes for the ontology.")
+    g = Graph()
+    g += _construct_resource_nodeshape(onto)
+    g += _construct_all_cardinalities(onto)
+    return g
+
+
+def _construct_resource_nodeshape(onto_graph: Graph) -> Graph:
+    logger.debug("Constructing resource NodeShape")
+    query_s = """
+    PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+    PREFIX sh: <http://www.w3.org/ns/shacl#>
+    PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+    PREFIX api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#>
+    PREFIX knora-api:  <http://api.knora.org/ontology/knora-api/v2#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
+    PREFIX dash: <http://datashapes.org/dash#>
+
+    CONSTRUCT {
+        ?class a sh:NodeShape ;
+                dash:closedByTypes true ;
+                sh:property api-shapes:hasPermissions_Cardinality ,
+                            [
+                              a sh:PropertyShape ;
+                              sh:path rdfs:label ;
+                              sh:minCount 1 ;
+                              sh:maxCount 1 ;
+                              sh:severity sh:Violation ;
+                              sh:message "A label is required" 
+                            ] ,
+                            [ 
+                              a sh:PropertyShape ;
+                              sh:path knora-api:hasStandoffLinkTo
+                            ] .
+    } WHERE {
+        ?class a owl:Class ;
+               knora-api:isResourceClass true ;
+               knora-api:canBeInstantiated true .
+    }
+    """
+    if results_graph := onto_graph.query(query_s).graph:
+        return results_graph
+    return Graph()
+
+
+def _construct_all_cardinalities(onto_graph: Graph) -> Graph:
+    g = Graph()
+    g += _construct_1_cardinality(onto_graph)
+    g += _construct_0_1_cardinality(onto_graph)
+    g += _construct_1_n_cardinality(onto_graph)
+    g += _construct_0_n_cardinality(onto_graph)
+    return g
+
+
+def _construct_1_cardinality(onto_graph: Graph) -> Graph:
+    logger.debug("Constructing cardinality: 1")
+    query_s = """
+    PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+    PREFIX sh: <http://www.w3.org/ns/shacl#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#>
+    PREFIX knora-api:  <http://api.knora.org/ontology/knora-api/v2#>
+    PREFIX dash: <http://datashapes.org/dash#>
+
+    CONSTRUCT {
+      ?class sh:property [
+          a sh:PropertyShape ;
+          sh:path ?propRestriction ;
+          sh:minCount 1 ;
+          sh:maxCount 1 ;
+          sh:severity sh:Violation ;
+          sh:message "Cardinality 1" 
+      ] .
+    } WHERE {
+      ?class a owl:Class ;
+          knora-api:isResourceClass true ;
+          knora-api:canBeInstantiated true ;
+          rdfs:subClassOf ?restriction .
+      ?restriction a owl:Restriction ;
+          owl:onProperty ?propRestriction ;
+          owl:cardinality 1 .
+          
+      ?propRestriction knora-api:isEditable true .
+      FILTER NOT EXISTS { ?propRestriction knora-api:isLinkValueProperty true }
+    }
+    """
+    if results_graph := onto_graph.query(query_s).graph:
+        return results_graph
+    return Graph()
+
+
+def _construct_0_1_cardinality(onto_graph: Graph) -> Graph:
+    logger.debug("Constructing cardinality: 0-1")
+    query_s = """
+    PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+    PREFIX sh: <http://www.w3.org/ns/shacl#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#>
+    PREFIX knora-api:  <http://api.knora.org/ontology/knora-api/v2#>
+    PREFIX dash: <http://datashapes.org/dash#>
+
+    CONSTRUCT {
+      ?class sh:property [
+          a sh:PropertyShape ;
+          sh:path ?propRestriction ;
+          sh:minCount 0 ;
+          sh:maxCount 1 ;
+          sh:severity sh:Violation ;
+          sh:message "Cardinality 0-1" 
+      ] .
+    } WHERE {
+      ?class a owl:Class ;
+          knora-api:isResourceClass true ;
+          knora-api:canBeInstantiated true ;
+          rdfs:subClassOf ?restriction .
+      ?restriction a owl:Restriction ;
+          owl:onProperty ?propRestriction ;
+          owl:maxCardinality 1 .
+          
+      ?propRestriction knora-api:isEditable true .
+      FILTER NOT EXISTS { ?propRestriction knora-api:isLinkValueProperty true }
+    }
+    """
+    if results_graph := onto_graph.query(query_s).graph:
+        return results_graph
+    return Graph()
+
+
+def _construct_1_n_cardinality(onto_graph: Graph) -> Graph:
+    logger.debug("Constructing cardinality: 1-n")
+    query_s = """
+    PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+    PREFIX sh: <http://www.w3.org/ns/shacl#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#>
+    PREFIX knora-api:  <http://api.knora.org/ontology/knora-api/v2#>
+    PREFIX dash: <http://datashapes.org/dash#>
+
+    CONSTRUCT {
+      ?class sh:property [
+          a sh:PropertyShape ;
+          sh:path ?propRestriction ;
+          sh:minCount 1 ;
+          sh:severity sh:Violation ;
+          sh:message "Cardinality 1-n" 
+      ] .
+    } WHERE {
+      ?class a owl:Class ;
+          knora-api:isResourceClass true ;
+          knora-api:canBeInstantiated true ;
+          rdfs:subClassOf ?restriction .
+      ?restriction a owl:Restriction ;
+          owl:onProperty ?propRestriction ;
+          owl:minCardinality 1 .
+          
+      ?propRestriction knora-api:isEditable true .
+      FILTER NOT EXISTS { ?propRestriction knora-api:isLinkValueProperty true }
+    }
+    """
+    if results_graph := onto_graph.query(query_s).graph:
+        return results_graph
+    return Graph()
+
+
+def _construct_0_n_cardinality(onto_graph: Graph) -> Graph:
+    logger.debug("Constructing cardinality: 0-n")
+    query_s = """
+    PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+    PREFIX sh: <http://www.w3.org/ns/shacl#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#>
+    PREFIX knora-api:  <http://api.knora.org/ontology/knora-api/v2#>
+    PREFIX dash: <http://datashapes.org/dash#>
+
+    CONSTRUCT {
+      ?class sh:property [
+          a sh:PropertyShape ;
+          sh:path ?propRestriction 
+      ] .
+    } WHERE {
+      ?class a owl:Class ;
+          knora-api:isResourceClass true ;
+          knora-api:canBeInstantiated true ;
+          rdfs:subClassOf ?restriction .
+      ?restriction a owl:Restriction ;
+          owl:onProperty ?propRestriction ;
+          owl:minCardinality 0 .
+          
+      ?propRestriction knora-api:isEditable true .
+      FILTER NOT EXISTS { ?propRestriction knora-api:isLinkValueProperty true }
+    }
+    """
+    if results_graph := onto_graph.query(query_s).graph:
+        return results_graph
+    return Graph()

dsp_tools/commands/validate_data/sparql/construct_shacl.py

@@ -0,0 +1,92 @@
+from loguru import logger
+from rdflib import Graph
+
+from dsp_tools.commands.validate_data.models.api_responses import ProjectDataFromApi
+from dsp_tools.commands.validate_data.models.validation import SHACLGraphs
+from dsp_tools.commands.validate_data.sparql.cardinality_shacl import construct_cardinality_node_shapes
+from dsp_tools.commands.validate_data.sparql.legal_info_shacl import construct_allowed_licenses_shape
+from dsp_tools.commands.validate_data.sparql.value_shacl import construct_property_shapes
+
+
+def construct_shapes_graphs(
+    onto: Graph,
+    knora_api: Graph,
+    project_info_from_api: ProjectDataFromApi,
+    permission_ids: list[str],
+) -> SHACLGraphs:
+    """
+    Constructs a shapes graph from a project ontology
+
+    Args:
+        onto: ontology as graph
+        knora_api: the knora-api ontology
+        project_info_from_api: information from a project from the api, for example lists
+        permission_ids: ids of permissions that were defined in the XML
+
+    Returns:
+        shapes graph
+    """
+    logger.debug("Constructing SHACL shapes from ontology.")
+    knora_subset = _get_all_relevant_knora_subset(knora_api)
+    graph_to_query = knora_subset + onto
+    cardinality = construct_cardinality_node_shapes(graph_to_query)
+    content = construct_property_shapes(graph_to_query, project_info_from_api.all_lists)
+    content += construct_allowed_licenses_shape(project_info_from_api.enabled_licenses)
+    content += _get_defined_permissions_shape(permission_ids)
+    return SHACLGraphs(cardinality=cardinality, content=content)
+
+
+def _get_all_relevant_knora_subset(knora_api: Graph) -> Graph:
+    logger.debug("Getting relevant knora-api subset")
+    props = ["isLinkProperty", "isEditable", "isLinkValueProperty"]
+    g = Graph()
+    for p in props:
+        g += _get_one_relevant_knora_subset(knora_api, p)
+    return g
+
+
+def _get_one_relevant_knora_subset(knora_api: Graph, knora_prop: str) -> Graph:
+    query_s = """
+    PREFIX owl: <http://www.w3.org/2002/07/owl#> 
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX knora-api:  <http://api.knora.org/ontology/knora-api/v2#>
+
+    CONSTRUCT {
+    
+      ?focusProp a owl:ObjectProperty ;
+                 knora-api:objectType ?type ;
+                 knora-api:%(knora_prop)s ?object .
+      
+    } WHERE {
+      
+      ?focusProp a owl:ObjectProperty ;
+                 knora-api:objectType ?type ;
+                 knora-api:%(knora_prop)s ?object .
+      
+    }
+    """ % {"knora_prop": knora_prop}  # noqa: UP031 (printf-string-formatting)
+    if results_graph := knora_api.query(query_s).graph:
+        return results_graph
+    return Graph()
+
+
+def _get_defined_permissions_shape(permission_ids: list[str]) -> Graph:
+    g = Graph()
+    permissions = " ".join([f'"{x}"' for x in permission_ids])
+    msg = f"You must reference one of the pre-defined permissions: {', '.join(permission_ids)}"
+    shape = f"""
+     @prefix sh:         <http://www.w3.org/ns/shacl#> .
+     @prefix owl:        <http://www.w3.org/2002/07/owl#> .
+     @prefix knora-api:  <http://api.knora.org/ontology/knora-api/v2#> .
+     @prefix api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#> .
+     
+     api-shapes:hasPermissions_Shape
+       a              sh:NodeShape ;
+       sh:targetClass knora-api:Value , knora-api:Resource ;
+       sh:path        knora-api:hasPermissions ;
+       sh:in          ( {permissions} ) ;
+       sh:message     "{msg}" ;
+       sh:severity    sh:Violation .
+     """
+    g.parse(data=shape, format="turtle")
+    return g

dsp_tools/commands/validate_data/sparql/legal_info_shacl.py

@@ -0,0 +1,36 @@
+from loguru import logger
+from rdflib import Graph
+
+from dsp_tools.commands.validate_data.models.api_responses import EnabledLicenseIris
+
+
+def construct_allowed_licenses_shape(license_iris: EnabledLicenseIris) -> Graph:
+    """Create a constraint detailing the allowed licences."""
+    formatted_iris = [f"<{x}>" for x in license_iris.enabled_licenses]
+    license_str = " ".join(formatted_iris)
+    if len(formatted_iris) == 0:
+        msg_str = '"You are only allowed to reference enabled licenses. No licenses are enabled for this project."'
+    else:
+        msg_str = (
+            '"Please only use enabled licenses in your data. Consult the project information for enabled licenses."'
+        )
+    logger.debug("Constructing allowed licenses shapes.")
+    ttl_str = """
+    @prefix sh:         <http://www.w3.org/ns/shacl#> .
+    @prefix knora-api:  <http://api.knora.org/ontology/knora-api/v2#> .
+    @prefix api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#> .
+    
+    api-shapes:FileValue_ClassShape
+      a sh:NodeShape ;
+      sh:targetClass knora-api:FileValue ;
+      sh:property [
+          a           sh:PropertyShape ;
+          sh:path     knora-api:hasLicense ;
+          sh:in       ( %(license_str)s ) ;
+          sh:message  %(msg)s ;
+          sh:severity  sh:Violation
+                  ] .
+    """ % {"license_str": license_str, "msg": msg_str}  # noqa: UP031 Use format specifiers instead of percent format
+    g = Graph()
+    g.parse(data=ttl_str, format="turtle")
+    return g