dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/validate_data/mappers.py

@@ -0,0 +1,143 @@
+from rdflib import RDF
+from rdflib import RDFS
+from rdflib import XSD
+
+from dsp_tools.commands.validate_data.constants import ARCHIVE_FILE_VALUE
+from dsp_tools.commands.validate_data.constants import AUDIO_FILE_VALUE
+from dsp_tools.commands.validate_data.constants import DOCUMENT_FILE_VALUE
+from dsp_tools.commands.validate_data.constants import IIIF_URI_VALUE
+from dsp_tools.commands.validate_data.constants import MOVING_IMAGE_FILE_VALUE
+from dsp_tools.commands.validate_data.constants import STILL_IMAGE_FILE_VALUE
+from dsp_tools.commands.validate_data.constants import TEXT_FILE_VALUE
+from dsp_tools.commands.validate_data.models.input_problems import ProblemType
+from dsp_tools.commands.validate_data.models.rdf_like_data import TripleObjectType
+from dsp_tools.commands.validate_data.models.rdf_like_data import TriplePropertyType
+from dsp_tools.commands.validate_data.models.validation import ViolationType
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import BOOLEAN_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import COLOR_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import DECIMAL_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import GEOMETRY_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import GEONAME_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import INT_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import RICHTEXT_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import SIMPLE_TEXT_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import TIME_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.make_rdf_graph.constants import URI_PROP_TYPE_INFO
+from dsp_tools.commands.xmlupload.models.rdf_models import RDFPropTypeInfo
+from dsp_tools.utils.rdf_constants import API_SHAPES
+from dsp_tools.utils.rdf_constants import KNORA_API
+from dsp_tools.utils.rdf_constants import KNORA_API_PREFIX
+from dsp_tools.utils.xml_parsing.models.parsed_resource import KnoraValueType
+
+FILE_TYPE_TO_PROP = {
+    KnoraValueType.ARCHIVE_FILE: f"{KNORA_API_PREFIX}hasArchiveFileValue",
+    KnoraValueType.AUDIO_FILE: f"{KNORA_API_PREFIX}hasAudioFileValue",
+    KnoraValueType.DOCUMENT_FILE: f"{KNORA_API_PREFIX}hasDocumentFileValue",
+    KnoraValueType.MOVING_IMAGE_FILE: f"{KNORA_API_PREFIX}hasMovingImageFileValue",
+    KnoraValueType.STILL_IMAGE_FILE: f"{KNORA_API_PREFIX}hasStillImageFileValue",
+    KnoraValueType.STILL_IMAGE_IIIF: f"{KNORA_API_PREFIX}hasStillImageFileValue",
+    KnoraValueType.TEXT_FILE: f"{KNORA_API_PREFIX}hasTextFileValue",
+}
+
+# Mappers from internal representation to API format
+XML_TAG_TO_VALUE_TYPE_MAPPER = {
+    "boolean-prop": KnoraValueType.BOOLEAN_VALUE,
+    "color-prop": KnoraValueType.COLOR_VALUE,
+    "date-prop": KnoraValueType.DATE_VALUE,
+    "decimal-prop": KnoraValueType.DECIMAL_VALUE,
+    "geometry-prop": KnoraValueType.GEOM_VALUE,
+    "geoname-prop": KnoraValueType.GEONAME_VALUE,
+    "list-prop": KnoraValueType.LIST_VALUE,
+    "integer-prop": KnoraValueType.INT_VALUE,
+    "resptr-prop": KnoraValueType.LINK_VALUE,
+    "time-prop": KnoraValueType.TIME_VALUE,
+    "uri-prop": KnoraValueType.URI_VALUE,
+}
+
+TRIPLE_PROP_TYPE_TO_IRI_MAPPER = {
+    TriplePropertyType.RDF_TYPE: RDF.type,
+    TriplePropertyType.RDFS_LABEL: RDFS.label,
+    TriplePropertyType.KNORA_COMMENT_ON_VALUE: KNORA_API.valueHasComment,
+    TriplePropertyType.KNORA_PERMISSIONS: KNORA_API.hasPermissions,
+    TriplePropertyType.KNORA_INTERVAL_START: KNORA_API.intervalValueHasStart,
+    TriplePropertyType.KNORA_INTERVAL_END: KNORA_API.intervalValueHasEnd,
+    TriplePropertyType.KNORA_STANDOFF_LINK: KNORA_API.hasStandoffLinkTo,
+    TriplePropertyType.KNORA_LICENSE: KNORA_API.hasLicense,
+    TriplePropertyType.KNORA_AUTHORSHIP: KNORA_API.hasAuthorship,
+    TriplePropertyType.KNORA_COPYRIGHT_HOLDER: KNORA_API.hasCopyrightHolder,
+    TriplePropertyType.KNORA_DATE_START: API_SHAPES.dateHasStart,
+    TriplePropertyType.KNORA_DATE_END: API_SHAPES.dateHasEnd,
+}
+
+VALUE_INFO_TO_RDF_MAPPER = {
+    KnoraValueType.BOOLEAN_VALUE: BOOLEAN_PROP_TYPE_INFO,
+    KnoraValueType.COLOR_VALUE: COLOR_PROP_TYPE_INFO,
+    KnoraValueType.DATE_VALUE: RDFPropTypeInfo(KNORA_API.DateValue, KNORA_API.valueAsString, XSD.string),
+    KnoraValueType.DECIMAL_VALUE: DECIMAL_PROP_TYPE_INFO,
+    KnoraValueType.GEONAME_VALUE: GEONAME_PROP_TYPE_INFO,
+    KnoraValueType.GEOM_VALUE: GEOMETRY_PROP_TYPE_INFO,
+    KnoraValueType.LIST_VALUE: RDFPropTypeInfo(KNORA_API.ListValue, KNORA_API.listValueAsListNode, XSD.string),
+    KnoraValueType.LINK_VALUE: RDFPropTypeInfo(KNORA_API.LinkValue, API_SHAPES.linkValueHasTargetID, XSD.string),
+    KnoraValueType.INT_VALUE: INT_PROP_TYPE_INFO,
+    KnoraValueType.INTERVAL_VALUE: RDFPropTypeInfo(KNORA_API.IntervalValue, KNORA_API.hasSegmentBounds),
+    KnoraValueType.SIMPLETEXT_VALUE: SIMPLE_TEXT_PROP_TYPE_INFO,
+    KnoraValueType.RICHTEXT_VALUE: RICHTEXT_PROP_TYPE_INFO,
+    KnoraValueType.TIME_VALUE: TIME_PROP_TYPE_INFO,
+    KnoraValueType.URI_VALUE: URI_PROP_TYPE_INFO,
+    KnoraValueType.ARCHIVE_FILE: ARCHIVE_FILE_VALUE,
+    KnoraValueType.AUDIO_FILE: AUDIO_FILE_VALUE,
+    KnoraValueType.DOCUMENT_FILE: DOCUMENT_FILE_VALUE,
+    KnoraValueType.MOVING_IMAGE_FILE: MOVING_IMAGE_FILE_VALUE,
+    KnoraValueType.STILL_IMAGE_FILE: STILL_IMAGE_FILE_VALUE,
+    KnoraValueType.STILL_IMAGE_IIIF: IIIF_URI_VALUE,
+    KnoraValueType.TEXT_FILE: TEXT_FILE_VALUE,
+}
+
+VALUE_INFO_TRIPLE_OBJECT_TYPE = {
+    KnoraValueType.BOOLEAN_VALUE: TripleObjectType.BOOLEAN,
+    KnoraValueType.COLOR_VALUE: TripleObjectType.STRING,
+    KnoraValueType.DATE_VALUE: TripleObjectType.STRING,
+    KnoraValueType.DECIMAL_VALUE: TripleObjectType.DECIMAL,
+    KnoraValueType.GEOM_VALUE: TripleObjectType.STRING,
+    KnoraValueType.GEONAME_VALUE: TripleObjectType.STRING,
+    KnoraValueType.LIST_VALUE: TripleObjectType.STRING,
+    KnoraValueType.LINK_VALUE: TripleObjectType.IRI,
+    KnoraValueType.INT_VALUE: TripleObjectType.INTEGER,
+    KnoraValueType.SIMPLETEXT_VALUE: TripleObjectType.STRING,
+    KnoraValueType.RICHTEXT_VALUE: TripleObjectType.STRING,
+    KnoraValueType.TIME_VALUE: TripleObjectType.DATETIME,
+    KnoraValueType.URI_VALUE: TripleObjectType.URI,
+    KnoraValueType.ARCHIVE_FILE: TripleObjectType.STRING,
+    KnoraValueType.AUDIO_FILE: TripleObjectType.STRING,
+    KnoraValueType.DOCUMENT_FILE: TripleObjectType.STRING,
+    KnoraValueType.MOVING_IMAGE_FILE: TripleObjectType.STRING,
+    KnoraValueType.STILL_IMAGE_FILE: TripleObjectType.STRING,
+    KnoraValueType.STILL_IMAGE_IIIF: TripleObjectType.URI,
+    KnoraValueType.TEXT_FILE: TripleObjectType.STRING,
+}
+
+TRIPLE_OBJECT_TYPE_TO_XSD = {
+    TripleObjectType.BOOLEAN: XSD.boolean,
+    TripleObjectType.DATETIME: XSD.dateTimeStamp,
+    TripleObjectType.DECIMAL: XSD.decimal,
+    TripleObjectType.INTEGER: XSD.int,
+    TripleObjectType.STRING: XSD.string,
+    TripleObjectType.URI: XSD.anyURI,
+    TripleObjectType.DATE_YYYY_MM_DD: XSD.date,
+}
+
+# validation results
+
+RESULT_TO_PROBLEM_MAPPER = {
+    ViolationType.SEQNUM_IS_PART_OF: ProblemType.GENERIC,
+    ViolationType.UNIQUE_VALUE: ProblemType.DUPLICATE_VALUE,
+    ViolationType.VALUE_TYPE: ProblemType.VALUE_TYPE_MISMATCH,
+    ViolationType.PATTERN: ProblemType.INPUT_REGEX,
+    ViolationType.GENERIC: ProblemType.GENERIC,
+    ViolationType.LINK_TARGET: ProblemType.LINK_TARGET_TYPE_MISMATCH,
+    ViolationType.MAX_CARD: ProblemType.MAX_CARD,
+    ViolationType.MIN_CARD: ProblemType.MIN_CARD,
+    ViolationType.NON_EXISTING_CARD: ProblemType.NON_EXISTING_CARD,
+    ViolationType.FILE_VALUE_PROHIBITED: ProblemType.FILE_VALUE_PROHIBITED,
+    ViolationType.FILE_VALUE_MISSING: ProblemType.FILE_VALUE_MISSING,
+}

dsp_tools/commands/validate_data/models/api_responses.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from rdflib import Graph
+from rdflib import URIRef
+
+from dsp_tools.clients.list_client import OneList
+
+
+@dataclass
+class SHACLValidationReport:
+    conforms: bool
+    validation_graph: Graph
+
+
+@dataclass
+class ProjectDataFromApi:
+    all_lists: list[OneList]
+    enabled_licenses: EnabledLicenseIris
+    resource_iris_in_db: list[InfoForResourceInDB]
+
+
+@dataclass
+class ListLookup:
+    lists: dict[tuple[str, str], str]
+
+
+@dataclass
+class SHACLListInfo:
+    list_iri: URIRef
+    sh_path: URIRef
+    sh_message: str
+    sh_in: list[str]
+
+
+@dataclass
+class EnabledLicenseIris:
+    enabled_licenses: list[str]
+
+
+@dataclass
+class InfoForResourceInDB:
+    res_iri: str
+    res_type: str

dsp_tools/commands/validate_data/models/input_problems.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from enum import StrEnum
+from enum import auto
+
+import pandas as pd
+import regex
+
+from dsp_tools.commands.validate_data.models.validation import UnexpectedComponent
+from dsp_tools.commands.validate_data.models.validation import ValidationReportGraphs
+
+
+@dataclass
+class ValidateDataResult:
+    no_problems: bool
+    problems: None | UnknownClassesInData | OntologyValidationProblem | SortedProblems
+    report_graphs: None | ValidationReportGraphs
+
+
+@dataclass
+class OntologyValidationProblem:
+    problems: list[OntologyResourceProblem]
+
+
+@dataclass
+class OntologyResourceProblem:
+    res_iri: str
+    msg: str
+
+
+@dataclass
+class UnknownClassesInData:
+    unknown_classes: set[str]
+    defined_classes: set[str]
+
+
+@dataclass
+class DuplicateFileWarning:
+    problems: list[InputProblem]
+
+
+@dataclass
+class AllProblems:
+    problems: list[InputProblem]
+    unexpected_results: list[UnexpectedComponent]
+
+
+@dataclass
+class SortedProblems:
+    unique_violations: list[InputProblem]
+    user_warnings: list[InputProblem]
+    user_info: list[InputProblem]
+    unexpected_shacl_validation_components: list[str]
+
+
+@dataclass
+class MessageComponents:
+    message_header: str
+    message_body: str | None
+    message_df: pd.DataFrame | None
+
+
+@dataclass
+class UserPrintMessages:
+    violations: MessageComponents | None
+    warnings: MessageComponents | None
+    infos: MessageComponents | None
+    unexpected_violations: MessageComponents | None
+
+
+@dataclass
+class InputProblem:
+    problem_type: ProblemType
+    res_id: str | None
+    res_type: str | None
+    prop_name: str
+    severity: Severity
+    message: str | None = None
+    input_value: str | None = None
+    input_type: str | None = None
+    expected: str | None = None
+
+    def __post_init__(self) -> None:
+        self.message = self._clean_input(self.message)
+        self.expected = self._clean_input(self.expected)
+
+    def _clean_input(self, to_clean: str | None) -> str | None:
+        if not to_clean:
+            return None
+        if not regex.search(r"\S+", to_clean):
+            return None
+        str_split = [content for x in to_clean.split(" ") if (content := x.strip())]
+        return " ".join(str_split)
+
+
+class Severity(Enum):
+    VIOLATION = auto()
+    WARNING = auto()
+    INFO = auto()
+
+
+class ProblemType(StrEnum):
+    GENERIC = "generic"
+    FILE_VALUE_MISSING = "file problem"
+    FILE_DUPLICATE = "Duplicate Filepath / IIIF-URI"
+    FILE_VALUE_PROHIBITED = "A file was added to the resource. This resource type must not have a file."
+    MAX_CARD = "Maximum Cardinality Violation"
+    MIN_CARD = "Minimum Cardinality Violation"
+    NON_EXISTING_CARD = "The resource class does not have a cardinality for this property."
+    VALUE_TYPE_MISMATCH = "Value Type Mismatch"
+    INPUT_REGEX = "Wrong Format of Input"
+    LINK_TARGET_TYPE_MISMATCH = "Linked Resource Type Mismatch"
+    LINK_TARGET_OF_ANOTHER_PROJECT = "Linked Resource is from another project"
+    LINK_TARGET_IS_IRI_OF_PROJECT = "Linked resource is an IRI of the project"
+    LINK_TARGET_NOT_FOUND_IN_DB = "Linked resource is an IRI and could not be found in the database."
+    INEXISTENT_LINKED_RESOURCE = "Linked Resource does not exist"
+    DUPLICATE_VALUE = "Your input is duplicated"

dsp_tools/commands/validate_data/models/rdf_like_data.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from enum import Enum
+from enum import auto
+from uuid import uuid4
+
+from dsp_tools.legacy_models.datetimestamp import DateTimeStamp
+from dsp_tools.utils.xml_parsing.models.parsed_resource import KnoraValueType
+
+
+@dataclass
+class RdfLikeData:
+    resources: list[RdfLikeResource]
+
+
+@dataclass
+class RdfLikeResource:
+    """
+    Represents a user facing project-specific resource.
+
+    Args:
+        res_id: resource ID provided by the user in the XML
+        property_objects: A list of properties and objects where the subject is the resource itself.
+            They are non-reified triples (not values).
+            For example, the label of a resource is here.
+        values: a list of user-facing values (e.g. BooleanValue)
+        migration_metadata: Metadata used for salsah migration
+    """
+
+    res_id: str
+    property_objects: list[PropertyObject]
+    values: list[RdfLikeValue]
+    migration_metadata: MigrationMetadata
+
+
+@dataclass
+class MigrationMetadata:
+    iri: str | None = None
+    ark: str | None = None
+    creation_date: DateTimeStamp | None = None
+
+    def any(self) -> bool:
+        return any([self.iri, self.ark, self.creation_date])
+
+
+@dataclass
+class PropertyObject:
+    """
+    Property and object of a triple.
+
+    Args:
+        property_type: maps to a specific knora-api or rdf(s) property
+        object_value: object of the triple, may be user facing (e.g. label) or metadata (e.g. permissions)
+        object_type: datatype for literals (e.g. boolean) or that it is an IRI (not a literal)
+    """
+
+    property_type: TriplePropertyType
+    object_value: str | None
+    object_type: TripleObjectType
+
+
+@dataclass
+class RdfLikeValue:
+    """
+    Contains information about a user-facing value, for example BooleanValue.
+
+    Args:
+        user_facing_prop: Absolute IRI of the property as defined in the ontology
+        user_facing_value: User-facing value, for example a number
+        knora_type: Maps to a knora value type (e.g. BooleanValue)
+        value_metadata: metadata of the value for example permissions, comments, etc.
+    """
+
+    user_facing_prop: str
+    user_facing_value: str | None
+    knora_type: KnoraValueType
+    value_metadata: list[PropertyObject]
+    value_uuid: str = field(default_factory=lambda: str(uuid4()))
+
+
+class TriplePropertyType(Enum):
+    """
+    Maps to a specific knora-api or rdf(s) property.
+    For example: KNORA_COMMENT_ON_VALUE -> knora-api:valueHasComment
+    """
+
+    RDFS_LABEL = auto()
+    RDF_TYPE = auto()
+    KNORA_PERMISSIONS = auto()
+    KNORA_COMMENT_ON_VALUE = auto()
+    KNORA_INTERVAL_START = auto()
+    KNORA_INTERVAL_END = auto()
+    KNORA_STANDOFF_LINK = auto()
+    KNORA_LICENSE = auto()
+    KNORA_AUTHORSHIP = auto()
+    KNORA_COPYRIGHT_HOLDER = auto()
+    KNORA_DATE_START = auto()
+    KNORA_DATE_END = auto()
+
+
+class TripleObjectType(Enum):
+    """
+    Maps to an xsd data type in case of literals, for example: BOOLEAN -> xsd:boolean
+    Or Indicates that it is an IRI, in which case it is not an RDF Literal
+    """
+
+    BOOLEAN = auto()
+    DATETIME = auto()
+    DECIMAL = auto()
+    INTEGER = auto()
+    IRI = auto()
+    INTERNAL_ID = auto()
+    STRING = auto()
+    URI = auto()
+    DATE_YYYY_MM_DD = auto()

dsp_tools/commands/validate_data/models/validation.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from enum import auto
+from pathlib import Path
+
+from rdflib import Graph
+
+from dsp_tools.utils.rdf_constants import SubjectObjectTypeAlias
+
+
+@dataclass
+class SHACLGraphs:
+    cardinality: Graph
+    content: Graph
+
+
+@dataclass
+class RDFGraphs:
+    data: Graph
+    ontos: Graph
+    cardinality_shapes: Graph
+    content_shapes: Graph
+    knora_api: Graph
+    resources_in_db_graph: Graph
+
+
+@dataclass
+class ValidationFilePaths:
+    directory: Path
+    data_file: str
+    shacl_file: str
+    report_file: str
+
+
+@dataclass
+class ValidationReportGraphs:
+    conforms: bool
+    validation_graph: Graph
+    shacl_graph: Graph
+    onto_graph: Graph
+    data_graph: Graph
+
+
+@dataclass
+class QueryInfo:
+    validation_bn: SubjectObjectTypeAlias
+    focus_iri: SubjectObjectTypeAlias
+    focus_rdf_type: SubjectObjectTypeAlias
+
+
+@dataclass
+class UnexpectedComponent:
+    component_type: str
+
+
+@dataclass
+class ValidationResultBaseInfo:
+    result_bn: SubjectObjectTypeAlias
+    source_constraint_component: SubjectObjectTypeAlias
+    focus_node_iri: SubjectObjectTypeAlias
+    focus_node_type: SubjectObjectTypeAlias
+    result_path: SubjectObjectTypeAlias
+    severity: SubjectObjectTypeAlias
+    detail: DetailBaseInfo | None = None
+
+
+@dataclass
+class DetailBaseInfo:
+    detail_bn: SubjectObjectTypeAlias
+    source_constraint_component: SubjectObjectTypeAlias
+
+
+@dataclass
+class ValidationResult:
+    violation_type: ViolationType
+    res_iri: SubjectObjectTypeAlias
+    res_class: SubjectObjectTypeAlias
+    severity: SubjectObjectTypeAlias
+    property: SubjectObjectTypeAlias | None = None
+    message: SubjectObjectTypeAlias | None = None
+    input_value: SubjectObjectTypeAlias | None = None
+    input_type: SubjectObjectTypeAlias | None = None
+    expected: SubjectObjectTypeAlias | None = None
+
+
+class ViolationType(Enum):
+    SEQNUM_IS_PART_OF = auto()
+    UNIQUE_VALUE = auto()
+    VALUE_TYPE = auto()
+    PATTERN = auto()
+    GENERIC = auto()
+    LINK_TARGET = auto()
+    MAX_CARD = auto()
+    MIN_CARD = auto()
+    NON_EXISTING_CARD = auto()
+    FILE_VALUE_PROHIBITED = auto()
+    FILE_VALUE_MISSING = auto()
+
+
+@dataclass
+class ReformattedIRI:
+    res_id: str
+    res_type: str
+    prop_name: str