dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/xmlupload/models/deserialise/xmlresource.py

@@ -1,171 +0,0 @@
-from __future__ import annotations
-
-import itertools
-from dataclasses import dataclass
-from typing import Optional
-
-from lxml import etree
-
-from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import IIIFUriInfo
-from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLBitstream
-from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLProperty
-from dsp_tools.commands.xmlupload.models.permission import Permissions
-from dsp_tools.models.datetimestamp import DateTimeStamp
-from dsp_tools.models.exceptions import XmlUploadError
-
-COMPOSITE_PROPS = (
-    "boolean-prop",
-    "color-prop",
-    "date-prop",
-    "decimal-prop",
-    "geometry-prop",
-    "geoname-prop",
-    "integer-prop",
-    "list-prop",
-    "resptr-prop",
-    "text-prop",
-    "time-prop",
-    "uri-prop",
-)
-
-
-@dataclass(frozen=True)
-class BitstreamInfo:
-    """
-    Represents a bitstream object,
-    consisting of its file name on the local file system,
-    the internal file name assigned by the ingest service
-    and optionally its permissions.
-    """
-
-    local_file: str
-    internal_file_name: str
-    permissions: Permissions | None = None
-
-
-@dataclass(frozen=True)
-class XMLResource:
-    """
-    Represents a resource in the XML used for data import.
-
-    Attributes:
-        res_id: The unique id of the resource
-        iri: The custom IRI of the resource
-        ark: The custom ARK of the resource
-        label: The label of the resource
-        restype: The type of the resource
-        permissions: The reference to the permissions set for this resource
-        creation_date: The creation date of the resource
-        bitstream: The bitstream object belonging to the resource
-        properties: The list of properties of the resource
-    """
-
-    res_id: str
-    iri: Optional[str]
-    ark: Optional[str]
-    label: str
-    restype: str
-    permissions: Optional[str]
-    creation_date: Optional[DateTimeStamp]
-    bitstream: Optional[XMLBitstream]
-    iiif_uri: Optional[IIIFUriInfo]
-    properties: list[XMLProperty]
-
-    @staticmethod
-    def from_node(node: etree._Element, default_ontology: str) -> XMLResource:
-        """
-        Factory that parses a resource node from the XML DOM
-
-        Args:
-            node: The DOM node to be processed representing a resource (which is a child of the `<knora>` element)
-            default_ontology: The default ontology (given in the attribute default-ontology of the `<knora>` element)
-
-        Returns:
-            An XMLResource object
-        """
-        bitstream, iiif_uri, properties = XMLResource._get_properties(node, default_ontology)
-        return XMLResource(
-            res_id=node.attrib["id"],
-            iri=node.attrib.get("iri"),
-            ark=node.attrib.get("ark"),
-            label=node.attrib["label"],
-            restype=XMLResource._get_restype(node, default_ontology),
-            permissions=node.attrib.get("permissions"),
-            creation_date=DateTimeStamp(x) if (x := node.attrib.get("creation_date")) else None,
-            bitstream=bitstream,
-            iiif_uri=iiif_uri,
-            properties=properties,
-        )
-
-    @staticmethod
-    def _get_restype(node: etree._Element, default_ontology: str) -> str:
-        # get the resource type which is in format namespace:resourcetype, p.ex. rosetta:Image
-        restype_orig = node.attrib["restype"]
-        if ":" not in restype_orig:
-            return f"knora-api:{restype_orig}"
-        elif restype_orig.startswith(":"):
-            # replace an empty namespace with the default ontology name
-            return f"{default_ontology}:{restype_orig[1:]}"
-        else:
-            return restype_orig
-
-    @staticmethod
-    def _get_properties(
-        node: etree._Element, default_ontology: str
-    ) -> tuple[XMLBitstream | None, IIIFUriInfo | None, list[XMLProperty]]:
-        bitstream: XMLBitstream | None = None
-        iiif_uri: IIIFUriInfo | None = None
-        ungrouped_properties: list[XMLProperty] = []
-        for subnode in node:
-            match subnode.tag:
-                case "bitstream":
-                    bitstream = XMLBitstream.from_node(subnode)
-                case "iiif-uri":
-                    iiif_uri = IIIFUriInfo.from_node(subnode)
-                case "isSegmentOf" | "relatesTo":
-                    ungrouped_properties.append(XMLProperty.from_node(subnode, "resptr", default_ontology))
-                case "hasSegmentBounds":
-                    ungrouped_properties.append(XMLProperty.from_node(subnode, "interval", default_ontology))
-                case "hasTitle" | "hasComment" | "hasDescription" | "hasKeyword":
-                    ungrouped_properties.append(XMLProperty.from_node(subnode, "text", default_ontology))
-                case str() as x if x in COMPOSITE_PROPS:
-                    # get the property type which is in format type-prop, p.ex. <decimal-prop>
-                    prop_type, _ = subnode.tag.split("-")
-                    ungrouped_properties.append(XMLProperty.from_node(subnode, prop_type, default_ontology))
-                case _:
-                    raise XmlUploadError(f"Unexpected tag '{subnode.tag}'")
-        grouped_properties = XMLResource._group_props(ungrouped_properties)
-        return bitstream, iiif_uri, grouped_properties
-
-    @staticmethod
-    def _group_props(ungrouped_properties: list[XMLProperty]) -> list[XMLProperty]:
-        properties = []
-        ungrouped_properties.sort(key=lambda x: x.name)
-        for _, xml_props in itertools.groupby(ungrouped_properties, lambda x: x.name):
-            if len(xml_props_list := list(xml_props)) == 1:
-                properties.append(xml_props_list[0])
-            else:
-                new_prop = xml_props_list.pop(0)
-                for xml_prop in xml_props_list:
-                    new_prop.values.extend(xml_prop.values)
-                properties.append(new_prop)
-        return properties
-
-    def get_props_with_links(self) -> list[XMLProperty]:
-        """
-        Get a list of all XMLProperties that have an outgoing link to another resource, be it a resptr-prop link
-        or a standoff link in a text.
-
-        Returns:
-            list of all XMLProperties
-        """
-        link_properties: list[XMLProperty] = []
-        for prop in self.properties:
-            if prop.valtype == "resptr":
-                link_properties.append(prop)
-            elif prop.valtype == "text":
-                for value in prop.values:
-                    if value.resrefs:
-                        link_properties.append(prop)
-                        break
-        return link_properties

dsp_tools/commands/xmlupload/models/namespace_context.py

@@ -1,39 +0,0 @@
-from rdflib.namespace import Namespace
-
-
-def get_default_json_ld_context() -> dict[str, str]:
-    """
-    Returns the JSON-LD context as a dictionary.
-
-    Returns:
-        JSON-LD context as a dictionary.
-    """
-    return {
-        "knora-api": "http://api.knora.org/ontology/knora-api/v2#",
-        "salsah-gui": "http://api.knora.org/ontology/salsah-gui/v2#",
-        "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
-        "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
-        "owl": "http://www.w3.org/2002/07/owl#",
-        "xsd": "http://www.w3.org/2001/XMLSchema#",
-    }
-
-
-def make_namespace_dict_from_onto_names(ontos: dict[str, str]) -> dict[str, Namespace]:
-    """Provided a dictionary of ontology names and IRIs, returns a dictionary of Namespace objects."""
-    ontos = correct_project_context_namespaces(ontos)
-    namespaces = {k: Namespace(v) for k, v in ontos.items()}
-    namespaces["knora-api"] = Namespace("http://api.knora.org/ontology/knora-api/v2#")
-    return namespaces
-
-
-def correct_project_context_namespaces(ontos: dict[str, str]) -> dict[str, str]:
-    """Add the hashtag to make it a valid namespace."""
-    return {k: f"{v}#" for k, v in ontos.items()}
-
-
-def get_json_ld_context_for_project(ontos: dict[str, str]) -> dict[str, str]:
-    """Provided a dictionary of ontology names and IRIs, returns a JSON-LD context for the project."""
-    context = get_default_json_ld_context()
-    project_context = correct_project_context_namespaces(ontos)
-    context.update(project_context)
-    return context

dsp_tools/commands/xmlupload/models/ontology_lookup_models.py

@@ -1,161 +0,0 @@
-from dataclasses import dataclass
-from dataclasses import field
-from typing import Any
-from typing import Literal
-
-import regex
-
-
-@dataclass(frozen=True)
-class OntoInfo:
-    """This class saves the properties and the classes from an ontology."""
-
-    classes: list[str] = field(default_factory=list)
-    properties: list[str] = field(default_factory=list)
-
-
-@dataclass
-class ProjectOntosInformation:
-    """This class saves information needed to check the consistency with the ontology."""
-
-    default_ontology_prefix: str
-    onto_lookup: dict[str, OntoInfo]
-
-
-def make_project_onto_information(
-    default_onto_prefix: str,
-    ontologies: dict[str, list[dict[str, Any]]],
-) -> ProjectOntosInformation:
-    """
-    This function formats ontologies returned by the dsp-api into a look-up model.
-
-    Args:
-        default_onto_prefix: prefix for the default ontology
-        ontologies: ontologies returned from the dsp-api
-
-    Returns:
-        A look-up that contains the property and class names from each ontology
-    """
-    onto_dict = {
-        onto_name: _extract_classes_and_properties_from_onto(onto_json) for onto_name, onto_json in ontologies.items()
-    }
-    return ProjectOntosInformation(default_onto_prefix, onto_dict)
-
-
-def _extract_classes_and_properties_from_onto(onto_json: list[dict[str, Any]]) -> OntoInfo:
-    """
-    This function takes an ontology response from the DSP-API.
-    It extracts the classes and properties.
-    And saves them in an instance of the class Ontology.
-
-    Args:
-        onto_json: response from DSP-API
-
-    Returns:
-        Ontology instance with the classes and properties
-    """
-    classes = _get_all_cleaned_classes_from_onto(onto_json)
-    properties = _get_all_cleaned_properties_from_onto(onto_json)
-    return OntoInfo(classes, properties)
-
-
-def _get_all_cleaned_classes_from_onto(onto_json: list[dict[str, Any]]) -> list[str]:
-    classes = _get_all_classes_from_onto(onto_json)
-    return _remove_prefixes(classes)
-
-
-def _get_all_classes_from_onto(onto_json: list[dict[str, Any]]) -> list[str]:
-    return [elem["@id"] for elem in onto_json if elem.get("knora-api:isResourceClass")]
-
-
-def _get_all_cleaned_properties_from_onto(onto_json: list[dict[str, Any]]) -> list[str]:
-    props = _get_all_properties_from_onto(onto_json)
-    return _remove_prefixes(props)
-
-
-def _get_all_properties_from_onto(onto_json: list[dict[str, Any]]) -> list[str]:
-    return [elem["@id"] for elem in onto_json if not elem.get("knora-api:isResourceClass")]
-
-
-def _remove_prefixes(ontology_elements: list[str]) -> list[str]:
-    return [x.split(":")[1] for x in ontology_elements]
-
-
-AllowedEncodings = Literal["utf8", "xml"]
-
-
-@dataclass
-class TextValueData:
-    resource_id: str
-    res_type: str
-    property_name: str
-    encoding: AllowedEncodings
-
-
-@dataclass
-class PropertyTextValueTypes:
-    """
-    This class contains the information
-    which properties have which type of encoding for a TextValue property in the ontology.
-    """
-
-    formatted_text_props: set[str] = field(default_factory=set)
-    unformatted_text_props: set[str] = field(default_factory=set)
-
-
-def get_text_value_types_of_properties_from_onto(
-    onto_json_dict: dict[str, list[dict[str, Any]]], default_onto: str
-) -> PropertyTextValueTypes:
-    """
-    This function takes a dict with the project ontologies in the format:
-        { ontology_name: ontology_json_from_api }
-    It retrieves the properties that are used with `knora-api:TextValue`.
-    They are separated into two categories: xml encoded ones, and utf8 encoded ones.
-
-    Args:
-        onto_json_dict: dict with the project ontologies
-        default_onto: name of the default ontology
-
-    Returns:
-        Look-up containing the properties separated according to the formatting
-    """
-    all_props = []
-    for onto_json in onto_json_dict.values():
-        all_props.extend(_get_all_text_value_types_properties_and_from_onto(onto_json))
-    return _make_text_value_property_type_lookup(all_props, default_onto)
-
-
-def _make_text_value_property_type_lookup(
-    prop_list: list[tuple[str, str]], default_onto: str
-) -> PropertyTextValueTypes:
-    formatted_text = {
-        _remove_default_prefix(p, default_onto) for p, _type in prop_list if _type == "salsah-gui:Richtext"
-    }
-    formatted_text.update(["hasComment", "hasDescription"])  # knora-api properties that can be used directly
-    unformatted_text = {
-        _remove_default_prefix(p, default_onto) for p, _type in prop_list if _type != "salsah-gui:Richtext"
-    }
-    unformatted_text.update(["hasTitle", "hasKeyword"])  # knora-api properties that can be used directly
-    return PropertyTextValueTypes(formatted_text, unformatted_text)
-
-
-def _remove_default_prefix(prop_str: str, default_onto: str) -> str:
-    return regex.sub(rf"^{default_onto}:", ":", prop_str)
-
-
-def _get_all_text_value_types_properties_and_from_onto(onto_json: list[dict[str, Any]]) -> list[tuple[str, str]]:
-    prop_id_list = [elem["@id"] for elem in onto_json if _check_if_text_value_property(elem)]
-    type_list = [elem["salsah-gui:guiElement"]["@id"] for elem in onto_json if _check_if_text_value_property(elem)]
-    return list(zip(prop_id_list, type_list))
-
-
-def _check_if_text_value_property(onto_ele: dict[str, Any]) -> bool:
-    if not onto_ele.get("knora-api:isResourceProperty"):
-        return False
-    if not (object_type := onto_ele.get("knora-api:objectType")):
-        return False
-    match object_type["@id"]:
-        case "knora-api:TextValue":
-            return True
-        case _:
-            return False

dsp_tools/commands/xmlupload/models/ontology_problem_models.py

@@ -1,178 +0,0 @@
-import itertools
-from dataclasses import dataclass
-
-import pandas as pd
-
-from dsp_tools.commands.xmlupload.models.ontology_lookup_models import TextValueData
-
-separator = "\n    "
-list_separator = "\n    - "
-medium_separator = "\n\n"
-maximum_prints = 50
-
-
-@dataclass(frozen=True)
-class InvalidOntologyElementsInData:
-    """This class saves and prints out the information regarding ontology classes and properties
-    that are in the XML but not the ontology."""
-
-    classes: list[tuple[str, list[str], str]]
-    properties: list[tuple[str, list[str], str]]
-    ontos_on_server: list[str]
-
-    def execute_problem_protocol(self) -> tuple[str, pd.DataFrame | None]:
-        """
-        If there are any elements in properties or classes,
-        this method composes an error message.
-
-        Returns:
-            the error message and a dataframe with the errors if they exceed 50 or None
-        """
-        base_msg = (
-            f"\nSome property and/or class type(s) used in the XML are unknown.\n"
-            f"The ontologies for your project on the server are:{list_separator}"
-            f"{list_separator.join(self.ontos_on_server)}{medium_separator}"
-        )
-        if cls_msg := self._compose_problem_string_for_cls():
-            base_msg += cls_msg + "\n"
-        if prop_msg := self._compose_problem_string_for_props():
-            base_msg += prop_msg + "\n"
-        if (
-            self._calculate_num_resources(self.classes) + self._calculate_num_resources(self.properties)
-            > maximum_prints
-        ):
-            df = self._get_problems_as_df()
-            return base_msg, df
-        return base_msg, None
-
-    def _get_problems_as_df(self) -> pd.DataFrame:
-        problems = [
-            [
-                {
-                    "problematic type": probs[0],
-                    "resource id": x,
-                    "problem": probs[2],
-                }
-                for x in probs[1]
-            ]
-            for probs in self.classes
-        ]
-        problems.extend(
-            [
-                [
-                    {
-                        "problematic type": probs[0],
-                        "resource id": x,
-                        "problem": probs[2],
-                    }
-                    for x in probs[1]
-                ]
-                for probs in self.properties
-            ]
-        )
-        unpacked: list[dict[str, str]] = list(itertools.chain(*problems))
-        return pd.DataFrame.from_records(unpacked)
-
-    def _calculate_num_resources(self, to_count: list[tuple[str, list[str], str]]) -> int:
-        return sum((len(x[1]) for x in to_count))
-
-    def _compose_problem_string_for_cls(self) -> str | None:
-        if self.classes:
-            if self._calculate_num_resources(self.classes) > maximum_prints:
-                return "Many resources have an invalid resource type.\nPlease consult the file for details."
-
-            def _format_cls(cls_tup: tuple[str, list[str], str]) -> str:
-                ids = list_separator + list_separator.join(cls_tup[1])
-                return (
-                    f"    Resource Type: '{cls_tup[0]}'{separator}"
-                    f"Problem: '{cls_tup[2]}'{separator}"
-                    f"Resource ID(s):{ids}"
-                )
-
-            problems = [_format_cls(x) for x in self.classes]
-
-            return (
-                f"The following resource(s) have an invalid resource type:{medium_separator}"
-                + medium_separator.join(problems)
-                + "\n"
-            )
-        else:
-            return None
-
-    def _compose_problem_string_for_props(self) -> str | None:
-        if self.properties:
-            if self._calculate_num_resources(self.properties) > maximum_prints:
-                return "Many properties have an invalid resource type.\nPlease consult the file for details."
-
-            def _format_prop(prop_tup: tuple[str, list[str], str]) -> str:
-                ids = list_separator + list_separator.join(prop_tup[1])
-                return (
-                    f"    Property Name: '{prop_tup[0]}'{separator}"
-                    f"Problem: '{prop_tup[2]}'{separator}"
-                    f"Resource ID(s):{ids}"
-                )
-
-            problems = [_format_prop(x) for x in self.properties]
-            return (
-                f"The following resource(s) have invalid property type(s):{medium_separator}"
-                + medium_separator.join(problems)
-                + "\n"
-            )
-        else:
-            return None
-
-
-@dataclass
-class InvalidTextValueEncodings:
-    """
-    This class contains information about resources and the respective properties that have invalid text encodings.
-
-    An invalid encoding would be a property that specifies `knora-api:Richtext` in the ontology,
-        but the `<text>` elements use: `<text encoding="utf8">`.
-    OR
-    A property that specifies `knora-api:Textarea` or `knora-api:SimpleText`
-        but the `<text>` elements use: `<text encoding="xml">`.
-    """
-
-    problematic_resources: list[TextValueData]
-
-    def execute_problem_protocol(self) -> tuple[str, pd.DataFrame | None]:
-        """
-        This method composes an error message for the user.
-
-        Returns:
-            the error message and a dataframe with the errors if they exceed the maximum allowed print statements
-        """
-        base_msg = (
-            "\nSome text encodings used in the XML data file are not conform with the gui_element "
-            "specified in the JSON ontology.\n"
-            "Please consult the ontology regarding the assigned gui_elements."
-        )
-        df = self._get_problems_as_df()
-        if len(df) > maximum_prints:
-            return base_msg, df
-        return base_msg + medium_separator + _make_msg_from_df(df), None
-
-    def _get_problems_as_df(self) -> pd.DataFrame:
-        df = pd.DataFrame(
-            {
-                "Resource ID": [x.resource_id for x in self.problematic_resources],
-                "Resource Type": [x.res_type for x in self.problematic_resources],
-                "Property Name": [x.property_name for x in self.problematic_resources],
-                "Encoding Used": [x.encoding for x in self.problematic_resources],
-            }
-        )
-        return df.sort_values(by=["Resource Type", "Resource ID", "Property Name"], ignore_index=True)
-
-
-def _make_msg_from_df(df: pd.DataFrame) -> str:
-    groups = df.groupby(by="Resource ID")
-    return medium_separator.join([_make_msg_for_one_resource(str(_id), res_df) for _id, res_df in groups])
-
-
-def _make_msg_for_one_resource(res_id: str, res_df: pd.DataFrame) -> str:
-    props = res_df["Property Name"].tolist()
-    encding = res_df["Encoding Used"].tolist()
-    restype = next(iter(res_df["Resource Type"]))
-    problems = [f"Property Name: '{p}' -> Encoding Used: '{e}'" for p, e in zip(props, encding)]
-    return f"Resource ID: '{res_id}' | Resource Type: '{restype}'{list_separator}{list_separator.join(problems)}"

dsp_tools/commands/xmlupload/models/serialise/jsonld_serialiser.py

@@ -1,40 +0,0 @@
-import json
-from typing import Any
-
-from pyld import jsonld
-from rdflib import Graph
-from rdflib import URIRef
-
-
-def serialise_property_graph(rdf_graph: Graph, prop_name: URIRef) -> dict[str, Any]:
-    """
-    It serialises an RDF-graph into json-ld,
-    which is conform to the format expected by the DSP-API.
-    It returns the information about the property and its value(s).
-    It is possible that the graph contains different properties.
-    The properties do not have to be from the same ontology.
-
-    Args:
-        rdf_graph: RDF graph
-        prop_name: One property name used in the graph.
-                    If there are several it does not matter which.
-
-    Returns:
-        A json-ld
-    """
-    json_graph = _make_json(rdf_graph)
-    return _frame_property(json_graph, prop_name)
-
-
-def _frame_property(serialised_json: list[dict[str, Any]], prop_name: URIRef) -> dict[str, Any]:
-    json_frame: dict[str, Any] = {
-        str(prop_name): {},
-    }
-    framed: dict[str, Any] = jsonld.frame(serialised_json, json_frame)
-    return framed
-
-
-def _make_json(rdf_graph: Graph) -> list[dict[str, Any]]:
-    graph_bytes = rdf_graph.serialize(format="json-ld", encoding="utf-8")
-    json_graph: list[dict[str, Any]] = json.loads(graph_bytes)
-    return json_graph

dsp_tools/commands/xmlupload/models/serialise/serialise_value.py

@@ -1,51 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Any
-from typing import Protocol
-
-
-@dataclass(frozen=True)
-class SerialiseProperty:
-    property_name: str
-    values: list[SerialiseValue]
-
-    def serialise(self) -> dict[str, Any]:
-        """Serialise the property and all its values."""
-        return {self.property_name: [value.serialise() for value in self.values]}
-
-
-class SerialiseValue(Protocol):
-    """A value to be serialised."""
-
-    value: str
-    permissions: str | None
-    comment: str | None
-
-    def serialise(self) -> dict[str, Any]:
-        """Serialise the value."""
-
-
-@dataclass(frozen=True)
-class SerialiseURI(SerialiseValue):
-    """A URI to be serialised."""
-
-    value: str
-    permissions: str | None
-    comment: str | None
-
-    def serialise(self) -> dict[str, Any]:
-        """Serialise the URI value."""
-
-        serialised = {
-            "@type": "knora-api:UriValue",
-            "knora-api:uriValueAsUri": {
-                "@type": "xsd:anyURI",
-                "@value": self.value,
-            },
-        }
-        if self.comment:
-            serialised["knora-api:valueHasComment"] = self.comment
-        if self.permissions:
-            serialised["knora-api:hasPermissions"] = self.permissions
-        return serialised