dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/xml_validate/xml_validate.py

@@ -1,151 +0,0 @@
-import warnings
-from copy import deepcopy
-from pathlib import Path
-
-from lxml import etree
-from rdflib import SH
-from rdflib import Graph
-from termcolor import cprint
-
-from dsp_tools.commands.xml_validate.api_connection import OntologyConnection
-from dsp_tools.commands.xml_validate.api_connection import ShaclValidator
-from dsp_tools.commands.xml_validate.deserialise_input import deserialise_xml
-from dsp_tools.commands.xml_validate.make_data_rdf import make_data_rdf
-from dsp_tools.commands.xml_validate.models.data_deserialised import ProjectDeserialised
-from dsp_tools.commands.xml_validate.models.data_rdf import DataRDF
-from dsp_tools.commands.xml_validate.models.validation import ValidationReport
-from dsp_tools.commands.xml_validate.reformat_validaton_result import reformat_validation_graph
-from dsp_tools.commands.xml_validate.sparql.construct_shapes import construct_shapes_graph
-from dsp_tools.models.custom_warnings import DspToolsUserWarning
-from dsp_tools.utils.xml_utils import parse_xml_file
-from dsp_tools.utils.xml_utils import remove_comments_from_element_tree
-from dsp_tools.utils.xml_utils import transform_into_localnames
-from dsp_tools.utils.xml_validation import validate_xml
-
-LIST_SEPARATOR = "\n    - "
-
-
-def xml_validate(filepath: Path, api_url: str, dev_route: bool, save_graphs: bool) -> bool:  # noqa: ARG001 (unused argument)
-    """
-    Takes a file and project information and validates it against the ontologies on the server.
-
-    Args:
-        filepath: path to the xml data file
-        api_url: url of the api host
-        dev_route: if this flag is set features that are still in development will be used
-        save_graphs: if this flag is set, all the graphs will be saved in a folder
-
-    Returns:
-        true unless it crashed
-    """
-    _inform_about_experimental_feature()
-    data_rdf, shortcode = _get_data_info_from_file(filepath, api_url)
-    onto_con = OntologyConnection(api_url, shortcode)
-    ontologies, shapes = _get_shacl(onto_con)
-    data_graph = data_rdf.make_graph()
-    generic_filepath = Path()
-    if save_graphs:
-        generic_filepath = _save_graphs(filepath, ontologies, shapes, data_graph)
-    # data_graph += ontologies
-    val = ShaclValidator(api_url)
-    report = _validate(val, shapes, data_graph)
-    if save_graphs:
-        report.validation_graph.serialize(f"{generic_filepath}_VALIDATION_REPORT.ttl")
-    if report.conforms:
-        cprint("\n   Validation passed!   ", color="green", attrs=["bold", "reverse"])
-    else:
-        reformatted = reformat_validation_graph(report.validation_graph, data_graph)
-        problem_msg = reformatted.get_msg(filepath)
-        cprint("\n   Validation errors found!   ", color="light_red", attrs=["bold", "reverse"])
-        print(problem_msg)
-        if reformatted.unexpected_results:
-            reformatted.unexpected_results.save_inform_user(
-                results_graph=report.validation_graph,
-                shacl=report.shacl_graph,
-                data=data_graph,
-            )
-    return True
-
-
-def _inform_about_experimental_feature() -> None:
-    what_is_validated = [
-        "This is an experimental feature, it will change and be extended continuously. "
-        "The following information of your data is being validated:",
-        "Cardinalities",
-    ]
-    warnings.warn(DspToolsUserWarning(LIST_SEPARATOR.join(what_is_validated)))
-
-
-def _save_graphs(filepath: Path, onto: Graph, shacl: Graph, data: Graph) -> Path:
-    parent_directory = filepath.parent
-    new_directory = parent_directory / "graphs"
-    new_directory.mkdir(exist_ok=True)
-    cprint(f"\n   Saving graphs to {new_directory}   ", color="light_blue", attrs=["bold", "reverse"])
-    generic_filepath = new_directory / filepath.stem
-    onto.serialize(f"{generic_filepath}_ONTO.ttl")
-    shacl.serialize(f"{generic_filepath}_SHACL.ttl")
-    data.serialize(f"{generic_filepath}_DATA.ttl")
-    onto_data = onto + data
-    onto_data.serialize(f"{generic_filepath}_ONTO_DATA.ttl")
-    return generic_filepath
-
-
-def _validate(validator: ShaclValidator, shapes_graph: Graph, data_graph: Graph) -> ValidationReport:
-    shape_str = shapes_graph.serialize(format="ttl")
-    data_str = data_graph.serialize(format="ttl")
-    results = validator.validate(data_str, shape_str)
-    conforms = bool(next(results.objects(None, SH.conforms)))
-    return ValidationReport(
-        conforms=conforms,
-        validation_graph=results,
-        shacl_graph=shapes_graph,
-        data_graph=data_graph,
-    )
-
-
-def _get_shacl(onto_con: OntologyConnection) -> tuple[Graph, Graph]:
-    ontologies = _get_project_ontos(onto_con)
-    knora_ttl = onto_con.get_knora_api()
-    kag = Graph()
-    kag.parse(data=knora_ttl, format="ttl")
-    onto_for_construction = deepcopy(ontologies) + kag
-    shapes = construct_shapes_graph(onto_for_construction)
-    shapes += ontologies
-    return ontologies, shapes
-
-
-def _get_project_ontos(onto_con: OntologyConnection) -> Graph:
-    all_ontos = onto_con.get_ontologies()
-    onto_g = Graph()
-    for onto in all_ontos:
-        og = Graph()
-        og.parse(data=onto, format="ttl")
-        onto_g += og
-    return onto_g
-
-
-def _get_data_info_from_file(file: Path, api_url: str) -> tuple[DataRDF, str]:
-    cleaned_root = _parse_and_clean_file(file, api_url)
-    deserialised: ProjectDeserialised = deserialise_xml(cleaned_root)
-    rdf_data: DataRDF = make_data_rdf(deserialised.data)
-    return rdf_data, deserialised.info.shortcode
-
-
-def _parse_and_clean_file(file: Path, api_url: str) -> etree._Element:
-    root = parse_xml_file(file)
-    root = remove_comments_from_element_tree(root)
-    validate_xml(root)
-    root = transform_into_localnames(root)
-    return _replace_namespaces(root, api_url)
-
-
-def _replace_namespaces(root: etree._Element, api_url: str) -> etree._Element:
-    with open("src/dsp_tools/resources/xml_validate/replace_namespace.xslt", "rb") as xslt_file:
-        xslt_data = xslt_file.read()
-    shortcode = root.attrib["shortcode"]
-    default_ontology = root.attrib["default-ontology"]
-    namespace = f"{api_url}/ontology/{shortcode}/{default_ontology}/v2#"
-    xslt_root = etree.XML(xslt_data)
-    transform = etree.XSLT(xslt_root)
-    replacement_value = etree.XSLT.strparam(namespace)
-    return transform(root, replacementValue=replacement_value).getroot()

dsp_tools/commands/xmlupload/check_consistency_with_ontology.py

@@ -1,253 +0,0 @@
-from collections import defaultdict
-from datetime import datetime
-from pathlib import Path
-from typing import cast
-
-import regex
-from lxml import etree
-from regex import Pattern
-
-from dsp_tools.commands.xmlupload.models.ontology_lookup_models import AllowedEncodings
-from dsp_tools.commands.xmlupload.models.ontology_lookup_models import ProjectOntosInformation
-from dsp_tools.commands.xmlupload.models.ontology_lookup_models import PropertyTextValueTypes
-from dsp_tools.commands.xmlupload.models.ontology_lookup_models import TextValueData
-from dsp_tools.commands.xmlupload.models.ontology_lookup_models import get_text_value_types_of_properties_from_onto
-from dsp_tools.commands.xmlupload.models.ontology_lookup_models import make_project_onto_information
-from dsp_tools.commands.xmlupload.models.ontology_problem_models import InvalidOntologyElementsInData
-from dsp_tools.commands.xmlupload.models.ontology_problem_models import InvalidTextValueEncodings
-from dsp_tools.commands.xmlupload.ontology_client import OntologyClient
-from dsp_tools.models.exceptions import InputError
-
-defaultOntologyColon: Pattern[str] = regex.compile(r"^:\w+$")
-knoraUndeclared: Pattern[str] = regex.compile(r"^\w+$")
-genericPrefixedOntology: Pattern[str] = regex.compile(r"^[\w\-]+:\w+$")
-KNORA_BASE_PROPERTIES = {
-    "bitstream",
-    "iiif-uri",
-    "isSegmentOf",
-    "hasSegmentBounds",
-    "hasTitle",
-    "hasComment",
-    "hasDescription",
-    "hasKeyword",
-    "relatesTo",
-}
-
-
-def do_xml_consistency_check_with_ontology(onto_client: OntologyClient, root: etree._Element) -> None:
-    """
-    This function takes an OntologyClient and the root of an XML.
-    It retrieves the ontologies from the server.
-    It analyses if any classes or properties are used that are not in the ontology.
-    It analyses if any properties have encodings that are not consistent with those specified in the ontology.
-
-     Args:
-         onto_client: client for the ontology retrieval
-         root: root of the XML
-
-     Raises:
-         InputError: if there are any invalid properties or classes and/or text values with wrong encodings.
-    """
-    cls_prop_lookup, text_value_encoding_lookup = _get_onto_lookups(onto_client)
-    classes_in_data, properties_in_data = _get_all_classes_and_properties_from_data(root)
-    problem_str = ""
-    problem_str += _check_all_classes_and_properties_in_onto(classes_in_data, properties_in_data, cls_prop_lookup)
-    problem_str += _check_correctness_all_text_value_encodings(root, text_value_encoding_lookup)
-    if problem_str:
-        raise InputError(problem_str)
-
-
-def _get_onto_lookups(onto_client: OntologyClient) -> tuple[ProjectOntosInformation, PropertyTextValueTypes]:
-    ontos = onto_client.get_all_project_ontologies_from_server()
-    text_value_encoding_lookup = get_text_value_types_of_properties_from_onto(ontos, onto_client.default_ontology)
-    ontos["knora-api"] = onto_client.get_knora_api_ontology_from_server()
-    return make_project_onto_information(onto_client.default_ontology, ontos), text_value_encoding_lookup
-
-
-def _check_all_classes_and_properties_in_onto(
-    classes_in_data: dict[str, list[str]],
-    properties_in_data: dict[str, list[str]],
-    onto_check_info: ProjectOntosInformation,
-) -> str:
-    class_problems = _find_all_class_types_in_onto(classes_in_data, onto_check_info)
-    property_problems = _find_all_properties_in_onto(properties_in_data, onto_check_info)
-    if not class_problems and not property_problems:
-        return ""
-    problems = InvalidOntologyElementsInData(
-        classes=class_problems, properties=property_problems, ontos_on_server=list(onto_check_info.onto_lookup.keys())
-    )
-    msg, df = problems.execute_problem_protocol()
-    if df is not None:
-        csv_file = f"XML_syntax_errors_{datetime.now().strftime('%Y-%m-%d_%H%M%S')}.csv"
-        df.to_csv(path_or_buf=Path(Path.cwd(), csv_file), index=False)
-        msg += (
-            "\n\n---------------------------------------\n\n"
-            f"\nAll the problems are listed in the file: '{Path.cwd()}/{csv_file}'\n"
-        )
-    return msg
-
-
-def _get_all_classes_and_properties_from_data(
-    root: etree._Element,
-) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
-    cls_dict = _get_all_class_types_and_ids_from_data(root)
-    prop_dict: defaultdict[str, list[str]] = defaultdict(list)
-    for resource in root.iterchildren(tag="resource"):
-        prop_dict = _get_all_property_names_and_resource_ids_one_resource(resource, prop_dict)
-    return cls_dict, prop_dict
-
-
-def _get_all_class_types_and_ids_from_data(root: etree._Element) -> dict[str, list[str]]:
-    cls_dict: dict[str, list[str]] = {}
-    for resource in root.iterchildren(tag="resource"):
-        restype = resource.attrib["restype"]
-        if restype in cls_dict:
-            cls_dict[restype].append(resource.attrib["id"])
-        else:
-            cls_dict[restype] = [resource.attrib["id"]]
-    return cls_dict
-
-
-def _get_all_property_names_and_resource_ids_one_resource(
-    resource: etree._Element, prop_dict: defaultdict[str, list[str]]
-) -> defaultdict[str, list[str]]:
-    for prop in resource.iterchildren():
-        if prop.tag in KNORA_BASE_PROPERTIES:
-            continue
-        prop_name = prop.attrib["name"]
-        prop_dict[prop_name].append(resource.attrib["id"])
-    return prop_dict
-
-
-def _find_all_class_types_in_onto(
-    classes: dict[str, list[str]], onto_check_info: ProjectOntosInformation
-) -> list[tuple[str, list[str], str]]:
-    problem_list = []
-    for cls_type, ids in classes.items():
-        if problem := _find_one_class_type_in_onto(cls_type, onto_check_info):
-            problem_list.append((cls_type, ids, problem))
-    return problem_list
-
-
-def _find_one_class_type_in_onto(cls_type: str, onto_check_info: ProjectOntosInformation) -> str | None:
-    prefix, cls_ = _get_separate_prefix_and_iri_from_onto_prop_or_cls(cls_type, onto_check_info.default_ontology_prefix)
-    if not prefix:
-        return "Class name does not follow a known ontology pattern"
-    if onto := onto_check_info.onto_lookup.get(prefix):
-        return "Invalid Class Type" if cls_ not in onto.classes else None
-    else:
-        return "Unknown ontology prefix"
-
-
-def _find_all_properties_in_onto(
-    properties: dict[str, list[str]], onto_check_info: ProjectOntosInformation
-) -> list[tuple[str, list[str], str]]:
-    problem_list = []
-    for prop_name, ids in properties.items():
-        if problem := _find_one_property_in_onto(prop_name, onto_check_info):
-            problem_list.append((prop_name, ids, problem))
-    return problem_list
-
-
-def _find_one_property_in_onto(prop_name: str, onto_check_info: ProjectOntosInformation) -> str | None:
-    prefix, prop = _get_separate_prefix_and_iri_from_onto_prop_or_cls(
-        prop_name, onto_check_info.default_ontology_prefix
-    )
-    if not prefix:
-        return "Property name does not follow a known ontology pattern"
-    if onto := onto_check_info.onto_lookup.get(prefix):
-        return "Invalid Property" if prop not in onto.properties else None
-    else:
-        return "Unknown ontology prefix"
-
-
-def _get_separate_prefix_and_iri_from_onto_prop_or_cls(
-    prop_or_cls: str, default_ontology_prefix: str
-) -> tuple[str, ...] | tuple[None, None]:
-    if defaultOntologyColon.match(prop_or_cls):
-        return default_ontology_prefix, prop_or_cls.lstrip(":")
-    elif knoraUndeclared.match(prop_or_cls):
-        return "knora-api", prop_or_cls
-    elif genericPrefixedOntology.match(prop_or_cls):
-        return tuple(prop_or_cls.split(":"))
-    else:
-        return None, None
-
-
-def _check_correctness_all_text_value_encodings(root: etree._Element, text_prop_look_up: PropertyTextValueTypes) -> str:
-    """
-    This function analyses if all the encodings for the `<text>` elements are correct
-    with respect to the specification in the ontology.
-
-    For example, if the ontology specifies that `:hasSimpleText` is without mark-up,
-    the encoding has to be `utf8`.
-
-
-    This is correct:
-    ```
-    <text-prop name=":hasSimpleText">
-        <text encoding="utf8">Dies ist ein einfacher Text ohne Markup</text>
-    </text-prop>
-    ```
-
-    This is wrong:
-    ```
-    <text-prop name=":hasSimpleText">
-        <text encoding="xml">Dies ist ein einfacher Text ohne Markup</text>
-    </text-prop>
-    ```
-
-    The accepted encodings are `xml` or `utf8`
-
-    Args:
-        root: root of the data xml document
-        text_prop_look_up: a lookup containing the property names and their specified types
-
-    Returns:
-        A string communicating the problem, if there are none the string is empty.
-    """
-    text_values_in_data = _get_all_ids_and_props_and_encodings_from_root(root)
-    invalid_text_values = [x for x in text_values_in_data if not _check_correctness_of_one_prop(x, text_prop_look_up)]
-    if not invalid_text_values:
-        return ""
-    msg, df = InvalidTextValueEncodings(invalid_text_values).execute_problem_protocol()
-    if df is not None:
-        csv_file = Path(f"text_value_encoding_errors_{datetime.now().strftime('%Y-%m-%d_%H%M%S')}.csv")
-        df.to_csv(path_or_buf=csv_file, index=False)
-        msg += (
-            "\n\n---------------------------------------\n\n"
-            f"All the problems are listed in the file: '{csv_file.absolute()}'"
-        )
-    return msg
-
-
-def _get_all_ids_and_props_and_encodings_from_root(root: etree._Element) -> list[TextValueData]:
-    res_list = []
-    for res_input in root.iterchildren(tag="resource"):
-        res_list.extend(_get_id_and_props_and_encodings_from_one_resource(res_input))
-    return res_list
-
-
-def _get_id_and_props_and_encodings_from_one_resource(resource: etree._Element) -> list[TextValueData]:
-    res_id = resource.attrib["id"]
-    res_type = resource.attrib["restype"]
-    return [
-        _get_prop_and_encoding_from_one_property(res_id, res_type, child)
-        for child in resource.iterchildren(tag="text-prop")
-    ]
-
-
-def _get_prop_and_encoding_from_one_property(res_id: str, res_type: str, prop: etree._Element) -> TextValueData:
-    prop_name = prop.attrib["name"]
-    encoding = cast(AllowedEncodings, prop[0].attrib["encoding"])
-    return TextValueData(res_id, res_type, prop_name, encoding)
-
-
-def _check_correctness_of_one_prop(text_val: TextValueData, text_prop_look_up: PropertyTextValueTypes) -> bool:
-    match text_val.encoding:
-        case "xml":
-            return text_val.property_name in text_prop_look_up.formatted_text_props
-        case "utf8":
-            return text_val.property_name in text_prop_look_up.unformatted_text_props
-        case _:
-            return False

dsp_tools/commands/xmlupload/models/deserialise/deserialise_value.py

@@ -1,236 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from dataclasses import field
-from typing import Optional
-from typing import Union
-from typing import cast
-
-import regex
-from lxml import etree
-
-from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue
-from dsp_tools.models.exceptions import XmlUploadError
-
-
-@dataclass(frozen=True)
-class XMLProperty:
-    """
-    Represents a property of a resource in the XML used for data import.
-
-    Attributes:
-        name: The name of the property
-        valtype: The type of the property
-        values: The list of values of the property
-    """
-
-    name: str
-    valtype: str
-    values: list[XMLValue]
-
-    @staticmethod
-    def from_node(node: etree._Element, valtype: str, default_ontology: str) -> XMLProperty:
-        """
-        The factory for the DSP property
-
-        Args:
-            node: the property node, p.ex. `<decimal-prop></decimal-prop>`
-            valtype: the type of value given by the name of the property node, p.ex. decimal in `<decimal-prop>`
-            default_ontology: the name of the ontology
-
-        Raises:
-            XmlUploadError: If an upload fails
-
-        Returns:
-            The DSP property
-        """
-        name = XMLProperty._get_name(node, default_ontology)
-        if node.tag.endswith("-prop"):
-            values = XMLProperty._get_values_from_normal_props(node, valtype)
-        else:
-            values = [XMLProperty._get_value_from_knora_base_prop(node)]
-        return XMLProperty(name, valtype, values)
-
-    @staticmethod
-    def _get_name(node: etree._Element, default_ontology: str) -> str:
-        # get the property name which is in format namespace:propertyname, p.ex. rosetta:hasName
-        orig = node.attrib.get("name")
-        if not orig:  # tags like <isSegmentOf> don't have a name attribute
-            return f"knora-api:{node.tag}"
-        elif ":" not in orig:
-            return f"knora-api:{orig}"
-        elif orig.startswith(":"):
-            # replace an empty namespace with the default ontology name
-            return f"{default_ontology}{orig}"
-        else:
-            return orig
-
-    @staticmethod
-    def _get_values_from_normal_props(node: etree._Element, valtype: str) -> list[XMLValue]:
-        # parse the subnodes of the property nodes which contain the actual values of the property
-        listname = node.attrib.get("list")  # save the list name if given (only for lists)
-        values: list[XMLValue] = []
-        for subnode in node:
-            if subnode.tag == valtype:  # the subnode must correspond to the expected value type
-                values.append(XMLValue.from_node(subnode, valtype, listname))
-            else:
-                raise XmlUploadError(f"ERROR Unexpected tag: '{subnode.tag}'. Property may contain only value tags!")
-        return values
-
-    @staticmethod
-    def _get_value_from_knora_base_prop(node: etree._Element) -> XMLValue:
-        resrefs = set()
-        if node.tag.endswith("hasSegmentBounds"):
-            value: str | FormattedTextValue = f"{node.attrib["segment_start"]}:{node.attrib["segment_end"]}"
-        elif node.tag.endswith(("hasDescription", "hasComment")):
-            value = _extract_formatted_text_from_node(node)
-            resrefs = value.find_internal_ids()
-        else:
-            str_orig = "".join(node.itertext())
-            value = _cleanup_unformatted_text(str_orig)
-        comment = node.attrib.get("comment")
-        permissions = node.attrib.get("permissions")
-        link_uuid = node.attrib.get("linkUUID")
-        return XMLValue(value=value, resrefs=resrefs, comment=comment, permissions=permissions, link_uuid=link_uuid)
-
-
-@dataclass
-class XMLValue:
-    """Represents a value of a resource property in the XML used for data import"""
-
-    value: Union[str, FormattedTextValue]
-    resrefs: set[str] = field(default_factory=set)
-    comment: Optional[str] = None
-    permissions: Optional[str] = None
-    link_uuid: Optional[str] = None
-
-    @staticmethod
-    def from_node(
-        node: etree._Element,
-        val_type: str,
-        listname: Optional[str] = None,
-    ) -> XMLValue:
-        """Factory method to create an XMLValue from an XML node"""
-        value: Union[str, FormattedTextValue] = ""
-        resrefs = set()
-        comment = node.get("comment")
-        permissions = node.get("permissions")
-        if val_type == "text" and node.get("encoding") == "xml":
-            value = _extract_formatted_text_from_node(node)
-            resrefs = value.find_internal_ids()
-        elif val_type == "text" and node.get("encoding") == "utf8":
-            str_orig = "".join(node.itertext())
-            value = _cleanup_unformatted_text(str_orig)
-        elif val_type == "list":
-            listname = cast(str, listname)
-            value = f"{listname}:" + "".join(node.itertext())
-        else:
-            value = "".join(node.itertext())
-        link_uuid = node.attrib.get("linkUUID")  # not all richtexts have a link, so this attribute is optional
-        return XMLValue(value=value, resrefs=resrefs, comment=comment, permissions=permissions, link_uuid=link_uuid)
-
-
-def _extract_formatted_text_from_node(node: etree._Element) -> FormattedTextValue:
-    xmlstr = etree.tostring(node, encoding="unicode", method="xml")
-    xmlstr = regex.sub(f"<{node.tag}.*?>|</{node.tag}>", "", xmlstr)
-    xmlstr = _cleanup_formatted_text(xmlstr)
-    return FormattedTextValue(xmlstr)
-
-
-def _cleanup_formatted_text(xmlstr_orig: str) -> str:
-    """
-    In a xml-encoded text value from the XML file,
-    there may be non-text characters that must be removed.
-    This function:
-        - replaces (multiple) line breaks by a space
-        - replaces multiple spaces or tabstops by a single space (except within `<code>` or `<pre>` tags)
-
-    Args:
-        xmlstr_orig: content of the tag from the XML file, in serialized form
-
-    Returns:
-        purged string, suitable to be sent to DSP-API
-    """
-    # replace (multiple) line breaks by a space
-    xmlstr = regex.sub("\n+", " ", xmlstr_orig)
-
-    # replace multiple spaces or tabstops by a single space (except within <code> or <pre> tags)
-    # the regex selects all spaces/tabstops not followed by </xyz> without <xyz in between.
-    # credits: https://stackoverflow.com/a/46937770/14414188
-    xmlstr = regex.sub("( {2,}|\t+)(?!(.(?!<(code|pre)))*</(code|pre)>)", " ", xmlstr)
-
-    # remove spaces after <br/> tags (except within <code> tags)
-    xmlstr = regex.sub("((?<=<br/?>) )(?!(.(?!<code))*</code>)", "", xmlstr)
-
-    # remove leading and trailing spaces
-    xmlstr = xmlstr.strip()
-
-    return xmlstr
-
-
-def _cleanup_unformatted_text(string_orig: str) -> str:
-    """
-    In a utf8-encoded text value from the XML file,
-    there may be non-text characters that must be removed.
-    This function:
-        - removes the `<text>` tags
-        - replaces multiple spaces or tabstops by a single space
-
-    Args:
-        string_orig: original string from the XML file
-
-    Returns:
-        purged string, suitable to be sent to DSP-API
-    """
-    # remove the <text> tags
-    string = regex.sub("<text.*?>", "", string_orig)
-    string = regex.sub("</text>", "", string)
-
-    # replace multiple spaces or tabstops by a single space
-    string = regex.sub(r" {2,}|\t+", " ", string)
-
-    # remove leading and trailing spaces (of every line, but also of the entire string)
-    string = "\n".join([s.strip() for s in string.split("\n")])
-    return string.strip()
-
-
-@dataclass(frozen=True)
-class XMLBitstream:
-    """
-    Represents a bitstream object (file) of a resource in the XML used for data import
-
-    Attributes:
-        value: The file path of the bitstream object
-        permissions: Reference to the set of permissions for the bitstream object
-    """
-
-    value: str
-    permissions: Optional[str] = None
-
-    @staticmethod
-    def from_node(node: etree._Element) -> XMLBitstream:
-        """Factory that parses a bitstream node from the XML DOM"""
-        if not node.text:
-            raise XmlUploadError("Empty bitstream tag")
-        return XMLBitstream(node.text, node.get("permissions"))
-
-
-@dataclass(frozen=True)
-class IIIFUriInfo:
-    """
-    Represents a IIIF URI of a resource in the XML used for data import
-
-    Attributes:
-        value: The IIIF URI of the object
-        permissions: Reference to the set of permissions for the IIIF URI
-    """
-
-    value: str
-    permissions: str | None = None
-
-    @staticmethod
-    def from_node(node: etree._Element) -> IIIFUriInfo:
-        """Factory that parses an IIIF URI node from the XML DOM"""
-        if not node.text:
-            raise XmlUploadError("Empty IIIF URI tag")
-        return IIIFUriInfo(node.text, node.get("permissions"))