dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/utils/xml_validation.py

@@ -1,197 +0,0 @@
-from __future__ import annotations
-
-import importlib.resources
-import warnings
-from datetime import datetime
-from pathlib import Path
-
-import regex
-from loguru import logger
-from lxml import etree
-
-from dsp_tools.models.custom_warnings import DspToolsUserWarning
-from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.xml_utils import parse_xml_file
-from dsp_tools.utils.xml_utils import remove_comments_from_element_tree
-from dsp_tools.utils.xml_utils import remove_namespaces_from_xml
-from dsp_tools.utils.xml_validation_models import InconsistentTextValueEncodings
-from dsp_tools.utils.xml_validation_models import TextValueData
-
-separator = "\n    "
-list_separator = "\n    - "
-medium_separator = "\n----------------------------\n"
-grand_separator = "\n\n---------------------------------------\n\n"
-
-
-def validate_xml_file(input_file: Path | str) -> bool:
-    """
-    Validates an XML file against the DSP XSD schema.
-
-    Args:
-        input_file: path to the XML file to be validated, or parsed ElementTree
-
-    Raises:
-        InputError: if the XML file is invalid
-
-    Returns:
-        True if the XML file is valid
-    """
-    root = parse_xml_file(input_file)
-    data_xml = remove_comments_from_element_tree(root)
-    return validate_xml(data_xml)
-
-
-def validate_xml(xml: etree._Element) -> bool:
-    """
-    Validates an XML element tree against the DSP XSD schema.
-
-    Args:
-        xml: the XML element tree to be validated
-
-    Raises:
-        InputError: if the XML file is invalid
-
-    Returns:
-        True if the XML file is valid
-    """
-    xml_no_namespace = remove_namespaces_from_xml(xml)
-
-    problems = []
-
-    problems.extend(_validate_xml_against_schema(xml))
-    problems.extend(_validate_xml_contents(xml_no_namespace))
-
-    if problems:
-        err_msg = grand_separator.join(problems)
-        logger.opt(exception=True).error(err_msg)
-        raise InputError(err_msg)
-
-    logger.info("The XML file is syntactically correct and passed validation.")
-    print(f"{datetime.now()}: The XML file is syntactically correct and passed validation.")
-
-    return True
-
-
-def _validate_xml_against_schema(data_xml: etree._Element) -> list[str]:
-    schema_res = importlib.resources.files("dsp_tools").joinpath("resources/schema/data.xsd")
-    with schema_res.open(encoding="utf-8") as schema_file:
-        xmlschema = etree.XMLSchema(etree.parse(schema_file))
-    if not xmlschema.validate(data_xml):
-        error_msg = "The XML file cannot be uploaded due to the following validation error(s):"
-        for error in xmlschema.error_log:
-            error_msg = f"{error_msg}{separator}Line {error.line}: {error.message}"
-        error_msg = error_msg.replace("{https://dasch.swiss/schema}", "")
-        return [error_msg]
-    return []
-
-
-def _validate_xml_contents(xml_no_namespace: etree._Element) -> list[str]:
-    problems = []
-    _find_xml_tags_in_simple_text_elements(xml_no_namespace)
-    problems.extend(_find_mixed_encodings_in_one_text_prop(xml_no_namespace))
-    _check_for_deprecated_syntax(xml_no_namespace)
-    return problems
-
-
-def _find_xml_tags_in_simple_text_elements(xml_no_namespace: etree._Element) -> None:
-    """
-    Checks if there are angular brackets in simple text.
-    It is possible that the user mistakenly added XML tags into a simple text field.
-    But it is also possible that an angular bracket should be displayed.
-    So that the user does not insert XML tags mistakenly into simple text fields,
-    the user is warned, if there is any present.
-
-    Args:
-        xml_no_namespace: parsed XML file with the namespaces removed
-    """
-    resources_with_potential_xml_tags = []
-    for text in xml_no_namespace.findall(path="resource/text-prop/text"):
-        regex_finds_tags = bool(regex.search(r'<([a-zA-Z/"]+|[^\s0-9].*[^\s0-9])>', str(text.text)))
-        etree_finds_tags = bool(list(text.iterchildren()))
-        has_tags = regex_finds_tags or etree_finds_tags
-        if text.attrib["encoding"] == "utf8" and has_tags:
-            sourceline = f"line {text.sourceline}: " if text.sourceline else " "
-            propname = text.getparent().attrib["name"]  # type: ignore[union-attr]
-            resname = text.getparent().getparent().attrib["id"]  # type: ignore[union-attr]
-            resources_with_potential_xml_tags.append(f"{sourceline}resource '{resname}', property '{propname}'")
-    if resources_with_potential_xml_tags:
-        err_msg = (
-            "Angular brackets in the format of <text> were found in text properties with encoding=utf8.\n"
-            "Please note that these will not be recognised as formatting in the text field, "
-            "but will be displayed as-is.\n"
-            f"The following resources of your XML file contain angular brackets:{list_separator}"
-            f"{list_separator.join(resources_with_potential_xml_tags)}"
-        )
-        warnings.warn(DspToolsUserWarning(err_msg))
-
-
-def _find_mixed_encodings_in_one_text_prop(xml_no_namespace: etree._Element) -> list[str]:
-    problems = check_if_only_one_encoding_is_used_per_prop_in_root(xml_no_namespace)
-    if not problems:
-        return []
-    msg, df = InconsistentTextValueEncodings(problems).execute_problem_protocol()
-    if df is not None:
-        csv_path = Path(f"XML_syntax_errors_{datetime.now().strftime('%Y-%m-%d_%H%M%S')}.csv")
-        msg = f"\nAll the problems are listed in the file: '{csv_path.absolute()}'{msg}"
-        df.to_csv(csv_path)
-    return [msg]
-
-
-def _check_for_deprecated_syntax(data_xml: etree._Element) -> None:
-    pass
-
-
-def check_if_only_one_encoding_is_used_per_prop_in_root(
-    root: etree._Element,
-) -> list[TextValueData]:
-    """
-    Check if all the encodings in the `<text>` elements are consistent within one `<text-prop>`
-
-    This is correct:
-    ```
-    <text-prop name=":hasSimpleText">
-        <text encoding="utf8">Text 1</text>
-        <text encoding="utf8">Text 2</text>
-    </text-prop>
-    ```
-
-    This is wrong:
-    ```
-    <text-prop name=":hasSimpleText">
-        <text encoding="utf8">Text 1</text>
-        <text encoding="xml">Text 2</text>
-    </text-prop>
-    ```
-
-    Args:
-        root: root of the data xml document
-
-    Returns:
-          A list of all the inconsistent `<text-props>`
-    """
-    text_props = _get_all_ids_and_encodings_from_root(root)
-    return _find_all_text_props_with_multiple_encodings(text_props)
-
-
-def _get_all_ids_and_encodings_from_root(
-    root: etree._Element,
-) -> list[TextValueData]:
-    res_list = []
-    for res_input in root.iterchildren(tag="resource"):
-        res_list.extend(_get_encodings_from_one_resource(res_input))
-    return res_list
-
-
-def _get_encodings_from_one_resource(resource: etree._Element) -> list[TextValueData]:
-    res_id = resource.attrib["id"]
-    return [_get_encodings_from_one_property(res_id, child) for child in list(resource.iterchildren(tag="text-prop"))]
-
-
-def _get_encodings_from_one_property(res_id: str, prop: etree._Element) -> TextValueData:
-    prop_name = prop.attrib["name"]
-    encodings = {x.attrib["encoding"] for x in prop.iterchildren()}
-    return TextValueData(res_id, prop_name, encodings)
-
-
-def _find_all_text_props_with_multiple_encodings(text_props: list[TextValueData]) -> list[TextValueData]:
-    return [x for x in text_props if len(x.encoding) != 1]

dsp_tools/utils/xml_validation_models.py

@@ -1,68 +0,0 @@
-from dataclasses import dataclass
-
-import pandas as pd
-
-list_separator = "\n    - "
-medium_separator = "\n----------------------------\n"
-grand_separator = "\n\n---------------------------------------\n\n"
-
-maximum_prints = 50
-
-
-@dataclass
-class TextValueData:
-    resource_id: str
-    property_name: str
-    encoding: set[str]
-
-
-@dataclass
-class InconsistentTextValueEncodings:
-    """
-    This class implements the `Problem` protocol
-    for resources and properties that contain invalid encodings.
-
-    An invalid encoding would be a `<text-prop>` element, that contains
-    `<text encoding="utf8">`
-    and
-    `<text encoding="xml">`
-    """
-
-    problematic_resources: list[TextValueData]
-
-    def execute_problem_protocol(self) -> tuple[str, pd.DataFrame | None]:
-        """
-        This method composes an error message for the user.
-        If the number of errors exceeds `maximum_prints`,
-        the errors are additionally returned as a dataframe that can be saved as a CSV file.
-
-        Returns:
-            the error message, and optionally a dataframe with the errors
-        """
-        base_msg = (
-            "\nSome <text-prop> elements contain <text> elements that use both 'xml' and 'utf8' encoding.\n"
-            "Only one encoding type can be used within one <text-prop> element."
-        )
-        df = self._get_problems_as_df()
-        if len(df) > maximum_prints:
-            return base_msg, df
-        return base_msg + grand_separator + _make_msg_from_df(df), None
-
-    def _get_problems_as_df(self) -> pd.DataFrame:
-        df = pd.DataFrame(
-            {
-                "Resource ID": [x.resource_id for x in self.problematic_resources],
-                "Property Name": [x.property_name for x in self.problematic_resources],
-            }
-        )
-        return df.sort_values(by=["Resource ID", "Property Name"], ignore_index=True)
-
-
-def _make_msg_from_df(df: pd.DataFrame) -> str:
-    groups = df.groupby(by="Resource ID")
-    return medium_separator.join([_make_msg_for_one_resource(str(_id), res_df) for _id, res_df in groups])
-
-
-def _make_msg_for_one_resource(res_id: str, res_df: pd.DataFrame) -> str:
-    problems = [f"Property Name: '{p}'" for p in res_df["Property Name"].tolist()]
-    return f"Resource ID: '{res_id}'{list_separator}{list_separator.join(problems)}"

dsp_tools/xmllib/models/file_values.py

@@ -1,78 +0,0 @@
-from __future__ import annotations
-
-import warnings
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-from typing import Protocol
-
-from lxml import etree
-
-from dsp_tools.models.custom_warnings import DspToolsUserWarning
-from dsp_tools.utils.uri_util import is_iiif_uri
-from dsp_tools.xmllib.value_checkers import is_string_like
-
-XML_NAMESPACE_MAP = {None: "https://dasch.swiss/schema", "xsi": "http://www.w3.org/2001/XMLSchema-instance"}
-DASCH_SCHEMA = "{https://dasch.swiss/schema}"
-
-
-class AbstractFileValue(Protocol):
-    value: str | Path
-    permissions: str | None
-    comment: str | None = None
-
-    def serialise(self) -> etree._Element:
-        raise NotImplementedError
-
-
-@dataclass
-class FileValue(AbstractFileValue):
-    value: str | Path
-    permissions: str | None = "prop-default"
-    comment: str | None = None
-    resource_id: str | None = None
-
-    def __post_init__(self) -> None:
-        if not is_string_like(str(self.value)):
-            _warn_type_mismatch(expected_type="file name", value=self.value, res_id=self.resource_id)
-
-    def serialise(self) -> etree._Element:
-        attribs = {}
-        if self.permissions:
-            attribs["permissions"] = self.permissions
-        if self.comment:
-            attribs["comment"] = self.comment
-        ele = etree.Element(f"{DASCH_SCHEMA}bitstream", attrib=attribs, nsmap=XML_NAMESPACE_MAP)
-        ele.text = str(self.value)
-        return ele
-
-
-@dataclass
-class IIIFUri(AbstractFileValue):
-    value: str
-    permissions: str | None = "prop-default"
-    comment: str | None = None
-    resource_id: str | None = None
-
-    def __post_init__(self) -> None:
-        if not is_iiif_uri(self.value):
-            _warn_type_mismatch(expected_type="IIIF uri", value=self.value, res_id=self.resource_id)
-
-    def serialise(self) -> etree._Element:
-        attribs = {}
-        if self.permissions:
-            attribs["permissions"] = self.permissions
-        if self.comment:
-            attribs["comment"] = self.comment
-        ele = etree.Element(f"{DASCH_SCHEMA}iiif-uri", attrib=attribs, nsmap=XML_NAMESPACE_MAP)
-        ele.text = str(self.value)
-        return ele
-
-
-def _warn_type_mismatch(expected_type: str, value: Any, res_id: str | None) -> None:
-    """Emits a warning if a values is not in the expected format."""
-    if res_id:
-        msg = f"The Resource '{res_id}' has an invalid input: The value '{value}' is not a valid {expected_type}."
-    else:
-        msg = f"The value '{value}' is not a valid {expected_type}."
-    warnings.warn(DspToolsUserWarning(msg))