dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/xmllib/internal/checkers.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+import regex
+
+from dsp_tools.error.xmllib_warnings import MessageInfo
+from dsp_tools.error.xmllib_warnings_util import emit_xmllib_input_info
+from dsp_tools.error.xmllib_warnings_util import emit_xmllib_input_warning
+
+
+def is_nonempty_value_internal(value: Any) -> bool:
+    """
+    This is a duplicate of value_checkers.is_nonempty_value(), to avoid circular imports.
+
+    Check if a value is not None-like
+    or that its string representation contains at least one of the following characters:
+
+    - ``\\p{S}`` = symbols and special characters
+    - ``\\p{P}`` = punctuation
+    - ``\\w`` = all Unicode letters, numbers, and _
+
+    Args:
+        value: value of any type
+
+    Returns:
+        True if the value is not None-like and contains at least one of the above-mentioned characters
+    """
+    if isinstance(value, (tuple, list, set)):
+        all_vals = [is_nonempty_value_internal(v) for v in value]
+        return all(all_vals)
+    if isinstance(value, dict):
+        all_vals = []
+        for k, v in value.items():
+            all_vals.append(is_nonempty_value_internal(k))
+            all_vals.append(is_nonempty_value_internal(v))
+        return all(all_vals)
+    if pd.isna(value):
+        return False
+    if regex.search(r"[\p{S}\p{P}\w]", str(value), flags=regex.UNICODE):
+        return True
+    return False
+
+
+def is_date_internal(value: Any) -> bool:
+    """
+    Is it a valid dsp-date.
+
+    Args:
+        value: value to check
+
+    Returns:
+        True if it conforms
+    """
+    calendar_optional = r"(?:(GREGORIAN|JULIAN|ISLAMIC):)?"
+    first_era_optional = r"(?:(CE|BCE|BC|AD):)?"
+    second_area_optional = r"(?::(CE|BCE|BC|AD))?"
+    date = r"\d{1,4}(?:-\d{1,2}){0,2}"
+    date_mandatory = rf"({date})"
+    date_optional = rf"(:{date})?"
+    full_date_pattern = (
+        rf"^{calendar_optional}{first_era_optional}{date_mandatory}{second_area_optional}{date_optional}$"
+    )
+    found = regex.search(full_date_pattern, str(value))
+    if not found:
+        return False
+    if found.group(1) == "ISLAMIC" and (found.group(2) or found.group(4)):
+        # eras are not supported yet for the islamic calendar
+        return False
+    return True
+
+
+def check_and_warn_potentially_empty_string(
+    *, value: Any, res_id: str | None, expected: str, prop_name: str | None = None, field: str | None = None
+) -> None:
+    """
+    If a user str() casts an input before using it in the xmllib we may get `None` values that are not recognised
+    as potentially empty.
+    This is to be used if the potentially erroneous input is not detectable through any other check.
+    For example a color or date value has its specific regex and it is not possible for such a value to slip through.
+
+    Args:
+        value: user input
+        res_id: Resource ID
+        expected: the type of value that is expected
+        prop_name: property name if used to check a property
+        field: if used to check a non-property field, for example a comment on a value
+
+    Warnings:
+        XmllibInputWarning: if it is an empty value or a string only with whitespaces
+        XmllibInputInfo: if it is a string containing a string value
+            that may be the result of str() casting an empty value
+    """
+    if not is_nonempty_value_internal(value):
+        msg_info = MessageInfo(
+            message=f"Your input '{value}' is empty. Please enter a valid {expected}.",
+            resource_id=res_id,
+            prop_name=prop_name,
+            field=field,
+        )
+        emit_xmllib_input_warning(msg_info)
+    else:
+        check_and_warn_if_a_string_contains_a_potentially_empty_value(
+            value=value, res_id=res_id, prop_name=prop_name, field=field
+        )
+
+
+def check_and_warn_if_a_string_contains_a_potentially_empty_value(
+    *, value: Any, res_id: str | None, prop_name: str | None = None, field: str | None = None
+) -> None:
+    """
+    If a user str() casts an input before using it in the xmllib we may get `None` values that are not recognised
+    as potentially empty.
+    This is to be used if the potentially erroneous input is not detectable through any other check.
+
+    Args:
+        value: user input
+        res_id: Resource ID
+        prop_name: property name if used to check a property
+        field: if used to check a non-property field, for example a comment on a value
+
+    Warnings:
+        XmllibInputInfo: if it is a string containing a string value
+            that may be the result of str() casting an empty value
+    """
+    empty_val_string = r"^(<NA>|nan|None)$"
+    if bool(regex.search(empty_val_string, str(value))):
+        type_lookup = {"<NA>": "pd.NA", "nan": "np.nan", "None": "None"}
+        msg = (
+            f"Your input '{value}' is a string but may be the result of `str({type_lookup[str(value)]})`. "
+            f"Please verify that the input is as expected."
+        )
+        msg_info = MessageInfo(
+            message=msg,
+            resource_id=res_id,
+            prop_name=prop_name,
+            field=field,
+        )
+        emit_xmllib_input_info(msg_info)
+
+
+def check_and_inform_about_angular_brackets(value: Any, res_id: str | None, prop_name: str | None = None) -> None:
+    """
+    Checks if a string value contains angular brackets.
+
+    Args:
+        value: String value
+        res_id: resource id
+        prop_name: property name
+    """
+    if bool(regex.search(r'<([a-zA-Z/"]+|[^\s0-9].*[^\s0-9])>', str(value))):
+        msg_info = MessageInfo(
+            message=(
+                f"Your input '{value}' contains angular brackets. "
+                f"Since this is a simpletext, please note that these will not be recognised as formatting "
+                f"in the text field, but will be displayed as-is."
+            ),
+            resource_id=res_id,
+            prop_name=prop_name,
+        )
+        emit_xmllib_input_info(msg_info)

dsp_tools/xmllib/internal/circumvent_circular_imports.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from lxml import etree
+
+from dsp_tools.error.xmllib_warnings import MessageInfo
+from dsp_tools.xmllib.general_functions import escape_reserved_xml_characters
+from dsp_tools.xmllib.internal.input_converters import numeric_entities
+
+
+def parse_richtext_as_xml(
+    input_str: str, resource_id: str | None = None, prop_name: str | None = None
+) -> etree._Element | MessageInfo:
+    """
+    Parses an input string as XML. If it contains a syntax error a MessageInfo is returned.
+    Else it returns the XML as etree
+
+    Args:
+        input_str: Richtext string
+        resource_id: ID of the resource for improved error message
+        prop_name: Property name for improved error message
+
+    Returns:
+        Parsed string or information for the user message.
+    """
+    escaped_text = escape_reserved_xml_characters(input_str)
+    num_ent = numeric_entities(escaped_text)
+    pseudo_xml = f"<ignore-this>{num_ent}</ignore-this>"
+    try:
+        return etree.fromstring(pseudo_xml)
+    except etree.XMLSyntaxError as err:
+        msg_str = (
+            f"The entered richtext value could not be converted to a valid XML.\n"
+            f"Original error message: {err.msg}\n"
+            f"Potential line/column numbers are relative to this text: {pseudo_xml}"
+        )
+        return MessageInfo(resource_id=resource_id, prop_name=prop_name, message=msg_str)

dsp_tools/xmllib/internal/constants.py

@@ -0,0 +1,46 @@
+# The accepted XML tags are defined at https://docs.dasch.swiss/latest/DSP-API/03-endpoints/api-v2/text/standard-standoff/
+_COMMON_BASE = [
+    "p",
+    "em",
+    "strong",
+    "u",
+    "sub",
+    "sup",
+    "strike",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "ol",
+    "ul",
+    "li",
+    "tbody",
+    "table",
+    "tr",
+    "td",
+    "br",
+    "hr",
+    "pre",
+    "cite",
+    "blockquote",
+    "code",
+]
+KNOWN_XML_TAG_REGEXES = [
+    *_COMMON_BASE,
+    "a( [^>]+?)?",  # <a> has attributes
+    "footnote( [^>]+?)?",  # the footnote text is in the attributes
+]
+KNOWN_XML_TAGS = [
+    *_COMMON_BASE,
+    "a",
+    "footnote",
+]
+XML_NAMESPACE_MAP = {None: "https://dasch.swiss/schema", "xsi": "http://www.w3.org/2001/XMLSchema-instance"}
+DASCH_SCHEMA = "{https://dasch.swiss/schema}"
+
+# These named entities are defined by the XML specification and are always expanded during parsing,
+# regardless of parser options.
+# Numeric character references (e.g., `&#34;` or `&#x22;`) are also always resolved
+PREDEFINED_XML_ENTITIES = ["&amp;", "&lt;", "&gt;", "&quot;", "&apos;"]

dsp_tools/xmllib/internal/input_converters.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from html.entities import html5
+from typing import Any
+
+import regex
+
+from dsp_tools.error.xmllib_warnings import MessageInfo
+from dsp_tools.error.xmllib_warnings_util import emit_xmllib_input_info
+from dsp_tools.error.xmllib_warnings_util import emit_xmllib_input_type_mismatch_warning
+from dsp_tools.error.xmllib_warnings_util import raise_xmllib_input_error
+from dsp_tools.xmllib.internal.checkers import check_and_warn_if_a_string_contains_a_potentially_empty_value
+from dsp_tools.xmllib.internal.checkers import is_nonempty_value_internal
+from dsp_tools.xmllib.internal.constants import PREDEFINED_XML_ENTITIES
+
+
+def check_and_get_corrected_comment(comment: Any, res_id: str | None, prop_name: str | None) -> str | None:
+    """The input of comments may also be pd.NA or such. In our models we only want a string or None."""
+    if is_nonempty_value_internal(comment):
+        check_and_warn_if_a_string_contains_a_potentially_empty_value(
+            value=comment,
+            res_id=res_id,
+            prop_name=prop_name,
+            field="comment on value",
+        )
+        return str(comment)
+    return None
+
+
+def check_and_fix_is_non_empty_string(
+    value: Any,
+    res_id: str | None = None,
+    prop_name: str | None = None,
+    value_field: str | None = None,
+    expected: str = "non empty string",
+) -> str:
+    """
+    Emits warnings if the string is or looks empty.
+    Converts a non-empty input into a string.
+    Returns an empty string if the input is empty (eg. pd.NA, None, etc.).
+
+    Args:
+        value: input to check
+        res_id: resource ID
+        prop_name: property name
+        value_field: field if it is not a property
+        expected: expected type
+
+    Returns:
+        The value as string, if it is empty an empty string.
+    """
+    if is_nonempty_value_internal(value):
+        check_and_warn_if_a_string_contains_a_potentially_empty_value(
+            value=value,
+            res_id=res_id,
+            prop_name=prop_name,
+            field=value_field,
+        )
+        return str(value)
+    else:
+        emit_xmllib_input_type_mismatch_warning(
+            expected_type=expected,
+            value=value,
+            res_id=res_id,
+            prop_name=prop_name,
+            value_field=value_field,
+        )
+        return ""
+
+
+def check_and_fix_collection_input(value: Any, prop_name: str, res_id: str) -> list[Any]:
+    """
+    To allow varied input but ensure consistent typing internally, collections are converted.
+    If a collection is empty, a warning is emitted for the user.
+
+    Args:
+        value: Input value
+        prop_name: Property name
+        res_id: Resource ID
+
+    Returns:
+        The input as a list
+
+    Raises:
+        XmllibInputError: if the input is a dictionary
+    """
+    match value:
+        case set() | list() | tuple():
+            if len(value) == 0:
+                msg_info = MessageInfo(
+                    message="The input is empty. Please note that no values will be added to the resource.",
+                    resource_id=res_id,
+                    prop_name=prop_name,
+                )
+                emit_xmllib_input_info(msg_info)
+            return list(value)
+        case dict():
+            msg_info = MessageInfo(
+                message="The input is a dictionary. Only collections (list, set, tuple) are permissible.",
+                resource_id=res_id,
+                prop_name=prop_name,
+            )
+            raise_xmllib_input_error(msg_info)
+        case _:
+            return [value]
+
+
+def unescape_reserved_xml_chars(richtext: str) -> str:
+    """
+    This function unescapes characters that are reserved in an XML.
+
+    Args:
+        richtext: Text to be escaped
+
+    Returns:
+        Escaped string
+    """
+    richtext = regex.sub(r"&lt;", "<", richtext or "")
+    richtext = regex.sub(r"&gt;", ">", richtext)
+    richtext = regex.sub(r"&amp;", "&", richtext)
+    return richtext
+
+
+def numeric_entities(text: str) -> str:
+    """
+    Make a string suitable to be written to an XML file,
+    by replacing all named HTML entities by their decimal numeric counterparts,
+    except the ones that are predefined in the XML specification.
+    Numeric HTML entities remain untouched.
+    Unescaped special characters remain untouched.
+
+    Args:
+        text: original text
+
+    Returns:
+        text with all named entities replaced
+
+    Examples:
+        ```python
+        result = xmllib.numeric_entities('&nbsp; &quot;')
+        # result == '&#160; &quot;'
+        ```
+    """
+    replacements: dict[str, str] = {}
+    for match in regex.findall(r"&[0-9A-Za-z]+;", text):
+        if match in PREDEFINED_XML_ENTITIES:
+            continue
+        char = html5[match[1:]]
+        replacements[match] = f"&#{ord(char)};"
+    text = regex.sub(
+        r"&[0-9A-Za-z]+;",
+        lambda x: replacements[x.group()] if x.group() not in PREDEFINED_XML_ENTITIES else x.group(),
+        text,
+    )
+    return text

dsp_tools/xmllib/internal/serialise_file_value.py

@@ -0,0 +1,57 @@
+from lxml import etree
+
+from dsp_tools.error.xmllib_errors import XmllibInternalError
+from dsp_tools.xmllib.internal.constants import DASCH_SCHEMA
+from dsp_tools.xmllib.internal.constants import XML_NAMESPACE_MAP
+from dsp_tools.xmllib.models.internal.file_values import AbstractFileValue
+from dsp_tools.xmllib.models.internal.file_values import FileValue
+from dsp_tools.xmllib.models.internal.file_values import IIIFUri
+from dsp_tools.xmllib.models.internal.file_values import Metadata
+from dsp_tools.xmllib.models.permissions import Permissions
+from dsp_tools.xmllib.value_checkers import is_nonempty_value
+
+
+def serialise_file_value(file_value: AbstractFileValue, authorship_id: str | None) -> etree._Element:
+    """
+    Serialise the file value of a resource.
+
+    Args:
+        file_value: File Value
+        authorship_id: ID referencing the authorship
+
+    Returns:
+        File Value as etree
+
+    Raises:
+        XmllibInternalError: in case of an unknown class
+    """
+    if isinstance(file_value, FileValue):
+        return _serialise_file_value(file_value, authorship_id, "bitstream")
+    elif isinstance(file_value, IIIFUri):
+        return _serialise_file_value(file_value, authorship_id, "iiif-uri")
+    else:
+        raise XmllibInternalError(
+            f"file_value must be either a FileValue or a IIIFUri, but you provided {file_value.__class__.__name__}"
+        )
+
+
+def _serialise_file_value(value: AbstractFileValue, authorship_id: str | None, tag_name: str) -> etree._Element:
+    attribs = _serialise_metadata(value.metadata, authorship_id)
+    if is_nonempty_value(value.comment):
+        attribs["comment"] = str(value.comment)
+    ele = etree.Element(f"{DASCH_SCHEMA}{tag_name}", attrib=attribs, nsmap=XML_NAMESPACE_MAP)
+    ele.text = str(value.value)
+    return ele
+
+
+def _serialise_metadata(metadata: Metadata, authorship_id: str | None) -> dict[str, str]:
+    attribs = {}
+    if metadata.license:
+        attribs["license"] = str(metadata.license)
+    if metadata.copyright_holder:
+        attribs["copyright-holder"] = metadata.copyright_holder
+    if authorship_id:
+        attribs["authorship-id"] = authorship_id
+    if metadata.permissions != Permissions.PROJECT_SPECIFIC_PERMISSIONS:
+        attribs["permissions"] = metadata.permissions.value
+    return attribs

dsp_tools/xmllib/internal/serialise_resource.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import os
+
+from dotenv import find_dotenv
+from dotenv import load_dotenv
+from lxml import etree
+
+from dsp_tools.error.xmllib_warnings import MessageInfo
+from dsp_tools.error.xmllib_warnings_util import emit_xmllib_input_warning
+from dsp_tools.error.xmllib_warnings_util import raise_xmllib_input_error
+from dsp_tools.xmllib.internal.constants import DASCH_SCHEMA
+from dsp_tools.xmllib.internal.constants import XML_NAMESPACE_MAP
+from dsp_tools.xmllib.internal.serialise_file_value import serialise_file_value
+from dsp_tools.xmllib.internal.serialise_values import serialise_values
+from dsp_tools.xmllib.internal.type_aliases import AnyResource
+from dsp_tools.xmllib.models.dsp_base_resources import AudioSegmentResource
+from dsp_tools.xmllib.models.dsp_base_resources import LinkResource
+from dsp_tools.xmllib.models.dsp_base_resources import RegionResource
+from dsp_tools.xmllib.models.dsp_base_resources import VideoSegmentResource
+from dsp_tools.xmllib.models.internal.file_values import AuthorshipLookup
+from dsp_tools.xmllib.models.internal.values import ColorValue
+from dsp_tools.xmllib.models.internal.values import LinkValue
+from dsp_tools.xmllib.models.internal.values import Richtext
+from dsp_tools.xmllib.models.internal.values import Value
+from dsp_tools.xmllib.models.permissions import Permissions
+from dsp_tools.xmllib.models.res import Resource
+
+load_dotenv(dotenv_path=find_dotenv(usecwd=True))
+
+
+def serialise_resources(resources: list[AnyResource], authorship_lookup: AuthorshipLookup) -> list[etree._Element]:
+    """
+    Serialise all the resources
+
+    Args:
+        resources: list of resources
+        authorship_lookup: lookup to map the authors to the corresponding IDs
+
+    Returns:
+        serialised resources
+    """
+    env_var = str(os.getenv("XMLLIB_SORT_RESOURCES")).lower()
+    if env_var == "true":
+        resources = sorted(resources, key=lambda x: x.res_id)
+    return [_serialise_one_resource(x, authorship_lookup) for x in resources]
+
+
+def _serialise_one_resource(res: AnyResource, authorship_lookup: AuthorshipLookup) -> etree._Element:
+    match res:
+        case Resource():
+            return _serialise_generic_resource(res, authorship_lookup)
+        case RegionResource():
+            return _serialise_region(res)
+        case LinkResource():
+            return _serialise_link(res)
+        case AudioSegmentResource():
+            return _serialise_segment(res, "audio-segment")
+        case VideoSegmentResource():
+            return _serialise_segment(res, "video-segment")
+        case _:
+            raise_xmllib_input_error(
+                MessageInfo(
+                    f"An unknown resource was added to the root. "
+                    f"Only Resource, RegionResource, LinkResource, VideoSegmentResource, AudioSegmentResource "
+                    f"are permitted."
+                    f"The input type is {res.__class__.__name__}"
+                )
+            )
+
+
+def _serialise_generic_resource(res: Resource, authorship_lookup: AuthorshipLookup) -> etree._Element:
+    ele = _make_generic_resource_element(res, "resource")
+    ele.attrib["restype"] = res.restype
+    if res.file_value:
+        auth_id = authorship_lookup.get_id(res.file_value.metadata.authorship)
+        ele.append(serialise_file_value(res.file_value, auth_id))
+    ele.extend(serialise_values(res.values))
+    return ele
+
+
+def _serialise_region(res: RegionResource) -> etree._Element:
+    ele = _make_generic_resource_element(res, "region")
+    ele.extend(_serialise_geometry_shape(res))
+    ele.extend(serialise_values(res.values))
+    return ele
+
+
+def _serialise_geometry_shape(res: RegionResource) -> list[etree._Element]:
+    prop_list: list[etree._Element] = []
+    if not res.geometry:
+        emit_xmllib_input_warning(
+            MessageInfo(
+                "The region resource does not have a geometry. Please note that an xmlupload will fail.", res.res_id
+            )
+        )
+        return prop_list
+    geo_prop = etree.Element(f"{DASCH_SCHEMA}geometry-prop", name="hasGeometry", nsmap=XML_NAMESPACE_MAP)
+    geo_attrib = (
+        {"permissions": res.permissions.value} if res.permissions != Permissions.PROJECT_SPECIFIC_PERMISSIONS else {}
+    )
+    ele = etree.Element(f"{DASCH_SCHEMA}geometry", nsmap=XML_NAMESPACE_MAP, attrib=geo_attrib)
+    ele.text = res.geometry.to_json_string()
+    geo_prop.append(ele)
+    prop_list.append(geo_prop)
+    prop_list.extend(
+        serialise_values([ColorValue(value=res.geometry.color, prop_name="hasColor", permissions=res.permissions)]),
+    )
+    return prop_list
+
+
+def _serialise_link(res: LinkResource) -> etree._Element:
+    problems = []
+    if not any([isinstance(x, Richtext) for x in res.values]):
+        problems.append("at least one comment")
+    if not any([isinstance(x, LinkValue) for x in res.values]):
+        problems.append("at least two links")
+    if problems:
+        msg = f"A link object requires {' and '.join(problems)}. Please note that an xmlupload will fail."
+        emit_xmllib_input_warning(MessageInfo(msg, res.res_id))
+    ele = _make_generic_resource_element(res, "link")
+    ele.extend(serialise_values(res.values))
+    return ele
+
+
+def _serialise_segment(res: AudioSegmentResource | VideoSegmentResource, segment_type: str) -> etree._Element:
+    seg = _make_generic_resource_element(res, segment_type)
+    seg.extend(_serialise_segment_children(res))
+    return seg
+
+
+def _make_generic_resource_element(res: AnyResource, res_type: str) -> etree._Element:
+    attribs = {"label": res.label, "id": res.res_id}
+    if res.permissions != Permissions.PROJECT_SPECIFIC_PERMISSIONS:
+        attribs["permissions"] = res.permissions.value
+    return etree.Element(f"{DASCH_SCHEMA}{res_type}", attrib=attribs, nsmap=XML_NAMESPACE_MAP)
+
+
+def _serialise_segment_children(segment: AudioSegmentResource | VideoSegmentResource) -> list[etree._Element]:
+    # The segment elements need to be in a specific order in order to pass the XSD validation
+    # Therefore, this filtering according to the properties is necessary
+    segment_elements = _serialise_according_to_prop_name(segment.values, "isSegmentOf")
+    bounds_attrib = {
+        "segment_start": str(segment.segment_bounds.segment_start),
+        "segment_end": str(segment.segment_bounds.segment_end),
+    }
+    if segment.permissions != Permissions.PROJECT_SPECIFIC_PERMISSIONS:
+        bounds_attrib["permissions"] = segment.permissions.value
+    segment_elements.append(
+        etree.Element(
+            f"{DASCH_SCHEMA}hasSegmentBounds",
+            attrib=bounds_attrib,
+            nsmap=XML_NAMESPACE_MAP,
+        )
+    )
+    segment_elements.extend(_serialise_according_to_prop_name(segment.values, "hasTitle"))
+    segment_elements.extend(_serialise_according_to_prop_name(segment.values, "hasComment"))
+    segment_elements.extend(_serialise_according_to_prop_name(segment.values, "hasDescription"))
+    segment_elements.extend(_serialise_according_to_prop_name(segment.values, "hasKeyword"))
+    segment_elements.extend(_serialise_according_to_prop_name(segment.values, "relatesTo"))
+    return segment_elements
+
+
+def _serialise_according_to_prop_name(values: list[Value], prop_name: str) -> list[etree._Element]:
+    to_serialise = (x for x in values if x.prop_name == prop_name)
+    return [_make_element_with_text(x) for x in to_serialise]
+
+
+def _make_element_with_text(value: Value) -> etree._Element:
+    attribs = {}
+    if value.permissions != Permissions.PROJECT_SPECIFIC_PERMISSIONS:
+        attribs["permissions"] = value.permissions.value
+    if value.comment is not None:
+        attribs["comment"] = str(value.comment)
+    ele = etree.Element(f"{DASCH_SCHEMA}{value.prop_name}", nsmap=XML_NAMESPACE_MAP, attrib=attribs)
+    ele.text = value.value
+    return ele