dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/utils/xml_parsing/parse_clean_validate_xml.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import importlib.resources
+from copy import deepcopy
+from pathlib import Path
+
+import pandas as pd
+import regex
+from loguru import logger
+from lxml import etree
+
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.error.exceptions import UserFilepathNotFoundError
+from dsp_tools.error.xsd_validation_error_msg import XSDValidationMessage
+from dsp_tools.error.xsd_validation_error_msg import get_xsd_validation_message_str
+from dsp_tools.utils.ansi_colors import BACKGROUND_BOLD_RED
+from dsp_tools.utils.ansi_colors import BOLD_RED
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+separator = "\n    "
+list_separator = "\n    - "
+
+
+def parse_and_clean_xml_file(input_file: Path) -> etree._Element:
+    root = parse_xml_file(input_file)
+    root = _remove_comments_from_element_tree(root)
+    if not validate_root_emit_user_message(root, Path(input_file).parent):
+        raise InputError("The XML file contains validation errors.")  # a detailed report has already been printed
+    print("The XML file is syntactically correct.")
+    return transform_into_localnames(root)
+
+
+def parse_and_validate_xml_file(input_file: Path | str) -> bool:
+    root = parse_xml_file(input_file)
+    data_xml = _remove_comments_from_element_tree(root)
+    return validate_root_emit_user_message(data_xml, Path(input_file).parent)
+
+
+def parse_xml_file(input_file: str | Path) -> etree._Element:
+    parser = etree.XMLParser(remove_comments=True, remove_pis=True)
+    if not Path(input_file).exists():
+        raise UserFilepathNotFoundError(input_file)
+    try:
+        return etree.parse(source=input_file, parser=parser).getroot()
+    except etree.XMLSyntaxError as err:
+        logger.error(f"The XML file contains the following syntax error: {err.msg}")
+        raise InputError(f"The XML file contains the following syntax error: {err.msg}") from None
+
+
+def transform_into_localnames(root: etree._Element) -> etree._Element:
+    """Removes the namespace of the tags."""
+    tree = deepcopy(root)
+    for elem in tree.iter():
+        elem.tag = etree.QName(elem).localname
+    return tree
+
+
+def _remove_comments_from_element_tree(input_tree: etree._Element) -> etree._Element:
+    """Removes comments and processing instructions."""
+    root = deepcopy(input_tree)
+    for c in root.xpath("//comment()"):
+        c.getparent().remove(c)
+    for c in root.xpath("//processing-instruction()"):
+        c.getparent().remove(c)
+    return root
+
+
+def _validate_xml_tree_against_schema(data_xml: etree._Element) -> etree.XMLSchema | None:
+    schema_res = importlib.resources.files("dsp_tools").joinpath("resources/schema/data.xsd")
+    with schema_res.open(encoding="utf-8") as schema_file:
+        xmlschema = etree.XMLSchema(etree.parse(schema_file))
+    if not xmlschema.validate(data_xml):
+        return xmlschema
+    return None
+
+
+def validate_root_emit_user_message(root: etree._Element, save_path: Path) -> bool:
+    validation_errors = _validate_root_get_validation_messages(root)
+    if validation_errors:
+        _emit_validation_errors(validation_errors, save_path)
+        return False
+    return True
+
+
+def _validate_root_get_validation_messages(data_xml: etree._Element) -> list[XSDValidationMessage] | None:
+    if errors := _validate_xml_tree_against_schema(data_xml):
+        return _reformat_validation_errors(errors.error_log)
+    return None
+
+
+def _emit_validation_errors(validation_errors: list[XSDValidationMessage], save_path: Path) -> None:
+    header_msg = f"During the XSD Schema validation the following {len(validation_errors)} error(s) were found: "
+    print(BACKGROUND_BOLD_RED, header_msg, RESET_TO_DEFAULT)
+    logger.error(header_msg)
+    if len(validation_errors) > 50:
+        save_path = save_path / "xsd_validation_errors.csv"
+        message_dicts = [vars(x) for x in validation_errors]
+        df = pd.DataFrame.from_records(message_dicts)
+        df.to_csv(save_path, index=False)
+        msg = f"Due to the large number of errors they are saved in the file '{save_path}'."
+        print(BOLD_RED + msg, RESET_TO_DEFAULT)
+        logger.error(msg)
+    else:
+        for one_msg in validation_errors:
+            msg_str = get_xsd_validation_message_str(one_msg)
+            print(BOLD_RED, msg_str, RESET_TO_DEFAULT)
+            logger.error(msg_str)
+
+
+def _reformat_validation_errors(log: etree._ListErrorLog) -> list[XSDValidationMessage]:
+    res = [_reformat_error_message_str(err.message, err.line) for err in log]
+    return [x for x in res if x]
+
+
+def _reformat_error_message_str(msg: str, line_number: int) -> XSDValidationMessage | None:
+    element, attrib = None, None
+    msg = msg.replace("{https://dasch.swiss/schema}", "")
+    first, message = msg.split(":", maxsplit=1)
+    if ele_found := regex.search(r"Element '(.*?)'", first):
+        element = ele_found.group(1)
+    if attrib_found := regex.search(r"attribute '(.*?)'", first):
+        attrib = attrib_found.group(1)
+    if "No precomputed value available, the value was either invalid or something strange happened" in message:
+        return None
+    if " is not a valid value of the atomic type 'xs:ID'." in message:
+        if found := regex.search(r"'.*?'", message):
+            id_ = found.group(0)
+        else:
+            id_ = ""
+        message = (
+            f"The provided resource id {id_} is either not a valid xsd:ID or not unique in the file."
+            "The function make_xsd_compatible_id() assists you in creating IDs."
+        )
+    else:
+        message = regex.sub(r"\[facet .+\] ", "", message)
+        message = regex.sub(r"pattern ('.+')", "pattern for this value", message).strip()
+    return XSDValidationMessage(line_number=line_number, element=element, attribute=attrib, message=message)

dsp_tools/xmllib/CLAUDE.md

@@ -0,0 +1,302 @@
+# CLAUDE.md - xmllib Module
+
+This file provides guidance to Claude Code when working with the `xmllib` module in dsp-tools.
+
+## Module Overview
+
+The `xmllib` module is the public API library for programmatic creation of DSP XML files.
+It provides a type-safe, validated approach to generating XML data that conforms to DSP (DaSCH Service Platform) requirements.
+
+## Architecture
+
+### Core Components
+
+#### Public API Layer (`__init__.py`)
+
+- **Primary Interface**: All public functions and classes are exported here
+- **Helper Functions**: Utilities for XML generation, validation, and formatting
+- **Value Processing**: Checkers and converters for different data types
+
+#### Models Layer (`models/`)
+
+- **XMLRoot**: Main container for XML documents (`models/root.py`)
+- **Resource**: Primary resource class for DSP resources (`models/res.py`)
+- **DSP Base Resources**: Specialized resource types (`models/dsp_base_resources.py`)
+- **Configuration**: Enums and options (`models/config_options.py`)
+- **Licenses**: License definitions (`models/licenses/`)
+- **Internal Models**: Value types and file models (`models/internal/`)
+
+#### Utilities Layer
+
+- **general_functions.py**: Public utility functions for XML creation
+- **value_checkers.py**: Validation functions for data types
+- **value_converters.py**: Data conversion utilities
+
+#### Internal Layer (`internal/`)
+
+- **checkers.py**: Type checking and input validation functions
+- **input_converters.py**: Data conversion utilities for internal use
+- **Serialization**: XML output generation (`serialise_resource.py`, `serialise_values.py`, `serialise_file_value.py`)
+- **constants.py**: Constants used in several other files
+- **type_aliases.py**: Type definitions and aliases
+- **circumvent_circular_imports.py**: Place for functions that would produce a circular import error in other places
+
+### Data Flow
+
+```text
+Raw Data → Validation → Resource Creation → XML Serialization
+    ↓           ↓              ↓               ↓
+Input Converters → Checkers → Models/Values → XML Output
+    (internal/)   (internal/)      (models/)      (internal/)
+```
+
+## Key Classes and Usage Patterns
+
+### XMLRoot - Document Container
+
+```python
+root = XMLRoot.create_new(shortcode="0000", default_ontology="onto")
+root.add_resource(resource)
+root.serialise_to_file("output.xml")
+```
+
+### Resource - Core Data Model
+
+```python
+resource = Resource.create_new(
+    res_id="resource_1",
+    restype=":Person",
+    label="John Doe"
+)
+resource.add_simpletext(prop_name=":hasName", value="John Doe")
+resource.add_date(prop_name=":hasBirthdate", value="GREGORIAN:CE:1990-01-01")
+resource.add_integer(prop_name=":hasAge", value=33)
+resource.add_link(prop_name=":hasPartner", value="resource_2")
+```
+
+### Specialized Resources
+
+```python
+# Region of Interest
+region = RegionResource.create_new(
+    res_id="region_1",
+    label="ROI 1",
+    region_of="image_1",
+).add_rectangle(
+    (0.5, 0.2),
+    (0.1, 0.3),
+)
+
+# Video Segment
+segment = VideoSegmentResource.create_new(
+    res_id="segment_1",
+    label="The second minute of the video",
+    segment_of="video_1",
+    segment_start="60",
+    segment_end="120"
+)
+```
+
+## Value Types and Validation
+
+### Supported Value Types
+
+- **SimpleText**: Plain text values
+- **Richtext**: Formatted text with standoff markup
+- **Integer**: Numeric integers
+- **Decimal**: Floating point numbers
+- **Boolean**: True/false values
+- **Date**: Calendar dates with era and precision. Always as data range with start and end.
+- **Time**: Timestamps
+- **Color**: Hex color values
+- **URI**: Web addresses and identifiers
+- **Geoname**: Geographic location references, as geoname.org identifiers
+- **List**: Controlled vocabulary values
+- **Link**: References to other resources
+- **File**: Multimedia file attachments
+
+### Validation Functions
+
+```python
+# Check data types
+date_value = reformat_date("1.1.1990", ".", None, DateFormat.DD_MM_YYYY)
+assert is_date(date_value)
+
+if is_bool_like("yes"):
+    bool_value = convert_to_bool_string("yes")
+
+if is_color("#FF0000"):
+    # Valid hex color
+    pass
+```
+
+## Configuration Options
+
+### Permissions
+
+- `PROJECT_SPECIFIC_PERMISSIONS`: Use project defaults
+- `OPEN`: Publicly accessible
+- `RESTRICTED`: Only visible for project members
+- `RESTRICTED_VIEW`: Only applicable for images: Publicly accessible,
+  but users who are not project members see the image in lower resolution or with a watermark
+
+### Date Formatting
+
+- `DateFormat.YYYY_MM_DD`: ISO format (2023-12-31)
+- `DateFormat.DD_MM_YYYY`: European format (31.12.2023)
+- `DateFormat.MM_DD_YYYY`: US format (12/31/2023)
+
+### Calendar Systems
+
+- `Calendar.GREGORIAN`: Modern standard calendar
+- `Calendar.JULIAN`: Julian calender, only used for historical dates
+- `Calendar.ISLAMIC`: Islamic calendar
+
+### Text Processing
+
+- `NewlineReplacement.PARAGRAPH`: Convert linebreaks to `<p>` tags
+- `NewlineReplacement.LINEBREAK`: Convert linebreaks to `<br>` tags
+- `NewlineReplacement.NONE`: Preserve linebreaks as-is
+
+## Rich Text and Standoff Markup
+
+### Creating Links
+
+```python
+# Link to another resource
+link_text = create_standoff_link_to_resource("target_resource_id", "link text")
+
+# Link to external URI
+uri_link = create_standoff_link_to_uri("https://example.com", "external link")
+
+# Add to richtext
+resource.add_richtext(prop_name=":hasDescription", value=f"See {link_text} for details")
+```
+
+### Footnotes
+
+```python
+# Create footnote
+footnote = create_footnote_string("This is a footnote")
+footnote_element = create_footnote_element("Footnote text")
+resource.add_richtext(prop_name=":hasDescription", value=f"Rich text with a footnote: {footnote}")
+```
+
+## Error Handling
+
+### Warning System
+
+- **Input Warnings**: Non-fatal data issues with input data
+- **Type Mismatches**: Automatic conversion with notification
+- **Validation Failures**: Clear error messages with resource context
+
+### Error Types
+
+```python
+# Warning for non-fatal issues
+emit_xmllib_input_warning("Non-standard boolean value converted")
+
+# Type mismatch warning
+if not is_color(val):
+    emit_xmllib_input_type_mismatch_warning(expected_type="color", value=val, res_id=resource_id, prop_name=prop_name)
+
+# Input validation errors
+raise_input_error(MessageInfo(f"Invalid date format: {date}", resource_id="res_1", prop_name=":hasDate"))
+```
+
+## Testing Approach
+
+### Unit Tests (`test/unittests/xmllib`)
+
+- Individual function validation
+- Type checking verification
+- Data conversion accuracy
+
+### Integration Tests (`test/integration/xmllib`)
+
+- Multi-resource XML generation
+- Cross-module compatibility
+- File I/O operations
+
+## Common Patterns
+
+### Resource Creation Pipeline
+
+1. **Validate Input**: Use checker functions
+2. **Convert Data**: Apply converters if needed
+3. **Create Resource**: Instantiate with validated data
+4. **Add Values**: Use typed `add_xyz()` methods
+5. **Serialize**: Generate XML output
+
+### Batch Processing
+
+```python
+# Process multiple resources
+for data_row in dataset:
+    resource = Resource.create_new(
+        res_id=make_xsd_compatible_id(data_row['id']),
+        restype=data_row['type'],
+        label=clean_whitespaces_from_string(data_row['label'])
+    )
+
+    # Add values with validation
+    for prop, value in data_row.items():
+        if is_nonempty_value(value):
+            resource.add_simpletext(prop_name=prop, value=str(value))
+
+    root.add_resource(resource)
+```
+
+### List Value Processing
+
+```python
+# Work with controlled vocabularies
+lookup = ListLookup(project_lists)
+string_with_list_labels = "Label 1; Label 2"
+nodes = get_list_nodes_from_string_via_list_name(
+    string_with_list_labels="Label 1; Label 2",
+    label_separator=";",
+    list_name="list1",
+    list_lookup=list_lookup,
+)
+assert nodes == ["node1", "node2"]
+resource.add_list_multiple(prop_name=":hasColor", list_name="list1", values=list_nodes)
+```
+
+## Important Guidelines
+
+### Type Safety
+
+- Always use type hints in new code
+- Leverage dataclass models for structured data
+- Use validation functions before creating values
+
+### Maintainability
+
+- Follow existing patterns for new value types
+- Add comprehensive docstrings for public functions
+- Include validation logic with clear error messages
+
+## Dependencies
+
+### External
+
+- `lxml`: XML processing and validation
+- `pandas`: Data manipulation (NA handling)
+- `regex`: Unicode-aware pattern matching
+- `loguru`: Structured logging
+
+### Internal
+
+- `dsp_tools.error`: Warning and error system
+- `dsp_tools.utils`: Data format utilities
+- Type definitions and constants
+
+## Migration Notes
+
+When updating xmllib:
+
+- Maintain backward compatibility in public API
+- Add deprecation warnings before removing features
+- Update type hints for new Python versions
+- Ensure XML schema compliance is preserved

dsp_tools/xmllib/__init__.py

@@ -0,0 +1,49 @@
+from dsp_tools.error.xmllib_warnings_util import initialise_warning_file
+
+from .general_functions import ListLookup as ListLookup
+from .general_functions import clean_whitespaces_from_string as clean_whitespaces_from_string
+from .general_functions import create_footnote_element as create_footnote_element
+from .general_functions import create_footnote_string as create_footnote_string
+from .general_functions import create_list_from_input as create_list_from_input
+from .general_functions import create_list_from_string as create_list_from_string
+from .general_functions import create_non_empty_list_from_string as create_non_empty_list_from_string
+from .general_functions import create_standoff_link_to_resource as create_standoff_link_to_resource
+from .general_functions import create_standoff_link_to_uri as create_standoff_link_to_uri
+from .general_functions import escape_reserved_xml_characters as escape_reserved_xml_characters
+from .general_functions import find_license_in_string as find_license_in_string
+from .general_functions import get_list_nodes_from_string_via_list_name as get_list_nodes_from_string_via_list_name
+from .general_functions import get_list_nodes_from_string_via_property as get_list_nodes_from_string_via_property
+from .general_functions import make_xsd_compatible_id as make_xsd_compatible_id
+from .general_functions import make_xsd_compatible_id_with_uuid as make_xsd_compatible_id_with_uuid
+from .models.config_options import NewlineReplacement as NewlineReplacement
+from .models.date_formats import Calendar as Calendar
+from .models.date_formats import DateFormat as DateFormat
+from .models.date_formats import Era as Era
+from .models.dsp_base_resources import AudioSegmentResource as AudioSegmentResource
+from .models.dsp_base_resources import LinkResource as LinkResource
+from .models.dsp_base_resources import RegionResource as RegionResource
+from .models.dsp_base_resources import VideoSegmentResource as VideoSegmentResource
+from .models.licenses.other import LicenseOther as LicenseOther
+from .models.licenses.recommended import LicenseRecommended as LicenseRecommended
+from .models.permissions import Permissions as Permissions
+from .models.res import Resource as Resource
+from .models.root import XMLRoot as XMLRoot
+from .value_checkers import check_richtext_syntax as check_richtext_syntax
+from .value_checkers import is_bool_like as is_bool_like
+from .value_checkers import is_color as is_color
+from .value_checkers import is_date as is_date
+from .value_checkers import is_decimal as is_decimal
+from .value_checkers import is_dsp_ark as is_dsp_ark
+from .value_checkers import is_dsp_iri as is_dsp_iri
+from .value_checkers import is_geoname as is_geoname
+from .value_checkers import is_integer as is_integer
+from .value_checkers import is_nonempty_value as is_nonempty_value
+from .value_checkers import is_timestamp as is_timestamp
+from .value_converters import convert_to_bool_string as convert_to_bool_string
+from .value_converters import find_dates_in_string as find_dates_in_string
+from .value_converters import reformat_date as reformat_date
+from .value_converters import replace_newlines_with_br_tags as replace_newlines_with_br_tags
+from .value_converters import replace_newlines_with_paragraph_tags as replace_newlines_with_paragraph_tags
+from .value_converters import replace_newlines_with_tags as replace_newlines_with_tags
+
+initialise_warning_file()