dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/update_legal/models.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from dataclasses import field
+
+from dsp_tools.error.exceptions import InvalidLicenseError
+from dsp_tools.xmllib.general_functions import find_license_in_string
+from dsp_tools.xmllib.models.licenses.recommended import License
+
+
+@dataclass
+class UpdateCounter:
+    resources_updated: int = 0
+    licenses_set: int = 0
+    copyrights_set: int = 0
+    authorships_set: int = 0
+    invalid_licenses_replaced: int = 0
+
+
+@dataclass(frozen=True)
+class LegalProperties:
+    """Property names in the XML, e.g. ':hasAuthor', ':hasCopyright', ':hasLicense'."""
+
+    authorship_prop: str | None = None
+    copyright_prop: str | None = None
+    license_prop: str | None = None
+
+    def has_any_property(self) -> bool:
+        return any([self.authorship_prop, self.copyright_prop, self.license_prop])
+
+
+@dataclass(frozen=True)
+class LegalMetadata:
+    """Represents legal metadata for a single resource, either from XML or CSV."""
+
+    license: str | None
+    copyright: str | None
+    authorships: Authorships
+
+    def any(self) -> bool:
+        return bool(self.license or self.copyright or (self.authorships and any(x for x in self.authorships.elems)))
+
+
+class LegalMetadataDefaults:
+    """Default values to use when legal metadata is missing from XML."""
+
+    authorship_default: str | None = None
+    copyright_default: str | None = None
+    license_default: License | None = None
+
+    def __init__(
+        self,
+        authorship_default: str | None = None,
+        copyright_default: str | None = None,
+        license_default: str | None = None,
+    ):
+        self.authorship_default = authorship_default
+        self.copyright_default = copyright_default
+        if license_default:
+            if lic := find_license_in_string(license_default):
+                self.license_default = lic
+            else:
+                raise InvalidLicenseError(license_default)
+
+
+@dataclass(frozen=True)
+class Problem:
+    """Represents a problem with legal metadata for a single resource."""
+
+    file_or_iiif_uri: str
+    res_id: str
+    license: str
+    copyright: str
+    authorships: list[str]
+
+
+@dataclass(frozen=True)
+class Authorships:
+    elems: frozenset[str] = field(default_factory=frozenset)
+
+    @classmethod
+    def from_iterable(cls, iterable: Iterable[str]) -> Authorships:
+        return cls(frozenset([x.strip() for x in iterable]))
+
+    def is_empty(self) -> bool:
+        return len(self.elems) == 0

dsp_tools/commands/update_legal/xml_operations.py

@@ -0,0 +1,247 @@
+from pathlib import Path
+
+import regex
+from lxml import etree
+
+from dsp_tools.commands.update_legal.csv_operations import is_fixme_value
+from dsp_tools.commands.update_legal.models import Authorships
+from dsp_tools.commands.update_legal.models import LegalMetadata
+from dsp_tools.commands.update_legal.models import LegalMetadataDefaults
+from dsp_tools.commands.update_legal.models import LegalProperties
+from dsp_tools.commands.update_legal.models import UpdateCounter
+from dsp_tools.xmllib.general_functions import find_license_in_string
+
+
+def collect_metadata(
+    res: etree._Element,
+    properties: LegalProperties,
+    defaults: LegalMetadataDefaults,
+    counter: UpdateCounter,
+    csv_metadata: LegalMetadata | None,
+    treat_invalid_licenses_as_unknown: bool = False,
+) -> LegalMetadata:
+    """Collect legal metadata from CSV corrections, XML properties, or defaults."""
+    license_val, copyright_val, authorships = _resolve_metadata_values(
+        res=res,
+        properties=properties,
+        defaults=defaults,
+        counter=counter,
+        csv_metadata=csv_metadata,
+        treat_invalid_licenses_as_unknown=treat_invalid_licenses_as_unknown,
+    )
+    return LegalMetadata(
+        license=license_val,
+        copyright=copyright_val,
+        authorships=authorships,
+    )
+
+
+def apply_metadata_to_resource(
+    res: etree._Element,
+    media_elem: etree._Element,
+    metadata: LegalMetadata,
+    properties: LegalProperties,
+    auth_text_to_id: dict[Authorships, int],
+) -> None:
+    """
+    Apply legal metadata to a resource's media element and remove old text properties.
+    This function modifies the XML tree in-place.
+
+    Args:
+        res: The resource element
+        media_elem: The bitstream or iiif-uri element to apply attributes to
+        metadata: The legal metadata to apply
+        properties: Configuration for property names (used for removal)
+        auth_text_to_id: Dictionary to track unique authorships (modified in-place)
+    """
+    _apply_metadata_to_element(
+        media_elem=media_elem,
+        license_val=metadata.license,
+        copyright_val=metadata.copyright,
+        authorships=metadata.authorships,
+        auth_text_to_id=auth_text_to_id,
+    )
+    _remove_text_properties(res, properties)
+
+
+def _resolve_metadata_values(
+    res: etree._Element,
+    properties: LegalProperties,
+    defaults: LegalMetadataDefaults,
+    counter: UpdateCounter,
+    csv_metadata: LegalMetadata | None,
+    treat_invalid_licenses_as_unknown: bool = False,
+) -> tuple[str | None, str | None, Authorships]:
+    """
+    Resolve metadata values using priority: CSV > XML > defaults.
+
+    Returns:
+        Tuple of (license_val, copyright_val, authorships)
+    """
+    # Start with CSV corrections if available
+    if csv_metadata:
+        license_val = csv_metadata.license
+        copyright_val = csv_metadata.copyright
+        authorships = csv_metadata.authorships
+    else:
+        license_val = None
+        copyright_val = None
+        authorships = Authorships()
+
+    # Collect license from XML, fall back to default
+    if license_val is None and properties.license_prop:
+        license_val = _extract_license_from_xml(
+            res, properties.license_prop, counter, treat_invalid_licenses_as_unknown
+        )
+    if license_val is None and defaults.license_default:
+        license_val = defaults.license_default.value
+
+    # Collect copyright XML, fall back to default
+    if copyright_val is None and properties.copyright_prop:
+        copyright_val = _extract_copyright_from_xml(res, properties.copyright_prop)
+    if copyright_val is None and defaults.copyright_default:
+        copyright_val = defaults.copyright_default
+
+    # Collect authorship from XML, fall back to default
+    if authorships.is_empty() and properties.authorship_prop:
+        authorships = _extract_authorships_from_xml(res, properties.authorship_prop)
+    if authorships.is_empty() and defaults.authorship_default:
+        authorships = Authorships.from_iterable({defaults.authorship_default})
+
+    return license_val, copyright_val, authorships
+
+
+def _extract_license_from_xml(
+    res: etree._Element,
+    license_prop: str,
+    counter: UpdateCounter,
+    treat_invalid_as_unknown: bool = False,
+) -> str | None:
+    """
+    Extract license from XML property.
+
+    - If one license is found and can be parsed, return its parsed value.
+    - If the property is absent or empty, return None -> will fall back to default.
+    - If multiple licenses are found, return a FIXME string.
+    - If the license is invalid and treat_invalid_as_unknown is True, return 'unknown' and increment counter.
+    - If the license is invalid and treat_invalid_as_unknown is False, return a FIXME string.
+
+    Returns:
+        License value, None, or FIXME string
+    """
+    license_elems: list[etree._Element] = res.xpath(f"./text-prop[@name='{license_prop}']/text")
+    if not license_elems:
+        return None
+    if len(license_elems) > 1:
+        license_texts = [elem.text.strip() for elem in license_elems if elem.text and elem.text.strip()]
+        return f"FIXME: Multiple licenses found. Choose one: {', '.join(license_texts)}"
+    license_elem = license_elems[0]
+    if not license_elem.text or not (license_text := license_elem.text.strip()):
+        return None
+    if not (lic := find_license_in_string(license_text)):
+        if treat_invalid_as_unknown:
+            counter.invalid_licenses_replaced += 1
+            return "http://rdfh.ch/licenses/unknown"
+        return f"FIXME: Invalid license: {license_text}"
+    return lic.value
+
+
+def _extract_copyright_from_xml(res: etree._Element, copy_prop: str) -> str | None:
+    copy_elems: list[etree._Element] = res.xpath(f"./text-prop[@name='{copy_prop}']/text")
+    if not copy_elems:
+        return None
+    if len(copy_elems) > 1:
+        copy_texts = [elem.text.strip() for elem in copy_elems if elem.text and elem.text.strip()]
+        return f"FIXME: Multiple copyrights found. Choose one: {', '.join(copy_texts)}"
+    copy_elem = copy_elems[0]
+    if not copy_elem.text or not (copy_text := copy_elem.text.strip()):
+        return None
+    return copy_text
+
+
+def _extract_authorships_from_xml(res: etree._Element, auth_prop: str) -> Authorships:
+    auth_elems: list[etree._Element] = res.xpath(f"./text-prop[@name='{auth_prop}']/text")
+    if not auth_elems:
+        return Authorships()
+    return Authorships.from_iterable([auth_elem.text.strip() for auth_elem in auth_elems if auth_elem.text])
+
+
+def _apply_metadata_to_element(
+    media_elem: etree._Element,
+    license_val: str | None,
+    copyright_val: str | None,
+    authorships: Authorships,
+    auth_text_to_id: dict[Authorships, int],
+) -> None:
+    """Apply legal metadata as attributes on the bitstream/iiif element."""
+    if license_val and not is_fixme_value(license_val):
+        media_elem.attrib["license"] = license_val
+    if copyright_val and not is_fixme_value(copyright_val):
+        media_elem.attrib["copyright-holder"] = copyright_val
+    if not any(is_fixme_value(x) for x in authorships.elems):
+        if (auth_id := auth_text_to_id.get(authorships)) is None:
+            auth_id = len(auth_text_to_id)
+            auth_text_to_id[authorships] = auth_id
+        media_elem.attrib["authorship-id"] = f"authorship_{auth_id}"
+
+
+def _remove_text_properties(res: etree._Element, properties: LegalProperties) -> None:
+    """Remove the text properties from XML (they're now attributes on media element)."""
+    if properties.authorship_prop:
+        for prop_elem in res.xpath(f"./text-prop[@name='{properties.authorship_prop}']"):
+            res.remove(prop_elem)
+    if properties.copyright_prop:
+        for prop_elem in res.xpath(f"./text-prop[@name='{properties.copyright_prop}']"):
+            res.remove(prop_elem)
+    if properties.license_prop:
+        for prop_elem in res.xpath(f"./text-prop[@name='{properties.license_prop}']"):
+            res.remove(prop_elem)
+
+
+def add_authorship_definitions_to_xml(root: etree._Element, auth_text_to_id: dict[Authorships, int]) -> None:
+    auth_defs = []
+    for auth_text, auth_id in auth_text_to_id.items():
+        auth_def = etree.Element("authorship", attrib={"id": f"authorship_{auth_id}"})
+        for single_auth in auth_text.elems:
+            auth_child = etree.Element("author")
+            auth_child.text = single_auth
+            auth_def.append(auth_child)
+        auth_defs.append(auth_def)
+    for auth_def in reversed(auth_defs):
+        root.insert(0, auth_def)
+
+
+def write_updated_xml(
+    input_file: Path,
+    root: etree._Element,
+    counter: UpdateCounter,
+    partial: bool = False,
+) -> None:
+    root_new = etree.ElementTree(root)
+
+    base_filename = regex.sub(r"(_PARTIALLY_updated|_updated)$", "", input_file.stem)
+    if partial and input_file.stem.endswith("_PARTIALLY_updated"):
+        # Overwrite the existing partial file
+        output_file = input_file
+    elif partial:
+        # Create new partial file
+        output_file = input_file.with_stem(f"{base_filename}_PARTIALLY_updated")
+    else:
+        # Success - create final updated file
+        output_file = input_file.with_stem(f"{base_filename}_updated")
+
+    etree.indent(root_new, space="    ")
+    root_new.write(output_file, pretty_print=True, encoding="utf-8", doctype='<?xml version="1.0" encoding="UTF-8"?>')
+
+    if partial:
+        print(f"\n⚠️  Partial update completed. Output written to: {output_file}")
+        print("   Some resources still have errors - check the CSV error file.\n")
+    else:
+        print(f"\n✓ Successfully updated all legal metadata. Output written to: {output_file}\n")
+
+    print(f" - Resources updated: {counter.resources_updated}\n")
+    print(f" - Licenses set: {counter.licenses_set}\n")
+    print(f" - Copyrights set: {counter.copyrights_set}\n")
+    print(f" - Authorships set: {counter.authorships_set}\n")
+    if counter.invalid_licenses_replaced > 0:
+        print(f" - Invalid licenses replaced with 'unknown': {counter.invalid_licenses_replaced}\n")

dsp_tools/commands/validate_data/CLAUDE.md

@@ -0,0 +1,159 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What is validate_data?
+
+The `validate_data` module is a CLI command that validates XML data files against ontologies stored on a DSP server. 
+It performs comprehensive validation using SHACL (Shapes Constraint Language) via a Docker-based CLI tool
+to ensure data conforms to ontological constraints before upload.
+
+## Key Components
+
+### Core Entry Points
+
+- **validate_data.py**: Main validation orchestrator with two public functions:
+    - `validate_data()`: Validates XML files from filesystem
+    - `validate_parsed_resources()`: Validates pre-parsed resources (used by the CLI commands `xmlupload` and `ingest-xmlupload`)
+
+### SHACL Validation Engine
+
+- **shacl_cli_validator.py**: Docker-based SHACL validation engine that:
+    - Runs SHACL validation using a containerized CLI tool
+    - Handles Docker communication and error handling
+    - Parses validation results back into Python objects
+    - Requires Docker Desktop to be running
+
+### Data Preparation Pipeline (`prepare_data/`)
+
+- **prepare_data.py**: Main data preparation coordinator with functions:
+    - `get_info_and_parsed_resources_from_file()`: Extracts resources from XML files
+    - `prepare_data_for_validation_from_parsed_resource()`: Prepares data for validation
+- **get_rdf_like_data.py**: Converts ParsedResource objects to RDF-like data structures
+- **make_data_graph.py**: Creates RDF graphs from RDF-like data
+
+### Validation Pipeline (`validation/`)
+
+- **get_validation_report.py**: Main validation orchestrator that coordinates SHACL validation
+- **check_for_unknown_classes.py**: Validates that all classes used in data are defined in the ontology
+- **validate_ontology.py**: Validates the ontology itself before data validation
+- **check_duplicate_files.py**: Checks for duplicate file references in the data
+
+### Validation Report Processing (`process_validation_report/`)
+
+- **query_validation_result.py**: Processes SHACL validation results into user-friendly formats
+- **get_user_validation_message.py**: Converts validation problems into user messages
+
+### API Clients (`api_clients.py`)
+
+- **OntologyClient**: Fetches project ontologies and knora-api ontology from DSP server
+- **ListClient**: Retrieves and reformats project lists for validation
+
+### Models
+
+- **models/validation.py**: Core data structures for RDF graphs and validation results
+- **models/api_responses.py**: API response models for ontologies, lists, and validation reports
+- **models/input_problems.py**: User-facing error/warning message structures
+- **models/rdf_like_data.py**: Intermediate data structures for RDF conversion
+
+### SHACL Shape Generation (`sparql/`)
+
+`sparql/` contains the shapes that are ontology specific and are generated during runtime. 
+
+- **construct_shacl.py**: Main SHACL shape construction coordinator
+- **cardinality_shacl.py**: Generates cardinality constraint shapes
+- **value_shacl.py**: Generates value content validation shapes
+- **legal_info_shacl.py**: Generates license validation shapes
+
+### Ontology Independent SHACL Shapes
+
+`src/dsp_tools/resources/validate_data/` contains RDF turtle files with SHACL shapes that apply to all ontologies.
+
+### Utility Functions
+
+- **constants.py**: Defines file paths for validation artifacts and RDF property type information
+- **utils.py**: Helper functions for:
+    - Temporary directory management (`get_temp_directory()`, `clean_up_temp_directory()`)
+    - IRI reformatting for user-friendly display
+    - Validation file cleanup and organization
+- **mappers.py**: Data transformation utilities for converting between different data representations
+
+### Validation Flow
+
+```text
+XML file → ParsedResource → RDF-like data → RDF Graph
+                                              ↓
+Project ontology → SHACL shapes ← ← ← ← ← ← ← ← 
+                     ↓
+           Docker SHACL CLI → Validation Report → User Messages
+```
+
+## Architecture Patterns
+
+### Four-Stage Validation Pipeline
+
+The validation process follows a strict sequential pipeline:
+
+1. **Unknown Classes Check** (`validation/check_for_unknown_classes.py`): 
+   - Ensures all classes used in data are defined in the ontology
+   - If unknown classes are found, validation terminates with an error
+2. **Ontology Validation** (`validation/validate_ontology.py`): 
+   - Validates the ontology itself for correctness
+   - If ontology errors are found, validation terminates with an error
+3. **Duplicate File Check** (`validation/check_duplicate_files.py`):
+   - Checks for duplicate file references in the data
+   - Generates warnings which will be added to potential warnings from the SHACL validation
+4. **SHACL Validation** (`validation/get_validation_report.py`):
+   - Performs comprehensive SHACL validation using Docker CLI
+   - Validates both cardinality constraints and content validation
+   - Generates detailed validation reports
+
+### Docker-Based Validation Architecture
+
+- **Containerized SHACL**: Uses a Docker container for SHACL validation to ensure consistency
+- **File-based Communication**: Writes RDF files to temporary directories for Docker processing
+- **Error Handling**: Robust error handling for Docker communication failures
+- **Temporary File Management**: Automatic cleanup of temporary validation files
+
+### Modular Data Processing Pipeline
+
+- **Data Preparation** (`prepare_data/`): Converts XML to RDF-ready data structures
+- **Validation Execution** (`validation/`): Performs validation checks in sequence
+- **Report Processing** (`process_validation_report/`): Converts validation results to user messages
+- **Clear Separation**: Each stage has dedicated modules with well-defined interfaces
+
+### Problem Categorization and Severity Levels
+
+Validation results are categorized into:
+
+- **Violations**: Critical errors that prevent xmlupload (always displayed)
+- **Warnings**: Issues that block upload on production servers (displayed based on severity setting)
+- **Info**: Potential problems that don't block upload (displayed only with INFO severity)
+- **Unexpected**: Unknown SHACL violations requiring dev team attention (always displayed)
+
+## Key Dependencies
+
+- **rdflib**: RDF graph manipulation and SPARQL queries
+- **Docker**: Required for containerized SHACL validation
+- **subprocess**: For executing Docker commands
+- **pandas**: For handling large validation result datasets
+- **yaml**: For parsing Docker configuration files
+- **loguru**: Structured logging throughout validation process
+
+## Testing Strategy
+
+- **Unit tests**: Test individual validation components
+- **Integration tests**: Test API client interactions
+- **E2E tests**: Test complete validation workflows with testcontainers
+
+## Important Notes
+
+- **Docker Requirement**: Validation requires Docker Desktop to be running for SHACL validation
+- **Local SHACL Validation**: Validation is performed locally using a Docker container, not server-side
+- **Temporary File Management**: Creates temporary files for Docker communication, automatically cleaned up
+- **Graph Saving**: Optional graph saving functionality for debugging complex validation failures  
+- **Production Behavior**: Production servers treat warnings as blockers, preventing upload
+- **Dual Interface**: Supports both direct XML file validation and pre-parsed resource validation (for xmlupload integration)
+- **Configuration Support**: Uses ValidateDataConfig for controlling validation behavior and output options
+- **CSV Output**: Large validation results are saved as CSV files for better handling
+- **Severity Levels**: Configurable severity levels control which validation messages are displayed

dsp_tools/commands/validate_data/constants.py

@@ -0,0 +1,59 @@
+from rdflib import XSD
+
+from dsp_tools.commands.xmlupload.models.rdf_models import RDFPropTypeInfo
+from dsp_tools.utils.rdf_constants import KNORA_API
+
+# paths for the validation files used by the docker container
+
+ONTOLOGIES_DATA_TTL = "ONTOLOGIES_DATA.ttl"
+ONTOLOGIES_SHACL_TTL = "ONTOLOGIES_SHACL.ttl"
+ONTOLOGIES_REPORT_TTL = "ONTOLOGIES_REPORT.ttl"
+
+CARDINALITY_DATA_TTL = "CARDINALITY_DATA.ttl"
+CARDINALITY_SHACL_TTL = "CARDINALITY_SHACL.ttl"
+CARDINALITY_REPORT_TTL = "CARDINALITY_REPORT.ttl"
+
+CONTENT_DATA_TTL = "CONTENT_DATA.ttl"
+CONTENT_SHACL_TTL = "CONTENT_SHACL.ttl"
+CONTENT_REPORT_TTL = "CONTENT_REPORT.ttl"
+
+
+ARCHIVE_FILE_VALUE = RDFPropTypeInfo(KNORA_API.ArchiveFileValue, KNORA_API.fileValueHasFilename, XSD.string)
+AUDIO_FILE_VALUE = RDFPropTypeInfo(KNORA_API.AudioFileValue, KNORA_API.fileValueHasFilename, XSD.string)
+DOCUMENT_FILE_VALUE = RDFPropTypeInfo(KNORA_API.DocumentFileValue, KNORA_API.fileValueHasFilename, XSD.string)
+MOVING_IMAGE_FILE_VALUE = RDFPropTypeInfo(KNORA_API.MovingImageFileValue, KNORA_API.fileValueHasFilename, XSD.string)
+STILL_IMAGE_FILE_VALUE = RDFPropTypeInfo(KNORA_API.StillImageFileValue, KNORA_API.fileValueHasFilename, XSD.string)
+TEXT_FILE_VALUE = RDFPropTypeInfo(KNORA_API.TextFileValue, KNORA_API.fileValueHasFilename, XSD.string)
+IIIF_URI_VALUE = RDFPropTypeInfo(
+    KNORA_API.StillImageExternalFileValue, KNORA_API.stillImageFileValueHasExternalUrl, XSD.anyURI
+)
+
+
+# validation results
+
+FILE_VALUE_PROPERTIES = {
+    KNORA_API.hasArchiveFileValue,
+    KNORA_API.hasAudioFileValue,
+    KNORA_API.hasDocumentFileValue,
+    KNORA_API.hasMovingImageFileValue,
+    KNORA_API.hasTextFileValue,
+    KNORA_API.hasStillImageFileValue,
+    KNORA_API.hasLicense,
+    KNORA_API.hasCopyrightHolder,
+    KNORA_API.hasAuthorship,
+}
+
+
+FILEVALUE_DETAIL_INFO = {
+    KNORA_API.hasArchiveFileValue: ("bitstream", "'zip', 'tar', 'gz', 'z', 'tgz', 'gzip', '7z'"),
+    KNORA_API.hasAudioFileValue: ("bitstream", "'mp3', 'wav'"),
+    KNORA_API.hasDocumentFileValue: ("bitstream", "'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'epub'"),
+    KNORA_API.hasMovingImageFileValue: ("bitstream", "'mp4'"),
+    KNORA_API.hasTextFileValue: ("bitstream", "'odd', 'rng', 'txt', 'xml', 'htm', 'html', 'xsd', 'xsl', 'csv', 'json'"),
+    KNORA_API.hasStillImageFileValue: (
+        "bitstream / iiif-uri",
+        "'jpg', 'jpeg', 'png', 'tif', 'tiff', 'jp2' or a IIIF-URI",
+    ),
+}
+
+LEGAL_INFO_PROPS = {KNORA_API.hasLicense, KNORA_API.hasCopyrightHolder, KNORA_API.hasAuthorship}