dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/xmlupload/stash/stash_circular_references.py

@@ -0,0 +1,68 @@
+from copy import deepcopy
+
+from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue
+from dsp_tools.commands.xmlupload.models.processed.res import ProcessedResource
+from dsp_tools.commands.xmlupload.models.processed.values import ProcessedLink
+from dsp_tools.commands.xmlupload.models.processed.values import ProcessedRichtext
+from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStash
+from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStashItem
+from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStash
+from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStashItem
+from dsp_tools.commands.xmlupload.stash.stash_models import Stash
+
+
+def stash_circular_references(resources: list[ProcessedResource], stash_lookup: dict[str, list[str]]) -> Stash | None:
+    """Stash the values that would create circular references and remove them from the Resources."""
+    stashed_link_values: list[LinkValueStashItem] = []
+    stashed_standoff_values: list[StandoffStashItem] = []
+
+    if not stash_lookup:
+        return None
+
+    for res in resources:
+        if res.res_id not in stash_lookup:
+            continue
+        links, standoff = _process_one_resource(res, stash_lookup)
+        stashed_link_values.extend(links)
+        stashed_standoff_values.extend(standoff)
+
+    standoff_stash = StandoffStash.make(stashed_standoff_values)
+    link_value_stash = LinkValueStash.make(stashed_link_values)
+    return Stash.make(standoff_stash=standoff_stash, link_value_stash=link_value_stash)
+
+
+def _process_one_resource(
+    resource: ProcessedResource,
+    stash_lookup: dict[str, list[str]],
+) -> tuple[list[LinkValueStashItem], list[StandoffStashItem]]:
+    stashed_link_values: list[LinkValueStashItem] = []
+    stashed_standoff_values: list[StandoffStashItem] = []
+
+    for val in resource.values.copy():
+        if isinstance(val, ProcessedLink):
+            if val.value_uuid not in stash_lookup[resource.res_id]:
+                continue
+            stashed_link_values.append(LinkValueStashItem(resource.res_id, resource.type_iri, val))
+            resource.values.remove(val)
+        elif isinstance(val, ProcessedRichtext):
+            if val.value_uuid not in stash_lookup[resource.res_id]:
+                continue
+            # val.value is a KnoraStandoffXml text with problematic links.
+            # stash it, then replace the problematic text with a UUID
+            stashed_standoff_values.append(_stash_standoff(val, resource.res_id, resource.type_iri))
+
+    return stashed_link_values, stashed_standoff_values
+
+
+def _stash_standoff(value: ProcessedRichtext, res_id: str, res_type: str) -> StandoffStashItem:
+    original_value = deepcopy(value)
+    # Replace the content with the UUID
+    value.value = FormattedTextValue(value.value_uuid)
+    # It is not necessary to add the permissions to the StandoffStashItem.
+    # Because when no new permissions are given during an update request,
+    # the permissions of the previous value are taken.
+    return StandoffStashItem(
+        res_id=res_id,
+        res_type=res_type,
+        value=original_value,
+    )

dsp_tools/commands/xmlupload/stash/stash_models.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from itertools import groupby
+
+from dsp_tools.commands.xmlupload.models.processed.values import ProcessedLink
+from dsp_tools.commands.xmlupload.models.processed.values import ProcessedRichtext
+
+
+@dataclass(frozen=True)
+class StandoffStashItem:
+    res_id: str
+    res_type: str
+    value: ProcessedRichtext
+
+
+@dataclass(frozen=True)
+class StandoffStash:
+    """Holds information about a number of stashed XML text values, organized by resource instance."""
+
+    res_2_stash_items: dict[str, list[StandoffStashItem]]
+
+    @staticmethod
+    def make(items: list[StandoffStashItem]) -> StandoffStash | None:
+        """
+        Factory method for StandoffStash.
+
+        Args:
+            items: A list of StandoffStashItem.
+
+        Returns:
+            StandoffStash | None: A StandoffStash object or None, if an empty list was passed.
+        """
+
+        def _get_res_id(x: StandoffStashItem) -> str:
+            return x.res_id
+
+        if not items:
+            return None
+        items = sorted(items, key=_get_res_id)
+        grouped_objects = {k: list(vals) for k, vals in groupby(items, key=_get_res_id)}
+        return StandoffStash(grouped_objects)
+
+
+@dataclass(frozen=True)
+class LinkValueStashItem:
+    """Holds information about a single stashed link value."""
+
+    res_id: str
+    res_type: str
+    value: ProcessedLink
+
+
+@dataclass(frozen=True)
+class LinkValueStash:
+    """Holds information about a number of stashed link values (resptr-props), organized by resource instance."""
+
+    res_2_stash_items: dict[str, list[LinkValueStashItem]]
+
+    @staticmethod
+    def make(items: list[LinkValueStashItem]) -> LinkValueStash | None:
+        """
+        Factory method for LinkValueStash.
+
+        Args:
+            items: A list of LinkValueStashItem.
+
+        Returns:
+            LinkValueStash | None: A LinkValueStash object or None, if an empty list was passed.
+        """
+
+        def _get_res_id(x: LinkValueStashItem) -> str:
+            return x.res_id
+
+        if not items:
+            return None
+        items = sorted(items, key=_get_res_id)
+        grouped_objects = {k: list(vals) for k, vals in groupby(items, key=_get_res_id)}
+        return LinkValueStash(grouped_objects)
+
+
+@dataclass(frozen=True)
+class Stash:
+    """Holds a standoff stash and a linkvalue stash"""
+
+    standoff_stash: StandoffStash | None
+    link_value_stash: LinkValueStash | None
+
+    @staticmethod
+    def make(standoff_stash: StandoffStash | None, link_value_stash: LinkValueStash | None) -> Stash | None:
+        """
+        Factory method for Stash.
+
+        Args:
+            standoff_stash: A StandoffStash object or None.
+            link_value_stash: A LinkValueStash object or None.
+
+        Returns:
+            Stash: A Stash object, or None if both inputs are None.
+        """
+        if standoff_stash or link_value_stash:
+            return Stash(standoff_stash, link_value_stash)
+        return None
+
+    def is_empty(self) -> bool:
+        """Check if there are any stashed items in this stash"""
+        standoff = not self.standoff_stash or not self.standoff_stash.res_2_stash_items
+        link = not self.link_value_stash or not self.link_value_stash.res_2_stash_items
+        return standoff and link

dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import cast
+
+from loguru import logger
+from rdflib import RDF
+from rdflib import BNode
+from rdflib import Graph
+from rdflib import URIRef
+from tqdm import tqdm
+
+from dsp_tools.clients.connection import Connection
+from dsp_tools.commands.xmlupload.make_rdf_graph.jsonld_utils import serialise_jsonld_for_value
+from dsp_tools.commands.xmlupload.make_rdf_graph.make_values import make_link_value_graph
+from dsp_tools.commands.xmlupload.models.upload_state import UploadState
+from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStash
+from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStashItem
+from dsp_tools.commands.xmlupload.stash.stash_models import Stash
+from dsp_tools.error.exceptions import BaseError
+
+
+def upload_stashed_resptr_props(
+    upload_state: UploadState,
+    con: Connection,
+) -> None:
+    """
+    After all resources are uploaded, the stashed resptr props must be applied to their resources in DSP.
+    The upload state is updated accordingly, as a side effect.
+
+    Args:
+        upload_state: the current state of the upload
+        con: connection to DSP
+    """
+    logger.info("Upload the stashed links...")
+    upload_state.pending_stash = cast(Stash, upload_state.pending_stash)
+    link_value_stash = cast(LinkValueStash, upload_state.pending_stash.link_value_stash)
+    progress_bar = tqdm(
+        link_value_stash.res_2_stash_items.copy().items(), desc="Upload the stashed links", dynamic_ncols=True
+    )
+    for res_id, stash_items in progress_bar:
+        res_iri = upload_state.iri_resolver.get(res_id)
+        if not res_iri:
+            # resource could not be uploaded to DSP, so the stash cannot be uploaded either
+            # no action necessary: this resource will remain in nonapplied_resptr_props,
+            # which will be handled by the caller
+            continue
+        logger.info(f"  Upload resptrs of resource '{res_id}'...")
+        for stash_item in reversed(stash_items):
+            # reversed avoids any problems caused by removing from the list we loop over at the same time
+            target_iri = upload_state.iri_resolver.get(stash_item.value.value)
+            if not target_iri:
+                continue
+            if _upload_stash_item(stash_item, res_iri, target_iri, con):
+                link_value_stash.res_2_stash_items[res_id].remove(stash_item)
+        # remove res_id if all stash items were uploaded
+        if not link_value_stash.res_2_stash_items[res_id]:
+            del link_value_stash.res_2_stash_items[res_id]
+
+
+def _upload_stash_item(
+    stash: LinkValueStashItem,
+    res_iri: str,
+    target_iri: str,
+    con: Connection,
+) -> bool:
+    """
+    Upload a single stashed link value to DSP.
+
+    Args:
+        stash: the stashed link value to upload
+        res_iri: the iri of the resource
+        target_iri: the iri of the target resource
+        con: connection to DSP
+
+    Returns:
+        True, if the upload was successful, False otherwise
+    """
+    graph = _make_link_value_create_graph(stash, res_iri, target_iri)
+    payload = serialise_jsonld_for_value(graph, res_iri)
+    try:
+        con.post(route="/v2/values", data=payload)
+    except BaseError as err:
+        _log_unable_to_upload_link_value(err.message, stash.res_id, stash.value.prop_iri)
+        return False
+    logger.debug(f'  Successfully uploaded resptr links of "{stash.value.prop_iri}"')
+    return True
+
+
+def _make_link_value_create_graph(
+    stash: LinkValueStashItem,
+    res_iri_str: str,
+    target_iri: str,
+) -> Graph:
+    """This function creates a JSON object that can be sent as an update request to the DSP-API."""
+    val_bn = BNode()
+    res_iri = URIRef(res_iri_str)
+    graph = make_link_value_graph(stash.value, val_bn, res_iri, URIRef(target_iri))
+    graph.add((res_iri, RDF.type, URIRef(stash.res_type)))
+    return graph
+
+
+def _log_unable_to_upload_link_value(msg: str, res_id: str, prop_name: str) -> None:
+    err_msg = f"Unable to upload the resptr prop of '{prop_name}' of resource '{res_id}'."
+    print(f"{datetime.now()}:     WARNING: {err_msg} Original error message: {msg}")
+    logger.error(err_msg)

dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+from typing import cast
+from urllib.parse import quote_plus
+
+from loguru import logger
+from rdflib import RDF
+from rdflib import Graph
+from rdflib import URIRef
+from tqdm import tqdm
+
+from dsp_tools.clients.connection import Connection
+from dsp_tools.commands.xmlupload.iri_resolver import IriResolver
+from dsp_tools.commands.xmlupload.make_rdf_graph.jsonld_utils import serialise_jsonld_for_value
+from dsp_tools.commands.xmlupload.make_rdf_graph.make_values import make_richtext_value_graph
+from dsp_tools.commands.xmlupload.models.upload_state import UploadState
+from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStash
+from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStashItem
+from dsp_tools.commands.xmlupload.stash.stash_models import Stash
+from dsp_tools.error.exceptions import BaseError
+
+
+def upload_stashed_xml_texts(upload_state: UploadState, con: Connection) -> None:
+    """
+    After all resources are uploaded, the stashed xml texts must be applied to their resources in DSP.
+    The upload state is updated accordingly, as a side effect.
+
+    Args:
+        upload_state: the current state of the upload
+        con: connection to DSP
+    """
+    logger.info("Upload the stashed XML texts...")
+    upload_state.pending_stash = cast(Stash, upload_state.pending_stash)
+    standoff_stash = cast(StandoffStash, upload_state.pending_stash.standoff_stash)
+    progress_bar = tqdm(
+        standoff_stash.res_2_stash_items.copy().items(), desc="Upload stashed XML texts", dynamic_ncols=True
+    )
+    for res_id, stash_items in progress_bar:
+        res_iri = upload_state.iri_resolver.get(res_id)
+        if not res_iri:
+            # resource could not be uploaded to DSP, so the stash cannot be uploaded either
+            # no action necessary: this resource will remain in the list of not uploaded stash items,
+            # which will be handled by the caller
+            continue
+        try:
+            resource_in_triplestore = con.get(f"/v2/resources/{quote_plus(res_iri)}")
+        except BaseError as err:
+            _log_unable_to_retrieve_resource(resource=res_id, received_error=err)
+            continue
+        logger.info(f"  Upload XML text(s) of resource '{res_id}'...")
+        for stash_item in stash_items:
+            value_iri = _get_value_iri(stash_item.value.prop_iri, resource_in_triplestore, stash_item.value.value_uuid)
+            if not value_iri:
+                continue
+            if _upload_stash_item(
+                stash_item=stash_item,
+                res_iri=res_iri,
+                value_iri=value_iri,
+                iri_resolver=upload_state.iri_resolver,
+                con=con,
+            ):
+                standoff_stash.res_2_stash_items[res_id].remove(stash_item)
+        if not standoff_stash.res_2_stash_items[res_id]:
+            standoff_stash.res_2_stash_items.pop(res_id)
+
+
+def _get_value_iri(
+    property_name: str,
+    resource: dict[str, Any],
+    uuid: str,
+) -> str | None:
+    prefixed_prop = _make_prefixed_prop_from_absolute_iri(property_name)
+    values_on_server = resource.get(prefixed_prop)
+    if not isinstance(values_on_server, list):
+        values_on_server = [values_on_server]
+
+    # get the IRI of the value that contains the UUID in its text
+    text_and_iris = ((v["knora-api:textValueAsXml"], v["@id"]) for v in values_on_server)
+    value_iri: str | None = next((iri for text, iri in text_and_iris if uuid in text), None)
+    # in case that "value_iri" is None, the value that contains the UUID in its text does not exist in DSP
+    # no action necessary: this resource will remain in nonapplied_xml_texts,
+    # which will be handled by the caller
+    return value_iri
+
+
+def _make_prefixed_prop_from_absolute_iri(absolute_iri: str) -> str:
+    _, onto, prop = absolute_iri.rsplit("/", 2)
+    local_name = prop.split("#")[-1]
+    return f"{onto}:{local_name}"
+
+
+def _upload_stash_item(
+    stash_item: StandoffStashItem,
+    res_iri: str,
+    value_iri: str,
+    iri_resolver: IriResolver,
+    con: Connection,
+) -> bool:
+    """
+    Upload a single stashed xml text to DSP.
+
+    Args:
+        stash_item: the stashed text value to upload
+        res_iri: the iri of the resource
+        value_iri: the iri of the value
+        iri_resolver: resolver to map ids from the XML file to IRIs in DSP
+        con: connection to DSP
+
+    Returns:
+        True, if the upload was successful, False otherwise
+    """
+    payload = _serialise_richtext_for_update(
+        stash_item=stash_item,
+        value_iri_str=value_iri,
+        res_iri_str=res_iri,
+        iri_resolver=iri_resolver,
+    )
+    try:
+        con.put(route="/v2/values", data=payload)
+    except BaseError as err:
+        _log_unable_to_upload_xml_resource(err, stash_item.res_id, stash_item.value.prop_iri)
+        return False
+    logger.debug(f'  Successfully uploaded xml text of "{stash_item.value.prop_iri}"')
+    return True
+
+
+def _serialise_richtext_for_update(
+    stash_item: StandoffStashItem, value_iri_str: str, res_iri_str: str, iri_resolver: IriResolver
+) -> dict[str, Any]:
+    graph = _make_richtext_update_graph(
+        stash_item=stash_item,
+        value_iri_str=value_iri_str,
+        res_iri_str=res_iri_str,
+        iri_resolver=iri_resolver,
+    )
+    return serialise_jsonld_for_value(graph, res_iri_str)
+
+
+def _make_richtext_update_graph(
+    stash_item: StandoffStashItem, value_iri_str: str, res_iri_str: str, iri_resolver: IriResolver
+) -> Graph:
+    res_iri = URIRef(res_iri_str)
+    value_iri = URIRef(value_iri_str)
+    val_graph = make_richtext_value_graph(
+        val=stash_item.value,
+        val_node=value_iri,
+        res_node=res_iri,
+        iri_resolver=iri_resolver,
+    )
+    val_graph.add((res_iri, RDF.type, URIRef(stash_item.res_type)))
+    return val_graph
+
+
+def _log_unable_to_retrieve_resource(
+    resource: str,
+    received_error: BaseError,
+) -> None:
+    """
+    This function logs the error if it is not possible to retrieve the resource.
+
+    Args:
+        resource: the resource id
+        received_error: the error
+    """
+    # print the message to keep track of the cause for the failure
+    # apart from that; no action is necessary:
+    # this resource will remain in nonapplied_xml_texts, which will be handled by the caller
+    err_msg = (
+        f"Unable to upload XML texts of resource '{resource}', "
+        "because the resource cannot be retrieved from the DSP server."
+    )
+    print(f"{datetime.now()}:   WARNING: {err_msg} Original error message: {received_error.message}")
+    logger.error(err_msg)
+
+
+def _log_unable_to_upload_xml_resource(
+    received_error: BaseError,
+    stashed_resource_id: str,
+    prop_name: str,
+) -> None:
+    """
+    This function logs if it is not possible to upload a xml resource.
+
+    Args:
+        received_error: Error received
+        stashed_resource_id: id of the resource
+        prop_name: name of the property
+    """
+    # print the message to keep track of the cause for the failure
+    # apart from that; no action is necessary:
+    # this resource will remain in nonapplied_xml_texts, which will be handled by the caller
+    err_msg = f"Unable to upload the xml text of '{prop_name}' of resource '{stashed_resource_id}'."
+    print(f"{datetime.now()}:     WARNING: {err_msg} Original error message: {received_error.message}")
+    logger.error(err_msg)

dsp_tools/commands/xmlupload/upload_config.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import dataclasses
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+
+import regex
+from loguru import logger
+
+from dsp_tools.cli.args import ValidationSeverity
+
+
+def _transform_server_url_to_foldername(server: str) -> str:
+    """
+    Take the server URL and transform it so that it can be used as foldername.
+
+    Args:
+        server: server, e.g. "https://api.test.dasch.swiss/" or "http://0.0.0.0:3333"
+
+    Returns:
+        simplified version, e.g. "test.dasch.swiss" or "localhost"
+    """
+    server_substitutions = {
+        r"https?://": "",
+        r"^api\.": "",
+        r":\d{2,5}/?$": "",
+        r"/$": "",
+        r"0.0.0.0": "localhost",  # noqa: S104 (hardcoded-bind-all-interfaces)
+    }
+    for pattern, repl in server_substitutions.items():
+        server = regex.sub(pattern, repl, server)
+    return server
+
+
+@dataclass(frozen=True)
+class DiagnosticsConfig:
+    """Configures all diagnostics for a given upload."""
+
+    server_as_foldername: str = "unknown"
+    save_location: Path = field(default=Path.home() / ".dsp-tools" / "xmluploads")
+
+
+@dataclass(frozen=True)
+class UploadConfig:
+    """Configuration for the upload process."""
+
+    media_previously_uploaded: bool = False
+    server: str = "unknown"
+    shortcode: str = "unknown"
+    diagnostics: DiagnosticsConfig = field(default_factory=DiagnosticsConfig)
+    interrupt_after: int | None = None
+    skip_iiif_validation: bool = False
+    skip_validation: bool = False
+    skip_ontology_validation: bool = False
+    ignore_duplicate_files_warning: bool = False
+    validation_severity: ValidationSeverity = field(default_factory=lambda: ValidationSeverity.INFO)
+    id2iri_file: str | None = None
+    do_not_request_resource_metadata_from_db: bool = False
+
+    def with_server_info(
+        self,
+        server: str,
+        shortcode: str,
+    ) -> UploadConfig:
+        """Create a new UploadConfig with the given server."""
+        server_as_foldername = _transform_server_url_to_foldername(server)
+        save_location = Path.home() / Path(".dsp-tools") / "xmluploads" / server_as_foldername / "resumable/latest.pkl"
+        save_location.parent.mkdir(parents=True, exist_ok=True)
+        logger.info(f"{save_location=:}")
+        diagnostics: DiagnosticsConfig = dataclasses.replace(
+            self.diagnostics,
+            server_as_foldername=server_as_foldername,
+            save_location=save_location,
+        )
+        return dataclasses.replace(self, server=server, shortcode=shortcode, diagnostics=diagnostics)

dsp_tools/commands/xmlupload/write_diagnostic_info.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from pathlib import Path
+
+from loguru import logger
+
+from dsp_tools.commands.xmlupload.upload_config import DiagnosticsConfig
+
+
+def write_id2iri_mapping(id2iri_mapping: dict[str, str], shortcode: str, diagnostics: DiagnosticsConfig) -> None:
+    """Writes the mapping of internal IDs to IRIs to a file."""
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S")
+    servername = diagnostics.server_as_foldername
+    json_str = json.dumps(id2iri_mapping, ensure_ascii=False, indent=4)
+    id2iri_filename_for_user = f"id2iri_{shortcode}_{servername}_{timestamp}.json"
+    with open(id2iri_filename_for_user, "w", encoding="utf-8") as f:
+        f.write(json_str)
+        print(f"{datetime.now()}: The mapping of internal IDs to IRIs was written to {id2iri_filename_for_user}")
+        logger.info(f"The mapping of internal IDs to IRIs was written to {id2iri_filename_for_user}")
+
+    id_2_iri_folder = Path.home() / ".dsp-tools" / "id2iri"
+    id_2_iri_folder.mkdir(parents=True, exist_ok=True)
+    id2iri_filename_for_user_home = f"id2iri_{timestamp}_{shortcode}_{servername}.json"
+    with open(id_2_iri_folder / id2iri_filename_for_user_home, "w", encoding="utf-8") as f:
+        f.write(json_str)