dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py

@@ -1,23 +1,28 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Any
 from typing import cast
 
 from loguru import logger
+from rdflib import RDF
+from rdflib import BNode
+from rdflib import Graph
+from rdflib import URIRef
+from tqdm import tqdm
 
+from dsp_tools.clients.connection import Connection
+from dsp_tools.commands.xmlupload.make_rdf_graph.jsonld_utils import serialise_jsonld_for_value
+from dsp_tools.commands.xmlupload.make_rdf_graph.make_values import make_link_value_graph
 from dsp_tools.commands.xmlupload.models.upload_state import UploadState
 from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStash
 from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStashItem
 from dsp_tools.commands.xmlupload.stash.stash_models import Stash
-from dsp_tools.models.exceptions import BaseError
-from dsp_tools.utils.connection import Connection
+from dsp_tools.error.exceptions import BaseError
 
 
 def upload_stashed_resptr_props(
     upload_state: UploadState,
     con: Connection,
-    context: dict[str, str],
 ) -> None:
     """
     After all resources are uploaded, the stashed resptr props must be applied to their resources in DSP.
@@ -26,28 +31,27 @@ def upload_stashed_resptr_props(
     Args:
         upload_state: the current state of the upload
         con: connection to DSP
-        context: the JSON-LD context of the resource
     """
-
-    print(f"{datetime.now()}: Upload the stashed resptrs...")
-    logger.info("Upload the stashed resptrs...")
+    logger.info("Upload the stashed links...")
     upload_state.pending_stash = cast(Stash, upload_state.pending_stash)
     link_value_stash = cast(LinkValueStash, upload_state.pending_stash.link_value_stash)
-    for res_id, stash_items in link_value_stash.res_2_stash_items.copy().items():
+    progress_bar = tqdm(
+        link_value_stash.res_2_stash_items.copy().items(), desc="Upload the stashed links", dynamic_ncols=True
+    )
+    for res_id, stash_items in progress_bar:
         res_iri = upload_state.iri_resolver.get(res_id)
         if not res_iri:
             # resource could not be uploaded to DSP, so the stash cannot be uploaded either
             # no action necessary: this resource will remain in nonapplied_resptr_props,
             # which will be handled by the caller
             continue
-        print(f"{datetime.now()}:   Upload resptrs of resource '{res_id}'...")
         logger.info(f"  Upload resptrs of resource '{res_id}'...")
         for stash_item in reversed(stash_items):
             # reversed avoids any problems caused by removing from the list we loop over at the same time
-            target_iri = upload_state.iri_resolver.get(stash_item.target_id)
+            target_iri = upload_state.iri_resolver.get(stash_item.value.value)
             if not target_iri:
                 continue
-            if _upload_stash_item(stash_item, res_iri, target_iri, con, context):
+            if _upload_stash_item(stash_item, res_iri, target_iri, con):
                 link_value_stash.res_2_stash_items[res_id].remove(stash_item)
         # remove res_id if all stash items were uploaded
         if not link_value_stash.res_2_stash_items[res_id]:
@@ -59,7 +63,6 @@ def _upload_stash_item(
     res_iri: str,
     target_iri: str,
     con: Connection,
-    context: dict[str, str],
 ) -> bool:
     """
     Upload a single stashed link value to DSP.
@@ -69,44 +72,35 @@ def _upload_stash_item(
         res_iri: the iri of the resource
         target_iri: the iri of the target resource
         con: connection to DSP
-        context: the JSON-LD context of the resource
 
     Returns:
         True, if the upload was successful, False otherwise
     """
-    payload = _create_resptr_prop_json_object_to_update(stash, res_iri, target_iri, context)
+    graph = _make_link_value_create_graph(stash, res_iri, target_iri)
+    payload = serialise_jsonld_for_value(graph, res_iri)
     try:
         con.post(route="/v2/values", data=payload)
     except BaseError as err:
-        _log_unable_to_upload_link_value(err.message, stash.res_id, stash.prop_name)
+        _log_unable_to_upload_link_value(err.message, stash.res_id, stash.value.prop_iri)
         return False
-    logger.debug(f'  Successfully uploaded resptr links of "{stash.prop_name}"')
+    logger.debug(f'  Successfully uploaded resptr links of "{stash.value.prop_iri}"')
     return True
 
 
-def _log_unable_to_upload_link_value(msg: str, res_id: str, prop_name: str) -> None:
-    err_msg = f"Unable to upload the resptr prop of '{prop_name}' of resource '{res_id}'."
-    print(f"{datetime.now()}:     WARNING: {err_msg} Original error message: {msg}")
-    logger.opt(exception=True).warning(err_msg)
-
-
-def _create_resptr_prop_json_object_to_update(
+def _make_link_value_create_graph(
     stash: LinkValueStashItem,
-    res_iri: str,
+    res_iri_str: str,
     target_iri: str,
-    context: dict[str, str],
-) -> dict[str, Any]:
+) -> Graph:
     """This function creates a JSON object that can be sent as an update request to the DSP-API."""
-    linkVal = {
-        "@type": "knora-api:LinkValue",
-        "knora-api:linkValueHasTargetIri": {"@id": target_iri},
-    }
-    if stash.permission:
-        linkVal["knora-api:hasPermissions"] = stash.permission
-    jsonobj = {
-        "@id": res_iri,
-        "@type": stash.res_type,
-        f"{stash.prop_name}Value": linkVal,
-        "@context": context,
-    }
-    return jsonobj
+    val_bn = BNode()
+    res_iri = URIRef(res_iri_str)
+    graph = make_link_value_graph(stash.value, val_bn, res_iri, URIRef(target_iri))
+    graph.add((res_iri, RDF.type, URIRef(stash.res_type)))
+    return graph
+
+
+def _log_unable_to_upload_link_value(msg: str, res_id: str, prop_name: str) -> None:
+    err_msg = f"Unable to upload the resptr prop of '{prop_name}' of resource '{res_id}'."
+    print(f"{datetime.now()}:     WARNING: {err_msg} Original error message: {msg}")
+    logger.error(err_msg)

dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py

@@ -6,94 +6,20 @@ from typing import cast
 from urllib.parse import quote_plus
 
 from loguru import logger
+from rdflib import RDF
+from rdflib import Graph
+from rdflib import URIRef
+from tqdm import tqdm
 
+from dsp_tools.clients.connection import Connection
 from dsp_tools.commands.xmlupload.iri_resolver import IriResolver
-from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue
+from dsp_tools.commands.xmlupload.make_rdf_graph.jsonld_utils import serialise_jsonld_for_value
+from dsp_tools.commands.xmlupload.make_rdf_graph.make_values import make_richtext_value_graph
 from dsp_tools.commands.xmlupload.models.upload_state import UploadState
 from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStash
 from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStashItem
 from dsp_tools.commands.xmlupload.stash.stash_models import Stash
-from dsp_tools.models.exceptions import BaseError
-from dsp_tools.utils.connection import Connection
-
-
-def _log_unable_to_retrieve_resource(
-    resource: str,
-    received_error: BaseError,
-) -> None:
-    """
-    This function logs the error if it is not possible to retrieve the resource.
-
-    Args:
-        resource: the resource id
-        received_error: the error
-    """
-    # print the message to keep track of the cause for the failure
-    # apart from that; no action is necessary:
-    # this resource will remain in nonapplied_xml_texts, which will be handled by the caller
-    err_msg = (
-        f"Unable to upload XML texts of resource '{resource}', "
-        "because the resource cannot be retrieved from the DSP server."
-    )
-    print(f"{datetime.now()}:   WARNING: {err_msg} Original error message: {received_error.message}")
-    logger.opt(exception=True).warning(err_msg)
-
-
-def _log_unable_to_upload_xml_resource(
-    received_error: BaseError,
-    stashed_resource_id: str,
-    prop_name: str,
-) -> None:
-    """
-    This function logs if it is not possible to upload a xml resource.
-
-    Args:
-        received_error: Error received
-        stashed_resource_id: id of the resource
-        prop_name: name of the property
-    """
-    # print the message to keep track of the cause for the failure
-    # apart from that; no action is necessary:
-    # this resource will remain in nonapplied_xml_texts, which will be handled by the caller
-    err_msg = f"Unable to upload the xml text of '{prop_name}' of resource '{stashed_resource_id}'."
-    print(f"{datetime.now()}:     WARNING: {err_msg} Original error message: {received_error.message}")
-    logger.opt(exception=True).warning(err_msg)
-
-
-def _create_XMLResource_json_object_to_update(
-    res_iri: str,
-    res_type: str,
-    link_prop_name: str,
-    value_iri: str,
-    new_xmltext: FormattedTextValue,
-    context: dict[str, str],
-) -> dict[str, Any]:
-    """
-    This function creates a JSON object that can be sent as update request to DSP-API.
-
-    Args:
-        res_iri: the iri of the resource
-        res_type: the type of the resource
-        link_prop_name: the name of the link property
-        value_iri: the iri of the value
-        new_xmltext: the new xml text to be uploaded
-        context: the JSON-LD context of the resource
-
-    Returns:
-        json string
-    """
-    jsonobj = {
-        "@id": res_iri,
-        "@type": res_type,
-        link_prop_name: {
-            "@id": value_iri,
-            "@type": "knora-api:TextValue",
-            "knora-api:textValueAsXml": new_xmltext.as_xml(),
-            "knora-api:textValueHasMapping": {"@id": "http://rdfh.ch/standoff/mappings/StandardMapping"},
-        },
-        "@context": context,
-    }
-    return jsonobj
+from dsp_tools.error.exceptions import BaseError
 
 
 def upload_stashed_xml_texts(upload_state: UploadState, con: Connection) -> None:
@@ -105,12 +31,13 @@ def upload_stashed_xml_texts(upload_state: UploadState, con: Connection) -> None
         upload_state: the current state of the upload
         con: connection to DSP
     """
-
-    print(f"{datetime.now()}: Upload the stashed XML texts...")
     logger.info("Upload the stashed XML texts...")
     upload_state.pending_stash = cast(Stash, upload_state.pending_stash)
     standoff_stash = cast(StandoffStash, upload_state.pending_stash.standoff_stash)
-    for res_id, stash_items in standoff_stash.res_2_stash_items.copy().items():
+    progress_bar = tqdm(
+        standoff_stash.res_2_stash_items.copy().items(), desc="Upload stashed XML texts", dynamic_ncols=True
+    )
+    for res_id, stash_items in progress_bar:
         res_iri = upload_state.iri_resolver.get(res_id)
         if not res_iri:
             # resource could not be uploaded to DSP, so the stash cannot be uploaded either
@@ -122,22 +49,17 @@ def upload_stashed_xml_texts(upload_state: UploadState, con: Connection) -> None
         except BaseError as err:
             _log_unable_to_retrieve_resource(resource=res_id, received_error=err)
             continue
-        print(f"{datetime.now()}:   Upload XML text(s) of resource '{res_id}'...")
         logger.info(f"  Upload XML text(s) of resource '{res_id}'...")
-        context = resource_in_triplestore["@context"]
         for stash_item in stash_items:
-            value_iri = _get_value_iri(stash_item.prop_name, resource_in_triplestore, stash_item.uuid)
+            value_iri = _get_value_iri(stash_item.value.prop_iri, resource_in_triplestore, stash_item.value.value_uuid)
             if not value_iri:
                 continue
             if _upload_stash_item(
                 stash_item=stash_item,
                 res_iri=res_iri,
-                res_type=stash_item.res_type,
-                res_id=res_id,
                 value_iri=value_iri,
                 iri_resolver=upload_state.iri_resolver,
                 con=con,
-                context=context,
             ):
                 standoff_stash.res_2_stash_items[res_id].remove(stash_item)
         if not standoff_stash.res_2_stash_items[res_id]:
@@ -149,7 +71,8 @@ def _get_value_iri(
     resource: dict[str, Any],
     uuid: str,
 ) -> str | None:
-    values_on_server = resource.get(property_name)
+    prefixed_prop = _make_prefixed_prop_from_absolute_iri(property_name)
+    values_on_server = resource.get(prefixed_prop)
     if not isinstance(values_on_server, list):
         values_on_server = [values_on_server]
 
@@ -162,15 +85,18 @@ def _get_value_iri(
     return value_iri
 
 
+def _make_prefixed_prop_from_absolute_iri(absolute_iri: str) -> str:
+    _, onto, prop = absolute_iri.rsplit("/", 2)
+    local_name = prop.split("#")[-1]
+    return f"{onto}:{local_name}"
+
+
 def _upload_stash_item(
     stash_item: StandoffStashItem,
     res_iri: str,
-    res_type: str,
-    res_id: str,
     value_iri: str,
     iri_resolver: IriResolver,
     con: Connection,
-    context: dict[str, str],
 ) -> bool:
     """
     Upload a single stashed xml text to DSP.
@@ -178,29 +104,93 @@ def _upload_stash_item(
     Args:
         stash_item: the stashed text value to upload
         res_iri: the iri of the resource
-        res_type: the type of the resource
-        res_id: the internal id of the resource
         value_iri: the iri of the value
         iri_resolver: resolver to map ids from the XML file to IRIs in DSP
         con: connection to DSP
-        context: the JSON-LD context of the resource
 
     Returns:
         True, if the upload was successful, False otherwise
     """
-    adjusted_text_value = stash_item.value.with_iris(iri_resolver)
-    payload = _create_XMLResource_json_object_to_update(
-        res_iri,
-        res_type,
-        stash_item.prop_name,
-        value_iri,
-        adjusted_text_value,
-        context,
+    payload = _serialise_richtext_for_update(
+        stash_item=stash_item,
+        value_iri_str=value_iri,
+        res_iri_str=res_iri,
+        iri_resolver=iri_resolver,
     )
     try:
         con.put(route="/v2/values", data=payload)
     except BaseError as err:
-        _log_unable_to_upload_xml_resource(err, res_id, stash_item.prop_name)
+        _log_unable_to_upload_xml_resource(err, stash_item.res_id, stash_item.value.prop_iri)
         return False
-    logger.debug(f'  Successfully uploaded xml text of "{stash_item.prop_name}"')
+    logger.debug(f'  Successfully uploaded xml text of "{stash_item.value.prop_iri}"')
     return True
+
+
+def _serialise_richtext_for_update(
+    stash_item: StandoffStashItem, value_iri_str: str, res_iri_str: str, iri_resolver: IriResolver
+) -> dict[str, Any]:
+    graph = _make_richtext_update_graph(
+        stash_item=stash_item,
+        value_iri_str=value_iri_str,
+        res_iri_str=res_iri_str,
+        iri_resolver=iri_resolver,
+    )
+    return serialise_jsonld_for_value(graph, res_iri_str)
+
+
+def _make_richtext_update_graph(
+    stash_item: StandoffStashItem, value_iri_str: str, res_iri_str: str, iri_resolver: IriResolver
+) -> Graph:
+    res_iri = URIRef(res_iri_str)
+    value_iri = URIRef(value_iri_str)
+    val_graph = make_richtext_value_graph(
+        val=stash_item.value,
+        val_node=value_iri,
+        res_node=res_iri,
+        iri_resolver=iri_resolver,
+    )
+    val_graph.add((res_iri, RDF.type, URIRef(stash_item.res_type)))
+    return val_graph
+
+
+def _log_unable_to_retrieve_resource(
+    resource: str,
+    received_error: BaseError,
+) -> None:
+    """
+    This function logs the error if it is not possible to retrieve the resource.
+
+    Args:
+        resource: the resource id
+        received_error: the error
+    """
+    # print the message to keep track of the cause for the failure
+    # apart from that; no action is necessary:
+    # this resource will remain in nonapplied_xml_texts, which will be handled by the caller
+    err_msg = (
+        f"Unable to upload XML texts of resource '{resource}', "
+        "because the resource cannot be retrieved from the DSP server."
+    )
+    print(f"{datetime.now()}:   WARNING: {err_msg} Original error message: {received_error.message}")
+    logger.error(err_msg)
+
+
+def _log_unable_to_upload_xml_resource(
+    received_error: BaseError,
+    stashed_resource_id: str,
+    prop_name: str,
+) -> None:
+    """
+    This function logs if it is not possible to upload a xml resource.
+
+    Args:
+        received_error: Error received
+        stashed_resource_id: id of the resource
+        prop_name: name of the property
+    """
+    # print the message to keep track of the cause for the failure
+    # apart from that; no action is necessary:
+    # this resource will remain in nonapplied_xml_texts, which will be handled by the caller
+    err_msg = f"Unable to upload the xml text of '{prop_name}' of resource '{stashed_resource_id}'."
+    print(f"{datetime.now()}:     WARNING: {err_msg} Original error message: {received_error.message}")
+    logger.error(err_msg)

dsp_tools/commands/xmlupload/upload_config.py

@@ -8,6 +8,8 @@ from pathlib import Path
 import regex
 from loguru import logger
 
+from dsp_tools.cli.args import ValidationSeverity
+
 
 def _transform_server_url_to_foldername(server: str) -> str:
     """
@@ -49,6 +51,12 @@ class UploadConfig:
     diagnostics: DiagnosticsConfig = field(default_factory=DiagnosticsConfig)
     interrupt_after: int | None = None
     skip_iiif_validation: bool = False
+    skip_validation: bool = False
+    skip_ontology_validation: bool = False
+    ignore_duplicate_files_warning: bool = False
+    validation_severity: ValidationSeverity = field(default_factory=lambda: ValidationSeverity.INFO)
+    id2iri_file: str | None = None
+    do_not_request_resource_metadata_from_db: bool = False
 
     def with_server_info(
         self,

dsp_tools/commands/xmlupload/write_diagnostic_info.py

@@ -2,21 +2,26 @@ from __future__ import annotations
 
 import json
 from datetime import datetime
+from pathlib import Path
 
 from loguru import logger
 
 from dsp_tools.commands.xmlupload.upload_config import DiagnosticsConfig
 
 
-def write_id2iri_mapping(
-    id2iri_mapping: dict[str, str],
-    diagnostics: DiagnosticsConfig,
-) -> None:
+def write_id2iri_mapping(id2iri_mapping: dict[str, str], shortcode: str, diagnostics: DiagnosticsConfig) -> None:
     """Writes the mapping of internal IDs to IRIs to a file."""
     timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S")
     servername = diagnostics.server_as_foldername
-    id2iri_filename = f"{timestamp}_id2iri_mapping_{servername}.json"
-    with open(id2iri_filename, "x", encoding="utf-8") as f:
-        json.dump(id2iri_mapping, f, ensure_ascii=False, indent=4)
-        print(f"{datetime.now()}: The mapping of internal IDs to IRIs was written to {id2iri_filename}")
-        logger.info(f"The mapping of internal IDs to IRIs was written to {id2iri_filename}")
+    json_str = json.dumps(id2iri_mapping, ensure_ascii=False, indent=4)
+    id2iri_filename_for_user = f"id2iri_{shortcode}_{servername}_{timestamp}.json"
+    with open(id2iri_filename_for_user, "w", encoding="utf-8") as f:
+        f.write(json_str)
+        print(f"{datetime.now()}: The mapping of internal IDs to IRIs was written to {id2iri_filename_for_user}")
+        logger.info(f"The mapping of internal IDs to IRIs was written to {id2iri_filename_for_user}")
+
+    id_2_iri_folder = Path.home() / ".dsp-tools" / "id2iri"
+    id_2_iri_folder.mkdir(parents=True, exist_ok=True)
+    id2iri_filename_for_user_home = f"id2iri_{timestamp}_{shortcode}_{servername}.json"
+    with open(id_2_iri_folder / id2iri_filename_for_user_home, "w", encoding="utf-8") as f:
+        f.write(json_str)