dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/id2iri.py

@@ -0,0 +1,258 @@
+import copy
+import json
+import warnings
+from collections.abc import Mapping
+from datetime import datetime
+from pathlib import Path
+
+import regex
+from loguru import logger
+from lxml import etree
+
+from dsp_tools.error.custom_warnings import DspToolsUserWarning
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_clean_xml_file
+
+
+def _check_input_parameters(
+    xml_file: str,
+    json_file: str,
+) -> tuple[Path, Path]:
+    """
+    Transform the input parameters into Path objects
+    and check if they are valid files.
+
+    Args:
+        xml_file: the XML file with the data to be replaced
+        json_file: the JSON file with the mapping (dict) of internal IDs to IRIs
+
+    Raises:
+        InputError: if one of the files could not be found
+
+    Returns:
+        path objects of the input parameters
+    """
+    xml_file_as_path = Path(xml_file)
+    if not xml_file_as_path.is_file():
+        logger.error(f"File {xml_file} could not be found.")
+        raise InputError(f"File {xml_file} could not be found.")
+
+    json_file_as_path = Path(json_file)
+    if not json_file_as_path.is_file():
+        logger.error(f"File {json_file} could not be found.")
+        raise InputError(f"File {json_file} could not be found.")
+
+    return xml_file_as_path, json_file_as_path
+
+
+def _parse_json_file(json_file: Path) -> dict[str, str]:
+    """
+    Read JSON file and parse it into a dictionary.
+
+    Args:
+        json_file: path to JSON file
+
+    Returns:
+        dictionary with the contents of the JSON file
+    """
+    with open(json_file, encoding="utf-8", mode="r") as file:
+        mapping: dict[str, str] = json.load(file)
+    return mapping
+
+
+def _replace_resptrs(
+    tree: etree._Element,
+    mapping: Mapping[str, str],
+    used_mapping_entries: set[str],
+) -> tuple[etree._Element, set[str]]:
+    """
+    Replace the internal IDs in the `<resptr>` tags by IRIs.
+
+    Args:
+        tree: parsed XML file
+        mapping: mapping of internal IDs to IRIs
+        used_mapping_entries: IDs of the mapping that have been found in the XML and have been replaced
+
+    Returns:
+        a tuple of the modified copy of the XML tree, and the set of the IDs that have been replaced
+    """
+    modified_tree = copy.deepcopy(tree)
+    xpaths = [f"/knora/{x}/resptr-prop/resptr" for x in ["resource", "link", "region"]]
+    xpaths.extend([f"/knora/{x}-segment/isSegmentOf" for x in ["video", "audio"]])
+    xpaths.extend([f"/knora/{x}-segment/relatesTo" for x in ["video", "audio"]])
+    resptr_xpath = "|".join(xpaths)
+    resptr_elems = modified_tree.xpath(resptr_xpath)
+    resptr_elems_replaced = 0
+    for resptr_elem in resptr_elems:
+        value_before = resptr_elem.text
+        if value_after := mapping.get(value_before):
+            resptr_elem.text = value_after
+            resptr_elems_replaced += 1
+            used_mapping_entries.add(value_before)
+
+    logger.info(f"Replaced {resptr_elems_replaced}/{len(resptr_elems)} resptr links in the XML file")
+    print(f"Replaced {resptr_elems_replaced}/{len(resptr_elems)} resptr links in the XML file")
+
+    return modified_tree, used_mapping_entries
+
+
+def _replace_salsah_links(
+    tree: etree._Element,
+    mapping: Mapping[str, str],
+    used_mapping_entries: set[str],
+) -> tuple[etree._Element, set[str]]:
+    """
+    Replace the internal IDs in the salsah-links of the `<text>` tags by IRIs.
+
+    Args:
+        tree: parsed XML file
+        mapping: mapping of internal IDs to IRIs
+        used_mapping_entries: IDs of the mapping that have been found in the XML and have been replaced
+
+    Returns:
+        a tuple of the modified copy of the XML tree, and the set of the IDs that have been replaced
+    """
+    modified_tree = copy.deepcopy(tree)
+    xpaths = [f"/knora/{x}/text-prop/text//a" for x in ["resource", "link", "region"]]
+    xpaths.extend([f"/knora/{x}-segment/hasComment//a" for x in ["video", "audio"]])
+    xpaths.extend([f"/knora/{x}-segment/hasDescription//a" for x in ["video", "audio"]])
+    salsah_xpath = "|".join(xpaths)
+    salsah_links = [x for x in modified_tree.xpath(salsah_xpath) if x.attrib.get("class") == "salsah-link"]
+    salsah_links_replaced = 0
+    for salsah_link in salsah_links:
+        value_before = regex.sub("IRI:|:IRI", "", salsah_link.attrib.get("href", ""))
+        if value_after := mapping.get(value_before):
+            salsah_link.attrib["href"] = value_after
+            salsah_links_replaced += 1
+            used_mapping_entries.add(value_before)
+
+    logger.info(f"Replaced {salsah_links_replaced}/{len(salsah_links)} salsah-links in the XML file")
+    print(f"Replaced {salsah_links_replaced}/{len(salsah_links)} salsah-links in the XML file")
+
+    return modified_tree, used_mapping_entries
+
+
+def _replace_ids_by_iris(
+    tree: etree._Element,
+    mapping: Mapping[str, str],
+) -> etree._Element:
+    """
+    Iterate over the `<resptr>` tags and the salsah-links of the `<text>` tags,
+    and replace the internal IDs by IRIs.
+    If an internal ID cannot be found in the mapping, the original ID is kept.
+
+    Args:
+        tree: parsed XML file
+        mapping: mapping of internal IDs to IRIs
+
+    Returns:
+        a modified copy of the XML tree
+    """
+    used_mapping_entries: set[str] = set()
+
+    tree, used_mapping_entries = _replace_resptrs(
+        tree=tree,
+        mapping=mapping,
+        used_mapping_entries=used_mapping_entries,
+    )
+
+    tree, used_mapping_entries = _replace_salsah_links(
+        tree=tree,
+        mapping=mapping,
+        used_mapping_entries=used_mapping_entries,
+    )
+
+    logger.info(f"Used {len(used_mapping_entries)}/{len(mapping)} entries from the mapping file")
+    print(f"Used {len(used_mapping_entries)}/{len(mapping)} entries from the mapping file")
+
+    return tree
+
+
+def _remove_resources_if_id_in_mapping(
+    tree: etree._Element,
+    mapping: Mapping[str, str],
+) -> etree._Element:
+    """
+    Remove all resources from the XML file if their ID is in the mapping.
+
+    Args:
+        tree: parsed XML file
+        mapping: mapping of internal IDs to IRIs
+
+    Returns:
+        a modified copy of the XML tree
+    """
+    modified_tree = copy.deepcopy(tree)
+    resources = modified_tree.xpath(
+        "|".join([f"/knora/{x}" for x in ["resource", "link", "region", "video-segment", "audio-segment"]])
+    )
+    resources_to_remove = [x for x in resources if x.attrib.get("id") in mapping]
+    for resource in resources_to_remove:
+        resource.getparent().remove(resource)
+
+    msg = (
+        f"Removed {len(resources_to_remove)}/{len(resources)} resources from the XML file, "
+        "because their ID was in the mapping"
+    )
+    logger.warning(msg)
+    warnings.warn(DspToolsUserWarning(msg))
+
+    return modified_tree
+
+
+def _write_output_file(
+    orig_xml_file: Path,
+    tree: etree._Element,
+) -> None:
+    """
+    Write modified XML file with replaced IDs to disk.
+
+    Args:
+        orig_xml_file: XML file that was provided as input
+        tree: modified XML tree with replaced IDs
+    """
+    timestamp_str = datetime.now().strftime("%Y%m%d-%H%M%S")
+    out_file = f"{orig_xml_file.stem}_replaced_{timestamp_str}.xml"
+    et = etree.ElementTree(tree)
+    et.write(out_file, pretty_print=True, xml_declaration=True, encoding="utf-8")
+    logger.info(f"XML with replaced IDs was written to file {out_file}.")
+    print(f"XML with replaced IDs was written to file {out_file}.")
+
+
+def id2iri(
+    xml_file: str,
+    json_file: str,
+    remove_resource_if_id_in_mapping: bool = False,
+) -> bool:
+    """
+    Replace internal IDs of an XML file
+    (`<resptr>` tags and salsah-links inside `<text>` tags)
+    by IRIs provided in a mapping file.
+    If an internal ID cannot be found in the mapping, the original ID is kept.
+    The output is written to a new XML file named "[original name]_replaced_[timestamp].xml".
+
+    Args:
+        xml_file: the XML file with the data to be replaced
+        json_file: the JSON file with the mapping (dict) of internal IDs to IRIs
+        remove_resource_if_id_in_mapping: if True, remove all resources from the XML file if their ID is in the mapping
+
+    Raises:
+        BaseError: if one of the two input files is not a valid file
+
+    Returns:
+        success status
+    """
+    xml_file_as_path, json_file_as_path = _check_input_parameters(xml_file=xml_file, json_file=json_file)
+    mapping = _parse_json_file(json_file_as_path)
+    tree = parse_and_clean_xml_file(xml_file_as_path)
+    tree = _replace_ids_by_iris(
+        tree=tree,
+        mapping=mapping,
+    )
+    if remove_resource_if_id_in_mapping:
+        tree = _remove_resources_if_id_in_mapping(
+            tree=tree,
+            mapping=mapping,
+        )
+    _write_output_file(orig_xml_file=xml_file_as_path, tree=tree)
+    return True

dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py

@@ -0,0 +1,178 @@
+import urllib.parse
+from collections.abc import Iterator
+from dataclasses import dataclass
+from dataclasses import field
+from http import HTTPStatus
+from pathlib import Path
+
+import regex
+from loguru import logger
+from requests import JSONDecodeError
+from requests import RequestException
+from requests import Session
+from requests.adapters import HTTPAdapter
+from requests.adapters import Retry
+
+from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.commands.ingest_xmlupload.upload_files.upload_failures import UploadFailure
+from dsp_tools.config.logger_config import LOGGER_SAVEPATH
+from dsp_tools.error.exceptions import BadCredentialsError
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.request_utils import RequestParameters
+from dsp_tools.utils.request_utils import log_request
+from dsp_tools.utils.request_utils import log_response
+
+
+@dataclass
+class BulkIngestClient:
+    """Client to upload multiple files to the ingest server and monitor the ingest process."""
+
+    dsp_ingest_url: str
+    authentication_client: AuthenticationClient
+    shortcode: str
+    imgdir: Path = field(default=Path.cwd())
+    session: Session = field(init=False)
+    retrieval_failures = 0
+
+    def __post_init__(self) -> None:
+        retries = 6
+        self.session = Session()
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=0.3,
+            allowed_methods=None,  # means all methods
+            status_forcelist=[HTTPStatus.INTERNAL_SERVER_ERROR, HTTPStatus.SERVICE_UNAVAILABLE],
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
+
+    def upload_file(
+        self,
+        filepath: Path,
+    ) -> UploadFailure | None:
+        """
+        Uploads a file to the ingest server.
+        The load balancer on DSP servers currently has a timeout of 10m,
+        so we need to use a slightly shorter timeout of 9m.
+        See https://linear.app/dasch/issue/INFRA-847/increase-traefik-readtimeout
+        # noqa: DAR101
+        # noqa: DAR201
+        """
+        logger.debug(f"Uploading file '{filepath}'")
+        timeout = 9 * 60
+        url = self._build_url_for_bulk_ingest_ingest_route(filepath)
+        headers = {
+            "Content-Type": "application/octet-stream",
+            "Authorization": f"Bearer {self.authentication_client.get_token()}",
+        }
+        err_msg = f"Failed to upload '{filepath}' to '{url}'."
+        params = RequestParameters("POST", url, timeout, headers=headers)
+        log_request(params)
+        try:
+            with open(self.imgdir / filepath, "rb") as binary_io:
+                res = self.session.post(
+                    url=params.url,
+                    headers=params.headers,
+                    data=binary_io,  # https://requests.readthedocs.io/en/latest/user/advanced/#streaming-uploads
+                    timeout=params.timeout,
+                )
+            log_response(res)
+        except RequestException as e:
+            logger.exception(err_msg)
+            return UploadFailure(filepath, f"Exception of requests library: {e}")
+        except OSError as e:
+            err_msg = f"Cannot bulk-ingest {filepath}, because the file could not be opened/read: {e.strerror}"
+            logger.error(err_msg)
+            return UploadFailure(filepath, err_msg)
+        if res.status_code != HTTPStatus.OK:
+            logger.error(f"{err_msg}: Response {res.status_code}: {res.text}")
+            return UploadFailure(filepath, res.reason, res.status_code, res.text)
+        return None
+
+    def _build_url_for_bulk_ingest_ingest_route(self, filepath: Path) -> str:
+        """
+        Remove the leading slash of absolute filepaths,
+        because the `/project/<shortcode>/bulk-ingest/ingest` route only accepts relative paths.
+        The leading slash has to be added again in the "ingest-xmlupload" step, when applying the ingest ID.
+
+        Args:
+            filepath: filepath
+
+        Returns:
+            url
+        """
+        quoted = regex.sub(r"^%2F", "", urllib.parse.quote(str(filepath), safe=""))
+        return f"{self.dsp_ingest_url}/projects/{self.shortcode}/bulk-ingest/ingest/{quoted}"
+
+    def trigger_ingest_process(self) -> None:
+        """Start the ingest process on the server."""
+        url = f"{self.dsp_ingest_url}/projects/{self.shortcode}/bulk-ingest"
+        timeout = 5
+        headers = {"Authorization": f"Bearer {self.authentication_client.get_token()}"}
+        params = RequestParameters("POST", url, timeout, headers=headers)
+        log_request(params)
+        res = self.session.post(params.url, timeout=params.timeout, headers=params.headers)
+        log_response(res)
+        if res.status_code == HTTPStatus.FORBIDDEN:
+            raise BadCredentialsError("Only ProjectAdmins or SystemAdmins can start the ingest process.")
+        if res.status_code == HTTPStatus.NOT_FOUND:
+            raise InputError(
+                f"No assets have been uploaded for project {self.shortcode}. "
+                "Before using the 'ingest-files' command, you must upload some files with the 'upload-files' command."
+            )
+        if res.status_code == HTTPStatus.CONFLICT:
+            msg = f"Ingest process on the server {self.dsp_ingest_url} is already running. Wait until it completes..."
+            print(msg)
+            logger.info(msg)
+            return
+        if res.status_code in [HTTPStatus.INTERNAL_SERVER_ERROR, HTTPStatus.SERVICE_UNAVAILABLE]:
+            raise InputError("Server is unavailable. Please try again later.")
+
+        try:
+            returned_shortcode = res.json().get("id")
+            failed: bool = returned_shortcode != self.shortcode
+        except JSONDecodeError:
+            failed = True
+        if failed:
+            raise InputError("Failed to trigger the ingest process. Please check the server logs, or try again later.")
+        print(f"Kicked off the ingest process on the server {self.dsp_ingest_url}. Wait until it completes...")
+        logger.info(f"Kicked off the ingest process on the server {self.dsp_ingest_url}. Wait until it completes...")
+
+    def retrieve_mapping_generator(self) -> Iterator[str | bool]:
+        """
+        Try to retrieve the mapping CSV from the server.
+
+        Yields:
+            True if the ingest process is still running.
+            False if there is a server error.
+            The mapping CSV if the ingest process has completed.
+
+        Raises:
+            InputError: if there are too many server errors in a row.
+        """
+        url = f"{self.dsp_ingest_url}/projects/{self.shortcode}/bulk-ingest/mapping.csv"
+        timeout = 5
+        while True:
+            headers = {"Authorization": f"Bearer {self.authentication_client.get_token()}"}
+            params = RequestParameters("GET", url, timeout, headers=headers)
+            log_request(params)
+            res = self.session.get(params.url, timeout=params.timeout, headers=params.headers)
+            log_response(res)
+            if res.status_code == HTTPStatus.CONFLICT:
+                self.retrieval_failures = 0
+                logger.info("Ingest process is still running. Wait until it completes...")
+                yield True
+            elif res.status_code != HTTPStatus.OK or not res.text.startswith("original,derivative"):
+                self.retrieval_failures += 1
+                if self.retrieval_failures > 15:
+                    raise InputError(f"There were too many server errors. Please check the logs at {LOGGER_SAVEPATH}.")
+                msg = "While retrieving the mapping CSV, the server responded with an unexpected status code/content."
+                logger.error(msg)
+                yield False
+            else:
+                logger.info("Ingest process completed.")
+                break
+        yield res.content.decode("utf-8")

dsp_tools/commands/ingest_xmlupload/create_resources/apply_ingest_id.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from pathlib import Path
+from typing import cast
+
+import pandas as pd
+from loguru import logger
+from lxml import etree
+
+from dsp_tools.commands.ingest_xmlupload.create_resources.user_information import IngestInformation
+from dsp_tools.error.exceptions import InputError
+
+
+def get_mapping_dict_from_file(shortcode: str) -> dict[str, str]:
+    """
+    This functions returns the information to replace the original filepaths with the identifier from dsp-ingest.
+
+    Args:
+        shortcode: Shortcode of the project
+
+    Returns:
+        dictionary with original: identifier from dsp-ingest
+
+    Raises:
+        InputError: if no file was found
+    """
+    filepath = Path(f"mapping-{shortcode}.csv")
+    if not filepath.is_file():
+        raise InputError(f"No mapping CSV file was found at {filepath}.")
+    df = pd.read_csv(filepath)
+    msg = f"The file '{filepath}' is used to map the internal original filepaths to the internal image IDs."
+    print(msg)
+    logger.info(msg)
+    return dict(zip(df["original"].tolist(), df["derivative"].tolist()))
+
+
+def replace_filepath_with_internal_filename(
+    xml_tree: etree._Element,
+    orig_path_2_asset_id: dict[str, str],
+) -> tuple[etree._Element, IngestInformation]:
+    """
+    Replace the original filepaths in the `<bitstream>` tags by the id filenames of the uploaded files.
+
+    Args:
+        xml_tree: The parsed original XML tree
+        orig_path_2_asset_id: Mapping from original filenames to asset IDs from the mapping.csv
+
+    Returns:
+        A copy of the XMl tree, with the replaced filepaths.
+        Message informing if all referenced files were uploaded or not.
+    """
+    no_id_found = []
+    used_media_file_paths = []
+    new_tree = deepcopy(xml_tree)
+    for elem in new_tree.iter():
+        if not etree.QName(elem).localname.endswith("bitstream") or not elem.text:
+            continue
+        img_path_str = elem.text.strip()
+        if img_path_str not in orig_path_2_asset_id and img_path_str.startswith("/"):
+            img_path_str = img_path_str[1:]
+        if img_path_str in orig_path_2_asset_id:
+            elem.text = orig_path_2_asset_id[img_path_str]
+            used_media_file_paths.append(img_path_str)
+        else:
+            no_id_found.append((cast("etree._Element", elem.getparent()).attrib["id"], str(elem.text)))
+
+    unused_media_paths = [x for x in orig_path_2_asset_id if x not in used_media_file_paths]
+    return new_tree, IngestInformation(unused_mediafiles=unused_media_paths, mediafiles_no_id=no_id_found)

dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from loguru import logger
+from lxml import etree
+
+from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.cli.args import ValidateDataConfig
+from dsp_tools.cli.args import ValidationSeverity
+from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.connection import Connection
+from dsp_tools.clients.connection_live import ConnectionLive
+from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
+from dsp_tools.clients.project_client_live import ProjectClientLive
+from dsp_tools.commands.ingest_xmlupload.create_resources.apply_ingest_id import get_mapping_dict_from_file
+from dsp_tools.commands.ingest_xmlupload.create_resources.apply_ingest_id import replace_filepath_with_internal_filename
+from dsp_tools.commands.validate_data.validate_data import validate_parsed_resources
+from dsp_tools.commands.xmlupload.models.ingest import BulkIngestedAssetClient
+from dsp_tools.commands.xmlupload.models.upload_clients import UploadClients
+from dsp_tools.commands.xmlupload.models.upload_state import UploadState
+from dsp_tools.commands.xmlupload.prepare_xml_input.get_processed_resources import get_processed_resources
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClientLive
+from dsp_tools.commands.xmlupload.prepare_xml_input.prepare_xml_input import get_parsed_resources_and_mappers
+from dsp_tools.commands.xmlupload.prepare_xml_input.prepare_xml_input import get_stash_and_upload_order
+from dsp_tools.commands.xmlupload.prepare_xml_input.read_validate_xml_file import validate_iiif_uris
+from dsp_tools.commands.xmlupload.upload_config import UploadConfig
+from dsp_tools.commands.xmlupload.xmlupload import enable_unknown_license_if_any_are_missing
+from dsp_tools.commands.xmlupload.xmlupload import execute_upload
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.ansi_colors import BOLD_RED
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+from dsp_tools.utils.data_formats.uri_util import is_prod_like_server
+from dsp_tools.utils.replace_id_with_iri import use_id2iri_mapping_to_replace_ids
+from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_clean_xml_file
+
+
+def ingest_xmlupload(
+    xml_file: Path,
+    creds: ServerCredentials,
+    interrupt_after: int | None = None,
+    skip_validation: bool = False,
+    skip_ontology_validation: bool = False,
+    id2iri_file: str | None = None,
+    do_not_request_resource_metadata_from_db: bool = False,
+) -> bool:
+    """
+    This function reads an XML file
+    and imports the data described in it onto the DSP server,
+    using the ingest XML upload method.
+    Before using this function,
+    the multimedia files must be ingested on the DSP server.
+    A mapping file with the internal IDs of the multimedia files must also be provided.
+
+    Args:
+        xml_file: path to XML file containing the resources
+        creds: credentials to access the DSP server
+        interrupt_after: if set, the upload will be interrupted after this number of resources
+        skip_validation: skip the SHACL validation
+        skip_ontology_validation: skip the ontology validation
+        id2iri_file: to replace internal IDs of an XML file by IRIs provided in this mapping file
+        do_not_request_resource_metadata_from_db: if true do not request metadata information from the api
+                                                  for existing resources
+
+    Returns:
+        True if all resources could be uploaded without errors; False if one of the resources could not be
+        uploaded because there is an error in it
+
+    Raises:
+        InputError: if any media was not uploaded or uploaded media was not referenced.
+    """
+    root = parse_and_clean_xml_file(xml_file)
+    shortcode = root.attrib["shortcode"]
+    root = _replace_filepaths_with_internal_filename_from_ingest(root, shortcode)
+
+    auth = AuthenticationClientLive(server=creds.server, email=creds.user, password=creds.password)
+    con = ConnectionLive(creds.server, auth)
+    config = UploadConfig(
+        media_previously_uploaded=True,
+        interrupt_after=interrupt_after,
+    ).with_server_info(
+        server=creds.server,
+        shortcode=shortcode,
+    )
+    clients = _get_live_clients(con, config, auth)
+
+    parsed_resources, lookups = get_parsed_resources_and_mappers(root, clients)
+    if id2iri_file:
+        parsed_resources = use_id2iri_mapping_to_replace_ids(parsed_resources, Path(id2iri_file))
+
+    validation_should_be_skipped = skip_validation
+    is_on_prod_like_server = is_prod_like_server(creds.server)
+    if is_on_prod_like_server and config.skip_validation:
+        msg = (
+            "You set the flag '--skip-validation' to circumvent the SHACL schema validation. "
+            "This means that the upload may fail due to undetected errors. "
+            "Do you wish to skip the validation (yes/no)? "
+        )
+        resp = ""
+        while resp not in ["yes", "no"]:
+            resp = input(BOLD_RED + msg + RESET_TO_DEFAULT)
+        if str(resp) == "no":
+            validation_should_be_skipped = False
+    if not validation_should_be_skipped:
+        v_severity = config.validation_severity
+        if is_on_prod_like_server:
+            v_severity = ValidationSeverity.INFO
+        validation_passed = validate_parsed_resources(
+            parsed_resources=parsed_resources,
+            authorship_lookup=lookups.authorships,
+            permission_ids=list(lookups.permissions.keys()),
+            shortcode=shortcode,
+            config=ValidateDataConfig(
+                xml_file,
+                save_graph_dir=None,
+                severity=v_severity,
+                ignore_duplicate_files_warning=True,
+                is_on_prod_server=is_on_prod_like_server,
+                skip_ontology_validation=skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=do_not_request_resource_metadata_from_db,
+            ),
+            auth=auth,
+        )
+        if not validation_passed:
+            return False
+    else:
+        logger.debug("SHACL validation was skipped.")
+
+    if not config.skip_iiif_validation:
+        validate_iiif_uris(root)
+
+    if not is_on_prod_like_server:
+        enable_unknown_license_if_any_are_missing(clients.legal_info_client, parsed_resources)
+
+    processed_resources = get_processed_resources(parsed_resources, lookups, is_on_prod_like_server)
+
+    sorted_resources, stash = get_stash_and_upload_order(processed_resources)
+
+    state = UploadState(
+        pending_resources=sorted_resources,
+        pending_stash=stash,
+        config=config,
+    )
+
+    return execute_upload(clients, state)
+
+
+def _replace_filepaths_with_internal_filename_from_ingest(root: etree._Element, shortcode: str) -> etree._Element:
+    orig_path_2_asset_id = get_mapping_dict_from_file(shortcode)
+    root, ingest_info = replace_filepath_with_internal_filename(root, orig_path_2_asset_id)
+    if ok := ingest_info.ok_msg():
+        print(ok)
+        logger.info(ok)
+    else:
+        err_msg = ingest_info.execute_error_protocol()
+        raise InputError(err_msg)
+    return root
+
+
+def _get_live_clients(con: Connection, config: UploadConfig, auth: AuthenticationClient) -> UploadClients:
+    ingest_client = BulkIngestedAssetClient()
+    project_client = ProjectClientLive(auth.server, auth)
+    list_client = ListClientLive(con, project_client.get_project_iri(config.shortcode))
+    legal_info_client = LegalInfoClientLive(config.server, config.shortcode, auth)
+    return UploadClients(ingest_client, list_client, legal_info_client)