dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/ingest_xmlupload/bulk_ingest_client.py

@@ -1,9 +1,11 @@
 import urllib.parse
+from collections.abc import Iterator
 from dataclasses import dataclass
 from dataclasses import field
+from http import HTTPStatus
 from pathlib import Path
-from typing import Iterator
 
+import regex
 from loguru import logger
 from requests import JSONDecodeError
 from requests import RequestException
@@ -11,18 +13,14 @@ from requests import Session
 from requests.adapters import HTTPAdapter
 from requests.adapters import Retry
 
+from dsp_tools.clients.authentication_client import AuthenticationClient
 from dsp_tools.commands.ingest_xmlupload.upload_files.upload_failures import UploadFailure
-from dsp_tools.models.exceptions import BadCredentialsError
-from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.logger_config import LOGGER_SAVEPATH
-
-STATUS_OK = 200
-STATUS_UNAUTHORIZED = 401
-STATUS_FORBIDDEN = 403
-STATUS_NOT_FOUND = 404
-STATUS_CONFLICT = 409
-STATUS_INTERNAL_SERVER_ERROR = 500
-STATUS_SERVER_UNAVAILABLE = 503
+from dsp_tools.config.logger_config import LOGGER_SAVEPATH
+from dsp_tools.error.exceptions import BadCredentialsError
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.request_utils import RequestParameters
+from dsp_tools.utils.request_utils import log_request
+from dsp_tools.utils.request_utils import log_response
 
 
 @dataclass
@@ -30,7 +28,7 @@ class BulkIngestClient:
     """Client to upload multiple files to the ingest server and monitor the ingest process."""
 
     dsp_ingest_url: str
-    token: str
+    authentication_client: AuthenticationClient
     shortcode: str
     imgdir: Path = field(default=Path.cwd())
     session: Session = field(init=False)
@@ -45,68 +43,93 @@ class BulkIngestClient:
             connect=retries,
             backoff_factor=0.3,
             allowed_methods=None,  # means all methods
-            status_forcelist=[STATUS_INTERNAL_SERVER_ERROR, STATUS_SERVER_UNAVAILABLE],
+            status_forcelist=[HTTPStatus.INTERNAL_SERVER_ERROR, HTTPStatus.SERVICE_UNAVAILABLE],
         )
         adapter = HTTPAdapter(max_retries=retry)
         self.session.mount("http://", adapter)
         self.session.mount("https://", adapter)
-        self.session.headers["Authorization"] = f"Bearer {self.token}"
 
     def upload_file(
         self,
         filepath: Path,
     ) -> UploadFailure | None:
-        """Uploads a file to the ingest server."""
-        url = f"{self.dsp_ingest_url}/projects/{self.shortcode}/bulk-ingest/ingest/{urllib.parse.quote(str(filepath))}"
-        headers = {"Content-Type": "application/octet-stream"}
-        timeout = 600
+        """
+        Uploads a file to the ingest server.
+        The load balancer on DSP servers currently has a timeout of 10m,
+        so we need to use a slightly shorter timeout of 9m.
+        See https://linear.app/dasch/issue/INFRA-847/increase-traefik-readtimeout
+        # noqa: DAR101
+        # noqa: DAR201
+        """
+        logger.debug(f"Uploading file '{filepath}'")
+        timeout = 9 * 60
+        url = self._build_url_for_bulk_ingest_ingest_route(filepath)
+        headers = {
+            "Content-Type": "application/octet-stream",
+            "Authorization": f"Bearer {self.authentication_client.get_token()}",
+        }
         err_msg = f"Failed to upload '{filepath}' to '{url}'."
+        params = RequestParameters("POST", url, timeout, headers=headers)
+        log_request(params)
         try:
             with open(self.imgdir / filepath, "rb") as binary_io:
-                content = binary_io.read()
-        except OSError as e:
-            logger.error(err_msg)
-            return UploadFailure(filepath, f"File could not be opened/read: {e.strerror}")
-        try:
-            logger.debug(f"REQUEST: POST to {url}, timeout: {timeout}, headers: {headers}")
-            res = self.session.post(
-                url=url,
-                headers=headers,
-                data=content,
-                timeout=timeout,
-            )
-            logger.debug(f"RESPONSE: {res.status_code}")
+                res = self.session.post(
+                    url=params.url,
+                    headers=params.headers,
+                    data=binary_io,  # https://requests.readthedocs.io/en/latest/user/advanced/#streaming-uploads
+                    timeout=params.timeout,
+                )
+            log_response(res)
         except RequestException as e:
-            logger.error(err_msg)
+            logger.exception(err_msg)
             return UploadFailure(filepath, f"Exception of requests library: {e}")
-        if res.status_code != STATUS_OK:
+        except OSError as e:
+            err_msg = f"Cannot bulk-ingest {filepath}, because the file could not be opened/read: {e.strerror}"
             logger.error(err_msg)
-            reason = f"Response {res.status_code}: {res.text}" if res.text else f"Response {res.status_code}"
-            return UploadFailure(filepath, reason)
-
+            return UploadFailure(filepath, err_msg)
+        if res.status_code != HTTPStatus.OK:
+            logger.error(f"{err_msg}: Response {res.status_code}: {res.text}")
+            return UploadFailure(filepath, res.reason, res.status_code, res.text)
         return None
 
+    def _build_url_for_bulk_ingest_ingest_route(self, filepath: Path) -> str:
+        """
+        Remove the leading slash of absolute filepaths,
+        because the `/project/<shortcode>/bulk-ingest/ingest` route only accepts relative paths.
+        The leading slash has to be added again in the "ingest-xmlupload" step, when applying the ingest ID.
+
+        Args:
+            filepath: filepath
+
+        Returns:
+            url
+        """
+        quoted = regex.sub(r"^%2F", "", urllib.parse.quote(str(filepath), safe=""))
+        return f"{self.dsp_ingest_url}/projects/{self.shortcode}/bulk-ingest/ingest/{quoted}"
+
     def trigger_ingest_process(self) -> None:
         """Start the ingest process on the server."""
         url = f"{self.dsp_ingest_url}/projects/{self.shortcode}/bulk-ingest"
         timeout = 5
-        logger.debug(f"REQUEST: POST to {url}, timeout: {timeout}")
-        res = self.session.post(url, timeout=timeout)
-        logger.debug(f"RESPONSE: {res.status_code}: {res.text}")
-        if res.status_code in [STATUS_UNAUTHORIZED, STATUS_FORBIDDEN]:
-            raise BadCredentialsError("Unauthorized to start the ingest process. Please check your credentials.")
-        if res.status_code == STATUS_NOT_FOUND:
-            raise UserError(
+        headers = {"Authorization": f"Bearer {self.authentication_client.get_token()}"}
+        params = RequestParameters("POST", url, timeout, headers=headers)
+        log_request(params)
+        res = self.session.post(params.url, timeout=params.timeout, headers=params.headers)
+        log_response(res)
+        if res.status_code == HTTPStatus.FORBIDDEN:
+            raise BadCredentialsError("Only ProjectAdmins or SystemAdmins can start the ingest process.")
+        if res.status_code == HTTPStatus.NOT_FOUND:
+            raise InputError(
                 f"No assets have been uploaded for project {self.shortcode}. "
                 "Before using the 'ingest-files' command, you must upload some files with the 'upload-files' command."
             )
-        if res.status_code == STATUS_CONFLICT:
+        if res.status_code == HTTPStatus.CONFLICT:
             msg = f"Ingest process on the server {self.dsp_ingest_url} is already running. Wait until it completes..."
             print(msg)
             logger.info(msg)
             return
-        if res.status_code in [STATUS_INTERNAL_SERVER_ERROR, STATUS_SERVER_UNAVAILABLE]:
-            raise UserError("Server is unavailable. Please try again later.")
+        if res.status_code in [HTTPStatus.INTERNAL_SERVER_ERROR, HTTPStatus.SERVICE_UNAVAILABLE]:
+            raise InputError("Server is unavailable. Please try again later.")
 
         try:
             returned_shortcode = res.json().get("id")
@@ -114,7 +137,7 @@ class BulkIngestClient:
         except JSONDecodeError:
             failed = True
         if failed:
-            raise UserError("Failed to trigger the ingest process. Please check the server logs, or try again later.")
+            raise InputError("Failed to trigger the ingest process. Please check the server logs, or try again later.")
         print(f"Kicked off the ingest process on the server {self.dsp_ingest_url}. Wait until it completes...")
         logger.info(f"Kicked off the ingest process on the server {self.dsp_ingest_url}. Wait until it completes...")
 
@@ -128,26 +151,28 @@ class BulkIngestClient:
             The mapping CSV if the ingest process has completed.
 
         Raises:
-            UserError: if there are too many server errors in a row.
+            InputError: if there are too many server errors in a row.
         """
         url = f"{self.dsp_ingest_url}/projects/{self.shortcode}/bulk-ingest/mapping.csv"
         timeout = 5
         while True:
-            logger.debug(f"REQUEST: GET to {url}, timeout: {timeout}")
-            res = self.session.get(url, timeout=timeout)
-            logger.debug(f"RESPONSE: {res.status_code}")
-            if res.status_code == STATUS_CONFLICT:
+            headers = {"Authorization": f"Bearer {self.authentication_client.get_token()}"}
+            params = RequestParameters("GET", url, timeout, headers=headers)
+            log_request(params)
+            res = self.session.get(params.url, timeout=params.timeout, headers=params.headers)
+            log_response(res)
+            if res.status_code == HTTPStatus.CONFLICT:
                 self.retrieval_failures = 0
                 logger.info("Ingest process is still running. Wait until it completes...")
                 yield True
-            elif res.status_code != STATUS_OK or not res.text.startswith("original,derivative"):
+            elif res.status_code != HTTPStatus.OK or not res.text.startswith("original,derivative"):
                 self.retrieval_failures += 1
                 if self.retrieval_failures > 15:
-                    raise UserError(f"There were too many server errors. Please check the logs at {LOGGER_SAVEPATH}.")
+                    raise InputError(f"There were too many server errors. Please check the logs at {LOGGER_SAVEPATH}.")
                 msg = "While retrieving the mapping CSV, the server responded with an unexpected status code/content."
                 logger.error(msg)
                 yield False
             else:
                 logger.info("Ingest process completed.")
                 break
-        yield res.text
+        yield res.content.decode("utf-8")

dsp_tools/commands/ingest_xmlupload/create_resources/apply_ingest_id.py

@@ -9,7 +9,7 @@ from loguru import logger
 from lxml import etree
 
 from dsp_tools.commands.ingest_xmlupload.create_resources.user_information import IngestInformation
-from dsp_tools.models.exceptions import InputError
+from dsp_tools.error.exceptions import InputError
 
 
 def get_mapping_dict_from_file(shortcode: str) -> dict[str, str]:
@@ -56,15 +56,9 @@ def replace_filepath_with_internal_filename(
     for elem in new_tree.iter():
         if not etree.QName(elem).localname.endswith("bitstream") or not elem.text:
             continue
-        img_path_str = elem.text
-        if img_path_str not in orig_path_2_asset_id:
-            img_path = Path(elem.text)
-            img_path_str = str(img_path.relative_to(Path.cwd()) if img_path.is_absolute() else img_path)
-            if img_path_str not in orig_path_2_asset_id:
-                img_path_str = str(img_path.with_suffix(img_path.suffix.lower()))
-            if img_path_str not in orig_path_2_asset_id:
-                img_path_str = str(img_path.with_suffix(img_path.suffix.upper()))
-
+        img_path_str = elem.text.strip()
+        if img_path_str not in orig_path_2_asset_id and img_path_str.startswith("/"):
+            img_path_str = img_path_str[1:]
         if img_path_str in orig_path_2_asset_id:
             elem.text = orig_path_2_asset_id[img_path_str]
             used_media_file_paths.append(img_path_str)

dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py

@@ -6,27 +6,44 @@ from loguru import logger
 from lxml import etree
 
 from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.cli.args import ValidateDataConfig
+from dsp_tools.cli.args import ValidationSeverity
+from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.connection import Connection
+from dsp_tools.clients.connection_live import ConnectionLive
+from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
+from dsp_tools.clients.project_client_live import ProjectClientLive
 from dsp_tools.commands.ingest_xmlupload.create_resources.apply_ingest_id import get_mapping_dict_from_file
 from dsp_tools.commands.ingest_xmlupload.create_resources.apply_ingest_id import replace_filepath_with_internal_filename
-from dsp_tools.commands.xmlupload.list_client import ListClientLive
+from dsp_tools.commands.validate_data.validate_data import validate_parsed_resources
 from dsp_tools.commands.xmlupload.models.ingest import BulkIngestedAssetClient
 from dsp_tools.commands.xmlupload.models.upload_clients import UploadClients
 from dsp_tools.commands.xmlupload.models.upload_state import UploadState
-from dsp_tools.commands.xmlupload.ontology_client import OntologyClientLive
-from dsp_tools.commands.xmlupload.project_client import ProjectClientLive
-from dsp_tools.commands.xmlupload.read_validate_xml_file import validate_and_parse
+from dsp_tools.commands.xmlupload.prepare_xml_input.get_processed_resources import get_processed_resources
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClientLive
+from dsp_tools.commands.xmlupload.prepare_xml_input.prepare_xml_input import get_parsed_resources_and_mappers
+from dsp_tools.commands.xmlupload.prepare_xml_input.prepare_xml_input import get_stash_and_upload_order
+from dsp_tools.commands.xmlupload.prepare_xml_input.read_validate_xml_file import validate_iiif_uris
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
+from dsp_tools.commands.xmlupload.xmlupload import enable_unknown_license_if_any_are_missing
 from dsp_tools.commands.xmlupload.xmlupload import execute_upload
-from dsp_tools.commands.xmlupload.xmlupload import prepare_upload
-from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.connection_live import ConnectionLive
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.ansi_colors import BOLD_RED
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+from dsp_tools.utils.data_formats.uri_util import is_prod_like_server
+from dsp_tools.utils.replace_id_with_iri import use_id2iri_mapping_to_replace_ids
+from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_clean_xml_file
 
 
 def ingest_xmlupload(
     xml_file: Path,
     creds: ServerCredentials,
     interrupt_after: int | None = None,
+    skip_validation: bool = False,
+    skip_ontology_validation: bool = False,
+    id2iri_file: str | None = None,
+    do_not_request_resource_metadata_from_db: bool = False,
 ) -> bool:
     """
     This function reads an XML file
@@ -40,6 +57,11 @@ def ingest_xmlupload(
         xml_file: path to XML file containing the resources
         creds: credentials to access the DSP server
         interrupt_after: if set, the upload will be interrupted after this number of resources
+        skip_validation: skip the SHACL validation
+        skip_ontology_validation: skip the ontology validation
+        id2iri_file: to replace internal IDs of an XML file by IRIs provided in this mapping file
+        do_not_request_resource_metadata_from_db: if true do not request metadata information from the api
+                                                  for existing resources
 
     Returns:
         True if all resources could be uploaded without errors; False if one of the resources could not be
@@ -48,11 +70,12 @@ def ingest_xmlupload(
     Raises:
         InputError: if any media was not uploaded or uploaded media was not referenced.
     """
-    default_ontology, root, shortcode = _parse_xml_and_replace_filepaths(xml_file)
-
-    con = ConnectionLive(creds.server)
-    con.login(creds.user, creds.password)
+    root = parse_and_clean_xml_file(xml_file)
+    shortcode = root.attrib["shortcode"]
+    root = _replace_filepaths_with_internal_filename_from_ingest(root, shortcode)
 
+    auth = AuthenticationClientLive(server=creds.server, email=creds.user, password=creds.password)
+    con = ConnectionLive(creds.server, auth)
     config = UploadConfig(
         media_previously_uploaded=True,
         interrupt_after=interrupt_after,
@@ -60,34 +83,70 @@ def ingest_xmlupload(
         server=creds.server,
         shortcode=shortcode,
     )
+    clients = _get_live_clients(con, config, auth)
+
+    parsed_resources, lookups = get_parsed_resources_and_mappers(root, clients)
+    if id2iri_file:
+        parsed_resources = use_id2iri_mapping_to_replace_ids(parsed_resources, Path(id2iri_file))
+
+    validation_should_be_skipped = skip_validation
+    is_on_prod_like_server = is_prod_like_server(creds.server)
+    if is_on_prod_like_server and config.skip_validation:
+        msg = (
+            "You set the flag '--skip-validation' to circumvent the SHACL schema validation. "
+            "This means that the upload may fail due to undetected errors. "
+            "Do you wish to skip the validation (yes/no)? "
+        )
+        resp = ""
+        while resp not in ["yes", "no"]:
+            resp = input(BOLD_RED + msg + RESET_TO_DEFAULT)
+        if str(resp) == "no":
+            validation_should_be_skipped = False
+    if not validation_should_be_skipped:
+        v_severity = config.validation_severity
+        if is_on_prod_like_server:
+            v_severity = ValidationSeverity.INFO
+        validation_passed = validate_parsed_resources(
+            parsed_resources=parsed_resources,
+            authorship_lookup=lookups.authorships,
+            permission_ids=list(lookups.permissions.keys()),
+            shortcode=shortcode,
+            config=ValidateDataConfig(
+                xml_file,
+                save_graph_dir=None,
+                severity=v_severity,
+                ignore_duplicate_files_warning=True,
+                is_on_prod_server=is_on_prod_like_server,
+                skip_ontology_validation=skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=do_not_request_resource_metadata_from_db,
+            ),
+            auth=auth,
+        )
+        if not validation_passed:
+            return False
+    else:
+        logger.debug("SHACL validation was skipped.")
 
-    ontology_client = OntologyClientLive(con=con, shortcode=shortcode, default_ontology=default_ontology)
-    resources, permissions_lookup, stash = prepare_upload(root, ontology_client)
-
-    clients = _get_live_clients(con, config)
-    state = UploadState(resources, stash, config, permissions_lookup)
-
-    return execute_upload(clients, state)
+    if not config.skip_iiif_validation:
+        validate_iiif_uris(root)
 
+    if not is_on_prod_like_server:
+        enable_unknown_license_if_any_are_missing(clients.legal_info_client, parsed_resources)
 
-def _parse_xml_and_replace_filepaths(xml_file: Path) -> tuple[str, etree._Element, str]:
-    """
-    Validate and parse an upload XML file.
-    The bulk-ingest must already have taken place, and the mapping CSV must be in the CWD.
+    processed_resources = get_processed_resources(parsed_resources, lookups, is_on_prod_like_server)
 
-    Args:
-        xml_file: file that will be parsed
+    sorted_resources, stash = get_stash_and_upload_order(processed_resources)
 
-    Returns:
-        The ontology name, the parsed XML file and the shortcode of the project
+    state = UploadState(
+        pending_resources=sorted_resources,
+        pending_stash=stash,
+        config=config,
+    )
 
-    Raises:
-        InputError: if replacing file paths with internal asset IDs failed
-    """
-    root, shortcode, default_ontology = validate_and_parse(xml_file)
+    return execute_upload(clients, state)
 
-    logger.info(f"Validated and parsed the XML. {shortcode=:} and {default_ontology=:}")
 
+def _replace_filepaths_with_internal_filename_from_ingest(root: etree._Element, shortcode: str) -> etree._Element:
     orig_path_2_asset_id = get_mapping_dict_from_file(shortcode)
     root, ingest_info = replace_filepath_with_internal_filename(root, orig_path_2_asset_id)
     if ok := ingest_info.ok_msg():
@@ -96,11 +155,12 @@ def _parse_xml_and_replace_filepaths(xml_file: Path) -> tuple[str, etree._Elemen
     else:
         err_msg = ingest_info.execute_error_protocol()
         raise InputError(err_msg)
-    return default_ontology, root, shortcode
+    return root
 
 
-def _get_live_clients(con: Connection, config: UploadConfig) -> UploadClients:
+def _get_live_clients(con: Connection, config: UploadConfig, auth: AuthenticationClient) -> UploadClients:
     ingest_client = BulkIngestedAssetClient()
-    project_client = ProjectClientLive(con, config.shortcode)
-    list_client = ListClientLive(con, project_client.get_project_iri())
-    return UploadClients(ingest_client, project_client, list_client)
+    project_client = ProjectClientLive(auth.server, auth)
+    list_client = ListClientLive(con, project_client.get_project_iri(config.shortcode))
+    legal_info_client = LegalInfoClientLive(config.server, config.shortcode, auth)
+    return UploadClients(ingest_client, list_client, legal_info_client)

dsp_tools/commands/ingest_xmlupload/create_resources/user_information.py

@@ -72,7 +72,7 @@ class IngestInformation:
                 "The data XML file contains references to multimedia files "
                 "which were not previously uploaded through dsp-ingest:\n"
                 f"    The file with the resource IDs and problematic filenames was saved at "
-                f"'{Path(self.csv_directory_path/self.mediafiles_no_id_csv)}'."
+                f"'{Path(self.csv_directory_path / self.mediafiles_no_id_csv)}'."
             )
         return None
 
@@ -87,7 +87,7 @@ class IngestInformation:
                 "The data XML file does not reference all the multimedia files which were previously "
                 "uploaded through dsp-ingest.\n"
                 f"    The file with the unused filenames was saved at "
-                f"'{Path(self.csv_directory_path/self.unused_mediafiles_csv)}'."
+                f"'{Path(self.csv_directory_path / self.unused_mediafiles_csv)}'."
             )
         return None
 

dsp_tools/commands/ingest_xmlupload/ingest_files/ingest_files.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from pathlib import Path
 from time import sleep
 from typing import cast
@@ -7,9 +6,8 @@ from loguru import logger
 from tqdm import tqdm
 
 from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
 from dsp_tools.commands.ingest_xmlupload.bulk_ingest_client import BulkIngestClient
-from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.connection_live import ConnectionLive
 
 
 def ingest_files(creds: ServerCredentials, shortcode: str) -> bool:
@@ -24,9 +22,8 @@ def ingest_files(creds: ServerCredentials, shortcode: str) -> bool:
     Returns:
         success status
     """
-    con: Connection = ConnectionLive(creds.server)
-    con.login(creds.user, creds.password)
-    bulk_ingest_client = BulkIngestClient(creds.dsp_ingest_url, con.get_token(), shortcode)
+    auth = AuthenticationClientLive(creds.server, creds.user, creds.password)
+    bulk_ingest_client = BulkIngestClient(creds.dsp_ingest_url, auth, shortcode)
     bulk_ingest_client.trigger_ingest_process()
     sleep(5)
     mapping = _retrieve_mapping(bulk_ingest_client)
@@ -35,7 +32,7 @@ def ingest_files(creds: ServerCredentials, shortcode: str) -> bool:
 
 
 def _retrieve_mapping(bulk_ingest_client: BulkIngestClient) -> str:
-    sleeping_time = 10
+    sleeping_time = 60
     desc = f"Wait until mapping CSV is ready. Ask server every {sleeping_time} seconds "
     progress_bar = tqdm(
         bulk_ingest_client.retrieve_mapping_generator(), desc=desc, bar_format="{desc}{elapsed}", dynamic_ncols=True
@@ -58,8 +55,10 @@ def _retrieve_mapping(bulk_ingest_client: BulkIngestClient) -> str:
 def _save_mapping(mapping: str, shortcode: str) -> None:
     filepath = Path(f"mapping-{shortcode}.csv")
     if filepath.exists():
-        timestamp = datetime.now().strftime("%Y-%m-%d_%H.%M.%S")
-        filepath = filepath.with_name(f"{filepath.stem}-{timestamp}.csv")
-    filepath.write_text(mapping)
+        i = 1
+        while (new_name_for_existing := Path(f"mapping-{shortcode}-{i}.csv")).exists():
+            i += 1
+        filepath.rename(new_name_for_existing)
+    filepath.write_text(mapping, encoding="utf-8")
     print(f"Saved mapping CSV to '{filepath}'")
     logger.info(f"Saved mapping CSV to '{filepath}'")

dsp_tools/commands/ingest_xmlupload/upload_files/filechecker.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Iterable
 
 from dsp_tools.commands.ingest_xmlupload.upload_files.input_error import FileProblems
 
 SUPPORTED_EXTENSIONS = (
-    "zip,tar,gz,z,tgz,gzip,7z,mp3,wav,pdf,doc,docx,xls,xlsx,ppt,pptx,"
-    "mp4,jpg,jpeg,jp2,png,tif,tiff,odd,rng,txt,xml,xsd,xsl,csv"
+    "zip,tar,gz,z,tgz,gzip,7z,mp3,wav,pdf,doc,docx,xls,xlsx,ppt,pptx,epub,"
+    "mp4,jpg,jpeg,jp2,json,png,tif,tiff,odd,rng,txt,xml,htm,html,xsd,xsl,csv"
 ).split(",")
 
 

dsp_tools/commands/ingest_xmlupload/upload_files/input_error.py

@@ -1,22 +1,14 @@
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Protocol
 
 import pandas as pd
 
+from dsp_tools.error.problems import Problem
+
 separator = "\n\n"
 list_separator = "\n - "
 
 
-class Problem(Protocol):
-    """Information about input errors."""
-
-    def execute_error_protocol(self) -> str:
-        """
-        This function initiates all the steps for successful problem communication with the user.
-        """
-
-
 @dataclass(frozen=True)
 class FileProblems(Problem):
     """Handle the error communication to the user in case that some files don't exist or are unsupported."""

dsp_tools/commands/ingest_xmlupload/upload_files/upload_failures.py

@@ -5,7 +5,8 @@ import pandas as pd
 import regex
 
 separator = "\n\n"
-list_separator = "\n - "
+list_separator = "\n- "
+list_separator_indented = "\n    - "
 
 
 @dataclass(frozen=True)
@@ -14,6 +15,8 @@ class UploadFailure:
 
     filepath: Path
     reason: str
+    status_code: int | None = None
+    response_text: str | None = None
 
 
 @dataclass(frozen=True)
@@ -44,13 +47,20 @@ class UploadFailures:
             msg += f"The full list of failed files has been saved to '{output_file}'."
         else:
             msg += f"Failed to upload the following {len(self.failures)} files:"
-            msg += list_separator + list_separator.join([f"{x.filepath}: {x.reason}" for x in self.failures])
+            for f in self.failures:
+                msg += list_separator + f"{f.filepath}: {f.reason}"
+                if f.status_code:
+                    msg += list_separator_indented + f"Status code: {f.status_code}"
+                if f.response_text:
+                    msg += list_separator_indented + f"Response text: {f.response_text}"
         return msg
 
     def _save_to_csv(self, output_file: Path) -> None:
         data = {
             "Filepath": [failure.filepath for failure in self.failures],
             "Reason": [failure.reason for failure in self.failures],
+            "Status code": [failure.status_code for failure in self.failures],
+            "Response text": [failure.response_text for failure in self.failures],
         }
         df = pd.DataFrame(data)
         df.to_csv(output_file, index=False)

dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py

@@ -5,14 +5,13 @@ from lxml import etree
 from tqdm import tqdm
 
 from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
 from dsp_tools.commands.ingest_xmlupload.bulk_ingest_client import BulkIngestClient
 from dsp_tools.commands.ingest_xmlupload.upload_files.filechecker import check_files
 from dsp_tools.commands.ingest_xmlupload.upload_files.upload_failures import UploadFailure
 from dsp_tools.commands.ingest_xmlupload.upload_files.upload_failures import UploadFailures
-from dsp_tools.commands.xmlupload.read_validate_xml_file import validate_and_parse
-from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.connection_live import ConnectionLive
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_clean_xml_file
 
 
 def upload_files(
@@ -32,14 +31,14 @@ def upload_files(
     Returns:
         success status
     """
-    root, shortcode, _ = validate_and_parse(xml_file)
+    root = parse_and_clean_xml_file(xml_file)
+    shortcode = root.attrib["shortcode"]
     paths = _get_validated_paths(root)
     print(f"Found {len(paths)} files to upload onto server {creds.dsp_ingest_url}.")
     logger.info(f"Found {len(paths)} files to upload onto server {creds.dsp_ingest_url}.")
 
-    con: Connection = ConnectionLive(creds.server)
-    con.login(creds.user, creds.password)
-    ingest_client = BulkIngestClient(creds.dsp_ingest_url, con.get_token(), shortcode, imgdir)
+    auth = AuthenticationClientLive(creds.server, creds.user, creds.password)
+    ingest_client = BulkIngestClient(creds.dsp_ingest_url, auth, shortcode, imgdir)
 
     failures: list[UploadFailure] = []
     progress_bar = tqdm(paths, desc="Uploading files", unit="file(s)", dynamic_ncols=True)
@@ -61,7 +60,7 @@ def upload_files(
 
 
 def _get_validated_paths(root: etree._Element) -> set[Path]:
-    paths = {Path(x.text) for x in root.xpath("//bitstream")}
+    paths = {Path(x.text.strip()) for x in root.xpath("//bitstream")}
     if problems := check_files(paths):
         msg = problems.execute_error_protocol()
         raise InputError(msg)