dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/ingest_xmlupload/create_resources/user_information.py

@@ -0,0 +1,121 @@
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+
+import pandas as pd
+
+separator = "\n    "
+list_separator = "\n    - "
+
+
+@dataclass(frozen=True)
+class IngestInformation:
+    """
+    This class stores the information about the mapping of ids provided by the dsp-ingest service
+    and the filepaths used in the XML file.
+    """
+
+    unused_mediafiles: list[str]
+    mediafiles_no_id: list[tuple[str, str]]
+    maximum_prints: int = 20
+    csv_directory_path: Path = field(default=Path.cwd())
+    unused_mediafiles_csv: str = "UnusedMediaUploaded.csv"
+    mediafiles_no_id_csv: str = "FilesNotUploaded.csv"
+
+    def ok_msg(self) -> str | None:
+        """
+        This function checks if no media was unused or not uploaded.
+        If that is the case it returns the message,
+        if not, it ends without an effect.
+
+        Returns:
+            Message if all went well.
+        """
+        if not self.unused_mediafiles and not self.mediafiles_no_id:
+            return (
+                "All multimedia files referenced in the XML file were uploaded through dsp-ingest.\n"
+                "All multimedia files uploaded through dsp-ingest were referenced in the XML file."
+            )
+        return None
+
+    def execute_error_protocol(self) -> str:
+        """
+        This function generates the user message and saves a file with the information
+        if a lot of resources are affected.
+
+        Returns:
+            User message
+        """
+        self._save_csv_if_applicable()
+        return self._get_error_msg()
+
+    def _get_error_msg(self) -> str:
+        msg_list = [
+            "The upload cannot continue as there are problems with the multimedia files referenced in the XML.",
+        ]
+        if has_msg := self._get_unused_mediafiles_msg():
+            msg_list.append(has_msg)
+        if has_msg := self._get_mediafiles_no_id_msg():
+            msg_list.append(has_msg)
+        return separator.join(msg_list)
+
+    def _get_mediafiles_no_id_msg(self) -> str | None:
+        if 0 < len(self.mediafiles_no_id) <= self.maximum_prints:
+            return (
+                "The data XML file contains references to the following multimedia files "
+                "which were not previously uploaded through dsp-ingest:"
+                + list_separator
+                + list_separator.join([f"Resource ID: '{x[0]}' | Filepath: '{x[1]}'" for x in self.mediafiles_no_id])
+            )
+        elif len(self.mediafiles_no_id) > self.maximum_prints:
+            return (
+                "The data XML file contains references to multimedia files "
+                "which were not previously uploaded through dsp-ingest:\n"
+                f"    The file with the resource IDs and problematic filenames was saved at "
+                f"'{Path(self.csv_directory_path / self.mediafiles_no_id_csv)}'."
+            )
+        return None
+
+    def _get_unused_mediafiles_msg(self) -> str | None:
+        if 0 < len(self.unused_mediafiles) <= self.maximum_prints:
+            return (
+                "The data XML file does not reference the following multimedia files which were previously "
+                "uploaded through dsp-ingest:" + list_separator + list_separator.join(self.unused_mediafiles)
+            )
+        elif len(self.unused_mediafiles) > self.maximum_prints:
+            return (
+                "The data XML file does not reference all the multimedia files which were previously "
+                "uploaded through dsp-ingest.\n"
+                f"    The file with the unused filenames was saved at "
+                f"'{Path(self.csv_directory_path / self.unused_mediafiles_csv)}'."
+            )
+        return None
+
+    def _save_csv_if_applicable(self) -> None:
+        if (unused_mediafiles_df := self._unused_mediafiles_to_df()) is not None:
+            _save_as_csv(unused_mediafiles_df, self.csv_directory_path, self.unused_mediafiles_csv)
+        if (no_id_df := self._mediafiles_no_id_to_df()) is not None:
+            _save_as_csv(no_id_df, self.csv_directory_path, self.mediafiles_no_id_csv)
+
+    def _unused_mediafiles_to_df(self) -> pd.DataFrame | None:
+        return (
+            pd.DataFrame({"Multimedia Filenames": self.unused_mediafiles})
+            if len(self.unused_mediafiles) > self.maximum_prints
+            else None
+        )
+
+    def _mediafiles_no_id_to_df(self) -> pd.DataFrame | None:
+        return (
+            pd.DataFrame(
+                {
+                    "Resource ID": [x[0] for x in self.mediafiles_no_id],
+                    "Filepath": [x[1] for x in self.mediafiles_no_id],
+                }
+            )
+            if len(self.mediafiles_no_id) > self.maximum_prints
+            else None
+        )
+
+
+def _save_as_csv(df: pd.DataFrame, directory_path: Path, filename: str) -> None:
+    df.to_csv(Path(directory_path, filename), index=False)

dsp_tools/commands/ingest_xmlupload/ingest_files/ingest_files.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+from time import sleep
+from typing import cast
+
+from loguru import logger
+from tqdm import tqdm
+
+from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.commands.ingest_xmlupload.bulk_ingest_client import BulkIngestClient
+
+
+def ingest_files(creds: ServerCredentials, shortcode: str) -> bool:
+    """
+    Kick off the ingest process on the server, and wait until it has finished.
+    Then, retrieve the mapping CSV from the server and save it in the CWD.
+
+    Args:
+        creds: credentials to log in on the server
+        shortcode: shortcode of the project
+
+    Returns:
+        success status
+    """
+    auth = AuthenticationClientLive(creds.server, creds.user, creds.password)
+    bulk_ingest_client = BulkIngestClient(creds.dsp_ingest_url, auth, shortcode)
+    bulk_ingest_client.trigger_ingest_process()
+    sleep(5)
+    mapping = _retrieve_mapping(bulk_ingest_client)
+    _save_mapping(mapping, shortcode)
+    return True
+
+
+def _retrieve_mapping(bulk_ingest_client: BulkIngestClient) -> str:
+    sleeping_time = 60
+    desc = f"Wait until mapping CSV is ready. Ask server every {sleeping_time} seconds "
+    progress_bar = tqdm(
+        bulk_ingest_client.retrieve_mapping_generator(), desc=desc, bar_format="{desc}{elapsed}", dynamic_ncols=True
+    )
+    num_of_attempts = 0
+    num_of_server_errors = 0
+    for result in progress_bar:
+        if result is False:
+            num_of_attempts += 1
+            num_of_server_errors += 1
+        elif result is True:
+            num_of_attempts += 1
+        elif isinstance(result, str):
+            break
+        progress_bar.set_description(f"{desc}(attempts: {num_of_attempts}, server errors: {num_of_server_errors})")
+        sleep(sleeping_time)
+    return cast(str, result)
+
+
+def _save_mapping(mapping: str, shortcode: str) -> None:
+    filepath = Path(f"mapping-{shortcode}.csv")
+    if filepath.exists():
+        i = 1
+        while (new_name_for_existing := Path(f"mapping-{shortcode}-{i}.csv")).exists():
+            i += 1
+        filepath.rename(new_name_for_existing)
+    filepath.write_text(mapping, encoding="utf-8")
+    print(f"Saved mapping CSV to '{filepath}'")
+    logger.info(f"Saved mapping CSV to '{filepath}'")

dsp_tools/commands/ingest_xmlupload/upload_files/filechecker.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from dsp_tools.commands.ingest_xmlupload.upload_files.input_error import FileProblems
+
+SUPPORTED_EXTENSIONS = (
+    "zip,tar,gz,z,tgz,gzip,7z,mp3,wav,pdf,doc,docx,xls,xlsx,ppt,pptx,epub,"
+    "mp4,jpg,jpeg,jp2,json,png,tif,tiff,odd,rng,txt,xml,htm,html,xsd,xsl,csv"
+).split(",")
+
+
+def check_files(files: Iterable[Path]) -> FileProblems | None:
+    """Check if the files exist and have supported extensions."""
+    unsupported_files = [file for file in files if file.suffix[1:].casefold() not in SUPPORTED_EXTENSIONS]
+    non_existing_files = [file for file in files if not file.exists() and file not in unsupported_files]
+    if non_existing_files or unsupported_files:
+        return FileProblems(non_existing_files, unsupported_files)
+    return None

dsp_tools/commands/ingest_xmlupload/upload_files/input_error.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+
+from dsp_tools.error.problems import Problem
+
+separator = "\n\n"
+list_separator = "\n - "
+
+
+@dataclass(frozen=True)
+class FileProblems(Problem):
+    """Handle the error communication to the user in case that some files don't exist or are unsupported."""
+
+    non_existing_files: list[Path]
+    unsupported_files: list[Path]
+    maximum_prints: int = 50
+
+    def __post_init__(self) -> None:
+        if not self.non_existing_files and not self.unsupported_files:
+            raise ValueError("It's not possible to create a FileProblems object without any problems.")
+
+    def execute_error_protocol(self) -> str:
+        """
+        Generate the error message to communicate the problems to the user.
+        If there are too many problems, save them to a file.
+
+        Returns:
+            error message
+        """
+        msg = "Some files referenced in the <bitstream> tags of your XML file cannot be uploaded to the server."
+        if len(self.non_existing_files) + len(self.unsupported_files) > self.maximum_prints:
+            output_file = Path("file_problems.csv")
+            self._save_to_csv(output_file)
+            msg += f" The full list of files with problems has been saved to '{output_file}'."
+            return msg
+        if self.non_existing_files:
+            msg += separator
+            msg += "The following files don't exist on your computer:"
+            msg += list_separator + list_separator.join([str(file) for file in self.non_existing_files])
+        if self.unsupported_files:
+            msg += separator
+            msg += "The following files have unsupported extensions:"
+            msg += list_separator + list_separator.join([str(file) for file in self.unsupported_files])
+        return msg
+
+    def _save_to_csv(self, output_file: Path) -> None:
+        problems = ["File doesn't exist"] * len(self.non_existing_files) + ["Extension not supported"] * len(
+            self.unsupported_files
+        )
+        data = {
+            "File": self.non_existing_files + self.unsupported_files,
+            "Problem": problems,
+        }
+        df = pd.DataFrame(data)
+        df.to_csv(output_file, index=False)

dsp_tools/commands/ingest_xmlupload/upload_files/upload_failures.py

@@ -0,0 +1,66 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+import regex
+
+separator = "\n\n"
+list_separator = "\n- "
+list_separator_indented = "\n    - "
+
+
+@dataclass(frozen=True)
+class UploadFailure:
+    """Information on why the upload of a file to the ingest server failed."""
+
+    filepath: Path
+    reason: str
+    status_code: int | None = None
+    response_text: str | None = None
+
+
+@dataclass(frozen=True)
+class UploadFailures:
+    """Aggregated information of all failed uploads."""
+
+    failures: list[UploadFailure]
+    num_of_initial_files: int
+    shortcode: str
+    dsp_ingest_url: str
+    maximum_prints: int = 50
+
+    def execute_error_protocol(self) -> str:
+        """
+        Generate the error message to communicate the problems to the user.
+        If there are too many problems, save them to a file.
+
+        Returns:
+            error message
+        """
+        ratio = f"{self.num_of_initial_files - len(self.failures)}/{self.num_of_initial_files}"
+        msg = f"Uploaded {ratio} files onto server {self.dsp_ingest_url}. "
+        if len(self.failures) > self.maximum_prints:
+            url = regex.sub(r"https?://", "", self.dsp_ingest_url)
+            output_file = Path(f"upload_failures_{self.shortcode}_{url}.csv")
+            self._save_to_csv(output_file)
+            msg += f"Failed to upload {len(self.failures)} files. "
+            msg += f"The full list of failed files has been saved to '{output_file}'."
+        else:
+            msg += f"Failed to upload the following {len(self.failures)} files:"
+            for f in self.failures:
+                msg += list_separator + f"{f.filepath}: {f.reason}"
+                if f.status_code:
+                    msg += list_separator_indented + f"Status code: {f.status_code}"
+                if f.response_text:
+                    msg += list_separator_indented + f"Response text: {f.response_text}"
+        return msg
+
+    def _save_to_csv(self, output_file: Path) -> None:
+        data = {
+            "Filepath": [failure.filepath for failure in self.failures],
+            "Reason": [failure.reason for failure in self.failures],
+            "Status code": [failure.status_code for failure in self.failures],
+            "Response text": [failure.response_text for failure in self.failures],
+        }
+        df = pd.DataFrame(data)
+        df.to_csv(output_file, index=False)

dsp_tools/commands/ingest_xmlupload/upload_files/upload_files.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+
+from loguru import logger
+from lxml import etree
+from tqdm import tqdm
+
+from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.commands.ingest_xmlupload.bulk_ingest_client import BulkIngestClient
+from dsp_tools.commands.ingest_xmlupload.upload_files.filechecker import check_files
+from dsp_tools.commands.ingest_xmlupload.upload_files.upload_failures import UploadFailure
+from dsp_tools.commands.ingest_xmlupload.upload_files.upload_failures import UploadFailures
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_clean_xml_file
+
+
+def upload_files(
+    xml_file: Path,
+    creds: ServerCredentials,
+    imgdir: Path = Path.cwd(),
+) -> bool:
+    """
+    Upload all files referenced in an XML file to the ingest server.
+    This involves no processing/ingesting of the files, just uploading them.
+
+    Args:
+        xml_file: XML file containing the resources and the references to the files to upload
+        creds: credentials to connect to the ingest server
+        imgdir: the bitstreams in the XML file are relative to this directory
+
+    Returns:
+        success status
+    """
+    root = parse_and_clean_xml_file(xml_file)
+    shortcode = root.attrib["shortcode"]
+    paths = _get_validated_paths(root)
+    print(f"Found {len(paths)} files to upload onto server {creds.dsp_ingest_url}.")
+    logger.info(f"Found {len(paths)} files to upload onto server {creds.dsp_ingest_url}.")
+
+    auth = AuthenticationClientLive(creds.server, creds.user, creds.password)
+    ingest_client = BulkIngestClient(creds.dsp_ingest_url, auth, shortcode, imgdir)
+
+    failures: list[UploadFailure] = []
+    progress_bar = tqdm(paths, desc="Uploading files", unit="file(s)", dynamic_ncols=True)
+    for path in progress_bar:
+        if res := ingest_client.upload_file(path):
+            failures.append(res)
+            progress_bar.set_description(f"Uploading files (failed: {len(failures)})")
+    if failures:
+        aggregated_failures = UploadFailures(failures, len(paths), shortcode, creds.dsp_ingest_url)
+        msg = aggregated_failures.execute_error_protocol()
+        logger.error(msg)
+        print(msg)
+        return False
+    else:
+        msg = f"Uploaded all {len(paths)} files onto server {creds.dsp_ingest_url}."
+        logger.info(msg)
+        print(msg)
+        return True
+
+
+def _get_validated_paths(root: etree._Element) -> set[Path]:
+    paths = {Path(x.text.strip()) for x in root.xpath("//bitstream")}
+    if problems := check_files(paths):
+        msg = problems.execute_error_protocol()
+        raise InputError(msg)
+    return paths

dsp_tools/commands/resume_xmlupload/resume_xmlupload.py

@@ -0,0 +1,96 @@
+import pickle
+import sys
+
+from loguru import logger
+
+from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.connection_live import ConnectionLive
+from dsp_tools.clients.legal_info_client import LegalInfoClient
+from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
+from dsp_tools.clients.project_client import ProjectClient
+from dsp_tools.clients.project_client_live import ProjectClientLive
+from dsp_tools.commands.xmlupload.models.ingest import AssetClient
+from dsp_tools.commands.xmlupload.models.ingest import BulkIngestedAssetClient
+from dsp_tools.commands.xmlupload.models.ingest import DspIngestClientLive
+from dsp_tools.commands.xmlupload.models.upload_clients import UploadClients
+from dsp_tools.commands.xmlupload.models.upload_state import UploadState
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClient
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClientLive
+from dsp_tools.commands.xmlupload.upload_config import UploadConfig
+from dsp_tools.commands.xmlupload.xmlupload import execute_upload
+from dsp_tools.utils.ansi_colors import RED
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+
+def resume_xmlupload(creds: ServerCredentials, skip_first_resource: bool = False) -> bool:
+    """
+    Resume an interrupted xmlupload.
+
+    Args:
+        creds: credentials to access the DSP server
+        skip_first_resource: if this flag is set, the first resource of the pending resources is removed
+
+    Returns:
+        True if all resources could be uploaded without errors; False if one of the resources could not be
+        uploaded because there is an error in it
+    """
+    server = creds.server
+    upload_state = _read_upload_state_from_disk(server)
+    if skip_first_resource:
+        _skip_first_resource(upload_state)
+
+    _print_and_log(upload_state, server)
+
+    auth = AuthenticationClientLive(server, creds.user, creds.password)
+    con = ConnectionLive(server, auth)
+
+    ingest_client: AssetClient
+    if upload_state.config.media_previously_uploaded:
+        ingest_client = BulkIngestedAssetClient()
+    else:
+        ingest_client = DspIngestClientLive(creds.dsp_ingest_url, auth, upload_state.config.shortcode, ".")
+
+    project_client: ProjectClient = ProjectClientLive(auth.server, auth)
+    list_client: ListClient = ListClientLive(con, project_client.get_project_iri(upload_state.config.shortcode))
+    legal_info_client: LegalInfoClient = LegalInfoClientLive(server, upload_state.config.shortcode, auth)
+    clients = UploadClients(ingest_client, list_client, legal_info_client)
+
+    return execute_upload(clients, upload_state)
+
+
+def _read_upload_state_from_disk(server: str) -> UploadState:
+    save_location = UploadConfig().with_server_info(server, "foo").diagnostics.save_location
+    with open(save_location, "rb") as f:
+        saved_state: UploadState = pickle.load(f)  # noqa: S301 (deserialization of untrusted data)
+    return saved_state
+
+
+def _skip_first_resource(upload_state: UploadState) -> None:
+    if len(upload_state.pending_resources) > 0:
+        upload_state.pending_resources.pop(0)
+    else:
+        msg = (
+            "The list of pending resources is empty.\n"
+            "It is not yet possible to skip the first item of the stashed properties.\n"
+            "Do you want to continue with the upload of the stashed properties anyway? [y/n]"
+        )
+        resp = None
+        while resp not in ["y", "n"]:
+            resp = input(RED + msg + RESET_TO_DEFAULT)
+        if resp == "n":
+            sys.exit(1)
+
+
+def _print_and_log(upload_state: UploadState, server: str) -> None:
+    previous_successful = len(upload_state.iri_resolver.lookup)
+    previous_failed = len(upload_state.failed_uploads)
+    previous_total = previous_successful + previous_failed
+    msg = (
+        f"Resuming upload for project {upload_state.config.shortcode} on server {server}. "
+        f"Number of resources uploaded until now: {previous_total}"
+    )
+    if previous_failed:
+        msg += f" ({previous_failed} of them failed)"
+    logger.info(msg)
+    print("\n==========================\n" + msg + "\n==========================\n")