dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/xmlupload/xmlupload.py

@@ -8,46 +8,56 @@ from pathlib import Path
 from typing import Never
 
 from loguru import logger
-from lxml import etree
+from rdflib import URIRef
 from tqdm import tqdm
 
 from dsp_tools.cli.args import ServerCredentials
-from dsp_tools.commands.xmlupload.check_consistency_with_ontology import do_xml_consistency_check_with_ontology
-from dsp_tools.commands.xmlupload.iiif_uri_validator import IIIFUriValidator
-from dsp_tools.commands.xmlupload.list_client import ListClient
-from dsp_tools.commands.xmlupload.list_client import ListClientLive
-from dsp_tools.commands.xmlupload.models.deserialise.xmlpermission import XmlPermission
-from dsp_tools.commands.xmlupload.models.deserialise.xmlresource import XMLResource
+from dsp_tools.cli.args import ValidateDataConfig
+from dsp_tools.cli.args import ValidationSeverity
+from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.connection import Connection
+from dsp_tools.clients.connection_live import ConnectionLive
+from dsp_tools.clients.fuseki_metrics import FusekiMetrics
+from dsp_tools.clients.legal_info_client import LegalInfoClient
+from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
+from dsp_tools.clients.project_client import ProjectClient
+from dsp_tools.clients.project_client_live import ProjectClientLive
+from dsp_tools.commands.validate_data.validate_data import validate_parsed_resources
+from dsp_tools.commands.xmlupload.make_rdf_graph.make_resource_and_values import create_resource_with_values
 from dsp_tools.commands.xmlupload.models.ingest import AssetClient
 from dsp_tools.commands.xmlupload.models.ingest import DspIngestClientLive
-from dsp_tools.commands.xmlupload.models.namespace_context import get_json_ld_context_for_project
-from dsp_tools.commands.xmlupload.models.permission import Permissions
+from dsp_tools.commands.xmlupload.models.lookup_models import IRILookups
+from dsp_tools.commands.xmlupload.models.lookup_models import XmlReferenceLookups
+from dsp_tools.commands.xmlupload.models.processed.res import ProcessedResource
 from dsp_tools.commands.xmlupload.models.upload_clients import UploadClients
 from dsp_tools.commands.xmlupload.models.upload_state import UploadState
-from dsp_tools.commands.xmlupload.ontology_client import OntologyClient
-from dsp_tools.commands.xmlupload.ontology_client import OntologyClientLive
-from dsp_tools.commands.xmlupload.project_client import ProjectClient
-from dsp_tools.commands.xmlupload.project_client import ProjectClientLive
-from dsp_tools.commands.xmlupload.read_validate_xml_file import check_if_bitstreams_exist
-from dsp_tools.commands.xmlupload.read_validate_xml_file import validate_and_parse
+from dsp_tools.commands.xmlupload.prepare_xml_input.get_processed_resources import get_processed_resources
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClient
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClientLive
+from dsp_tools.commands.xmlupload.prepare_xml_input.prepare_xml_input import get_parsed_resources_and_mappers
+from dsp_tools.commands.xmlupload.prepare_xml_input.prepare_xml_input import get_stash_and_upload_order
+from dsp_tools.commands.xmlupload.prepare_xml_input.read_validate_xml_file import check_if_bitstreams_exist
+from dsp_tools.commands.xmlupload.prepare_xml_input.read_validate_xml_file import validate_iiif_uris
 from dsp_tools.commands.xmlupload.resource_create_client import ResourceCreateClient
-from dsp_tools.commands.xmlupload.stash.stash_circular_references import identify_circular_references
-from dsp_tools.commands.xmlupload.stash.stash_circular_references import stash_circular_references
-from dsp_tools.commands.xmlupload.stash.stash_models import Stash
 from dsp_tools.commands.xmlupload.stash.upload_stashed_resptr_props import upload_stashed_resptr_props
 from dsp_tools.commands.xmlupload.stash.upload_stashed_xml_texts import upload_stashed_xml_texts
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
 from dsp_tools.commands.xmlupload.write_diagnostic_info import write_id2iri_mapping
-from dsp_tools.models.custom_warnings import DspToolsUserWarning
-from dsp_tools.models.exceptions import BaseError
-from dsp_tools.models.exceptions import PermanentConnectionError
-from dsp_tools.models.exceptions import PermanentTimeOutError
-from dsp_tools.models.exceptions import UserError
-from dsp_tools.models.exceptions import XmlUploadInterruptedError
-from dsp_tools.models.projectContext import ProjectContext
-from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.connection_live import ConnectionLive
-from dsp_tools.utils.logger_config import WARNINGS_SAVEPATH
+from dsp_tools.config.logger_config import WARNINGS_SAVEPATH
+from dsp_tools.error.custom_warnings import DspToolsUserWarning
+from dsp_tools.error.exceptions import BaseError
+from dsp_tools.error.exceptions import PermanentConnectionError
+from dsp_tools.error.exceptions import PermanentTimeOutError
+from dsp_tools.error.exceptions import XmlUploadInterruptedError
+from dsp_tools.utils.ansi_colors import BOLD_RED
+from dsp_tools.utils.ansi_colors import BOLD_YELLOW
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+from dsp_tools.utils.data_formats.uri_util import is_prod_like_server
+from dsp_tools.utils.fuseki_bloating import communicate_fuseki_bloating
+from dsp_tools.utils.replace_id_with_iri import use_id2iri_mapping_to_replace_ids
+from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedResource
+from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_clean_xml_file
 
 
 def xmlupload(
@@ -67,7 +77,7 @@ def xmlupload(
 
     Raises:
         BaseError: in case of permanent network or software failure
-        UserError: in case of permanent network or software failure, or if the XML file is invalid
+        InputError: in case of permanent network or software failure, or if the XML file is invalid
         InputError: in case of permanent network or software failure, or if the XML file is invalid
 
     Returns:
@@ -75,66 +85,143 @@ def xmlupload(
         uploaded because there is an error in it
     """
 
-    default_ontology, root, shortcode = _parse_xml(input_file=input_file, imgdir=imgdir)
-
-    if not config.skip_iiif_validation:
-        _validate_iiif_uris(root)
+    root = parse_and_clean_xml_file(input_file)
+    shortcode = root.attrib["shortcode"]
 
-    con = ConnectionLive(creds.server)
-    con.login(creds.user, creds.password)
+    auth = AuthenticationClientLive(server=creds.server, email=creds.user, password=creds.password)
+    con = ConnectionLive(creds.server, auth)
     config = config.with_server_info(server=creds.server, shortcode=shortcode)
+    clients = _get_live_clients(con, auth, creds, shortcode, imgdir)
 
-    ontology_client = OntologyClientLive(con=con, shortcode=shortcode, default_ontology=default_ontology)
-    resources, permissions_lookup, stash = prepare_upload(root, ontology_client)
+    parsed_resources, lookups = get_parsed_resources_and_mappers(root, clients)
+    if config.id2iri_file:
+        parsed_resources = use_id2iri_mapping_to_replace_ids(parsed_resources, Path(config.id2iri_file))
 
-    clients = _get_live_clients(con, creds, shortcode, imgdir)
-    state = UploadState(resources, stash, config, permissions_lookup)
+    is_on_prod_like_server = is_prod_like_server(creds.server)
 
-    return execute_upload(clients, state)
+    validation_ok = _handle_validation(
+        parsed_resources=parsed_resources,
+        lookups=lookups,
+        config=config,
+        is_on_prod_like_server=is_on_prod_like_server,
+        auth=auth,
+        input_file=input_file,
+    )
+    if not validation_ok:
+        return False
 
+    check_if_bitstreams_exist(root, imgdir)
+    if not config.skip_iiif_validation:
+        validate_iiif_uris(root)
 
-def _parse_xml(imgdir: str, input_file: Path) -> tuple[str, etree._Element, str]:
-    """
-    This function takes a path to an XML file.
-    It validates the file against the XML schema.
-    It checks if all the mentioned bitstream files are in the specified location.
-    It retrieves the shortcode and default ontology from the XML file.
+    if not is_on_prod_like_server:
+        enable_unknown_license_if_any_are_missing(clients.legal_info_client, parsed_resources)
 
-    Args:
-        imgdir: directory to the bitstream files
-        input_file: file that will be pased
+    processed_resources = get_processed_resources(parsed_resources, lookups, is_on_prod_like_server)
 
-    Returns:
-        The ontology name, the parsed XML file and the shortcode of the project
-    """
-    root, shortcode, default_ontology = validate_and_parse(input_file)
-    check_if_bitstreams_exist(root=root, imgdir=imgdir)
-    logger.info(f"Validated and parsed the XML file. {shortcode=:} and {default_ontology=:}")
-    return default_ontology, root, shortcode
+    sorted_resources, stash = get_stash_and_upload_order(processed_resources)
+    state = UploadState(
+        pending_resources=sorted_resources,
+        pending_stash=stash,
+        config=config,
+    )
+
+    return execute_upload(clients, state)
+
+
+def _handle_validation(
+    parsed_resources: list[ParsedResource],
+    lookups: XmlReferenceLookups,
+    config: UploadConfig,
+    is_on_prod_like_server: bool,
+    auth: AuthenticationClient,
+    input_file: Path,
+) -> bool:
+    validation_should_be_skipped = config.skip_validation
+    if is_on_prod_like_server and config.skip_validation:
+        msg = (
+            "You set the flag '--skip-validation' to circumvent the SHACL schema validation. "
+            "This means that the upload may fail due to undetected errors. "
+            "Do you wish to skip the validation (yes/no)? "
+        )
+        resp = ""
+        while resp not in ["yes", "no"]:
+            resp = input(BOLD_RED + msg + RESET_TO_DEFAULT)
+        if str(resp) == "no":
+            validation_should_be_skipped = False
+    if not validation_should_be_skipped:
+        ignore_duplicates = config.ignore_duplicate_files_warning
+        if is_on_prod_like_server and ignore_duplicates:
+            msg = (
+                "You set the flag '--ignore-duplicate-files-warning'. "
+                "This means that duplicate multimedia files will not be detected. "
+                "Are you sure you want to exclude this from the validation? (yes/no)"
+            )
+            resp = ""
+            while resp not in ["yes", "no"]:
+                resp = input(BOLD_RED + msg + RESET_TO_DEFAULT)
+            if str(resp) == "no":
+                ignore_duplicates = False
+        v_severity = config.validation_severity
+        if is_on_prod_like_server:
+            v_severity = ValidationSeverity.INFO
+        validation_passed = validate_parsed_resources(
+            parsed_resources=parsed_resources,
+            authorship_lookup=lookups.authorships,
+            permission_ids=list(lookups.permissions.keys()),
+            shortcode=config.shortcode,
+            config=ValidateDataConfig(
+                input_file,
+                save_graph_dir=None,
+                severity=v_severity,
+                ignore_duplicate_files_warning=ignore_duplicates,
+                is_on_prod_server=is_on_prod_like_server,
+                skip_ontology_validation=config.skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=config.do_not_request_resource_metadata_from_db,
+            ),
+            auth=auth,
+        )
+        if not validation_passed:
+            return False
+    else:
+        logger.debug("SHACL validation was skipped.")
+    return True
 
 
 def _get_live_clients(
     con: Connection,
+    auth: AuthenticationClient,
     creds: ServerCredentials,
     shortcode: str,
     imgdir: str,
 ) -> UploadClients:
     ingest_client: AssetClient
-    ingest_client = DspIngestClientLive(
-        dsp_ingest_url=creds.dsp_ingest_url,
-        token=con.get_token(),
-        shortcode=shortcode,
-        imgdir=imgdir,
-    )
-    project_client: ProjectClient = ProjectClientLive(con, shortcode)
-    list_client: ListClient = ListClientLive(con, project_client.get_project_iri())
+    ingest_client = DspIngestClientLive(creds.dsp_ingest_url, auth, shortcode, imgdir)
+    project_client: ProjectClient = ProjectClientLive(auth.server, auth)
+    list_client: ListClient = ListClientLive(con, project_client.get_project_iri(shortcode))
+    legal_info_client: LegalInfoClient = LegalInfoClientLive(creds.server, shortcode, auth)
     return UploadClients(
         asset_client=ingest_client,
-        project_client=project_client,
         list_client=list_client,
+        legal_info_client=legal_info_client,
     )
 
 
+def enable_unknown_license_if_any_are_missing(
+    legal_info_client: LegalInfoClient, parsed_resources: list[ParsedResource]
+) -> None:
+    all_license_infos = [x.file_value.metadata.license_iri for x in parsed_resources if x.file_value]
+    if not all(all_license_infos):
+        legal_info_client.enable_unknown_license()
+        msg = (
+            "The files or iiif-uris in your data are missing some legal information. "
+            "To facilitate an upload on a test environment we are adding dummy information.\n"
+            "In order to be able to use the license 'unknown' in place of missing licenses, "
+            "we are enabling it for your project."
+        )
+        print(BOLD_YELLOW, msg, RESET_TO_DEFAULT)
+
+
 def execute_upload(clients: UploadClients, upload_state: UploadState) -> bool:
     """Execute an upload from an upload state, and clean up afterwards.
 
@@ -145,29 +232,33 @@ def execute_upload(clients: UploadClients, upload_state: UploadState) -> bool:
     Returns:
         True if all resources could be uploaded without errors; False if any resource could not be uploaded
     """
+    logger.debug("Start uploading data")
+    db_metrics = None
+    if clients.legal_info_client.server == "http://0.0.0.0:3333":
+        db_metrics = FusekiMetrics()
+        db_metrics.try_get_start_size()
+    _upload_copyright_holders(upload_state.pending_resources, clients.legal_info_client)
     _upload_resources(clients, upload_state)
+    if db_metrics is not None:
+        db_metrics.try_get_end_size()
+        communicate_fuseki_bloating(db_metrics)
     return _cleanup_upload(upload_state)
 
 
-def prepare_upload(
-    root: etree._Element,
-    ontology_client: OntologyClient,
-) -> tuple[list[XMLResource], dict[str, Permissions], Stash | None]:
-    """Do the consistency check, resolve circular references, and return the resources and permissions."""
-    do_xml_consistency_check_with_ontology(onto_client=ontology_client, root=root)
-    return _resolve_circular_references(
-        root=root,
-        con=ontology_client.con,
-        default_ontology=ontology_client.default_ontology,
-    )
+def _upload_copyright_holders(resources: list[ProcessedResource], legal_info_client: LegalInfoClient) -> None:
+    logger.debug("Get and upload copyright holders")
+    copyright_holders = _get_copyright_holders(resources)
+    legal_info_client.post_copyright_holders(copyright_holders)
 
 
-def _validate_iiif_uris(root: etree._Element) -> None:
-    uris = [uri for node in root.iter(tag="iiif-uri") if (uri := node.text)]
-    if problems := IIIFUriValidator(uris).validate():
-        msg = problems.get_msg()
-        warnings.warn(DspToolsUserWarning(msg))
-        logger.warning(msg)
+def _get_copyright_holders(resources: list[ProcessedResource]) -> list[str]:
+    copyright_holders = set()
+    for res in resources:
+        if res.file_value:
+            copyright_holders.add(res.file_value.metadata.copyright_holder)
+        elif res.iiif_uri:
+            copyright_holders.add(res.iiif_uri.metadata.copyright_holder)
+    return [x for x in copyright_holders if x]
 
 
 def _cleanup_upload(upload_state: UploadState) -> bool:
@@ -180,7 +271,11 @@ def _cleanup_upload(upload_state: UploadState) -> bool:
     Returns:
         success status (deduced from failed_uploads and non-applied stash)
     """
-    write_id2iri_mapping(upload_state.iri_resolver.lookup, upload_state.config.diagnostics)
+    write_id2iri_mapping(
+        id2iri_mapping=upload_state.iri_resolver.lookup,
+        shortcode=upload_state.config.shortcode,
+        diagnostics=upload_state.config.diagnostics,
+    )
     has_stash_failed = upload_state.pending_stash and not upload_state.pending_stash.is_empty()
     if not upload_state.failed_uploads and not has_stash_failed:
         success = True
@@ -205,28 +300,6 @@ def _cleanup_upload(upload_state: UploadState) -> bool:
     return success
 
 
-def _resolve_circular_references(
-    root: etree._Element,
-    con: Connection,
-    default_ontology: str,
-) -> tuple[list[XMLResource], dict[str, Permissions], Stash | None]:
-    logger.info("Checking resources for circular references...")
-    print(f"{datetime.now()}: Checking resources for circular references...")
-    stash_lookup, upload_order = identify_circular_references(root)
-    logger.info("Get data from XML...")
-    resources, permissions_lookup = _get_data_from_xml(
-        con=con,
-        root=root,
-        default_ontology=default_ontology,
-    )
-    sorting_lookup = {res.res_id: res for res in resources}
-    resources = [sorting_lookup[res_id] for res_id in upload_order]
-    logger.info("Stashing circular references...")
-    print(f"{datetime.now()}: Stashing circular references...")
-    stash = stash_circular_references(resources, stash_lookup, permissions_lookup)
-    return resources, permissions_lookup, stash
-
-
 def _upload_resources(clients: UploadClients, upload_state: UploadState) -> None:
     """
     Iterates through all resources and tries to upload them to DSP.
@@ -241,20 +314,16 @@ def _upload_resources(clients: UploadClients, upload_state: UploadState) -> None
         BaseException: in case of an unhandled exception during resource creation
         XmlUploadInterruptedError: if the number of resources created is equal to the interrupt_after value
     """
-    project_iri = clients.project_client.get_project_iri()
-    project_onto_dict = clients.project_client.get_ontology_name_dict()
-    listnode_lookup = clients.list_client.get_list_node_id_to_iri_lookup()
+    project_iri = clients.list_client.project_iri
 
-    resource_create_client = ResourceCreateClient(
-        con=clients.project_client.con,
-        project_iri=project_iri,
-        iri_resolver=upload_state.iri_resolver,
-        project_onto_dict=project_onto_dict,
-        permissions_lookup=upload_state.permissions_lookup,
-        listnode_lookup=listnode_lookup,
-        media_previously_ingested=upload_state.config.media_previously_uploaded,
+    iri_lookup = IRILookups(
+        project_iri=URIRef(project_iri),
+        id_to_iri=upload_state.iri_resolver,
     )
 
+    resource_create_client = ResourceCreateClient(
+        con=clients.list_client.con,
+    )
     progress_bar = tqdm(upload_state.pending_resources.copy(), desc="Creating Resources", dynamic_ncols=True)
     try:
         for creation_attempts_of_this_round, resource in enumerate(progress_bar):
@@ -263,104 +332,61 @@ def _upload_resources(clients: UploadClients, upload_state: UploadState) -> None
                 resource=resource,
                 ingest_client=clients.asset_client,
                 resource_create_client=resource_create_client,
+                iri_lookups=iri_lookup,
                 creation_attempts_of_this_round=creation_attempts_of_this_round,
             )
             progress_bar.set_description(f"Creating Resources (failed: {len(upload_state.failed_uploads)})")
         if upload_state.pending_stash:
-            _upload_stash(upload_state, clients.project_client)
+            _upload_stash(upload_state, clients.list_client.con)
     except XmlUploadInterruptedError as err:
         _handle_upload_error(err, upload_state)
 
 
-def _get_data_from_xml(
-    con: Connection,
-    root: etree._Element,
-    default_ontology: str,
-) -> tuple[list[XMLResource], dict[str, Permissions]]:
-    proj_context = _get_project_context_from_server(connection=con, shortcode=root.attrib["shortcode"])
-    permissions = _extract_permissions_from_xml(root, proj_context)
-    resources = _extract_resources_from_xml(root, default_ontology)
-    permissions_lookup = {name: perm.get_permission_instance() for name, perm in permissions.items()}
-    return resources, permissions_lookup
-
-
 def _upload_stash(
     upload_state: UploadState,
-    project_client: ProjectClient,
+    con: Connection,
 ) -> None:
     if upload_state.pending_stash and upload_state.pending_stash.standoff_stash:
-        upload_stashed_xml_texts(upload_state, project_client.con)
-    context = get_json_ld_context_for_project(project_client.get_ontology_name_dict())
+        upload_stashed_xml_texts(upload_state, con)
     if upload_state.pending_stash and upload_state.pending_stash.link_value_stash:
-        upload_stashed_resptr_props(upload_state, project_client.con, context)
-
-
-def _get_project_context_from_server(connection: Connection, shortcode: str) -> ProjectContext:
-    """
-    This function retrieves the project context previously uploaded on the server (json file)
-
-    Args:
-        connection: connection to the server
-        shortcode: shortcode of the project
-
-    Returns:
-        Project context
-
-    Raises:
-        UserError: If the project was not previously uploaded on the server
-    """
-    try:
-        proj_context = ProjectContext(con=connection, shortcode=shortcode)
-    except BaseError:
-        logger.opt(exception=True).error("Unable to retrieve project context from DSP server")
-        raise UserError("Unable to retrieve project context from DSP server") from None
-    return proj_context
-
-
-def _extract_permissions_from_xml(root: etree._Element, proj_context: ProjectContext) -> dict[str, XmlPermission]:
-    permission_ele = list(root.iter(tag="permissions"))
-    permissions = [XmlPermission(permission, proj_context) for permission in permission_ele]
-    return {permission.permission_id: permission for permission in permissions}
-
-
-def _extract_resources_from_xml(root: etree._Element, default_ontology: str) -> list[XMLResource]:
-    resources = list(root.iter(tag="resource"))
-    return [XMLResource.from_node(res, default_ontology) for res in resources]
+        upload_stashed_resptr_props(upload_state, con)
 
 
 def _upload_one_resource(
     upload_state: UploadState,
-    resource: XMLResource,
+    resource: ProcessedResource,
     ingest_client: AssetClient,
     resource_create_client: ResourceCreateClient,
+    iri_lookups: IRILookups,
     creation_attempts_of_this_round: int,
 ) -> None:
-    try:
-        if resource.bitstream:
-            success, media_info = ingest_client.get_bitstream_info(
-                resource.bitstream, upload_state.permissions_lookup, resource.label, resource.res_id
-            )
-        else:
-            success, media_info = True, None
-
-        if not success:
+    media_info = None
+    if resource.file_value:
+        try:
+            ingest_result = ingest_client.get_bitstream_info(resource.file_value)
+        except PermanentConnectionError as err:
+            _handle_permanent_connection_error(err)
+        except KeyboardInterrupt:
+            _handle_keyboard_interrupt()
+        if not ingest_result:
             upload_state.failed_uploads.append(resource.res_id)
             return
-    except PermanentConnectionError as err:
-        _handle_permanent_connection_error(err)
-    except KeyboardInterrupt:
-        _handle_keyboard_interrupt()
+        media_info = ingest_result
 
     iri = None
     try:
-        iri = resource_create_client.create_resource(resource, media_info)
+        serialised_resource = create_resource_with_values(
+            resource=resource, bitstream_information=media_info, lookups=iri_lookups
+        )
+        logger.info(f"Attempting to create resource {resource.res_id} (label: {resource.label})...")
+        iri = resource_create_client.create_resource(serialised_resource, resource_has_bitstream=bool(media_info))
     except (PermanentTimeOutError, KeyboardInterrupt) as err:
         _handle_permanent_timeout_or_keyboard_interrupt(err, resource.res_id)
     except PermanentConnectionError as err:
         _handle_permanent_connection_error(err)
     except Exception as err:  # noqa: BLE001 (blind-except)
         err_msg = err.message if isinstance(err, BaseError) else None
-        _handle_resource_creation_failure(resource, err_msg)
+        _inform_about_resource_creation_failure(resource, err_msg)
 
     try:
         _tidy_up_resource_creation_idempotent(upload_state, iri, resource)
@@ -374,6 +400,7 @@ def _handle_permanent_connection_error(err: PermanentConnectionError) -> Never:
     msg = "Lost connection to DSP server, probably because the server is down. "
     msg += f"Please continue later with 'resume-xmlupload'. Reason for this failure: {err.message}"
     logger.error(msg)
+    msg += f"\nSee {WARNINGS_SAVEPATH} for more information."
     raise XmlUploadInterruptedError(msg) from None
 
 
@@ -410,7 +437,7 @@ def _interrupt_if_indicated(upload_state: UploadState, creation_attempts_of_this
 def _tidy_up_resource_creation_idempotent(
     upload_state: UploadState,
     iri: str | None,
-    resource: XMLResource,
+    resource: ProcessedResource,
 ) -> None:
     previous_successful = len(upload_state.iri_resolver.lookup)
     previous_failed = len(upload_state.failed_uploads)
@@ -431,15 +458,10 @@ def _tidy_up_resource_creation_idempotent(
         upload_state.pending_resources.remove(resource)
 
 
-def _handle_resource_creation_failure(resource: XMLResource, err_msg: str | None) -> None:
-    msg = f"{datetime.now()}: WARNING: Unable to create resource '{resource.label}' (ID: '{resource.res_id}')"
+def _inform_about_resource_creation_failure(resource: ProcessedResource, err_msg: str | None) -> None:
+    log_msg = f"Unable to create resource '{resource.label}' ({resource.res_id})\n"
     if err_msg:
-        msg = f"{msg}: {err_msg}"
-    log_msg = (
-        f"Unable to create resource '{resource.label}' ({resource.res_id})\n"
-        f"Resource details:\n{vars(resource)}\n"
-        f"Property details:\n" + "\n".join([str(vars(prop)) for prop in resource.properties])
-    )
+        log_msg += err_msg
     logger.exception(log_msg)
 
 
@@ -476,7 +498,7 @@ def _handle_upload_error(err: BaseException, upload_state: UploadState) -> None:
         msg += f"Independently from this, there were some resources that could not be uploaded: {failed}\n"
 
     if exit_code == 1:
-        logger.exception(msg)
+        logger.error(msg)
     else:
         logger.info(msg)
     print(msg)

dsp_tools/config/logger_config.py

@@ -0,0 +1,69 @@
+import os
+from datetime import datetime
+from pathlib import Path
+
+from dotenv import find_dotenv
+from dotenv import load_dotenv
+from loguru import logger
+
+load_dotenv(dotenv_path=find_dotenv(usecwd=True))
+
+
+def _make_and_get_logs_directory() -> Path:
+    """Get the base .dsp-tools directory, creating it if it doesn't exist."""
+    base_dir = Path.home() / ".dsp-tools" / "logs"
+    base_dir.mkdir(exist_ok=True, parents=True)
+    return base_dir
+
+
+timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S-%f")[:-3]
+
+LOGGER_SAVEPATH = (_make_and_get_logs_directory() / f"{timestamp}_logging.log").absolute()
+WARNINGS_SAVEPATH = Path("warnings.log")
+
+
+def logger_config() -> None:
+    """
+    This function configures the log files.
+    Currently, there are three sinks:
+    - timestamp_logging.log in ~/.dsp-tools/logs/ contains the entire stack-trace
+    - warnings.log in the cwd only with level warning and higher for the user (no stack-trace)
+      OR a complete logging.log file with the stack-trace if configured in the .env
+    - print output on the terminal, formatted the same as the warnings.log
+    """
+    # If this is not removed, the default formatting is also printed out on the terminal
+    logger.remove()
+
+    text_format = "<level>{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}</level>"
+    rotation_size = "100 MB"
+
+    logger.add(
+        sink=LOGGER_SAVEPATH,
+        format=text_format,
+        backtrace=True,
+        diagnose=True,
+        delay=True,
+    )
+
+    additional_log = str(os.getenv("DSP_TOOLS_SAVE_ADDITIONAL_LOG_FILE_IN_CWD"))
+    if additional_log.lower() == "true":
+        logger.add(
+            sink=Path("logging.log"),
+            format=text_format,
+            backtrace=True,
+            diagnose=True,
+            rotation=rotation_size,
+            retention=2,
+            delay=True,
+        )
+    else:
+        logger.add(
+            sink=WARNINGS_SAVEPATH,
+            level="WARNING",
+            format=text_format,
+            backtrace=False,
+            diagnose=False,
+            rotation=rotation_size,
+            retention=2,
+            delay=True,
+        )

dsp_tools/{utils → config}/warnings_config.py

@@ -1,7 +1,8 @@
 import warnings
 from typing import TextIO
 
-from dsp_tools.models.custom_warnings import DspToolsWarning
+from dsp_tools.error.custom_warnings import DspToolsWarning
+from dsp_tools.error.xmllib_warnings import XmllibUserInfoBase
 
 
 def initialize_warnings() -> None:
@@ -23,6 +24,8 @@ def initialize_warnings() -> None:
     ) -> None:
         if issubclass(category, DspToolsWarning):
             category.showwarning(str(message))
+        elif issubclass(category, XmllibUserInfoBase):
+            category.showwarning(str(message))
         else:
             built_in_showwarning(message, category, filename, lineno, file, line)