dsp-tools 17.0.0.post26__tar.gz → 17.0.0.post29__tar.gz

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dsp-tools
-Version: 17.0.0.post26
+Version: 17.0.0.post29
 Summary: DSP-TOOLS is a Python package with a command line interface that helps you interact with a DaSCH service platform (DSP) server.
 Author: DaSCH - Swiss National Data and Service Center for the Humanities
 Author-email: DaSCH - Swiss National Data and Service Center for the Humanities <info@dasch.swiss>

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/pyproject.toml

@@ -2,7 +2,7 @@
 
 [project]
 name = "dsp-tools"
-version = "17.0.0.post26"
+version = "17.0.0.post29"
 description = "DSP-TOOLS is a Python package with a command line interface that helps you interact with a DaSCH service platform (DSP) server."
 authors = [
     { name = "DaSCH - Swiss National Data and Service Center for the Humanities", email = "info@dasch.swiss" },

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/cli/args.py

@@ -25,6 +25,7 @@ class ValidateDataConfig:
     ignore_duplicate_files_warning: bool
     is_on_prod_server: bool
     skip_ontology_validation: bool
+    do_not_request_resource_metadata_from_db: bool
 
 
 class ValidationSeverity(Enum):

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/cli/call_action.py

@@ -211,6 +211,7 @@ def _call_ingest_xmlupload(args: argparse.Namespace) -> bool:
         skip_validation=args.skip_validation,
         skip_ontology_validation=args.skip_ontology_validation,
         id2iri_replacement_file=args.id2iri_replacement_with_file,
+        do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
     )
 
 
@@ -250,6 +251,7 @@ def _call_xmlupload(args: argparse.Namespace) -> bool:
                 ignore_duplicate_files_warning=args.ignore_duplicate_files_warning,
                 validation_severity=severity,
                 skip_ontology_validation=args.skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
                 id2iri_replacement_file=id_2_iri_file,
             ),
         )
@@ -266,6 +268,7 @@ def _call_validate_data(args: argparse.Namespace) -> bool:
         ignore_duplicate_files_warning=args.ignore_duplicate_files_warning,
         skip_ontology_validation=args.skip_ontology_validation,
         id2iri_replacement_file=args.id2iri_replacement_with_file,
+        do_not_request_resource_metadata_from_db=args.do_not_request_resource_metadata_from_db,
     )
 
 

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/cli/create_parsers.py

@@ -242,6 +242,13 @@ def _add_ingest_xmlupload(
         "--id2iri-replacement-with-file",
         help="replaces internal IDs of an XML file by IRIs provided in this mapping file",
     )
+    subparser.add_argument(
+        "--do-not-request-resource-metadata-from-db",
+        action="store_true",
+        help=(
+            "Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
+        ),
+    )
 
 
 def _add_xmlupload(
@@ -279,6 +286,13 @@ def _add_xmlupload(
         help="Which severity level of validation message should be printed out",
         default="info",
     )
+    subparser.add_argument(
+        "--do-not-request-resource-metadata-from-db",
+        action="store_true",
+        help=(
+            "Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
+        ),
+    )
     subparser.add_argument(
         "--id2iri-replacement-with-file",
         help="replaces internal IDs of an XML file by IRIs provided in this mapping file",
@@ -309,6 +323,13 @@ def _add_validate_data(
         "--id2iri-replacement-with-file",
         help="replaces internal IDs of an XML file by IRIs provided in this mapping file",
     )
+    subparser.add_argument(
+        "--do-not-request-resource-metadata-from-db",
+        action="store_true",
+        help=(
+            "Do not request IRIs of existing resources from the db (references to existing resources won't be checked)"
+        ),
+    )
     subparser.add_argument(
         "--save-graphs", action="store_true", help="Save the data, onto and shacl graph as ttl files."
     )

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/clients/metadata_client.py

@@ -6,9 +6,9 @@ from typing import Protocol
 from dsp_tools.clients.authentication_client import AuthenticationClient
 
 
-class MetadataRetrieval(Enum):
-    SUCCESS = auto()
-    FAILURE = auto()
+class ExistingResourcesRetrieved(Enum):
+    TRUE = auto()
+    FALSE = auto()
 
 
 @dataclass
@@ -20,5 +20,5 @@ class MetadataClient(Protocol):
     server: str
     authentication_client: AuthenticationClient
 
-    def get_resource_metadata(self, shortcode: str) -> tuple[MetadataRetrieval, list[dict[str, str]]]:
+    def get_resource_metadata(self, shortcode: str) -> tuple[ExistingResourcesRetrieved, list[dict[str, str]]]:
         """Get all resource metadata from one project."""

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/clients/metadata_client_live.py

@@ -4,8 +4,8 @@ import requests
 from loguru import logger
 
 from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.clients.metadata_client import ExistingResourcesRetrieved
 from dsp_tools.clients.metadata_client import MetadataClient
-from dsp_tools.clients.metadata_client import MetadataRetrieval
 from dsp_tools.utils.request_utils import RequestParameters
 from dsp_tools.utils.request_utils import log_request
 from dsp_tools.utils.request_utils import log_response
@@ -18,7 +18,7 @@ class MetadataClientLive(MetadataClient):
     server: str
     authentication_client: AuthenticationClient
 
-    def get_resource_metadata(self, shortcode: str) -> tuple[MetadataRetrieval, list[dict[str, str]]]:
+    def get_resource_metadata(self, shortcode: str) -> tuple[ExistingResourcesRetrieved, list[dict[str, str]]]:
         url = f"{self.server}/v2/metadata/projects/{shortcode}/resources?format=JSON"
         header = {"Authorization": f"Bearer {self.authentication_client.get_token()}"}
         params = RequestParameters(method="GET", url=url, timeout=TIMEOUT, headers=header)
@@ -33,10 +33,11 @@ class MetadataClientLive(MetadataClient):
             if response.ok:
                 # we log the response separately because if it was successful it will be too big
                 log_response(response, include_response_content=False)
-                return MetadataRetrieval.SUCCESS, response.json()
+                logger.debug(f"{len(response.json())} NUMBER OF RESOURCES RETRIEVED")
+                return ExistingResourcesRetrieved.TRUE, response.json()
             # here the response text is important
             log_response(response)
-            return MetadataRetrieval.FAILURE, []
+            return ExistingResourcesRetrieved.FALSE, []
         except Exception as err:  # noqa: BLE001 (blind exception)
             logger.error(err)
-            return MetadataRetrieval.FAILURE, []
+            return ExistingResourcesRetrieved.FALSE, []

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py

@@ -43,6 +43,7 @@ def ingest_xmlupload(
     skip_validation: bool = False,
     skip_ontology_validation: bool = False,
     id2iri_replacement_file: str | None = None,
+    do_not_request_resource_metadata_from_db: bool = False,
 ) -> bool:
     """
     This function reads an XML file
@@ -59,6 +60,8 @@ def ingest_xmlupload(
         skip_validation: skip the SHACL validation
         skip_ontology_validation: skip the ontology validation
         id2iri_replacement_file: to replace internal IDs of an XML file by IRIs provided in this mapping file
+        do_not_request_resource_metadata_from_db: if true do not request metadata information from the api
+                                                  for existing resources
 
     Returns:
         True if all resources could be uploaded without errors; False if one of the resources could not be
@@ -115,6 +118,7 @@ def ingest_xmlupload(
                 ignore_duplicate_files_warning=True,
                 is_on_prod_server=is_on_prod_like_server,
                 skip_ontology_validation=skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=do_not_request_resource_metadata_from_db,
             ),
             auth=auth,
         )

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/commands/validate_data/models/input_problems.py

@@ -114,5 +114,6 @@ class ProblemType(StrEnum):
     LINK_TARGET_TYPE_MISMATCH = "Linked Resource Type Mismatch"
     LINK_TARGET_OF_ANOTHER_PROJECT = "Linked Resource is from another project"
     LINK_TARGET_IS_IRI_OF_PROJECT = "Linked resource is an IRI of the project"
+    LINK_TARGET_NOT_FOUND_IN_DB = "Linked resource is an IRI and could not be found in the database."
     INEXISTENT_LINKED_RESOURCE = "Linked Resource does not exist"
     DUPLICATE_VALUE = "Your input is duplicated"

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/commands/validate_data/prepare_data/prepare_data.py

@@ -8,7 +8,7 @@ from rdflib import URIRef
 
 from dsp_tools.clients.authentication_client import AuthenticationClient
 from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
-from dsp_tools.clients.metadata_client import MetadataRetrieval
+from dsp_tools.clients.metadata_client import ExistingResourcesRetrieved
 from dsp_tools.clients.metadata_client_live import MetadataClientLive
 from dsp_tools.commands.validate_data.api_clients import ListClient
 from dsp_tools.commands.validate_data.api_clients import OntologyClient
@@ -47,13 +47,16 @@ def prepare_data_for_validation_from_parsed_resource(
     permission_ids: list[str],
     auth: AuthenticationClient,
     shortcode: str,
-) -> tuple[RDFGraphs, set[str], MetadataRetrieval]:
+    do_not_request_resource_metadata_from_db: bool,
+) -> tuple[RDFGraphs, set[str], ExistingResourcesRetrieved]:
     used_iris = {x.res_type for x in parsed_resources}
-    proj_info, metadata_retrieval_success = _get_project_specific_information_from_api(auth, shortcode)
+    proj_info, existing_resources_retrieved = _get_project_specific_information_from_api(
+        auth, shortcode, do_not_request_resource_metadata_from_db
+    )
     list_lookup = _make_list_lookup(proj_info.all_lists)
     data_rdf = _make_data_graph_from_parsed_resources(parsed_resources, authorship_lookup, list_lookup)
     rdf_graphs = _create_graphs(data_rdf, shortcode, auth, proj_info, permission_ids)
-    return rdf_graphs, used_iris, metadata_retrieval_success
+    return rdf_graphs, used_iris, existing_resources_retrieved
 
 
 def _make_list_lookup(project_lists: list[OneList]) -> ListLookup:
@@ -66,18 +69,22 @@ def _make_list_lookup(project_lists: list[OneList]) -> ListLookup:
 
 
 def _get_project_specific_information_from_api(
-    auth: AuthenticationClient, shortcode: str
-) -> tuple[ProjectDataFromApi, MetadataRetrieval]:
+    auth: AuthenticationClient, shortcode: str, do_not_request_resource_metadata_from_db: bool
+) -> tuple[ProjectDataFromApi, ExistingResourcesRetrieved]:
     list_client = ListClient(auth.server, shortcode)
     all_lists = list_client.get_lists()
     enabled_licenses = _get_license_iris(shortcode, auth)
-    retrieval_status, formatted_metadata = _get_metadata_info(auth, shortcode)
-    return ProjectDataFromApi(all_lists, enabled_licenses, formatted_metadata), retrieval_status
+    if do_not_request_resource_metadata_from_db:
+        existing_resources_retrieved = ExistingResourcesRetrieved.FALSE
+        formatted_metadata: list[InfoForResourceInDB] = []
+    else:
+        existing_resources_retrieved, formatted_metadata = _get_metadata_info(auth, shortcode)
+    return ProjectDataFromApi(all_lists, enabled_licenses, formatted_metadata), existing_resources_retrieved
 
 
 def _get_metadata_info(
     auth: AuthenticationClient, shortcode: str
-) -> tuple[MetadataRetrieval, list[InfoForResourceInDB]]:
+) -> tuple[ExistingResourcesRetrieved, list[InfoForResourceInDB]]:
     metadata_client = MetadataClientLive(auth.server, auth)
     retrieval_status, metadata = metadata_client.get_resource_metadata(shortcode)
     formatted_metadata = [InfoForResourceInDB(x["resourceIri"], x["resourceClassIri"]) for x in metadata]

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/commands/validate_data/process_validation_report/get_user_validation_message.py

@@ -3,6 +3,7 @@ from collections import defaultdict
 import pandas as pd
 
 from dsp_tools.cli.args import ValidationSeverity
+from dsp_tools.clients.metadata_client import ExistingResourcesRetrieved
 from dsp_tools.commands.validate_data.models.input_problems import AllProblems
 from dsp_tools.commands.validate_data.models.input_problems import DuplicateFileWarning
 from dsp_tools.commands.validate_data.models.input_problems import InputProblem
@@ -20,9 +21,14 @@ PROBLEM_TYPES_IGNORE_STR_ENUM_INFO = {ProblemType.GENERIC, ProblemType.FILE_VALU
 
 
 def sort_user_problems(
-    all_problems: AllProblems, duplicate_file_warnings: DuplicateFileWarning | None, shortcode: str
+    all_problems: AllProblems,
+    duplicate_file_warnings: DuplicateFileWarning | None,
+    shortcode: str,
+    existing_resources_retrieved: ExistingResourcesRetrieved,
 ) -> SortedProblems:
-    iris_removed, links_level_info = _separate_resource_links_to_iris_of_own_project(all_problems.problems, shortcode)
+    iris_removed, links_level_info = _separate_resource_links_to_iris_of_own_project(
+        all_problems.problems, shortcode, existing_resources_retrieved
+    )
     filtered_problems = _filter_out_duplicate_problems(iris_removed)
     violations, warnings, info = _separate_according_to_severity(filtered_problems)
     if duplicate_file_warnings:
@@ -47,37 +53,61 @@ def _separate_according_to_severity(
 
 
 def _separate_resource_links_to_iris_of_own_project(
-    problems: list[InputProblem], shortcode: str
+    problems: list[InputProblem], shortcode: str, existing_resources_retrieved: ExistingResourcesRetrieved
 ) -> tuple[list[InputProblem], list[InputProblem]]:
     link_level_info = []
     all_others = []
-    resource_iri_start = "http://rdfh.ch/"
-    project_resource_iri = f"{resource_iri_start}{shortcode}/"
     for prblm in problems:
         if prblm.problem_type != ProblemType.INEXISTENT_LINKED_RESOURCE:
             all_others.append(prblm)
-            continue
-        if not prblm.input_value:
-            all_others.append(prblm)
-        elif prblm.input_value.startswith(project_resource_iri):
-            prblm.message = (
-                "You used an absolute IRI to reference an existing resource in the DB. "
-                "If this resource does not exist or is not of the correct type, an xmlupload will fail."
-            )
-            prblm.problem_type = ProblemType.LINK_TARGET_IS_IRI_OF_PROJECT
-            link_level_info.append(prblm)
-        elif prblm.input_value.startswith(resource_iri_start):
-            prblm.message = (
-                "You used an absolute IRI to reference an existing resource of another project in the DB. "
-                "Cross-Project resource links are not permitted."
-            )
-            prblm.problem_type = ProblemType.LINK_TARGET_OF_ANOTHER_PROJECT
-            all_others.append(prblm)
         else:
-            all_others.append(prblm)
+            is_violation, triaged_problem = _determined_link_value_message_and_level(
+                prblm, shortcode, existing_resources_retrieved
+            )
+            if is_violation:
+                all_others.append(triaged_problem)
+            else:
+                link_level_info.append(triaged_problem)
     return all_others, link_level_info
 
 
+def _determined_link_value_message_and_level(
+    problem: InputProblem, shortcode: str, existing_resources_retrieved: ExistingResourcesRetrieved
+) -> tuple[bool, InputProblem]:
+    is_violation = True
+    resource_iri_start = "http://rdfh.ch/"
+    project_resource_iri = f"{resource_iri_start}{shortcode}/"
+    if not problem.input_value:
+        return is_violation, problem
+    if problem.input_value.startswith(project_resource_iri):
+        # case IRI and matches those of the projects itself
+        if existing_resources_retrieved == ExistingResourcesRetrieved.TRUE:
+            # if metadata was sucessfully retrieved, then the IRI is wrong
+            problem.problem_type = ProblemType.LINK_TARGET_NOT_FOUND_IN_DB
+            problem.message = (
+                "You used an absolute IRI to reference an existing resource in the DB. "
+                "We could not find a reference to this resource in the database."
+            )
+            return is_violation, problem
+        # if we could not retrieve the metadata, then we cannot verify if it exists or not, so it is only an info
+        problem.problem_type = ProblemType.LINK_TARGET_IS_IRI_OF_PROJECT
+        problem.message = (
+            "You used an absolute IRI to reference an existing resource in the DB. "
+            "If this resource does not exist or is not of the correct type, an xmlupload will fail."
+        )
+        return not is_violation, problem
+    if problem.input_value.startswith(resource_iri_start):
+        # case IRI, but does not contain the shortcode of the project
+        problem.message = (
+            "You used an absolute IRI to reference an existing resource of another project in the DB. "
+            "Cross-Project resource links are not permitted."
+        )
+        problem.problem_type = ProblemType.LINK_TARGET_OF_ANOTHER_PROJECT
+        return is_violation, problem
+    # all other cases, it is not an IRI and must be an internal ID that does not exist in the XML
+    return is_violation, problem
+
+
 def _filter_out_duplicate_problems(problems: list[InputProblem]) -> list[InputProblem]:
     grouped, without_res_id = _group_problems_by_resource(problems)
     filtered = without_res_id

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/commands/validate_data/validate_data.py

@@ -10,6 +10,7 @@ from dsp_tools.cli.args import ValidateDataConfig
 from dsp_tools.cli.args import ValidationSeverity
 from dsp_tools.clients.authentication_client import AuthenticationClient
 from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.metadata_client import ExistingResourcesRetrieved
 from dsp_tools.commands.validate_data.models.input_problems import OntologyValidationProblem
 from dsp_tools.commands.validate_data.models.input_problems import SortedProblems
 from dsp_tools.commands.validate_data.models.input_problems import UnknownClassesInData
@@ -51,6 +52,7 @@ def validate_data(
     save_graphs: bool,
     skip_ontology_validation: bool,
     id2iri_replacement_file: str | None,
+    do_not_request_resource_metadata_from_db: bool,
 ) -> bool:
     """
     Takes a file and project information and validates it against the ontologies on the server.
@@ -62,6 +64,7 @@ def validate_data(
         save_graphs: if this flag is set, all the graphs will be saved in a folder
         skip_ontology_validation: skip the ontology validation
         id2iri_replacement_file: to replace internal IDs of an XML file by IRIs provided in this mapping file
+        do_not_request_resource_metadata_from_db: true if no metadata for existing resources should be requested
 
     Returns:
         True if no errors that impede an xmlupload were found.
@@ -78,6 +81,7 @@ def validate_data(
         ignore_duplicate_files_warning=ignore_duplicate_files_warning,
         is_on_prod_server=is_prod_like_server(creds.server),
         skip_ontology_validation=skip_ontology_validation,
+        do_not_request_resource_metadata_from_db=do_not_request_resource_metadata_from_db,
     )
     auth = AuthenticationClientLive(server=creds.server, email=creds.user, password=creds.password)
 
@@ -104,14 +108,17 @@ def validate_parsed_resources(
     config: ValidateDataConfig,
     auth: AuthenticationClient,
 ) -> bool:
-    rdf_graphs, used_iris, _ = prepare_data_for_validation_from_parsed_resource(
+    rdf_graphs, used_iris, existing_resources_retrieved = prepare_data_for_validation_from_parsed_resource(
         parsed_resources=parsed_resources,
         authorship_lookup=authorship_lookup,
         permission_ids=permission_ids,
         auth=auth,
         shortcode=shortcode,
+        do_not_request_resource_metadata_from_db=config.do_not_request_resource_metadata_from_db,
+    )
+    validation_result = _validate_data(
+        rdf_graphs, used_iris, parsed_resources, config, shortcode, existing_resources_retrieved
     )
-    validation_result = _validate_data(rdf_graphs, used_iris, parsed_resources, config, shortcode)
     if validation_result.no_problems:
         logger.debug("No validation errors found.")
         print(NO_VALIDATION_ERRORS_FOUND_MSG)
@@ -143,6 +150,7 @@ def _validate_data(
     parsed_resources: list[ParsedResource],
     config: ValidateDataConfig,
     shortcode: str,
+    existing_resources_retrieved: ExistingResourcesRetrieved,
 ) -> ValidateDataResult:
     logger.debug(f"Validate-data called with the following config: {vars(config)}")
     # Check if unknown classes are used
@@ -171,7 +179,7 @@ def _validate_data(
             )
             return ValidateDataResult(False, sorted_problems, report)
     reformatted = reformat_validation_graph(report)
-    sorted_problems = sort_user_problems(reformatted, duplicate_file_warnings, shortcode)
+    sorted_problems = sort_user_problems(reformatted, duplicate_file_warnings, shortcode, existing_resources_retrieved)
     return ValidateDataResult(False, sorted_problems, report)
 
 

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/commands/xmlupload/upload_config.py

@@ -56,6 +56,7 @@ class UploadConfig:
     ignore_duplicate_files_warning: bool = False
     validation_severity: ValidationSeverity = field(default_factory=lambda: ValidationSeverity.INFO)
     id2iri_replacement_file: str | None = None
+    do_not_request_resource_metadata_from_db: bool = False
 
     def with_server_info(
         self,

{dsp_tools-17.0.0.post26 → dsp_tools-17.0.0.post29}/src/dsp_tools/commands/xmlupload/xmlupload.py

@@ -177,6 +177,7 @@ def _handle_validation(
                 ignore_duplicate_files_warning=ignore_duplicates,
                 is_on_prod_server=is_on_prod_like_server,
                 skip_ontology_validation=config.skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=config.do_not_request_resource_metadata_from_db,
             ),
             auth=auth,
         )