dsp-tools 17.0.0.post26__py3-none-any.whl → 18.0.0__py3-none-any.whl

dsp_tools/clients/{ontology_client.py → ontology_clients.py}

@@ -6,9 +6,9 @@ from rdflib import Literal
 from dsp_tools.clients.authentication_client import AuthenticationClient
 
 
-class OntologyClient(Protocol):
+class OntologyCreateClient(Protocol):
     """
-    Protocol class/interface for the ontology endpoint in the API.
+    Protocol class/interface to create / update the ontology through the API.
     """
 
     server: str
@@ -19,3 +19,18 @@ class OntologyClient(Protocol):
 
     def post_resource_cardinalities(self, cardinality_graph: dict[str, Any]) -> Literal | None:
         """Add cardinalities to an existing resource class."""
+
+
+class OntologyGetClient(Protocol):
+    """
+    Protocol class/interface to get ontologies from the API.
+    """
+
+    api_url: str
+    shortcode: str
+
+    def get_knora_api(self) -> str:
+        """Get the knora-api ontology."""
+
+    def get_ontologies(self) -> tuple[list[str], list[str]]:
+        """Get all project ontologies."""

dsp_tools/clients/{ontology_client_live.py → ontology_create_client_live.py}

@@ -11,7 +11,7 @@ from requests import ReadTimeout
 from requests import Response
 
 from dsp_tools.clients.authentication_client import AuthenticationClient
-from dsp_tools.clients.ontology_client import OntologyClient
+from dsp_tools.clients.ontology_clients import OntologyCreateClient
 from dsp_tools.error.exceptions import BadCredentialsError
 from dsp_tools.error.exceptions import UnexpectedApiResponseError
 from dsp_tools.utils.rdflib_constants import KNORA_API
@@ -24,7 +24,7 @@ TIMEOUT = 60
 
 
 @dataclass
-class OntologyClientLive(OntologyClient):
+class OntologyCreateClientLive(OntologyCreateClient):
     """
     Client for the ontology endpoint in the API.
     """

dsp_tools/clients/ontology_get_client_live.py

@@ -0,0 +1,65 @@
+from dataclasses import dataclass
+from typing import Any
+from typing import cast
+
+import requests
+
+from dsp_tools.clients.ontology_clients import OntologyGetClient
+from dsp_tools.error.exceptions import InternalError
+from dsp_tools.utils.request_utils import RequestParameters
+from dsp_tools.utils.request_utils import log_request
+from dsp_tools.utils.request_utils import log_response
+
+
+@dataclass
+class OntologyGetClientLive(OntologyGetClient):
+    api_url: str
+    shortcode: str
+
+    def get_knora_api(self) -> str:
+        url = f"{self.api_url}/ontology/knora-api/v2#"
+        headers = {"Accept": "text/turtle"}
+        timeout = 60
+        log_request(RequestParameters("GET", url, timeout=timeout, headers=headers))
+        response = requests.get(url=url, headers=headers, timeout=timeout)
+        log_response(response, include_response_content=False)
+        if not response.ok:
+            raise InternalError(f"Failed Request: {response.status_code} {response.text}")
+        return response.text
+
+    def get_ontologies(self) -> tuple[list[str], list[str]]:
+        """
+        Returns a list of project ontologies as a string in turtle format.
+        And a list of the ontology IRIs
+
+        Returns:
+            list of ontologies and IRIs
+        """
+        ontology_iris = self._get_ontology_iris()
+        ontologies = [self._get_one_ontology(x) for x in ontology_iris]
+        return ontologies, ontology_iris
+
+    def _get_ontology_iris(self) -> list[str]:
+        url = f"{self.api_url}/admin/projects/shortcode/{self.shortcode}"
+        timeout = 10
+        log_request(RequestParameters("GET", url, timeout=timeout))
+        response = requests.get(url=url, timeout=timeout)
+        log_response(response)
+        if not response.ok:
+            raise InternalError(f"Failed Request: {response.status_code} {response.text}")
+        response_json = cast(dict[str, Any], response.json())
+        if not (ontos := response_json.get("project", {}).get("ontologies")):
+            raise InternalError(f"The response from the API does not contain any ontologies.\nResponse:{response.text}")
+        output = cast(list[str], ontos)
+        return output
+
+    def _get_one_ontology(self, ontology_iri: str) -> str:
+        url = ontology_iri
+        headers = {"Accept": "text/turtle"}
+        timeout = 30
+        log_request(RequestParameters("GET", url, timeout=timeout, headers=headers))
+        response = requests.get(url=url, headers=headers, timeout=timeout)
+        log_response(response, include_response_content=False)
+        if not response.ok:
+            raise InternalError(f"Failed Request: {response.status_code} {response.text}")
+        return response.text

dsp_tools/clients/project_client.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass
+class ProjectInfoClient(Protocol):
+    api_url: str
+
+    def get_project_iri(self, shortcode: str) -> str | None:
+        """Get the IRI of a project via shortcode."""

dsp_tools/clients/project_client_live.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from http import HTTPStatus
+from typing import cast
+
+import requests
+
+from dsp_tools.clients.project_client import ProjectInfoClient
+from dsp_tools.error.exceptions import UnexpectedApiResponseError
+from dsp_tools.utils.request_utils import RequestParameters
+from dsp_tools.utils.request_utils import log_request
+from dsp_tools.utils.request_utils import log_response
+
+
+@dataclass
+class ProjectInfoClientLive(ProjectInfoClient):
+    api_url: str
+
+    def get_project_iri(self, shortcode: str) -> str | None:
+        url = f"{self.api_url}/admin/projects/shortcode/{shortcode}"
+        timeout = 30
+        params = RequestParameters("GET", url, timeout)
+        log_request(params)
+        response = requests.get(url, timeout=timeout)
+        log_response(response)
+        if response.ok:
+            result = response.json()
+            return cast(str, result["project"]["id"])
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            return None
+        raise UnexpectedApiResponseError(f"Encountered unexpected API response with the code {response.status_code}")

dsp_tools/commands/create/create_on_server/cardinalities.py

@@ -3,9 +3,10 @@ from typing import Any
 from loguru import logger
 from rdflib import Literal
 from rdflib import URIRef
+from tqdm import tqdm
 
-from dsp_tools.clients.ontology_client import OntologyClient
-from dsp_tools.clients.ontology_client_live import OntologyClientLive
+from dsp_tools.clients.ontology_clients import OntologyCreateClient
+from dsp_tools.clients.ontology_create_client_live import OntologyCreateClientLive
 from dsp_tools.commands.create.models.input_problems import CollectedProblems
 from dsp_tools.commands.create.models.input_problems import CreateProblem
 from dsp_tools.commands.create.models.input_problems import ProblemType
@@ -25,7 +26,7 @@ def add_all_cardinalities(
     ontologies: list[ParsedOntology],
     project_iri_lookup: ProjectIriLookup,
     created_iris: CreatedIriCollection,
-    onto_client: OntologyClientLive,
+    onto_client: OntologyCreateClientLive,
 ) -> CollectedProblems | None:
     all_problems = []
     for onto in ontologies:
@@ -36,25 +37,30 @@ def add_all_cardinalities(
             problems = _add_all_cardinalities_for_one_onto(
                 cardinalities=onto.cardinalities,
                 onto_iri=URIRef(onto_iri),
+                onto_name=onto.name,
                 last_modification_date=last_mod_date,
                 onto_client=onto_client,
                 created_iris=created_iris,
             )
             all_problems.extend(problems)
     if all_problems:
-        return CollectedProblems("During the cardinality creation the following problems occurred:", all_problems)
+        return CollectedProblems("    While adding cardinalities the following problems occurred:", all_problems)
     return None
 
 
 def _add_all_cardinalities_for_one_onto(
     cardinalities: list[ParsedClassCardinalities],
     onto_iri: URIRef,
+    onto_name: str,
     last_modification_date: Literal,
-    onto_client: OntologyClient,
+    onto_client: OntologyCreateClient,
     created_iris: CreatedIriCollection,
 ) -> list[CreateProblem]:
     problems: list[CreateProblem] = []
-    for c in cardinalities:
+    progress_bar = tqdm(
+        cardinalities, desc=f"    Adding cardinalities to the ontology '{onto_name}'", dynamic_ncols=True
+    )
+    for c in progress_bar:
         # we do not inform about classes failures here, as it will have been done upstream
         if c.class_iri not in created_iris.classes:
             logger.warning(f"CARDINALITY: Class '{c.class_iri}' not in successes, no cardinalities added.")
@@ -74,7 +80,7 @@ def _add_cardinalities_for_one_class(
     resource_card: ParsedClassCardinalities,
     onto_iri: URIRef,
     last_modification_date: Literal,
-    onto_client: OntologyClient,
+    onto_client: OntologyCreateClient,
     successful_props: set[str],
 ) -> tuple[Literal, list[UploadProblem]]:
     res_iri = URIRef(resource_card.class_iri)
@@ -96,7 +102,7 @@ def _add_one_cardinality(
     res_iri: URIRef,
     onto_iri: URIRef,
     last_modification_date: Literal,
-    onto_client: OntologyClient,
+    onto_client: OntologyCreateClient,
 ) -> tuple[Literal, UploadProblem | None]:
     card_serialised = _serialise_card(card, res_iri, onto_iri, last_modification_date)
     new_mod_date = onto_client.post_resource_cardinalities(card_serialised)

dsp_tools/commands/create/create_on_server/lists.py

@@ -0,0 +1,150 @@
+from typing import Any
+
+from loguru import logger
+from tqdm import tqdm
+
+from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.clients.list_client import ListCreateClient
+from dsp_tools.clients.list_client_live import ListCreateClientLive
+from dsp_tools.clients.list_client_live import ListGetClientLive
+from dsp_tools.commands.create.models.input_problems import CollectedProblems
+from dsp_tools.commands.create.models.input_problems import CreateProblem
+from dsp_tools.commands.create.models.input_problems import ProblemType
+from dsp_tools.commands.create.models.input_problems import UploadProblem
+from dsp_tools.commands.create.models.input_problems import UserInformation
+from dsp_tools.commands.create.models.input_problems import UserInformationMessage
+from dsp_tools.commands.create.models.parsed_project import ParsedList
+from dsp_tools.commands.create.models.parsed_project import ParsedListNode
+from dsp_tools.commands.create.models.parsed_project import ParsedNodeInfo
+from dsp_tools.commands.create.models.server_project_info import ListNameToIriLookup
+from dsp_tools.utils.ansi_colors import BOLD
+from dsp_tools.utils.ansi_colors import BOLD_CYAN
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+
+def create_lists(
+    parsed_lists: list[ParsedList], shortcode: str, auth: AuthenticationClient, project_iri: str
+) -> tuple[ListNameToIriLookup, CollectedProblems | None]:
+    print(BOLD + "Processing list section:" + RESET_TO_DEFAULT)
+    name2iri = get_existing_lists_on_server(shortcode, auth)
+    if not parsed_lists:
+        return name2iri, None
+    lists_to_create, existing_info = _filter_out_existing_lists(parsed_lists, name2iri)
+    if existing_info:
+        _print_existing_list_info(existing_info)
+    if not lists_to_create:
+        msg = "    All lists defined in the project are already on the server, no list was uploaded."
+        logger.warning(msg)
+        print(BOLD_CYAN + msg + RESET_TO_DEFAULT)
+        return name2iri, None
+
+    create_client = ListCreateClientLive(auth.server, auth, project_iri)
+
+    all_problems: list[CreateProblem] = []
+    progress_bar = tqdm(lists_to_create, desc="    Creating lists", dynamic_ncols=True)
+    for new_lst in progress_bar:
+        list_iri, problems = _create_new_list(new_lst, create_client, project_iri)
+        if list_iri is None:
+            problems.extend(problems)
+        else:
+            name2iri.add_iri(new_lst.list_info.name, list_iri)
+
+    create_problems = None
+    if all_problems:
+        create_problems = CollectedProblems("The following problems occurred during list creation:", all_problems)
+    return name2iri, create_problems
+
+
+def _print_existing_list_info(existing_lists: list[UserInformation]) -> None:
+    lists = ", ".join([x.focus_object for x in existing_lists])
+    msg = f"    The following lists already exist on the server and will be skipped: {lists}"
+    logger.info(msg)
+    print(BOLD_CYAN + msg + RESET_TO_DEFAULT)
+
+
+def get_existing_lists_on_server(shortcode: str, auth: AuthenticationClient) -> ListNameToIriLookup:
+    client = ListGetClientLive(auth.server, shortcode)
+    name2iri_dict = client.get_all_list_iris_and_names()
+    return ListNameToIriLookup(name2iri_dict)
+
+
+def _filter_out_existing_lists(
+    parsed_lists: list[ParsedList], name2iri: ListNameToIriLookup
+) -> tuple[list[ParsedList], list[UserInformation]]:
+    lists_to_create: list[ParsedList] = []
+    existing_info: list[UserInformation] = []
+
+    for parsed_list in parsed_lists:
+        if name2iri.check_list_exists(parsed_list.list_info.name):
+            existing_info.append(
+                UserInformation(parsed_list.list_info.name, UserInformationMessage.LIST_EXISTS_ON_SERVER)
+            )
+        else:
+            lists_to_create.append(parsed_list)
+
+    return lists_to_create, existing_info
+
+
+def _create_new_list(
+    parsed_list: ParsedList, create_client: ListCreateClient, project_iri: str
+) -> tuple[str | None, list[UploadProblem]]:
+    serialised = _serialise_list(parsed_list.list_info, project_iri)
+    new_iri = create_client.create_new_list(serialised)
+
+    if new_iri is None:
+        problems: list[UploadProblem] = [
+            UploadProblem(parsed_list.list_info.name, ProblemType.LIST_COULD_NOT_BE_CREATED)
+        ]
+        return None, problems
+
+    node_problems = []
+    if parsed_list.children:
+        node_problems = _create_node_tree(parsed_list.children, new_iri, create_client, project_iri)
+
+    return new_iri, node_problems
+
+
+def _serialise_list(parsed_list_info: ParsedNodeInfo, project_iri: str) -> dict[str, Any]:
+    node_dict = {
+        "projectIri": project_iri,
+        "name": parsed_list_info.name,
+        "labels": _convert_to_api_format(parsed_list_info.labels),
+    }
+    if parsed_list_info.comments:
+        node_dict["comments"] = _convert_to_api_format(parsed_list_info.comments)
+    return node_dict
+
+
+def _create_node_tree(
+    nodes: list[ParsedListNode], parent_iri: str, create_client: ListCreateClient, project_iri: str
+) -> list[UploadProblem]:
+    problems: list[UploadProblem] = []
+
+    for node in nodes:
+        serialised = _serialise_node(node.node_info, parent_iri, project_iri)
+        node_iri = create_client.add_list_node(serialised, parent_iri)
+        if node_iri is None:
+            problems.append(UploadProblem(node.node_info.name, ProblemType.LIST_NODE_COULD_NOT_BE_CREATED))
+            continue
+
+        if node.children:
+            child_problems = _create_node_tree(node.children, node_iri, create_client, project_iri)
+            problems.extend(child_problems)
+
+    return problems
+
+
+def _serialise_node(node_info: ParsedNodeInfo, parent_iri: str, project_iri: str) -> dict[str, Any]:
+    node_dict = {
+        "parentNodeIri": parent_iri,
+        "projectIri": project_iri,
+        "name": node_info.name,
+        "labels": _convert_to_api_format(node_info.labels),
+    }
+    if node_info.comments:
+        node_dict["comments"] = _convert_to_api_format(node_info.comments)
+    return node_dict
+
+
+def _convert_to_api_format(lang_dict: dict[str, str]) -> list[dict[str, str]]:
+    return [{"value": value, "language": lang} for lang, value in lang_dict.items()]

dsp_tools/commands/create/lists_only.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.project_client_live import ProjectInfoClientLive
+from dsp_tools.commands.create.communicate_problems import print_problem_collection
+from dsp_tools.commands.create.create_on_server.lists import create_lists
+from dsp_tools.commands.create.models.input_problems import CollectedProblems
+from dsp_tools.commands.create.parsing.parse_project import parse_lists_only
+from dsp_tools.error.exceptions import ProjectNotFoundError
+from dsp_tools.utils.ansi_colors import BACKGROUND_BOLD_YELLOW
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+
+def create_lists_only(project_file_as_path_or_parsed: str | Path | dict[str, Any], creds: ServerCredentials) -> bool:
+    result = parse_lists_only(project_file_as_path_or_parsed)
+    if isinstance(result, CollectedProblems):
+        print_problem_collection(result)
+        return False
+    project_metadata, parsed_lists = result
+
+    if not parsed_lists:
+        msg = "Your file did not contain any lists, therefore no lists were created on the server."
+        logger.info(msg)
+        print(BACKGROUND_BOLD_YELLOW + msg + RESET_TO_DEFAULT)
+        return True
+
+    project_info = ProjectInfoClientLive(creds.server)
+    project_iri = project_info.get_project_iri(project_metadata.shortcode)
+    if not project_iri:
+        raise ProjectNotFoundError(
+            f"This commands adds lists to an existing project. "
+            f"The project with the shortcode {project_metadata.shortcode} does not exist on this server. "
+            f"If you wish to create an entire project use the `create` command without the flag."
+        )
+
+    auth = AuthenticationClientLive(creds.server, creds.user, creds.password)
+    _, problems = create_lists(parsed_lists, project_metadata.shortcode, auth, project_iri)
+    if problems:
+        print_problem_collection(problems)
+        return False
+    return True

dsp_tools/commands/create/models/input_problems.py

@@ -10,6 +10,16 @@ class CollectedProblems:
     problems: list[CreateProblem]
 
 
+@dataclass
+class UserInformation:
+    focus_object: str
+    msg: UserInformationMessage
+
+
+class UserInformationMessage(StrEnum):
+    LIST_EXISTS_ON_SERVER = "The list already exists on the server, therefore it is skipped entirely."
+
+
 @dataclass
 class CreateProblem:
     problematic_object: str
@@ -30,3 +40,6 @@ class ProblemType(StrEnum):
         "nor does it belong to one of the project ontologies."
     )
     CARDINALITY_COULD_NOT_BE_ADDED = "The cardinality could not be added."
+    DUPLICATE_LIST_NAME = "You have lists in your project with the same name, this is not permitted."
+    LIST_COULD_NOT_BE_CREATED = "The list could not be created on the server."
+    LIST_NODE_COULD_NOT_BE_CREATED = "The list node could not be created on the server."

dsp_tools/commands/create/models/parsed_project.py

@@ -45,5 +45,18 @@ class ParsedUser:
 
 @dataclass
 class ParsedList:
+    list_info: ParsedNodeInfo
+    children: list[ParsedListNode]
+
+
+@dataclass
+class ParsedNodeInfo:
     name: str
-    info: dict[str, Any]
+    labels: dict[str, str]
+    comments: dict[str, str] | None
+
+
+@dataclass
+class ParsedListNode:
+    node_info: ParsedNodeInfo
+    children: list[ParsedListNode]

dsp_tools/commands/create/models/rdf_ontology.py

@@ -6,13 +6,6 @@ from rdflib import Literal
 from rdflib import URIRef
 
 
-@dataclass
-class RdfResourceCardinality:
-    resource_iri: URIRef
-    on_property: URIRef
-    cardinality: RdfCardinalityRestriction
-
-
 @dataclass
 class RdfCardinalityRestriction:
     owl_property: URIRef

dsp_tools/commands/create/models/server_project_info.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from dataclasses import field
 
 from dsp_tools.commands.create.constants import KNORA_API_STR
+from dsp_tools.error.exceptions import InternalError
 
 
 @dataclass
@@ -12,9 +13,6 @@ class ProjectIriLookup:
     def add_onto(self, name: str, iri: str) -> None:
         self.onto_iris[name] = iri
 
-    def get_onto_iri(self, name: str) -> str | None:
-        return self.onto_iris.get(name)
-
 
 @dataclass
 class CreatedIriCollection:
@@ -23,3 +21,19 @@ class CreatedIriCollection:
 
     def __post_init__(self) -> None:
         self.properties.update({f"{KNORA_API_STR}seqnum", f"{KNORA_API_STR}isPartOf"})
+
+
+@dataclass
+class ListNameToIriLookup:
+    name2iri: dict[str, str]
+
+    def check_list_exists(self, name: str) -> bool:
+        return name in self.name2iri.keys()
+
+    def add_iri(self, name: str, iri: str) -> None:
+        if self.check_list_exists(name):
+            raise InternalError(f"List with the name '{name}' already exists in the lookup.")
+        self.name2iri[name] = iri
+
+    def get_iri(self, name: str) -> str | None:
+        return self.name2iri.get(name)

dsp_tools/commands/create/parsing/parse_lists.py

@@ -0,0 +1,45 @@
+from collections import Counter
+from typing import Any
+
+from dsp_tools.commands.create.models.input_problems import CollectedProblems
+from dsp_tools.commands.create.models.input_problems import CreateProblem
+from dsp_tools.commands.create.models.input_problems import InputProblem
+from dsp_tools.commands.create.models.input_problems import ProblemType
+from dsp_tools.commands.create.models.parsed_project import ParsedList
+from dsp_tools.commands.create.models.parsed_project import ParsedListNode
+from dsp_tools.commands.create.models.parsed_project import ParsedNodeInfo
+
+
+def parse_list_section(lists: list[dict[str, Any]]) -> list[ParsedList] | CollectedProblems:
+    list_section = [_parse_one_list(one_list) for one_list in lists]
+    list_names = [parsed_list.list_info.name for parsed_list in list_section]
+    duplicates = {name for name, count in Counter(list_names).items() if count > 1}
+    if duplicates:
+        problems: list[CreateProblem] = [
+            InputProblem(problematic_object=name, problem=ProblemType.DUPLICATE_LIST_NAME) for name in duplicates
+        ]
+        return CollectedProblems(header="The following problems were found in the list section:", problems=problems)
+    return list_section
+
+
+def _parse_one_list(one_list: dict[str, Any]) -> ParsedList:
+    list_info = _parse_node_info(one_list)
+    children = _parse_nodes_and_children(one_list["nodes"]) if "nodes" in one_list else []
+    return ParsedList(list_info=list_info, children=children)
+
+
+def _parse_node_info(one_node: dict[str, Any]) -> ParsedNodeInfo:
+    return ParsedNodeInfo(
+        name=one_node["name"],
+        labels=one_node["labels"],
+        comments=one_node.get("comments"),
+    )
+
+
+def _parse_nodes_and_children(list_nodes: list[dict[str, Any]]) -> list[ParsedListNode]:
+    parsed_nodes = []
+    for node in list_nodes:
+        node_info = _parse_node_info(node)
+        children = _parse_nodes_and_children(node["nodes"]) if "nodes" in node else []
+        parsed_nodes.append(ParsedListNode(node_info=node_info, children=children))
+    return parsed_nodes

dsp_tools/commands/create/parsing/parse_project.py

@@ -11,12 +11,25 @@ from dsp_tools.commands.create.models.parsed_project import ParsedPermissions
 from dsp_tools.commands.create.models.parsed_project import ParsedProject
 from dsp_tools.commands.create.models.parsed_project import ParsedProjectMetadata
 from dsp_tools.commands.create.models.parsed_project import ParsedUser
+from dsp_tools.commands.create.parsing.parse_lists import parse_list_section
 from dsp_tools.commands.create.parsing.parse_ontology import parse_ontology
 from dsp_tools.commands.create.parsing.parsing_utils import create_prefix_lookup
 from dsp_tools.commands.project.create.project_validate import validate_project
 from dsp_tools.utils.json_parsing import parse_json_input
 
 
+def parse_lists_only(
+    project_file_as_path_or_parsed: str | Path | dict[str, Any],
+) -> tuple[ParsedProjectMetadata, list[ParsedList]] | CollectedProblems:
+    complete_json = _parse_and_validate(project_file_as_path_or_parsed)
+    project_json = complete_json["project"]
+    project_metadata = _parse_metadata(project_json)
+    parsed_lists, problems = _parse_lists(project_json)
+    if isinstance(problems, CollectedProblems):
+        return problems
+    return project_metadata, parsed_lists
+
+
 def parse_project(
     project_file_as_path_or_parsed: str | Path | dict[str, Any], api_url: str
 ) -> ParsedProject | list[CollectedProblems]:
@@ -28,6 +41,9 @@ def _parse_project(complete_json: dict[str, Any], api_url: str) -> ParsedProject
     prefix_lookup = create_prefix_lookup(complete_json, api_url)
     project_json = complete_json["project"]
     ontologies, failures = _parse_all_ontologies(project_json, prefix_lookup)
+    parsed_lists, problems = _parse_lists(project_json)
+    if isinstance(problems, CollectedProblems):
+        failures.append(problems)
     if failures:
         return failures
     return ParsedProject(
@@ -36,7 +52,7 @@ def _parse_project(complete_json: dict[str, Any], api_url: str) -> ParsedProject
         permissions=_parse_permissions(project_json),
         groups=_parse_groups(project_json),
         users=_parse_users(project_json),
-        lists=_parse_lists(project_json),
+        lists=parsed_lists,
         ontologies=ontologies,
     )
 
@@ -79,10 +95,13 @@ def _parse_users(project_json: dict[str, Any]) -> list[ParsedUser]:
     return [ParsedUser(x) for x in found]
 
 
-def _parse_lists(project_json: dict[str, Any]) -> list[ParsedList]:
+def _parse_lists(project_json: dict[str, Any]) -> tuple[list[ParsedList], CollectedProblems | None]:
     if not (found := project_json.get("lists")):
-        return []
-    return [ParsedList(x["name"], x) for x in found]
+        return [], None
+    result = parse_list_section(found)
+    if isinstance(result, CollectedProblems):
+        return [], result
+    return result, None
 
 
 def _parse_all_ontologies(

dsp_tools/commands/ingest_xmlupload/create_resources/upload_xml.py

@@ -43,6 +43,7 @@ def ingest_xmlupload(
     skip_validation: bool = False,
     skip_ontology_validation: bool = False,
     id2iri_replacement_file: str | None = None,
+    do_not_request_resource_metadata_from_db: bool = False,
 ) -> bool:
     """
     This function reads an XML file
@@ -59,6 +60,8 @@ def ingest_xmlupload(
         skip_validation: skip the SHACL validation
         skip_ontology_validation: skip the ontology validation
         id2iri_replacement_file: to replace internal IDs of an XML file by IRIs provided in this mapping file
+        do_not_request_resource_metadata_from_db: if true do not request metadata information from the api
+                                                  for existing resources
 
     Returns:
         True if all resources could be uploaded without errors; False if one of the resources could not be
@@ -115,6 +118,7 @@ def ingest_xmlupload(
                 ignore_duplicate_files_warning=True,
                 is_on_prod_server=is_on_prod_like_server,
                 skip_ontology_validation=skip_ontology_validation,
+                do_not_request_resource_metadata_from_db=do_not_request_resource_metadata_from_db,
             ),
             auth=auth,
         )