dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/clients/project_client_live.py

@@ -0,0 +1,66 @@
+from dataclasses import dataclass
+from http import HTTPStatus
+from typing import Any
+from typing import cast
+
+import requests
+from requests import RequestException
+
+from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.clients.project_client import ProjectClient
+from dsp_tools.error.exceptions import BadCredentialsError
+from dsp_tools.error.exceptions import FatalNonOkApiResponseCode
+from dsp_tools.error.exceptions import ProjectNotFoundError
+from dsp_tools.utils.request_utils import RequestParameters
+from dsp_tools.utils.request_utils import ResponseCodeAndText
+from dsp_tools.utils.request_utils import log_and_raise_request_exception
+from dsp_tools.utils.request_utils import log_request
+from dsp_tools.utils.request_utils import log_response
+
+
+@dataclass
+class ProjectClientLive(ProjectClient):
+    server: str
+    auth: AuthenticationClient
+
+    def get_project_iri(self, shortcode: str) -> str:
+        url = f"{self.server}/admin/projects/shortcode/{shortcode}"
+        timeout = 30
+        params = RequestParameters("GET", url, timeout)
+        log_request(params)
+        try:
+            response = requests.get(url, timeout=timeout)
+        except RequestException as err:
+            log_and_raise_request_exception(err)
+
+        log_response(response)
+        if response.ok:
+            result = response.json()
+            return cast(str, result["project"]["id"])
+        if response.status_code == HTTPStatus.NOT_FOUND:
+            raise ProjectNotFoundError(f"The project with the shortcode {shortcode} does not exist on this server. ")
+        raise FatalNonOkApiResponseCode(url, response.status_code, response.text)
+
+    def post_new_project(self, project_info: dict[str, Any]) -> str | ResponseCodeAndText:
+        url = f"{self.server}/admin/projects"
+        timeout = 10
+        headers = {"Authorization": f"Bearer {self.auth.get_token()}"}
+        params = RequestParameters("POST", url, timeout, headers=headers, data=project_info)
+        log_request(params)
+        try:
+            response = requests.post(
+                params.url, timeout=params.timeout, headers=params.headers, data=params.data_serialized
+            )
+        except RequestException as err:
+            log_and_raise_request_exception(err)
+
+        log_response(response)
+        if response.ok:
+            result = response.json()
+            return cast(str, result["project"]["id"])
+        if response.status_code == HTTPStatus.FORBIDDEN:
+            raise BadCredentialsError(
+                "Only a SystemAdmin can create a project, your permissions are insufficient for this action. "
+                "Please contact support@dasch.swiss if you require a new project."
+            )
+        return ResponseCodeAndText(response.status_code, response.text)

dsp_tools/commands/create/communicate_problems.py

@@ -0,0 +1,24 @@
+from loguru import logger
+
+from dsp_tools.commands.create.models.create_problems import CollectedProblems
+from dsp_tools.commands.create.models.create_problems import CreateProblem
+from dsp_tools.utils.ansi_colors import BOLD_RED
+from dsp_tools.utils.ansi_colors import RED
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+
+def print_all_problem_collections(problems: list[CollectedProblems]) -> None:
+    for prob in problems:
+        print_problem_collection(prob)
+
+
+def print_problem_collection(problem_collection: CollectedProblems) -> None:
+    individual_problems = _create_individual_problem_strings(problem_collection.problems)
+    logger.error(problem_collection.header + individual_problems)
+    print(BOLD_RED, problem_collection.header, RESET_TO_DEFAULT)
+    print(RED, individual_problems, RESET_TO_DEFAULT)
+
+
+def _create_individual_problem_strings(problems: list[CreateProblem]) -> str:
+    str_list = [f"{p.problematic_object}: {p.problem!s}" for p in problems]
+    return "   - " + "\n    - ".join(str_list)

dsp_tools/commands/create/create.py

@@ -0,0 +1,134 @@
+from pathlib import Path
+
+from dotenv import find_dotenv
+from dotenv import load_dotenv
+from loguru import logger
+
+from dsp_tools.cli.args import ServerCredentials
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.group_user_clients_live import GroupClientLive
+from dsp_tools.clients.permissions_client import PermissionsClient
+from dsp_tools.commands.create.communicate_problems import print_all_problem_collections
+from dsp_tools.commands.create.communicate_problems import print_problem_collection
+from dsp_tools.commands.create.create_on_server.complete_ontologies import create_ontologies
+from dsp_tools.commands.create.create_on_server.default_permissions import create_default_permissions
+from dsp_tools.commands.create.create_on_server.group_users import create_groups
+from dsp_tools.commands.create.create_on_server.group_users import create_users
+from dsp_tools.commands.create.create_on_server.group_users import get_existing_group_to_iri_lookup
+from dsp_tools.commands.create.create_on_server.lists import create_lists
+from dsp_tools.commands.create.create_on_server.lists import get_existing_lists_on_server
+from dsp_tools.commands.create.create_on_server.project import create_project
+from dsp_tools.commands.create.models.parsed_project import ParsedProject
+from dsp_tools.commands.create.project_validate import parse_and_validate_project
+from dsp_tools.error.exceptions import InternalError
+from dsp_tools.utils.ansi_colors import BOLD_GREEN
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+load_dotenv(dotenv_path=find_dotenv(usecwd=True))
+
+
+def create(project_file: Path, creds: ServerCredentials, exit_if_exists: bool) -> bool:
+    parsing_result = parse_and_validate_project(project_file, creds.server)
+    match parsing_result:
+        case ParsedProject():
+            return _execute_create(parsing_result, creds, exit_if_exists)
+        case list():
+            print_all_problem_collections(parsing_result)
+            return False
+        case _:
+            raise InternalError("Unreachable result of project parsing.")
+
+
+def _execute_create(parsed_project: ParsedProject, creds: ServerCredentials, exit_if_exists: bool) -> bool:
+    msg = "JSON project file is syntactically correct and passed validation."
+    print(BOLD_GREEN + "    JSON project file is syntactically correct and passed validation." + RESET_TO_DEFAULT)
+    logger.info(msg)
+    overall_success = True
+    auth = AuthenticationClientLive(creds.server, creds.user, creds.password)
+
+    # create project
+    project_iri = create_project(parsed_project.project_metadata, auth, exit_if_exists)
+
+    # create the lists
+    if parsed_project.lists:
+        list_name_2_iri, list_problems = create_lists(
+            parsed_lists=parsed_project.lists,
+            shortcode=parsed_project.project_metadata.shortcode,
+            auth=auth,
+            project_iri=project_iri,
+        )
+        if list_problems:
+            overall_success = False
+            print_problem_collection(list_problems)
+    else:
+        list_name_2_iri = get_existing_lists_on_server(parsed_project.project_metadata.shortcode, auth)
+
+    # create the groups
+    group_client = GroupClientLive(creds.server, auth)
+    group_lookup = get_existing_group_to_iri_lookup(
+        group_client, project_iri, parsed_project.project_metadata.shortname
+    )
+    if parsed_project.groups:
+        group_lookup, group_problems = create_groups(
+            groups=parsed_project.groups,
+            group_client=group_client,
+            project_iri=project_iri,
+            group_lookup=group_lookup,
+        )
+        if group_problems:
+            print_problem_collection(group_problems)
+            overall_success = False
+
+    if parsed_project.users:
+        user_problems = create_users(
+            users=parsed_project.users,
+            user_memberships=parsed_project.user_memberships,
+            group_lookup=group_lookup,
+            auth=auth,
+            project_iri=project_iri,
+        )
+        if user_problems:
+            print_problem_collection(user_problems)
+            overall_success = False
+
+    # create the ontologies
+    success = create_ontologies(
+        parsed_ontologies=parsed_project.ontologies,
+        list_name_2_iri=list_name_2_iri,
+        project_iri=project_iri,
+        shortcode=parsed_project.project_metadata.shortcode,
+        auth=auth,
+    )
+    if not success:
+        overall_success = False
+
+    # create the default permissions (DOAPs)
+    perm_client = PermissionsClient(auth, project_iri)
+    success = create_default_permissions(
+        perm_client=perm_client,
+        default_permissions=parsed_project.permissions.default_permissions,
+        default_permissions_overrule=parsed_project.permissions.default_permissions_overrule,
+        shortcode=parsed_project.project_metadata.shortcode,
+    )
+    if not success:
+        overall_success = False
+
+    # final steps
+    if overall_success:
+        msg = (
+            f"Successfully created project '{parsed_project.project_metadata.shortname}' "
+            f"({parsed_project.project_metadata.shortcode}) with all its ontologies. "
+            f"There were no problems during the creation process."
+        )
+        print(f"========================================================\n{msg}")
+        logger.info(msg)
+    else:
+        msg = (
+            f"The project '{parsed_project.project_metadata.shortname}' ({parsed_project.project_metadata.shortcode}) "
+            f"with its ontologies could be created, "
+            f"but during the creation process, some problems occurred. Please carefully check the console output."
+        )
+        print(f"========================================================\nWARNING: {msg}")
+        logger.warning(msg)
+
+    return overall_success

dsp_tools/commands/create/create_on_server/cardinalities.py

@@ -0,0 +1,111 @@
+from loguru import logger
+from rdflib import Literal
+from rdflib import URIRef
+from tqdm import tqdm
+
+from dsp_tools.clients.ontology_clients import OntologyCreateClient
+from dsp_tools.commands.create.models.create_problems import CollectedProblems
+from dsp_tools.commands.create.models.create_problems import CreateProblem
+from dsp_tools.commands.create.models.create_problems import UploadProblem
+from dsp_tools.commands.create.models.create_problems import UploadProblemType
+from dsp_tools.commands.create.models.parsed_ontology import ParsedClassCardinalities
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+from dsp_tools.commands.create.models.parsed_ontology import ParsedPropertyCardinality
+from dsp_tools.commands.create.models.server_project_info import CreatedIriCollection
+from dsp_tools.commands.create.models.server_project_info import ProjectIriLookup
+from dsp_tools.commands.create.serialisation.ontology import serialise_cardinality_graph_for_request
+from dsp_tools.utils.data_formats.iri_util import from_dsp_iri_to_prefixed_iri
+
+
+def add_all_cardinalities(
+    ontologies: list[ParsedOntology],
+    project_iri_lookup: ProjectIriLookup,
+    created_iris: CreatedIriCollection,
+    onto_client: OntologyCreateClient,
+) -> CollectedProblems | None:
+    all_problems = []
+    for onto in ontologies:
+        onto_iri = project_iri_lookup.onto_iris.get(onto.name)
+        # we do not inform about onto failures here, as it will have been done upstream
+        if onto_iri:
+            last_mod_date = onto_client.get_last_modification_date(project_iri_lookup.project_iri, onto_iri)
+            problems = _add_all_cardinalities_for_one_onto(
+                cardinalities=onto.cardinalities,
+                onto_iri=URIRef(onto_iri),
+                onto_name=onto.name,
+                last_modification_date=last_mod_date,
+                onto_client=onto_client,
+                created_iris=created_iris,
+            )
+            all_problems.extend(problems)
+    if all_problems:
+        return CollectedProblems("    While adding cardinalities the following problems occurred:", all_problems)
+    return None
+
+
+def _add_all_cardinalities_for_one_onto(
+    cardinalities: list[ParsedClassCardinalities],
+    onto_iri: URIRef,
+    onto_name: str,
+    last_modification_date: Literal,
+    onto_client: OntologyCreateClient,
+    created_iris: CreatedIriCollection,
+) -> list[CreateProblem]:
+    problems: list[CreateProblem] = []
+    progress_bar = tqdm(
+        cardinalities, desc=f"    Adding cardinalities to the ontology '{onto_name}'", dynamic_ncols=True
+    )
+    for c in progress_bar:
+        # we do not inform about classes failures here, as it will have been done upstream
+        if c.class_iri not in created_iris.created_classes:
+            logger.warning(f"CARDINALITY: Class '{c.class_iri}' not in successes, no cardinalities added.")
+            continue
+        last_modification_date, creation_problems = _add_cardinalities_for_one_class(
+            resource_card=c,
+            onto_iri=onto_iri,
+            last_modification_date=last_modification_date,
+            onto_client=onto_client,
+            successful_props=created_iris.created_properties,
+        )
+        problems.extend(creation_problems)
+    return problems
+
+
+def _add_cardinalities_for_one_class(
+    resource_card: ParsedClassCardinalities,
+    onto_iri: URIRef,
+    last_modification_date: Literal,
+    onto_client: OntologyCreateClient,
+    successful_props: set[str],
+) -> tuple[Literal, list[UploadProblem]]:
+    res_iri = URIRef(resource_card.class_iri)
+    problems = []
+    for one_card in resource_card.cards:
+        if one_card.propname not in successful_props:
+            problems.append(UploadProblem(one_card.propname, UploadProblemType.CARDINALITY_PROPERTY_NOT_FOUND))
+            continue
+        last_modification_date, problem = _add_one_cardinality(
+            one_card, res_iri, onto_iri, last_modification_date, onto_client
+        )
+        if problem:
+            problems.append(problem)
+    return last_modification_date, problems
+
+
+def _add_one_cardinality(
+    card: ParsedPropertyCardinality,
+    res_iri: URIRef,
+    onto_iri: URIRef,
+    last_modification_date: Literal,
+    onto_client: OntologyCreateClient,
+) -> tuple[Literal, UploadProblem | None]:
+    card_serialised = serialise_cardinality_graph_for_request(card, res_iri, onto_iri, last_modification_date)
+    new_mod_date = onto_client.post_resource_cardinalities(card_serialised)
+    if not new_mod_date:
+        prefixed_cls = from_dsp_iri_to_prefixed_iri(str(res_iri))
+        prefixed_prop = from_dsp_iri_to_prefixed_iri(card.propname)
+        return last_modification_date, UploadProblem(
+            f"{prefixed_cls} / {prefixed_prop}",
+            UploadProblemType.CARDINALITY_COULD_NOT_BE_ADDED,
+        )
+    return new_mod_date, None

dsp_tools/commands/create/create_on_server/classes.py

@@ -0,0 +1,99 @@
+from http import HTTPStatus
+from typing import Any
+
+import rustworkx as rx
+from loguru import logger
+from rdflib import Literal
+from rdflib import URIRef
+from tqdm import tqdm
+
+from dsp_tools.clients.ontology_clients import OntologyCreateClient
+from dsp_tools.commands.create.create_on_server.onto_utils import get_modification_date_onto_lookup
+from dsp_tools.commands.create.create_on_server.onto_utils import should_retry_request
+from dsp_tools.commands.create.create_on_server.onto_utils import sort_for_upload
+from dsp_tools.commands.create.models.create_problems import CollectedProblems
+from dsp_tools.commands.create.models.create_problems import CreateProblem
+from dsp_tools.commands.create.models.create_problems import UploadProblem
+from dsp_tools.commands.create.models.create_problems import UploadProblemType
+from dsp_tools.commands.create.models.parsed_ontology import ParsedClass
+from dsp_tools.commands.create.models.server_project_info import CreatedIriCollection
+from dsp_tools.commands.create.models.server_project_info import OntoLastModDateLookup
+from dsp_tools.commands.create.models.server_project_info import ProjectIriLookup
+from dsp_tools.commands.create.serialisation.ontology import serialise_class_graph_for_request
+
+
+def create_all_classes(
+    classes: list[ParsedClass],
+    project_iri_lookup: ProjectIriLookup,
+    created_iris: CreatedIriCollection,
+    client: OntologyCreateClient,
+) -> tuple[CreatedIriCollection, CollectedProblems | None]:
+    upload_order = _get_class_create_order(classes)
+    logger.debug(f"Class creation order: {upload_order}")
+    cls_lookup = {c.name: c for c in classes}
+    all_problems: list[CreateProblem] = []
+    onto_lookup = get_modification_date_onto_lookup(project_iri_lookup, client)
+    progress_bar = tqdm(upload_order, desc="    Creating classes", dynamic_ncols=True)
+    logger.debug("Starting class creation")
+    for c in progress_bar:
+        cls = cls_lookup[c]
+        if previous_blocker := _is_class_blocked(cls.name, set(cls.supers), created_iris):
+            all_problems.append(previous_blocker)
+            created_iris.failed_classes.add(cls.name)
+        else:
+            create_result = _create_one_class(cls, onto_lookup, client)
+            if isinstance(create_result, Literal):
+                onto_lookup.update_last_mod_date(cls.onto_iri, create_result)
+                created_iris.created_classes.add(cls.name)
+            else:
+                all_problems.append(create_result)
+                created_iris.failed_classes.add(cls.name)
+    upload_problems = None
+    if all_problems:
+        upload_problems = CollectedProblems("    While creating classes the following problems occurred:", all_problems)
+    return created_iris, upload_problems
+
+
+def _get_class_create_order(classes: list[ParsedClass]) -> list[str]:
+    logger.debug("Creating class upload order.")
+    graph, node_to_iri = _make_graph_to_sort(classes)
+    return sort_for_upload(graph, node_to_iri)
+
+
+def _make_graph_to_sort(classes: list[ParsedClass]) -> tuple[rx.PyDiGraph, dict[int, str]]:
+    graph: rx.PyDiGraph[Any, Any] = rx.PyDiGraph()
+    cls_iris = [x.name for x in classes]
+    node_indices = list(graph.add_nodes_from(cls_iris))
+    iri_to_node = dict(zip(cls_iris, node_indices))
+    node_to_iri = dict(zip(node_indices, cls_iris))
+    for i, p in enumerate(classes):
+        for super_cls in p.supers:
+            if super_cls in cls_iris:
+                graph.add_edge(iri_to_node[p.name], iri_to_node[super_cls], i)
+    return graph, node_to_iri
+
+
+def _is_class_blocked(cls_name: str, supers: set[str], created_iris: CreatedIriCollection) -> CreateProblem | None:
+    if created_iris.any_classes_failed(supers):
+        return UploadProblem(cls_name, UploadProblemType.CLASS_SUPER_FAILED)
+    return None
+
+
+def _create_one_class(
+    cls: ParsedClass, onto_lookup: OntoLastModDateLookup, client: OntologyCreateClient
+) -> Literal | CreateProblem:
+    cls_serialised = serialise_class_graph_for_request(
+        cls, URIRef(cls.onto_iri), onto_lookup.get_last_mod_date(cls.onto_iri)
+    )
+    request_result = client.post_new_class(cls_serialised)
+    if isinstance(request_result, Literal):
+        return request_result
+    if should_retry_request(request_result):
+        new_mod_date = client.get_last_modification_date(onto_lookup.project_iri, cls.onto_iri)
+        cls_serialised = serialise_class_graph_for_request(cls, URIRef(cls.onto_iri), new_mod_date)
+        request_result = client.post_new_class(cls_serialised)
+        if isinstance(request_result, Literal):
+            return request_result
+    if request_result.status_code == HTTPStatus.BAD_REQUEST:
+        return UploadProblem(cls.name, request_result.text)
+    return UploadProblem(cls.name, UploadProblemType.CLASS_COULD_NOT_BE_CREATED)

dsp_tools/commands/create/create_on_server/complete_ontologies.py

@@ -0,0 +1,116 @@
+from loguru import logger
+
+from dsp_tools.clients.authentication_client import AuthenticationClient
+from dsp_tools.clients.ontology_clients import OntologyCreateClient
+from dsp_tools.clients.ontology_create_client_live import OntologyCreateClientLive
+from dsp_tools.commands.create.communicate_problems import print_problem_collection
+from dsp_tools.commands.create.create_on_server.cardinalities import add_all_cardinalities
+from dsp_tools.commands.create.create_on_server.classes import create_all_classes
+from dsp_tools.commands.create.create_on_server.onto_utils import get_project_iri_lookup
+from dsp_tools.commands.create.create_on_server.ontology import create_all_ontologies
+from dsp_tools.commands.create.create_on_server.properties import create_all_properties
+from dsp_tools.commands.create.models.create_problems import CollectedProblems
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+from dsp_tools.commands.create.models.server_project_info import CreatedIriCollection
+from dsp_tools.commands.create.models.server_project_info import ListNameToIriLookup
+from dsp_tools.commands.create.models.server_project_info import ProjectIriLookup
+from dsp_tools.utils.ansi_colors import BOLD
+from dsp_tools.utils.ansi_colors import BOLD_RED
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+
+def create_ontologies(
+    parsed_ontologies: list[ParsedOntology],
+    list_name_2_iri: ListNameToIriLookup,
+    project_iri: str,
+    shortcode: str,
+    auth: AuthenticationClient,
+) -> bool:
+    success_collection = CreatedIriCollection()
+    onto_client = OntologyCreateClientLive(auth.server, auth)
+    overall_success = True
+
+    logger.info("Processing Ontology Section")
+    print(BOLD + "Processing Ontology Section:" + RESET_TO_DEFAULT)
+    project_iri_lookup, ontology_problems = _create_ontologies_on_server(
+        ontologies=parsed_ontologies,
+        project_iri=project_iri,
+        shortcode=shortcode,
+        client=onto_client,
+    )
+    if ontology_problems:
+        overall_success = False
+        print_problem_collection(ontology_problems)
+    if not project_iri_lookup.onto_iris:
+        msg = "No ontologies could be created on the server."
+        logger.error(msg)
+        print(BOLD_RED + "    " + msg + RESET_TO_DEFAULT)
+        return False
+
+    all_classes = []
+    for onto in parsed_ontologies:
+        all_classes.extend(onto.classes)
+    if all_classes:
+        print(BOLD + "Processing Classes:" + RESET_TO_DEFAULT)
+        success_collection, cls_problems = create_all_classes(
+            classes=all_classes,
+            project_iri_lookup=project_iri_lookup,
+            created_iris=success_collection,
+            client=onto_client,
+        )
+        if cls_problems:
+            overall_success = False
+            print_problem_collection(cls_problems)
+    else:
+        logger.info("No classes found in the ontology.")
+
+    all_props = []
+    for onto in parsed_ontologies:
+        all_props.extend(onto.properties)
+    if all_props:
+        print(BOLD + "Processing Properties:" + RESET_TO_DEFAULT)
+        success_collection, property_problems = create_all_properties(
+            properties=all_props,
+            project_iri_lookup=project_iri_lookup,
+            created_iris=success_collection,
+            list_lookup=list_name_2_iri,
+            client=onto_client,
+        )
+        if property_problems:
+            overall_success = False
+            print_problem_collection(property_problems)
+    else:
+        logger.info("No properties found in the ontology.")
+
+    print(BOLD + "Processing Cardinalities:" + RESET_TO_DEFAULT)
+    problems = add_all_cardinalities(
+        ontologies=parsed_ontologies,
+        project_iri_lookup=project_iri_lookup,
+        created_iris=success_collection,
+        onto_client=onto_client,
+    )
+    if problems:
+        overall_success = False
+        print_problem_collection(problems)
+
+    return overall_success
+
+
+def _create_ontologies_on_server(
+    ontologies: list[ParsedOntology], project_iri: str, shortcode: str, client: OntologyCreateClient
+) -> tuple[ProjectIriLookup, CollectedProblems | None]:
+    project_iri_lookup = get_project_iri_lookup(client.server, shortcode, project_iri)
+    new_ontologies = []
+    for parsed_onto in ontologies:
+        if not project_iri_lookup.does_onto_exist(parsed_onto.name):
+            new_ontologies.append(parsed_onto)
+    if not new_ontologies:
+        if len(ontologies) > 0:
+            msg = (
+                "No new ontologies were found in the project. "
+                "All ontologies already exist on the server. Continuing adding information to the ontologies."
+            )
+            logger.info(msg)
+            print(BOLD + "    " + msg + RESET_TO_DEFAULT)
+        return project_iri_lookup, None
+    return create_all_ontologies(new_ontologies, project_iri_lookup, client)