dsp-tools 0.9.13__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/create/parsing/parse_ontology.py

@@ -0,0 +1,243 @@
+from typing import Any
+from typing import cast
+
+from dsp_tools.commands.create.models.create_problems import CollectedProblems
+from dsp_tools.commands.create.models.create_problems import CreateProblem
+from dsp_tools.commands.create.models.create_problems import InputProblem
+from dsp_tools.commands.create.models.create_problems import InputProblemType
+from dsp_tools.commands.create.models.parsed_ontology import Cardinality
+from dsp_tools.commands.create.models.parsed_ontology import GuiElement
+from dsp_tools.commands.create.models.parsed_ontology import KnoraObjectType
+from dsp_tools.commands.create.models.parsed_ontology import ParsedClass
+from dsp_tools.commands.create.models.parsed_ontology import ParsedClassCardinalities
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+from dsp_tools.commands.create.models.parsed_ontology import ParsedProperty
+from dsp_tools.commands.create.models.parsed_ontology import ParsedPropertyCardinality
+from dsp_tools.commands.create.parsing.parsing_utils import resolve_to_absolute_iri
+
+CARDINALITY_MAPPER = {
+    "0-1": Cardinality.C_0_1,
+    "1": Cardinality.C_1,
+    "0-n": Cardinality.C_0_N,
+    "1-n": Cardinality.C_1_N,
+}
+
+OBJECT_TYPE_MAPPER = {
+    "BooleanValue": KnoraObjectType.BOOLEAN,
+    "ColorValue": KnoraObjectType.COLOR,
+    "DateValue": KnoraObjectType.DATE,
+    "DecimalValue": KnoraObjectType.DECIMAL,
+    "GeonameValue": KnoraObjectType.GEONAME,
+    "IntValue": KnoraObjectType.INT,
+    "ListValue": KnoraObjectType.LIST,
+    "TextValue": KnoraObjectType.TEXT,
+    "TimeValue": KnoraObjectType.TIME,
+    "UriValue": KnoraObjectType.URI,
+}
+
+GUI_ELEMENT_MAPPER = {
+    "Checkbox": GuiElement.CHECKBOX,
+    "Colorpicker": GuiElement.COLORPICKER,
+    "Date": GuiElement.DATE,
+    "Spinbox": GuiElement.SPINBOX,
+    "Geonames": GuiElement.GEONAMES,
+    "List": GuiElement.LIST,
+    "SimpleText": GuiElement.SIMPLETEXT,
+    "Textarea": GuiElement.TEXTAREA,
+    "Richtext": GuiElement.RICHTEXT,
+    "Searchbox": GuiElement.SEARCHBOX,
+    "TimeStamp": GuiElement.TIME_STAMP,
+}
+
+
+def parse_ontology(ontology_json: dict[str, Any], prefixes: dict[str, str]) -> ParsedOntology | CollectedProblems:
+    onto_name = ontology_json["name"]
+    current_onto = prefixes[onto_name]
+    fails = []
+    props, prop_fails = _parse_properties(ontology_json["properties"], current_onto, prefixes)
+    fails.extend(prop_fails)
+    classes, cls_fails = _parse_classes(ontology_json["resources"], current_onto, prefixes)
+    fails.extend(cls_fails)
+    cards, card_fails = _parse_cardinalities(ontology_json["resources"], current_onto, prefixes)
+    fails.extend(card_fails)
+    if fails:
+        return CollectedProblems(
+            f"During the parsing of the ontology '{onto_name}' the following errors occurred", fails
+        )
+    return ParsedOntology(
+        name=onto_name,
+        label=ontology_json["label"],
+        comment=ontology_json.get("comment"),
+        classes=classes,
+        properties=props,
+        cardinalities=cards,
+    )
+
+
+def _parse_properties(
+    properties_list: list[dict[str, Any]],
+    current_onto_prefix: str,
+    prefixes: dict[str, str],
+) -> tuple[list[ParsedProperty], list[CreateProblem]]:
+    problems: list[CreateProblem] = []
+    parsed = []
+    for prop in properties_list:
+        result = _parse_one_property(prop, current_onto_prefix, prefixes)
+        if isinstance(result, ParsedProperty):
+            parsed.append(result)
+        else:
+            problems.extend(result)
+    return parsed, problems
+
+
+def _parse_one_property(
+    prop: dict[str, Any], current_onto_prefix: str, prefixes: dict[str, str]
+) -> ParsedProperty | list[CreateProblem]:
+    problems: list[CreateProblem] = []
+    prop_name = f"{current_onto_prefix}{prop['name']}"
+    labels = prop["labels"]
+    comments = prop.get("comments")
+    object_str = prop["object"]
+    gui_element = GUI_ELEMENT_MAPPER[prop["gui_element"]]
+    subject = prop.get("subject")
+
+    supers = []
+    for super_prop in prop["super"]:
+        if not (resolved := resolve_to_absolute_iri(super_prop, current_onto_prefix, prefixes)):
+            problems.append(
+                InputProblem(
+                    f'At property "{prop["name"]}" / Super: "{super_prop}"',
+                    InputProblemType.PREFIX_COULD_NOT_BE_RESOLVED,
+                )
+            )
+        else:
+            supers.append(resolved)
+
+    if gui_element == GuiElement.SEARCHBOX:
+        if not (obj_iri := resolve_to_absolute_iri(object_str, current_onto_prefix, prefixes)):
+            problems.append(
+                InputProblem(
+                    f'At property "{prop["name"]}" / Object: "{object_str}"',
+                    InputProblemType.PREFIX_COULD_NOT_BE_RESOLVED,
+                )
+            )
+            object_value = ""
+        else:
+            object_value = obj_iri
+    else:
+        object_value = OBJECT_TYPE_MAPPER[object_str]
+
+    if subject:
+        if not (resolved_subject := resolve_to_absolute_iri(prop["subject"], current_onto_prefix, prefixes)):
+            problems.append(
+                InputProblem(
+                    f'At property "{prop["name"]}" / Subject: "{subject}"',
+                    InputProblemType.PREFIX_COULD_NOT_BE_RESOLVED,
+                )
+            )
+        subject = resolved_subject
+
+    list_name = None
+    if object_value == KnoraObjectType.LIST:
+        list_name = prop["gui_attributes"]["hlist"]
+
+    if problems:
+        return problems
+    return ParsedProperty(
+        name=prop_name,
+        labels=labels,
+        comments=comments,
+        supers=supers,
+        object=object_value,
+        subject=subject,
+        gui_element=gui_element,
+        node_name=list_name,
+        onto_iri=current_onto_prefix.rstrip("#"),
+    )
+
+
+def _parse_classes(
+    classes_list: list[dict[str, Any]], current_onto_prefix: str, prefixes: dict[str, str]
+) -> tuple[list[ParsedClass], list[CreateProblem]]:
+    parsed = []
+    problems = []
+    for cls in classes_list:
+        result = _parse_one_class(cls, current_onto_prefix, prefixes)
+        if isinstance(result, ParsedClass):
+            parsed.append(result)
+        else:
+            problems.extend(result)
+    return parsed, problems
+
+
+def _parse_one_class(
+    cls: dict[str, Any], current_onto_prefix: str, prefixes: dict[str, str]
+) -> ParsedClass | list[CreateProblem]:
+    problems: list[CreateProblem] = []
+    supers = [cls["super"]] if isinstance(cls["super"], str) else cls["super"]
+    resolved_supers = []
+    for s in supers:
+        if not (resolved_super := resolve_to_absolute_iri(s, current_onto_prefix, prefixes)):
+            problems.append(
+                InputProblem(
+                    f'At class "{cls["name"]}" / Super: "{s}"',
+                    InputProblemType.PREFIX_COULD_NOT_BE_RESOLVED,
+                )
+            )
+        else:
+            resolved_supers.append(resolved_super)
+    if problems:
+        return problems
+    return ParsedClass(
+        name=f"{current_onto_prefix}{cls['name']}",
+        labels=cls["labels"],
+        comments=cls.get("comments"),
+        supers=resolved_supers,
+        onto_iri=current_onto_prefix.rstrip("#"),
+    )
+
+
+def _parse_cardinalities(
+    classes_list: list[dict[str, Any]], current_onto_prefix: str, prefixes: dict[str, str]
+) -> tuple[list[ParsedClassCardinalities], list[CreateProblem]]:
+    parsed = []
+    failures = []
+    for c in classes_list:
+        if c.get("cardinalities"):
+            result = _parse_one_class_cardinality(c, current_onto_prefix, prefixes)
+            if isinstance(result, ParsedClassCardinalities):
+                parsed.append(result)
+            else:
+                failures.extend(result)
+    return parsed, failures
+
+
+def _parse_one_class_cardinality(
+    cls_json: dict[str, Any], current_onto_prefix: str, prefixes: dict[str, str]
+) -> ParsedClassCardinalities | list[CreateProblem]:
+    failures = []
+    parsed = []
+    for c in cls_json["cardinalities"]:
+        result = _parse_one_cardinality(c, current_onto_prefix, prefixes)
+        if isinstance(result, ParsedPropertyCardinality):
+            parsed.append(result)
+        else:
+            failures.append(result)
+    if failures:
+        return failures
+    cls_iri = f"{current_onto_prefix}{cls_json['name']}"
+    return ParsedClassCardinalities(cls_iri, parsed)
+
+
+def _parse_one_cardinality(
+    card_json: dict[str, str | int], current_onto_prefix: str, prefixes: dict[str, str]
+) -> ParsedPropertyCardinality | CreateProblem:
+    prp_name = cast(str, card_json["propname"])
+    if not (resolved := resolve_to_absolute_iri(prp_name, current_onto_prefix, prefixes)):
+        return CreateProblem(prp_name, InputProblemType.PREFIX_COULD_NOT_BE_RESOLVED)
+    gui = cast(int | None, card_json.get("gui_order"))
+    return ParsedPropertyCardinality(
+        propname=resolved,
+        cardinality=CARDINALITY_MAPPER[cast(str, card_json["cardinality"])],
+        gui_order=gui,
+    )

dsp_tools/commands/create/parsing/parse_project.py

@@ -0,0 +1,149 @@
+import os
+from typing import Any
+
+from dotenv import find_dotenv
+from dotenv import load_dotenv
+
+from dsp_tools.commands.create.models.create_problems import CollectedProblems
+from dsp_tools.commands.create.models.create_problems import CreateProblem
+from dsp_tools.commands.create.models.create_problems import InputProblem
+from dsp_tools.commands.create.models.create_problems import InputProblemType
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+from dsp_tools.commands.create.models.parsed_project import ParsedGroup
+from dsp_tools.commands.create.models.parsed_project import ParsedGroupDescription
+from dsp_tools.commands.create.models.parsed_project import ParsedList
+from dsp_tools.commands.create.models.parsed_project import ParsedPermissions
+from dsp_tools.commands.create.models.parsed_project import ParsedProject
+from dsp_tools.commands.create.models.parsed_project import ParsedProjectMetadata
+from dsp_tools.commands.create.models.parsed_project import ParsedUser
+from dsp_tools.commands.create.models.parsed_project import ParsedUserMemberShipInfo
+from dsp_tools.commands.create.parsing.parse_lists import parse_list_section
+from dsp_tools.commands.create.parsing.parse_ontology import parse_ontology
+from dsp_tools.commands.create.parsing.parsing_utils import create_prefix_lookup
+
+# Load .env file only if it exists in the current working directory
+# This allows CI to set environment variables directly without interference
+dotenv_file = find_dotenv(usecwd=True)
+if dotenv_file:
+    load_dotenv(dotenv_path=dotenv_file, override=False)
+
+
+def parse_project(complete_json: dict[str, Any], api_url: str) -> ParsedProject | list[CollectedProblems]:
+    prefix_lookup = create_prefix_lookup(complete_json, api_url)
+    project_json = complete_json["project"]
+    ontologies, failures = _parse_all_ontologies(project_json, prefix_lookup)
+    parsed_lists, list_problems = _parse_lists(project_json)
+    if list_problems:
+        failures.append(list_problems)
+    users, memberships, user_problems = _parse_users(project_json)
+    if user_problems:
+        failures.append(user_problems)
+    if failures:
+        return failures
+    return ParsedProject(
+        prefixes=prefix_lookup,
+        project_metadata=_parse_metadata(project_json),
+        permissions=_parse_permissions(project_json),
+        groups=_parse_groups(project_json),
+        users=users,
+        user_memberships=memberships,
+        lists=parsed_lists,
+        ontologies=ontologies,
+    )
+
+
+def _parse_metadata(project_json: dict[str, Any]) -> ParsedProjectMetadata:
+    return ParsedProjectMetadata(
+        shortcode=project_json["shortcode"],
+        shortname=project_json["shortname"],
+        longname=project_json["longname"],
+        descriptions=project_json["descriptions"],
+        keywords=project_json["keywords"],
+        enabled_licenses=project_json.get("enabled_licenses", []),
+    )
+
+
+def _parse_permissions(project_json: dict[str, Any]) -> ParsedPermissions:
+    return ParsedPermissions(
+        default_permissions=project_json["default_permissions"],
+        default_permissions_overrule=project_json.get("default_permissions_overrule"),
+    )
+
+
+def _parse_groups(project_json: dict[str, Any]) -> list[ParsedGroup]:
+    if not (found := project_json.get("groups")):
+        return []
+    return [_parse_one_group(x) for x in found]
+
+
+def _parse_one_group(group_dict: dict[str, Any]) -> ParsedGroup:
+    descriptions = [ParsedGroupDescription(lang=lang, text=text) for lang, text in group_dict["descriptions"].items()]
+    return ParsedGroup(name=group_dict["name"], descriptions=descriptions)
+
+
+def _parse_users(
+    project_json: dict[str, Any],
+) -> tuple[list[ParsedUser], list[ParsedUserMemberShipInfo], CollectedProblems | None]:
+    if not (found := project_json.get("users")):
+        return [], [], None
+    users, memberships = [], []
+    input_problems: list[CreateProblem] = []
+    for u in found:
+        result = _parse_one_user(u)
+        if isinstance(result, InputProblem):
+            input_problems.append(result)
+        else:
+            usr, mbmr = result
+            users.append(usr)
+            memberships.append(mbmr)
+    if input_problems:
+        problem_collection = CollectedProblems(
+            "When parsing the user section the following problems occurred:", input_problems
+        )
+    else:
+        problem_collection = None
+    return users, memberships, problem_collection
+
+
+def _parse_one_user(user_dict: dict[str, Any]) -> tuple[ParsedUser, ParsedUserMemberShipInfo] | InputProblem:
+    projects = user_dict.get("projects", [])
+    is_admin = ":admin" in projects
+    groups = [g.removeprefix(":") for g in user_dict.get("groups", [])]
+    pw = user_dict["password"]
+    if not pw:
+        pw = os.getenv("DSP_USER_PASSWORD")
+        if not pw:
+            return InputProblem(user_dict["username"], InputProblemType.USER_PASSWORD_NOT_SET)
+    usr = ParsedUser(
+        username=user_dict["username"],
+        email=user_dict["email"],
+        given_name=user_dict["givenName"],
+        family_name=user_dict["familyName"],
+        password=pw,
+        lang=user_dict.get("lang", "en"),
+    )
+    memberships = ParsedUserMemberShipInfo(user_dict["username"], is_admin, groups)
+    return usr, memberships
+
+
+def _parse_lists(project_json: dict[str, Any]) -> tuple[list[ParsedList], CollectedProblems | None]:
+    if not (found := project_json.get("lists")):
+        return [], None
+    result = parse_list_section(found)
+    if isinstance(result, CollectedProblems):
+        return [], result
+    return result, None
+
+
+def _parse_all_ontologies(
+    project_json: dict[str, Any], prefix_lookup: dict[str, str]
+) -> tuple[list[ParsedOntology], list[CollectedProblems]]:
+    ontos = []
+    failures = []
+    for o in project_json["ontologies"]:
+        result = parse_ontology(o, prefix_lookup)
+        if isinstance(result, ParsedOntology):
+            ontos.append(result)
+        else:
+            failures.append(result)
+    return ontos, failures

dsp_tools/commands/create/parsing/parsing_utils.py

@@ -0,0 +1,40 @@
+from typing import Any
+
+import regex
+
+from dsp_tools.utils.data_formats.iri_util import make_dsp_ontology_prefix
+from dsp_tools.utils.data_formats.uri_util import is_uri
+from dsp_tools.utils.rdf_constants import DSP_NAME_TO_PREFIX
+from dsp_tools.utils.rdf_constants import KNORA_API_PREFIX
+
+
+def resolve_to_absolute_iri(prefixed: str, current_onto: str, prefix_lookup: dict[str, str]) -> str | None:
+    if is_uri(prefixed):
+        return prefixed
+    if prefixed.startswith(":"):
+        return f"{current_onto}{prefixed.lstrip(':')}"
+    segments = prefixed.split(":", maxsplit=1)
+    if len(segments) == 1:
+        return f"{KNORA_API_PREFIX}{segments[0]}"
+    if not (found := prefix_lookup.get(segments[0])):
+        return None
+    return f"{found}{segments[1]}"
+
+
+def create_prefix_lookup(project_json: dict[str, Any], api_url: str) -> dict[str, str]:
+    defined_prefixes = project_json.get("prefixes", {})
+    defined_prefixes = defined_prefixes | DSP_NAME_TO_PREFIX
+    defined_prefixes = _correct_external_prefix(defined_prefixes)
+    shortcode = project_json["project"]["shortcode"]
+    for onto in project_json["project"]["ontologies"]:
+        onto_name = onto["name"]
+        defined_prefixes[onto_name] = make_dsp_ontology_prefix(api_url, shortcode, onto_name)
+    return defined_prefixes
+
+
+def _correct_external_prefix(prefixes: dict[str, str]) -> dict[str, str]:
+    for prfx, namespace in prefixes.items():
+        if regex.search(r"(#|\/)$", namespace):
+            continue
+        prefixes[prfx] = f"{namespace}/"
+    return prefixes