dsp-tools 17.0.0.post10__tar.gz → 17.0.0.post14__tar.gz

{dsp_tools-17.0.0.post10 → dsp_tools-17.0.0.post14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dsp-tools
-Version: 17.0.0.post10
+Version: 17.0.0.post14
 Summary: DSP-TOOLS is a Python package with a command line interface that helps you interact with a DaSCH service platform (DSP) server.
 Author: DaSCH - Swiss National Data and Service Center for the Humanities
 Author-email: DaSCH - Swiss National Data and Service Center for the Humanities <info@dasch.swiss>

{dsp_tools-17.0.0.post10 → dsp_tools-17.0.0.post14}/pyproject.toml

@@ -2,7 +2,7 @@
 
 [project]
 name = "dsp-tools"
-version = "17.0.0.post10"
+version = "17.0.0.post14"
 description = "DSP-TOOLS is a Python package with a command line interface that helps you interact with a DaSCH service platform (DSP) server."
 authors = [
     { name = "DaSCH - Swiss National Data and Service Center for the Humanities", email = "info@dasch.swiss" },
@@ -203,5 +203,6 @@ exclude = [
     "src/dsp_tools/error/xmllib_warnings_util.py",
     "src/dsp_tools/commands/excel2xml/",
     "test/",
+    "src/dsp_tools/commands/create/", # to be ignored to enable small steps untill the intensive work is completed
 ]
 sort_by_size = true  # sort unused classes and functions by size 

{dsp_tools-17.0.0.post10 → dsp_tools-17.0.0.post14}/src/dsp_tools/cli/call_action.py

@@ -2,6 +2,7 @@ import argparse
 import subprocess
 from pathlib import Path
 
+import requests
 from loguru import logger
 
 from dsp_tools.cli.args import ServerCredentials
@@ -27,9 +28,13 @@ from dsp_tools.commands.start_stack import StackHandler
 from dsp_tools.commands.validate_data.validate_data import validate_data
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
 from dsp_tools.commands.xmlupload.xmlupload import xmlupload
+from dsp_tools.error.exceptions import DockerNotReachableError
+from dsp_tools.error.exceptions import DspApiNotReachableError
 from dsp_tools.error.exceptions import InputError
 from dsp_tools.utils.xml_parsing.parse_clean_validate_xml import parse_and_validate_xml_file
 
+LOCALHOST_API = "http://0.0.0.0:3333"
+
 
 def call_requested_action(args: argparse.Namespace) -> bool:  # noqa: PLR0912 (too many branches)
     """
@@ -41,20 +46,36 @@ def call_requested_action(args: argparse.Namespace) -> bool:  # noqa: PLR0912 (t
     Raises:
         BaseError: from the called function
         InputError: from the called function
+        DockerNotReachableError: from the called function
+        LocalDspApiNotReachableError: from the called function
         unexpected errors from the called methods and underlying libraries
 
     Returns:
         success status
     """
     match args.action:
+        # commands with Docker / API interactions
+        case "start-stack":
+            result = _call_start_stack(args)
+        case "stop-stack":
+            result = _call_stop_stack()
         case "create":
             result = _call_create(args)
-        case "xmlupload":
-            result = _call_xmlupload(args)
+        case "get":
+            result = _call_get(args)
         case "validate-data":
             result = _call_validate_data(args)
+        case "xmlupload":
+            result = _call_xmlupload(args)
         case "resume-xmlupload":
             result = _call_resume_xmlupload(args)
+        case "upload-files":
+            result = _call_upload_files(args)
+        case "ingest-files":
+            result = _call_ingest_files(args)
+        case "ingest-xmlupload":
+            result = _call_ingest_xmlupload(args)
+        # commands that do not require docker
         case "excel2json":
             result = _call_excel2json(args)
         case "old-excel2json":
@@ -69,18 +90,6 @@ def call_requested_action(args: argparse.Namespace) -> bool:  # noqa: PLR0912 (t
             result = _call_excel2properties(args)
         case "id2iri":
             result = _call_id2iri(args)
-        case "start-stack":
-            result = _call_start_stack(args)
-        case "stop-stack":
-            result = _call_stop_stack()
-        case "get":
-            result = _call_get(args)
-        case "upload-files":
-            result = _call_upload_files(args)
-        case "ingest-files":
-            result = _call_ingest_files(args)
-        case "ingest-xmlupload":
-            result = _call_ingest_xmlupload(args)
         case _:
             print(f"ERROR: Unknown action '{args.action}'")
             logger.error(f"Unknown action '{args.action}'")
@@ -89,6 +98,7 @@ def call_requested_action(args: argparse.Namespace) -> bool:  # noqa: PLR0912 (t
 
 
 def _call_stop_stack() -> bool:
+    _check_docker_health()
     stack_handler = StackHandler(StackConfiguration())
     return stack_handler.stop_stack()
 
@@ -164,7 +174,7 @@ def _call_old_excel2json(args: argparse.Namespace) -> bool:
 
 
 def _call_upload_files(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    _check_health_with_docker_on_localhost(args.server)
     return upload_files(
         xml_file=Path(args.xml_file),
         creds=_get_creds(args),
@@ -173,12 +183,12 @@ def _call_upload_files(args: argparse.Namespace) -> bool:
 
 
 def _call_ingest_files(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    _check_health_with_docker_on_localhost(args.server)
     return ingest_files(creds=_get_creds(args), shortcode=args.shortcode)
 
 
 def _call_ingest_xmlupload(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    _check_health_with_docker(args.server)
     interrupt_after = args.interrupt_after if args.interrupt_after > 0 else None
     return ingest_xmlupload(
         xml_file=Path(args.xml_file),
@@ -191,7 +201,7 @@ def _call_ingest_xmlupload(args: argparse.Namespace) -> bool:
 
 
 def _call_xmlupload(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    _check_health_with_docker(args.server)
     if args.validate_only:
         success = parse_and_validate_xml_file(Path(args.xmlfile))
         print("The XML file is syntactically correct.")
@@ -227,7 +237,7 @@ def _call_xmlupload(args: argparse.Namespace) -> bool:
 
 
 def _call_validate_data(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    _check_health_with_docker(args.server)
     return validate_data(
         filepath=Path(args.xmlfile),
         creds=_get_creds(args),
@@ -239,7 +249,8 @@ def _call_validate_data(args: argparse.Namespace) -> bool:
 
 
 def _call_resume_xmlupload(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    # this does not need docker if not on localhost, as does not need to validate
+    _check_health_with_docker_on_localhost(args.server)
     return resume_xmlupload(
         creds=_get_creds(args),
         skip_first_resource=args.skip_first_resource,
@@ -247,7 +258,7 @@ def _call_resume_xmlupload(args: argparse.Namespace) -> bool:
 
 
 def _call_get(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    _check_health_with_docker_on_localhost(args.server)
     return get_project(
         project_identifier=args.project,
         outfile_path=args.project_definition,
@@ -257,7 +268,7 @@ def _call_get(args: argparse.Namespace) -> bool:
 
 
 def _call_create(args: argparse.Namespace) -> bool:
-    _check_docker_health_if_on_localhost(args.server)
+    _check_health_with_docker_on_localhost(args.server)
     success = False
     match args.lists_only, args.validate_only:
         case True, True:
@@ -289,11 +300,43 @@ def _get_creds(args: argparse.Namespace) -> ServerCredentials:
     )
 
 
-def _check_docker_health_if_on_localhost(api_url: str) -> None:
-    if api_url == "http://0.0.0.0:3333":
+def _check_health_with_docker_on_localhost(api_url: str) -> None:
+    if api_url == LOCALHOST_API:
         _check_docker_health()
+    _check_api_health(api_url)
+
+
+def _check_health_with_docker(api_url: str) -> None:
+    # validate always needs docker running
+    _check_docker_health()
+    _check_api_health(api_url)
 
 
 def _check_docker_health() -> None:
     if subprocess.run("docker stats --no-stream".split(), check=False, capture_output=True).returncode != 0:
-        raise InputError("Docker is not running properly. Please start Docker and try again.")
+        raise DockerNotReachableError()
+
+
+def _check_api_health(api_url: str) -> None:
+    health_url = f"{api_url}/health"
+    msg = (
+        "The DSP-API could not be reached. Please check if your stack is healthy "
+        "or start a stack with 'dsp-tools start-stack' if none is running."
+    )
+    try:
+        response = requests.get(health_url, timeout=2)
+        if not response.ok:
+            if api_url != LOCALHOST_API:
+                msg = (
+                    f"The DSP-API could not be reached (returned status {response.status_code}). "
+                    f"Please contact the DaSCH engineering team for help."
+                )
+            logger.error(msg)
+            raise DspApiNotReachableError(msg)
+        logger.debug(f"DSP API health check passed: {health_url}")
+    except requests.exceptions.RequestException as e:
+        logger.error(e)
+        if api_url != LOCALHOST_API:
+            msg = "The DSP-API responded with a request exception. Please contact the DaSCH engineering team for help."
+        logger.error(msg)
+        raise DspApiNotReachableError(msg) from None

dsp_tools-17.0.0.post14/src/dsp_tools/commands/create/constants.py

@@ -0,0 +1,4 @@
+KNORA_API = "http://api.knora.org/ontology/knora-api/v2#"
+SALSAH_GUI = "http://api.knora.org/ontology/salsah-gui/v2#"
+
+UNIVERSAL_PREFIXES = {"knora-api": KNORA_API, "salsah-gui": SALSAH_GUI}

dsp_tools-17.0.0.post14/src/dsp_tools/commands/create/models/input_problems.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+@dataclass
+class CollectedProblems:
+    header: str
+    problems: list[InputProblem]
+
+
+@dataclass
+class InputProblem:
+    problematic_object: str
+    problem: ProblemType
+
+
+class ProblemType(StrEnum):
+    PREFIX_COULD_NOT_BE_RESOLVED = (
+        "The prefix used is not defined in the 'prefix' section of the file, "
+        "nor does it belong to one of the project ontologies."
+    )

dsp_tools-17.0.0.post14/src/dsp_tools/commands/create/models/parsed_ontology.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from enum import auto
+from typing import Any
+
+
+@dataclass
+class ParsedOntology:
+    name: str
+    label: str
+    comment: str | None
+    classes: list[ParsedClass]
+    properties: list[ParsedProperty]
+    cardinalities: list[ParsedClassCardinalities]
+
+
+@dataclass
+class ParsedClass:
+    name: str
+    info: dict[str, Any]
+
+
+@dataclass
+class ParsedProperty:
+    name: str
+    info: dict[str, Any]
+
+
+@dataclass
+class ParsedClassCardinalities:
+    class_iri: str
+    cards: list[ParsedPropertyCardinality]
+
+
+@dataclass
+class ParsedPropertyCardinality:
+    propname: str
+    cardinality: Cardinality
+    gui_order: int | None
+
+
+class Cardinality(Enum):
+    C_0_1 = auto()
+    C_1 = auto()
+    C_0_N = auto()
+    C_1_N = auto()

dsp_tools-17.0.0.post14/src/dsp_tools/commands/create/models/parsed_project.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+
+
+@dataclass
+class ParsedProject:
+    prefixes: dict[str, str]
+    project_metadata: ParsedProjectMetadata
+    permissions: ParsedPermissions
+    groups: list[ParsedGroup]
+    users: list[ParsedUser]
+    lists: list[ParsedList]
+    ontologies: list[ParsedOntology]
+
+
+@dataclass
+class ParsedProjectMetadata:
+    shortcode: str
+    shortname: str
+    longname: str
+    descriptions: dict[str, str]
+    keywords: list[str]
+    enabled_licenses: list[str]
+
+
+@dataclass
+class ParsedPermissions:
+    default_permissions: str
+    default_permissions_overrule: dict[str, Any] | None
+
+
+@dataclass
+class ParsedGroup:
+    info: dict[str, Any]
+
+
+@dataclass
+class ParsedUser:
+    info: dict[str, Any]
+
+
+@dataclass
+class ParsedList:
+    name: str
+    info: dict[str, Any]

dsp_tools-17.0.0.post14/src/dsp_tools/commands/create/parsing/parse_ontology.py

@@ -0,0 +1,108 @@
+from typing import Any
+from typing import cast
+
+from dsp_tools.commands.create.models.input_problems import CollectedProblems
+from dsp_tools.commands.create.models.input_problems import InputProblem
+from dsp_tools.commands.create.models.input_problems import ProblemType
+from dsp_tools.commands.create.models.parsed_ontology import Cardinality
+from dsp_tools.commands.create.models.parsed_ontology import ParsedClass
+from dsp_tools.commands.create.models.parsed_ontology import ParsedClassCardinalities
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+from dsp_tools.commands.create.models.parsed_ontology import ParsedProperty
+from dsp_tools.commands.create.models.parsed_ontology import ParsedPropertyCardinality
+from dsp_tools.commands.create.parsing.parsing_utils import resolve_to_absolute_iri
+
+CARDINALITY_MAPPER = {
+    "0-1": Cardinality.C_0_1,
+    "1": Cardinality.C_1,
+    "0-n": Cardinality.C_0_N,
+    "1-n": Cardinality.C_1_N,
+}
+
+
+def parse_ontology(ontology_json: dict[str, Any], prefixes: dict[str, str]) -> ParsedOntology | CollectedProblems:
+    onto_name = ontology_json["name"]
+    current_onto = prefixes[onto_name]
+    fails = []
+    props, prop_fails = _parse_properties(ontology_json["properties"], current_onto)
+    fails.extend(prop_fails)
+    classes, cls_fails = _parse_classes(ontology_json["resources"], current_onto)
+    fails.extend(cls_fails)
+    cards, card_fails = _parse_cardinalities(ontology_json["resources"], current_onto, prefixes)
+    fails.extend(card_fails)
+    if fails:
+        return CollectedProblems(
+            f"During the parsing of the ontology '{onto_name}' the following errors occurred", fails
+        )
+    return ParsedOntology(
+        name=onto_name,
+        label=ontology_json["label"],
+        comment=ontology_json.get("comment"),
+        classes=classes,
+        properties=props,
+        cardinalities=cards,
+    )
+
+
+def _parse_properties(
+    properties_list: list[dict[str, Any]], current_onto_prefix: str
+) -> tuple[list[ParsedProperty], list[InputProblem]]:
+    parsed = []
+    for prop in properties_list:
+        parsed.append(ParsedProperty(f"{current_onto_prefix}{prop['name']}", prop))
+    return parsed, []
+
+
+def _parse_classes(
+    classes_list: list[dict[str, Any]], current_onto_prefix: str
+) -> tuple[list[ParsedClass], list[InputProblem]]:
+    parsed = []
+    for cls in classes_list:
+        parsed.append(ParsedClass(f"{current_onto_prefix}{cls['name']}", cls))
+    return parsed, []
+
+
+def _parse_cardinalities(
+    classes_list: list[dict[str, Any]], current_onto_prefix: str, prefixes: dict[str, str]
+) -> tuple[list[ParsedClassCardinalities], list[InputProblem]]:
+    parsed = []
+    failures = []
+    for c in classes_list:
+        if c.get("cardinalities"):
+            result = _parse_one_class_cardinality(c, current_onto_prefix, prefixes)
+            if isinstance(result, ParsedClassCardinalities):
+                parsed.append(result)
+            else:
+                failures.extend(result)
+    return parsed, failures
+
+
+def _parse_one_class_cardinality(
+    cls_json: dict[str, Any], current_onto_prefix: str, prefixes: dict[str, str]
+) -> ParsedClassCardinalities | list[InputProblem]:
+    failures = []
+    parsed = []
+    for c in cls_json["cardinalities"]:
+        result = _parse_one_cardinality(c, current_onto_prefix, prefixes)
+        if isinstance(result, ParsedPropertyCardinality):
+            parsed.append(result)
+        else:
+            failures.append(result)
+    if failures:
+        return failures
+    cls_iri = f"{current_onto_prefix}{cls_json['name']}"
+    return ParsedClassCardinalities(cls_iri, parsed)
+
+
+def _parse_one_cardinality(
+    card_json: dict[str, str | int], current_onto_prefix: str, prefixes: dict[str, str]
+) -> ParsedPropertyCardinality | InputProblem:
+    prp_name = cast(str, card_json["propname"])
+    if not (resolved := resolve_to_absolute_iri(prp_name, current_onto_prefix, prefixes)):
+        return InputProblem(prp_name, ProblemType.PREFIX_COULD_NOT_BE_RESOLVED)
+    gui = cast(int | None, card_json.get("gui_order"))
+    return ParsedPropertyCardinality(
+        propname=resolved,
+        cardinality=CARDINALITY_MAPPER[cast(str, card_json["cardinality"])],
+        gui_order=gui,
+    )

dsp_tools-17.0.0.post14/src/dsp_tools/commands/create/parsing/parse_project.py

@@ -0,0 +1,99 @@
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from dsp_tools.commands.create.models.input_problems import CollectedProblems
+from dsp_tools.commands.create.models.parsed_ontology import ParsedOntology
+from dsp_tools.commands.create.models.parsed_project import ParsedGroup
+from dsp_tools.commands.create.models.parsed_project import ParsedList
+from dsp_tools.commands.create.models.parsed_project import ParsedPermissions
+from dsp_tools.commands.create.models.parsed_project import ParsedProject
+from dsp_tools.commands.create.models.parsed_project import ParsedProjectMetadata
+from dsp_tools.commands.create.models.parsed_project import ParsedUser
+from dsp_tools.commands.create.parsing.parse_ontology import parse_ontology
+from dsp_tools.commands.create.parsing.parsing_utils import create_prefix_lookup
+from dsp_tools.commands.project.create.project_validate import validate_project
+from dsp_tools.utils.json_parsing import parse_json_input
+
+
+def parse_project(
+    project_file_as_path_or_parsed: str | Path | dict[str, Any], api_url: str
+) -> ParsedProject | list[CollectedProblems]:
+    complete_json = _parse_and_validate(project_file_as_path_or_parsed)
+    return _parse_project(complete_json, api_url)
+
+
+def _parse_project(complete_json: dict[str, Any], api_url: str) -> ParsedProject | list[CollectedProblems]:
+    prefix_lookup = create_prefix_lookup(complete_json, api_url)
+    project_json = complete_json["project"]
+    ontologies, failures = _parse_all_ontologies(project_json, prefix_lookup)
+    if failures:
+        return failures
+    return ParsedProject(
+        prefixes=prefix_lookup,
+        project_metadata=_parse_metadata(project_json),
+        permissions=_parse_permissions(project_json),
+        groups=_parse_groups(project_json),
+        users=_parse_users(project_json),
+        lists=_parse_lists(project_json),
+        ontologies=ontologies,
+    )
+
+
+def _parse_and_validate(project_file_as_path_or_parsed: str | Path | dict[str, Any]) -> dict[str, Any]:
+    project_json = parse_json_input(project_file_as_path_or_parsed=project_file_as_path_or_parsed)
+    validate_project(project_json)
+    print("    JSON project file is syntactically correct and passed validation.")
+    logger.info("JSON project file is syntactically correct and passed validation.")
+    return project_json
+
+
+def _parse_metadata(project_json: dict[str, Any]) -> ParsedProjectMetadata:
+    return ParsedProjectMetadata(
+        shortcode=project_json["shortcode"],
+        shortname=project_json["shortname"],
+        longname=project_json["longname"],
+        descriptions=project_json["descriptions"],
+        keywords=project_json["keywords"],
+        enabled_licenses=project_json["enabled_licenses"],
+    )
+
+
+def _parse_permissions(project_json: dict[str, Any]) -> ParsedPermissions:
+    return ParsedPermissions(
+        default_permissions=project_json["default_permissions"],
+        default_permissions_overrule=project_json.get("default_permissions_overrule"),
+    )
+
+
+def _parse_groups(project_json: dict[str, Any]) -> list[ParsedGroup]:
+    if not (found := project_json.get("groups")):
+        return []
+    return [ParsedGroup(x) for x in found]
+
+
+def _parse_users(project_json: dict[str, Any]) -> list[ParsedUser]:
+    if not (found := project_json.get("users")):
+        return []
+    return [ParsedUser(x) for x in found]
+
+
+def _parse_lists(project_json: dict[str, Any]) -> list[ParsedList]:
+    if not (found := project_json.get("lists")):
+        return []
+    return [ParsedList(x["name"], x) for x in found]
+
+
+def _parse_all_ontologies(
+    project_json: dict[str, Any], prefix_lookup: dict[str, str]
+) -> tuple[list[ParsedOntology], list[CollectedProblems]]:
+    ontos = []
+    failures = []
+    for o in project_json["ontologies"]:
+        result = parse_ontology(o, prefix_lookup)
+        if isinstance(result, ParsedOntology):
+            ontos.append(result)
+        else:
+            failures.append(result)
+    return ontos, failures

dsp_tools-17.0.0.post14/src/dsp_tools/commands/create/parsing/parsing_utils.py

@@ -0,0 +1,43 @@
+from typing import Any
+
+import regex
+
+from dsp_tools.commands.create.constants import KNORA_API
+from dsp_tools.commands.create.constants import UNIVERSAL_PREFIXES
+from dsp_tools.utils.data_formats.uri_util import is_uri
+
+
+def resolve_to_absolute_iri(prefixed: str, current_onto: str, prefix_lookup: dict[str, str]) -> str | None:
+    if is_uri(prefixed):
+        return prefixed
+    if prefixed.startswith(":"):
+        return f"{current_onto}{prefixed.lstrip(':')}"
+    segments = prefixed.split(":", maxsplit=1)
+    if len(segments) == 1:
+        return f"{KNORA_API}{segments[0]}"
+    if not (found := prefix_lookup.get(segments[0])):
+        return None
+    return f"{found}{segments[1]}"
+
+
+def create_prefix_lookup(project_json: dict[str, Any], api_url: str) -> dict[str, str]:
+    defined_prefixes = project_json.get("prefixes", {})
+    defined_prefixes = defined_prefixes | UNIVERSAL_PREFIXES
+    defined_prefixes = _correct_external_prefix(defined_prefixes)
+    shortcode = project_json["project"]["shortcode"]
+    for onto in project_json["project"]["ontologies"]:
+        onto_name = onto["name"]
+        defined_prefixes[onto_name] = _make_dsp_ontology_prefix(api_url, shortcode, onto_name)
+    return defined_prefixes
+
+
+def _correct_external_prefix(prefixes: dict[str, str]) -> dict[str, str]:
+    for prfx, namespace in prefixes.items():
+        if regex.search(r"(#|\/)$", namespace):
+            continue
+        prefixes[prfx] = f"{namespace}/"
+    return prefixes
+
+
+def _make_dsp_ontology_prefix(api_url: str, shortcode: str, onto_name: str) -> str:
+    return f"{api_url}/ontology/{shortcode}/{onto_name}/v2#"

{dsp_tools-17.0.0.post10 → dsp_tools-17.0.0.post14}/src/dsp_tools/error/exceptions.py

@@ -45,6 +45,18 @@ class InternalError(BaseError):
                 super().__init__(default_msg)
 
 
+class DockerNotReachableError(BaseError):
+    """This error is raised when docker is not running."""
+
+    def __init__(self) -> None:
+        msg = "Docker is not running properly. Please start Docker and try again."
+        super().__init__(msg)
+
+
+class DspApiNotReachableError(BaseError):
+    """This error is raised when the DSP-API could not be reached on localhost."""
+
+
 class InputError(BaseError):
     """This error is raised when the user input is invalid. The message should be as user-friendly as possible."""