dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/resources/validate_data/shacl-cli-image.yml

@@ -0,0 +1,4 @@
+---
+# Take the latest release from:
+# https://github.com/dasch-swiss/shacl-cli/releases
+image: "daschswiss/shacl-cli:v0.0.5"

dsp_tools/resources/validate_data/validate-ontology.ttl

@@ -0,0 +1,99 @@
+@prefix sh:         <http://www.w3.org/ns/shacl#> .
+@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> .
+@prefix owl:        <http://www.w3.org/2002/07/owl#> .
+@prefix knora-api:  <http://api.knora.org/ontology/knora-api/v2#> .
+
+@prefix api-shapes: <http://api.knora.org/ontology/knora-api/shapes/v2#> .
+
+
+api-shapes:FindMissingMaxCardinality_Seqnum_OntologyShape
+  a              sh:NodeShape ;
+  sh:targetClass owl:Class ;
+  sh:sparql      [
+                   a          sh:SPARQLConstraint ;
+                   sh:select  """
+                    PREFIX owl: <http://www.w3.org/2002/07/owl#>
+                    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+                    PREFIX knora-api: <http://api.knora.org/ontology/knora-api/v2#>
+
+                    SELECT $this ?path WHERE {
+                        $this a owl:Class ;
+                              rdfs:subClassOf ?restriction .
+                        ?restriction a owl:Restriction ;
+                                     owl:onProperty ?path .
+
+                        ?path rdfs:subPropertyOf* knora-api:seqnum .
+
+                        FILTER NOT EXISTS {
+                            ?restriction owl:maxCardinality|owl:cardinality 1
+                        }
+                    }
+                    """ ;
+                   sh:message "seqnum must either have cardinality 1 or 0-1."
+                 ] ;
+  sh:severity    sh:Violation .
+
+
+api-shapes:FindMissingSeqnum_OntologyShape
+  a              sh:NodeShape ;
+  sh:targetClass owl:Class ;
+  sh:sparql      [
+                   a          sh:SPARQLConstraint ;
+                   sh:select  """
+                    PREFIX owl: <http://www.w3.org/2002/07/owl#>
+                    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+                    PREFIX knora-api: <http://api.knora.org/ontology/knora-api/v2#>
+
+                    SELECT $this ?path WHERE {
+
+                        $this a owl:Class ;
+                              rdfs:subClassOf ?restrictionIsPartOf .
+                        ?restrictionIsPartOf a owl:Restriction ;
+                                             owl:onProperty ?path .
+                        ?path rdfs:subPropertyOf* knora-api:isPartOf .
+
+                        FILTER NOT EXISTS {
+                            $this a owl:Class ;
+                                  rdfs:subClassOf ?restrictionSeqnum .
+                            ?restrictionSeqnum a owl:Restriction ;
+                                               owl:onProperty ?seqnumProp .
+                            ?seqnumProp rdfs:subPropertyOf* knora-api:seqnum .
+                        }
+                    }
+                    """ ;
+                   sh:message "A class with a cardinality for isPartOf also requires a cardinality for seqnum."
+                 ] ;
+  sh:severity    sh:Violation .
+
+
+api-shapes:FindMissingIsPartOf_OntologyShape
+  a              sh:NodeShape ;
+  sh:targetClass owl:Class ;
+  sh:sparql      [
+                   a          sh:SPARQLConstraint ;
+                   sh:select  """
+                    PREFIX owl: <http://www.w3.org/2002/07/owl#>
+                    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+                    PREFIX knora-api: <http://api.knora.org/ontology/knora-api/v2#>
+
+                    SELECT $this ?path WHERE {
+
+                        $this a owl:Class ;
+                              rdfs:subClassOf ?restrictionSeqnum .
+                        ?restrictionSeqnum a owl:Restriction ;
+                                           owl:onProperty ?path .
+                        ?path rdfs:subPropertyOf* knora-api:seqnum .
+
+                        FILTER NOT EXISTS {
+                            $this a owl:Class ;
+                                  rdfs:subClassOf ?restrictionIsPartOf .
+                            ?restrictionIsPartOf a owl:Restriction ;
+                                                 owl:onProperty ?isPartOfProp .
+                            ?isPartOfProp rdfs:subPropertyOf* knora-api:isPartOf .
+                        }
+                    }
+                    """ ;
+                   sh:message "A class with a cardinality for seqnum also requires a cardinality for isPartOf."
+                 ] ;
+  sh:severity    sh:Violation .

dsp_tools/utils/ansi_colors.py

@@ -0,0 +1,32 @@
+# These are ANSI escape codes which can be used to configure the print output on the terminal
+# http://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html
+# The semicolon separates different configurations
+
+# All codes must start with an escape sequence, it may differ in different languages, in Python "\u001b[" works
+SEQUENCE_START = "\u001b["
+
+# the "m" at the end signals,
+# that the configuration code is finished and after that the string that should be printed starts
+SEQUENCE_END = "m"
+
+# reset to the default setting of the console
+RESET_TO_DEFAULT = f"{SEQUENCE_START}0{SEQUENCE_END}"
+
+BOLD = f"{SEQUENCE_START}1{SEQUENCE_END}"  # 1 (bold) ; 32 (green)
+
+# If you want to change for example both the text color and the background color you can combine the sequences
+# for example: BACKGROUND_BOLD_MAGENTA + YELLOW -> magenta background with yellow text
+
+# Colored Text
+BOLD_GREEN = f"{SEQUENCE_START}1;32{SEQUENCE_END}"  # 1 (bold) ; 32 (green)
+BOLD_RED = f"{SEQUENCE_START}1;31{SEQUENCE_END}"  # 1 (bold) ; 31 (red)
+BOLD_CYAN = f"{SEQUENCE_START}1;36{SEQUENCE_END}"  # 1 (bold) ; 36 (cyan)
+BOLD_YELLOW = f"{SEQUENCE_START}1;33{SEQUENCE_END}"  # 1 (bold) ; 33 (yellow)
+YELLOW = f"{SEQUENCE_START}0;33{SEQUENCE_END}"  # 0 (normal font) ; 33 (yellow)
+RED = f"{SEQUENCE_START}0;31{SEQUENCE_END}"  # 0 (normal font) ; 31 (red)
+
+# Colored Background
+BACKGROUND_BOLD_RED = f"{SEQUENCE_START}1;41{SEQUENCE_END}"  # 1 (bold) ; 41 (background red)
+BACKGROUND_BOLD_YELLOW = f"{SEQUENCE_START}1;43{SEQUENCE_END}"  # 1 (bold) ; 43 (background yellow)
+BACKGROUND_BOLD_GREEN = f"{SEQUENCE_START}1;42{SEQUENCE_END}"  # 1 (bold) ; 42 (background green)
+BACKGROUND_BOLD_CYAN = f"{SEQUENCE_START}1;46{SEQUENCE_END}"  # 1 (bold) ; 46 (background cyan)

dsp_tools/utils/{date_util.py → data_formats/date_util.py}

@@ -6,7 +6,7 @@ from typing import cast
 
 import regex
 
-from dsp_tools.models.exceptions import BaseError
+from dsp_tools.error.exceptions import BaseError
 
 _calendar = r"GREGORIAN|JULIAN|ISLAMIC"
 _era = r"CE|BCE|BC|AD"
@@ -70,6 +70,18 @@ class Era(Enum):
                 raise BaseError(f"Invalid era type: {s}")
 
 
+class DayMonthYearEra(Enum):
+    DAY = "Day"
+    MONTH = "Month"
+    YEAR = "Year"
+    ERA = "Era"
+
+
+class StartEnd(Enum):
+    START = "Start"
+    END = "End"
+
+
 @dataclass(frozen=True)
 class SingleDate:
     """Information about a single date."""

dsp_tools/utils/data_formats/iri_util.py

@@ -0,0 +1,30 @@
+import regex
+
+_iri_pattern = r"^(http)s?://[\w\.\-~]*(:\d{,6})?(/[\w\-~]+)*(#[\w\-~]*)?"
+_resource_iri_pattern = r"https?://rdfh.ch/[a-fA-F0-9]{4}/[\w-]{22}"
+
+
+def is_iri(s: str) -> bool:
+    """Checks whether a string is a valid IRI."""
+    return regex.fullmatch(_iri_pattern, s) is not None
+
+
+def is_resource_iri(s: str) -> bool:
+    """Checks whether a string is a valid resource IRI."""
+    return regex.fullmatch(_resource_iri_pattern, s) is not None
+
+
+def from_dsp_iri_to_prefixed_iri(iri: str) -> str:
+    dsp_iri_re = r".+\/(.+?)\/v2#(.+)$"
+    if not (found := regex.search(dsp_iri_re, iri)):
+        return iri
+    return f"{found.group(1)}:{found.group(2)}"
+
+
+def make_dsp_ontology_prefix(api_url: str, shortcode: str, onto_name: str) -> str:
+    api_fixed = convert_api_url_for_correct_iri_namespace_construction(api_url)
+    return f"{api_fixed}/ontology/{shortcode}/{onto_name}/v2#"
+
+
+def convert_api_url_for_correct_iri_namespace_construction(api_url: str) -> str:
+    return regex.sub(r"^https", "http", api_url)

dsp_tools/utils/{shared.py → data_formats/shared.py}

@@ -1,18 +1,14 @@
 from __future__ import annotations
 
-import json
 import unicodedata
-from pathlib import Path
 from typing import Any
 from typing import Optional
 from typing import TypeGuard
-from typing import Union
 
 import pandas as pd
 import regex
 
 from dsp_tools.commands.excel2xml.propertyelement import PropertyElement
-from dsp_tools.models.exceptions import BaseError
 
 
 def simplify_name(value: str) -> str:
@@ -73,7 +69,7 @@ def check_notna(value: Optional[Any]) -> TypeGuard[Any]:
     if isinstance(value, PropertyElement):
         value = value.value
 
-    if isinstance(value, (bool, int)) or (
+    if isinstance(value, bool | int) or (
         isinstance(value, float) and pd.notna(value)
     ):  # necessary because isinstance(np.nan, float)
         return True
@@ -83,33 +79,3 @@ def check_notna(value: Optional[Any]) -> TypeGuard[Any]:
         )
     else:
         return False
-
-
-def parse_json_input(project_file_as_path_or_parsed: Union[str, Path, dict[str, Any]]) -> dict[str, Any]:
-    """
-    Check the input for a method that expects a JSON project definition, either as file path or as parsed JSON object:
-    If it is parsed already, return it unchanged.
-    If the input is a file path, parse it.
-
-    Args:
-        project_file_as_path_or_parsed: path to the JSON project definition, or parsed JSON object
-
-    Raises:
-        BaseError: if the input is invalid
-
-    Returns:
-        the parsed JSON object
-    """
-    project_definition: dict[str, Any] = {}
-    if isinstance(project_file_as_path_or_parsed, dict):
-        project_definition = project_file_as_path_or_parsed
-    elif isinstance(project_file_as_path_or_parsed, (str, Path)) and Path(project_file_as_path_or_parsed).exists():
-        with open(project_file_as_path_or_parsed, encoding="utf-8") as f:
-            try:
-                project_definition = json.load(f)
-            except json.JSONDecodeError as e:
-                msg = f"The input file '{project_file_as_path_or_parsed}' cannot be parsed to a JSON object."
-                raise BaseError(msg) from e
-    else:
-        raise BaseError("Invalid input: The input must be a path to a JSON file or a parsed JSON object.")
-    return project_definition

dsp_tools/utils/{uri_util.py → data_formats/uri_util.py}

@@ -1,10 +1,20 @@
 import regex
 
 
+def is_prod_like_server(server: str) -> bool:
+    return server in [
+        "https://api.dasch.swiss",
+        "https://api.rdu.dasch.swiss",
+        "https://api.ls-prod-server.dasch.swiss",
+        "https://api.ls-test-server.dasch.swiss",
+        "https://api.stage.dasch.swiss",
+    ]
+
+
 def is_uri(s: str) -> bool:
     """Checks if the given string is a valid URI."""
     # URI = scheme ":" ["//" host [":" port]] path ["?" query] ["#" fragment]
-    chars_for_path_query_fragment = r"\w_.\-:~%()!@,;/=*&"
+    chars_for_path_query_fragment = r"\w_.\-:~%()!@,;/=*&'+"
     scheme = r"(?<scheme>[a-z][a-z0-9+.\-]*)"
     host = r"(?<host>[\w_.\-:~\[\]]+)"
     port = r"(?<port>:\d{0,6})"
@@ -50,7 +60,7 @@ def is_iiif_uri(uri: str) -> bool:
         r"^((\^|%5e)?max|"  # max | ^max
         r"(\^|%5e)?full|"  # full | ^full
         r"(\^|%5e)?pct:\d+(\.\d+)?|"  # pct:n | ^pct:n
-        r"(\^|%5e)?(\d+(\.\d+)?)+,|"  # w, | ^w,
+        r"(\^|%5e)?(\d+(\.\d+)?)+(,|%2c)|"  # w, | ^w,
         r"(\^|%5e)?,\d+(\.\d+)?|"  # ,h | ^,h
         r"(\^|%5e)?!?\d+(\.\d+)?,\d+(\.\d+)?)$"  # w,h | ^w,h | !w,h | ^!w,h
     )

dsp_tools/utils/fuseki_bloating.py

@@ -0,0 +1,63 @@
+from typing import assert_never
+
+from loguru import logger
+
+from dsp_tools.clients.fuseki_metrics import FusekiBloatingLevel
+from dsp_tools.clients.fuseki_metrics import FusekiMetrics
+from dsp_tools.config.logger_config import LOGGER_SAVEPATH
+from dsp_tools.utils.ansi_colors import BACKGROUND_BOLD_RED
+from dsp_tools.utils.ansi_colors import BOLD_YELLOW
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
+
+_1_GB_IN_BYTES = 1_000_000_000
+WARNING_BLOATING = 10 * _1_GB_IN_BYTES
+CRITICAL_BLOATING = 20 * _1_GB_IN_BYTES
+
+
+def communicate_fuseki_bloating(fuseki_metrics: FusekiMetrics) -> None:
+    size_diff = _analyse_fuseki_sizes(fuseki_metrics)
+    bloating_level = _get_bloating_level(size_diff)
+    if size_diff is not None:
+        rounded = round(size_diff / _1_GB_IN_BYTES, 2)
+    else:
+        rounded = None
+    msg = (
+        f"The xmlupload caused the database to use {rounded} GB disk space. "
+        f"Please check that your test server has enough disk space for an upload. "
+        f"If you have any questions contact the dsp-tools developers at support@dasch.swiss."
+    )
+    match bloating_level:
+        case FusekiBloatingLevel.OK:
+            logger.debug(msg)
+        case FusekiBloatingLevel.WARNING:
+            logger.warning(msg)
+            print(f"{BOLD_YELLOW}WARNING: {msg}{RESET_TO_DEFAULT}")
+        case FusekiBloatingLevel.CRITICAL:
+            logger.warning(msg)
+            print(f"{BACKGROUND_BOLD_RED}WARNING: {msg}{RESET_TO_DEFAULT}")
+        case FusekiBloatingLevel.CALCULATION_FAILURE:
+            msg = (
+                "The database bloating size could not be calculated. "
+                f"Please contact the dsp-tools developers (at support@dasch.swiss) "
+                f"with your logs saved at {LOGGER_SAVEPATH}."
+            )
+            print(f"{BACKGROUND_BOLD_RED}{msg}{RESET_TO_DEFAULT}")
+            logger.error(msg)
+        case _:
+            assert_never(bloating_level)
+
+
+def _get_bloating_level(size_diff: int | None) -> FusekiBloatingLevel:
+    if size_diff is None:
+        return FusekiBloatingLevel.CALCULATION_FAILURE
+    if size_diff <= WARNING_BLOATING:
+        return FusekiBloatingLevel.OK
+    if size_diff <= CRITICAL_BLOATING:
+        return FusekiBloatingLevel.WARNING
+    return FusekiBloatingLevel.CRITICAL
+
+
+def _analyse_fuseki_sizes(fuseki_metrics: FusekiMetrics) -> int | None:
+    if fuseki_metrics.end_size is not None and fuseki_metrics.start_size is not None:
+        return fuseki_metrics.end_size - fuseki_metrics.start_size
+    return None

dsp_tools/utils/json_parsing.py

@@ -0,0 +1,22 @@
+import json
+from pathlib import Path
+from typing import Any
+from typing import cast
+
+from loguru import logger
+
+from dsp_tools.error.exceptions import JSONFileParsingError
+from dsp_tools.error.exceptions import UserFilepathNotFoundError
+
+
+def parse_json_file(filepath: Path) -> dict[str, Any]:
+    if not filepath.exists():
+        raise UserFilepathNotFoundError(filepath)
+    with open(filepath, encoding="utf-8") as f:
+        try:
+            loaded = json.load(f)
+            return cast(dict[str, Any], loaded)
+        except json.JSONDecodeError as e:
+            logger.error(e)
+            msg = f"The input file '{filepath}' cannot be parsed to a JSON object."
+            raise JSONFileParsingError(msg) from None

dsp_tools/utils/rdf_constants.py

@@ -0,0 +1,42 @@
+from typing import Union
+
+from rdflib import IdentifiedNode
+from rdflib import Literal
+from rdflib import Namespace
+from rdflib import Variable
+from rdflib.term import Node
+
+# naming conventions of prefixes and namespaces
+
+# a prefix as a string is called [...]_PREFIX
+# a prefix as a rdflib Namespace has no further suffix
+
+#####
+# rdflib typing
+type PropertyTypeAlias = Union[IdentifiedNode, Variable]
+type SubjectObjectTypeAlias = Union[IdentifiedNode, Literal, Variable, Node]
+
+#####
+# DSP-Namespaces
+KNORA_API_PREFIX = "http://api.knora.org/ontology/knora-api/v2#"
+KNORA_API = Namespace(KNORA_API_PREFIX)
+
+KNORA_ADMIN_PREFIX = "http://www.knora.org/ontology/knora-admin#"
+
+SALSAH_GUI_PREFIX = "http://api.knora.org/ontology/salsah-gui/v2#"
+SALSAH_GUI = Namespace(SALSAH_GUI_PREFIX)
+
+
+#####
+# Mappers
+
+DSP_NAME_TO_PREFIX = {"knora-api": KNORA_API_PREFIX, "salsah-gui": SALSAH_GUI_PREFIX}
+
+
+#####
+# For validate-data
+API_SHAPES_PREFIX = "http://api.knora.org/ontology/knora-api/shapes/v2#"
+API_SHAPES = Namespace(API_SHAPES_PREFIX)
+DATA = Namespace("http://data/")
+DASH_PREFIX = "http://datashapes.org/dash#"
+DASH = Namespace(DASH_PREFIX)

dsp_tools/utils/rdflib_utils.py

@@ -0,0 +1,10 @@
+import json
+from typing import Any
+
+from rdflib import Graph
+
+
+def serialise_json(rdf_graph: Graph) -> list[dict[str, Any]]:
+    graph_bytes = rdf_graph.serialize(format="json-ld", encoding="utf-8")
+    json_graph: list[dict[str, Any]] = json.loads(graph_bytes)
+    return json_graph

dsp_tools/utils/replace_id_with_iri.py

@@ -0,0 +1,66 @@
+from copy import deepcopy
+from pathlib import Path
+
+from dsp_tools.commands.xmlupload.iri_resolver import IriResolver
+from dsp_tools.commands.xmlupload.richtext_id2iri import replace_ids_if_found
+from dsp_tools.error.exceptions import DuplicateIdsInXmlAndId2IriMapping
+from dsp_tools.utils.json_parsing import parse_json_file
+from dsp_tools.utils.xml_parsing.models.parsed_resource import KnoraValueType
+from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedResource
+from dsp_tools.utils.xml_parsing.models.parsed_resource import ParsedValue
+
+
+def use_id2iri_mapping_to_replace_ids(resources: list[ParsedResource], id2iri_file: Path) -> list[ParsedResource]:
+    lookup = parse_json_file(id2iri_file)
+    iri_lookup = IriResolver(lookup)
+    return _replace_all_ids_with_iris(resources, iri_lookup)
+
+
+def _replace_all_ids_with_iris(resources: list[ParsedResource], iri_lookup: IriResolver) -> list[ParsedResource]:
+    new_ids = {x.res_id for x in resources}
+    _raise_error_if_resource_ids_are_reused(new_ids, set(iri_lookup.lookup.keys()))
+    return [_process_one_resource(r, iri_lookup) for r in resources]
+
+
+def _raise_error_if_resource_ids_are_reused(
+    resources_ids_from_xml: set[str], resource_ids_from_mapping: set[str]
+) -> None:
+    overlap = resources_ids_from_xml.intersection(resource_ids_from_mapping)
+    if overlap:
+        msg = (
+            "You provided a id2iri mapping file to replace IDs referenced in links and "
+            "richtext with IRIs of previously uploaded resources. "
+            "This means that the new XML file must not contain IDs that were previously used.\n"
+            f"The following ID(s) are both in the id2iri mapping and the new data: {', '.join(overlap)}"
+        )
+        raise DuplicateIdsInXmlAndId2IriMapping(msg)
+
+
+def _process_one_resource(res: ParsedResource, iri_lookup: IriResolver) -> ParsedResource:
+    new_vals = []
+    previous_vals = deepcopy(res.values)
+    for v in previous_vals:
+        if v.value_type == KnoraValueType.LINK_VALUE:
+            new_vals.append(_process_link_value(v, iri_lookup))
+        elif v.value_type == KnoraValueType.RICHTEXT_VALUE:
+            new_vals.append(_process_richtext_value(v, iri_lookup))
+        else:
+            new_vals.append(v)
+    res.values = new_vals
+    return res
+
+
+def _process_link_value(val: ParsedValue, iri_lookup: IriResolver) -> ParsedValue:
+    # later code deals with cases when it is not a string, we can safely ignore this here
+    if isinstance(val.value, str):
+        if found := iri_lookup.get(val.value):
+            val.value = found
+    return val
+
+
+def _process_richtext_value(val: ParsedValue, iri_lookup: IriResolver) -> ParsedValue:
+    # later code deals with cases when it is not a string, we can safely ignore this here
+    if isinstance(val.value, str):
+        replaced, _ = replace_ids_if_found(val.value, iri_lookup)
+        val.value = replaced
+    return val