dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/utils/connection_live.py

@@ -1,383 +0,0 @@
-import json
-import os
-import time
-from dataclasses import dataclass
-from dataclasses import field
-from datetime import datetime
-from functools import partial
-from importlib.metadata import version
-from typing import Any
-from typing import Literal
-from typing import Never
-from typing import Optional
-from typing import cast
-
-import regex
-from loguru import logger
-from requests import ReadTimeout
-from requests import RequestException
-from requests import Response
-from requests import Session
-
-from dsp_tools.models.exceptions import BadCredentialsError
-from dsp_tools.models.exceptions import BaseError
-from dsp_tools.models.exceptions import InvalidInputError
-from dsp_tools.models.exceptions import PermanentConnectionError
-from dsp_tools.models.exceptions import PermanentTimeOutError
-from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.logger_config import WARNINGS_SAVEPATH
-from dsp_tools.utils.set_encoder import SetEncoder
-
-HTTP_OK = 200
-HTTP_UNAUTHORIZED = 401
-
-
-@dataclass
-class RequestParameters:
-    method: Literal["POST", "GET", "PUT", "DELETE"]
-    url: str
-    timeout: int
-    data: dict[str, Any] | None = None
-    data_serialized: bytes | None = field(init=False, default=None)
-    headers: dict[str, str] | None = None
-    files: dict[str, tuple[str, Any]] | None = None
-
-    def __post_init__(self) -> None:
-        self.data_serialized = self._serialize_payload(self.data)
-
-    def _serialize_payload(self, payload: dict[str, Any] | None) -> bytes | None:
-        # If data is not encoded as bytes, issues can occur with non-ASCII characters,
-        # where the content-length of the request will turn out to be different from the actual length.
-        return json.dumps(payload, cls=SetEncoder, ensure_ascii=False).encode("utf-8") if payload else None
-
-    def as_kwargs(self) -> dict[str, Any]:
-        return {
-            "method": self.method,
-            "url": self.url,
-            "timeout": self.timeout,
-            "data": self.data_serialized,
-            "headers": self.headers,
-            "files": self.files,
-        }
-
-
-@dataclass
-class ConnectionLive(Connection):
-    """
-    A Connection instance represents a connection to a DSP server.
-
-    Attributes:
-        server: address of the server, e.g https://api.dasch.swiss
-        token: session token received by the server after login
-    """
-
-    server: str
-    token: Optional[str] = None
-    session: Session = field(init=False, default=Session())
-    # downtimes of server-side services -> API still processes request
-    # -> retry too early has side effects (e.g. duplicated resources)
-    timeout_put_post: int = field(init=False, default=30 * 60)
-    timeout_get_delete: int = field(init=False, default=20)
-
-    def __post_init__(self) -> None:
-        self.session.headers["User-Agent"] = f'DSP-TOOLS/{version("dsp-tools")}'
-        if self.server.endswith("/"):
-            self.server = self.server[:-1]
-
-    def login(self, email: str, password: str) -> None:
-        """
-        Retrieve a session token and store it as class attribute.
-
-        Args:
-            email: email address of the user
-            password: password of the user
-
-        Raises:
-            UserError: if DSP-API returns no token with the provided user credentials
-        """
-        try:
-            response = self.post(
-                route="/v2/authentication",
-                data={"email": email, "password": password},
-                timeout=10,
-            )
-        except BadCredentialsError:
-            raise UserError(f"Username and/or password are not valid on server '{self.server}'") from None
-        except PermanentConnectionError as e:
-            raise UserError(e.message) from None
-        if not response.get("token"):
-            raise UserError("Unable to retrieve a token from the server with the provided credentials.")
-        self.token = response["token"]
-        self.session.headers["Authorization"] = f"Bearer {self.token}"
-
-    def logout(self) -> None:
-        """
-        Delete the token on the server and in this class.
-        """
-        if self.token:
-            self.delete(route="/v2/authentication")
-            self.token = None
-            del self.session.headers["Authorization"]
-
-    def get_token(self) -> str:
-        """
-        Return the token of this connection.
-
-        Returns:
-            token
-
-        Raises:
-            BaseError: if no token is available
-        """
-        if not self.token:
-            raise BaseError("No token available.")
-        return self.token
-
-    def post(
-        self,
-        route: str,
-        data: dict[str, Any] | None = None,
-        files: dict[str, tuple[str, Any]] | None = None,
-        headers: dict[str, str] | None = None,
-        timeout: int | None = None,
-    ) -> dict[str, Any]:
-        """
-        Make an HTTP POST request to the server to which this connection has been established.
-
-        Args:
-            route: route that will be called on the server
-            data: payload of the HTTP request
-            files: files to be uploaded, if any
-            headers: headers for the HTTP request
-            timeout: timeout of the HTTP request, or None if the default should be used
-
-        Returns:
-            response from server
-
-        Raises:
-            PermanentConnectionError: if all attempts have failed
-            InvalidInputError: if the API responds with a permanent error because of invalid input data
-        """
-        if data:
-            headers = headers or {}
-            if "Content-Type" not in headers:
-                headers["Content-Type"] = "application/json; charset=UTF-8"
-        params = RequestParameters(
-            "POST", self._make_url(route), timeout or self.timeout_put_post, data, headers, files
-        )
-        response = self._try_network_action(params)
-        return cast(dict[str, Any], response.json())
-
-    def get(
-        self,
-        route: str,
-        headers: dict[str, str] | None = None,
-    ) -> dict[str, Any]:
-        """
-        Make an HTTP GET request to the server to which this connection has been established.
-
-        Args:
-            route: route that will be called on the server
-            headers: headers for the HTTP request
-
-        Returns:
-            response from server
-
-        Raises:
-            PermanentConnectionError: if all attempts have failed
-            InvalidInputError: if the API responds with a permanent error because of invalid input data
-        """
-        params = RequestParameters("GET", self._make_url(route), self.timeout_get_delete, headers=headers)
-        response = self._try_network_action(params)
-        return cast(dict[str, Any], response.json())
-
-    def put(
-        self,
-        route: str,
-        data: dict[str, Any] | None = None,
-        headers: dict[str, str] | None = None,
-    ) -> dict[str, Any]:
-        """
-        Make an HTTP GET request to the server to which this connection has been established.
-
-        Args:
-            route: route that will be called on the server
-            data: payload of the HTTP request
-            headers: headers of the HTTP request
-
-        Returns:
-            response from server
-
-        Raises:
-            PermanentConnectionError: if all attempts have failed
-            InvalidInputError: if the API responds with a permanent error because of invalid input data
-        """
-        if data:
-            headers = headers or {}
-            if "Content-Type" not in headers:
-                headers["Content-Type"] = "application/json; charset=UTF-8"
-        params = RequestParameters("PUT", self._make_url(route), self.timeout_put_post, data, headers)
-        response = self._try_network_action(params)
-        return cast(dict[str, Any], response.json())
-
-    def delete(
-        self,
-        route: str,
-        headers: dict[str, str] | None = None,
-    ) -> dict[str, Any]:
-        """
-        Make an HTTP GET request to the server to which this connection has been established.
-
-        Args:
-            route: route that will be called on the server
-            headers: headers for the HTTP request
-
-        Returns:
-            response from server
-
-        Raises:
-            PermanentConnectionError: if all attempts have failed
-            InvalidInputError: if the API responds with a permanent error because of invalid input data
-        """
-        params = RequestParameters("DELETE", self._make_url(route), self.timeout_get_delete, headers=headers)
-        response = self._try_network_action(params)
-        return cast(dict[str, Any], response.json())
-
-    def _make_url(self, route: str) -> str:
-        if not route.startswith("/"):
-            route = f"/{route}"
-        return self.server + route
-
-    def _try_network_action(self, params: RequestParameters) -> Response:
-        """
-        Try 7 times to execute an HTTP request.
-        If a timeout error, a ConnectionError, or a requests.RequestException occur,
-        or if the response indicates that there is a non-permanent server-side problem,
-        this function waits and retries the HTTP request.
-        The waiting times are 1, 2, 4, 8, 16, 32, 64 seconds.
-
-        Args:
-            params: keyword arguments for the HTTP request
-
-        Raises:
-            BadCredentialsError: if the server returns a 401 status code on the route /v2/authentication
-            PermanentConnectionError: if all attempts have failed
-            InvalidInputError: if the API responds with a permanent error because of invalid input data
-            unexpected exceptions: if the action fails with an unexpected exception
-
-        Returns:
-            the return value of action
-        """
-        action = partial(self.session.request, **params.as_kwargs())
-        for i in range(7):
-            try:
-                self._log_request(params)
-                response = action()
-            except (TimeoutError, ReadTimeout) as err:
-                self._log_and_raise_timeouts(err)
-            except (ConnectionError, RequestException):
-                self._renew_session()
-                self._log_and_sleep(reason="Connection Error raised", retry_counter=i, exc_info=True)
-                continue
-
-            self._log_response(response)
-            if response.status_code == HTTP_OK:
-                return response
-
-            self._handle_non_ok_responses(response, params.url, i)
-
-        # if all attempts have failed, raise error
-        msg = f"Permanently unable to execute the network action. See {WARNINGS_SAVEPATH} for more information."
-        raise PermanentConnectionError(msg)
-
-    def _handle_non_ok_responses(self, response: Response, request_url: str, retry_counter: int) -> None:
-        if "v2/authentication" in request_url and response.status_code == HTTP_UNAUTHORIZED:
-            raise BadCredentialsError("Bad credentials")
-        in_500_range = 500 <= response.status_code < 600
-        try_again_later = "try again later" in response.text.lower()
-        should_retry = (try_again_later or in_500_range) and not self._in_testing_environment()
-        if should_retry:
-            self._log_and_sleep("Transient Error", retry_counter, exc_info=False)
-            return None
-        else:
-            msg = "Permanently unable to execute the network action. "
-            if original_str := regex.search(r'{"knora-api:error":"dsp\.errors\.(.*)","@context', str(response.content)):
-                msg += f"\n{' '*37}Original Message: {original_str.group(1)}\n"
-                if original_str.group(1).startswith("OntologyConstraintException"):
-                    msg += f"See {WARNINGS_SAVEPATH} for more information."
-                    raise InvalidInputError(msg)
-            msg += f"See {WARNINGS_SAVEPATH} for more information."
-            raise PermanentConnectionError(msg)
-
-    def _renew_session(self) -> None:
-        self.session.close()
-        self.session = Session()
-        self.session.headers["User-Agent"] = f'DSP-TOOLS/{version("dsp-tools")}'
-        if self.token:
-            self.session.headers["Authorization"] = f"Bearer {self.token}"
-
-    def _log_and_sleep(self, reason: str, retry_counter: int, exc_info: bool) -> None:
-        msg = f"{reason}: Try reconnecting to DSP server, next attempt in {2 ** retry_counter} seconds..."
-        print(f"{datetime.now()}: {msg}")
-        if exc_info:
-            logger.opt(exception=True).error(f"{msg} ({retry_counter=:})")
-        else:
-            logger.error(f"{msg} ({retry_counter=:})")
-        time.sleep(2**retry_counter)
-
-    def _log_and_raise_timeouts(self, error: TimeoutError | ReadTimeout) -> Never:
-        msg = f"A '{error.__class__.__name__}' occurred during the connection to the DSP server."
-        print(f"{datetime.now()}: {msg}")
-        logger.exception(msg)
-        raise PermanentTimeOutError(msg) from None
-
-    def _log_response(self, response: Response) -> None:
-        dumpobj: dict[str, Any] = {
-            "status_code": response.status_code,
-            "headers": self._anonymize(dict(response.headers)),
-        }
-        try:
-            dumpobj["content"] = self._anonymize(json.loads(response.text))
-        except json.JSONDecodeError:
-            dumpobj["content"] = response.text if "token" not in response.text else "***"
-        logger.debug(f"RESPONSE: {json.dumps(dumpobj)}")
-
-    def _anonymize(self, data: dict[str, Any] | None) -> dict[str, Any] | None:
-        if not data:
-            return data
-        data = data.copy()
-        if "token" in data:
-            data["token"] = self._mask(data["token"])
-        if "Set-Cookie" in data:
-            data["Set-Cookie"] = self._mask(data["Set-Cookie"])
-        if "Authorization" in data:
-            if match := regex.search(r"^Bearer (.+)", data["Authorization"]):
-                data["Authorization"] = f"Bearer {self._mask(match.group(1))}"
-        if "password" in data:
-            data["password"] = "*" * len(data["password"])
-        return data
-
-    def _mask(self, sensitive_info: str) -> str:
-        unmasked_until = 5
-        if len(sensitive_info) <= unmasked_until * 2:
-            return "*" * len(sensitive_info)
-        else:
-            return f"{sensitive_info[:unmasked_until]}[+{len(sensitive_info) - unmasked_until}]"
-
-    def _in_testing_environment(self) -> bool:
-        in_testing_env = os.getenv("DSP_TOOLS_TESTING")  # set in .github/workflows/tests-on-push.yml
-        return in_testing_env == "true"
-
-    def _log_request(self, params: RequestParameters) -> None:
-        dumpobj = {
-            "method": params.method,
-            "url": params.url,
-            "headers": self._anonymize(dict(self.session.headers) | (params.headers or {})),
-            "timeout": params.timeout,
-        }
-        if params.data:
-            dumpobj["data"] = self._anonymize(params.data)
-        if params.files:
-            dumpobj["files"] = params.files["file"][0]
-        logger.debug(f"REQUEST: {json.dumps(dumpobj, cls=SetEncoder)}")

dsp_tools/utils/iri_util.py

@@ -1,14 +0,0 @@
-import regex
-
-_iri_pattern = r"^(http)s?://[\w\.\-~]*(:\d{,6})?(/[\w\-~]+)*(#[\w\-~]*)?"
-_resource_iri_pattern = r"https?://rdfh.ch/[a-fA-F0-9]{4}/[\w-]{22}"
-
-
-def is_iri(s: str) -> bool:
-    """Checks whether a string is a valid IRI."""
-    return regex.fullmatch(_iri_pattern, s) is not None
-
-
-def is_resource_iri(s: str) -> bool:
-    """Checks whether a string is a valid resource IRI."""
-    return regex.fullmatch(_resource_iri_pattern, s) is not None

dsp_tools/utils/logger_config.py

@@ -1,41 +0,0 @@
-from pathlib import Path
-
-from loguru import logger
-
-LOGGER_SAVEPATH = (Path.home() / ".dsp-tools" / "logging.log").absolute()
-WARNINGS_SAVEPATH = Path("warnings.log")
-
-
-def logger_config() -> None:
-    """
-    This function configures the log files.
-    Currently, there are three sinks:
-    - logger.log in the home directory for development purposes
-    - warnings.log in the cwd only with level warning and higher for the user (no stack-trace)
-    - print output on the terminal, formatted the same as the warnings.log
-    """
-    # If this is not removed, the default formatting is also printed out on the terminal
-    logger.remove()
-
-    text_format = "<level>{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {message}</level>"
-    rotation_size = "100 MB"
-    retention_number = 30
-
-    logger.add(
-        sink=LOGGER_SAVEPATH,
-        format=text_format,
-        backtrace=True,
-        diagnose=True,
-        rotation=rotation_size,
-        retention=retention_number,
-    )
-
-    logger.add(
-        sink=WARNINGS_SAVEPATH,
-        level="WARNING",
-        format=text_format,
-        backtrace=False,
-        diagnose=False,
-        rotation=rotation_size,
-        retention=10,
-    )

dsp_tools/utils/set_encoder.py

@@ -1,20 +0,0 @@
-import json
-from typing import Any
-from typing import Union
-
-from dsp_tools.commands.project.models.context import Context
-from dsp_tools.commands.project.models.helpers import OntoIri
-
-
-class SetEncoder(json.JSONEncoder):
-    """Encoder used to serialize objects to JSON that would by default not be serializable"""
-
-    def default(self, o: Union[set[Any], Context, OntoIri]) -> Any:
-        """Return a serializable object for o"""
-        if isinstance(o, set):
-            return list(o)
-        elif isinstance(o, Context):
-            return o.toJsonObj()
-        elif isinstance(o, OntoIri):
-            return {"iri": o.iri, "hashtag": o.hashtag}
-        return json.JSONEncoder.default(self, o)

dsp_tools/utils/xml_utils.py

@@ -1,145 +0,0 @@
-from __future__ import annotations
-
-import copy
-from copy import deepcopy
-from pathlib import Path
-
-from loguru import logger
-from lxml import etree
-
-from dsp_tools.models.exceptions import InputError
-
-
-def parse_and_clean_xml_file(input_file: Path) -> etree._Element:
-    """
-    Parse an XML file with DSP-conform data,
-    remove namespace URI from the elements' names,
-    and transform the special tags `<annotation>`, `<region>`, `<link>`, `<video-segment>`, `<audio-segment>`
-    to their technically correct form
-    `<resource restype="Annotation">`, `<resource restype="Region">`, `<resource restype="LinkObj">`,
-    `<resource restype="VideoSegment">`, `<resource restype="AudioSegment">`.
-
-    Args:
-        input_file: path to the XML file
-
-    Returns:
-        the root element of the parsed XML file
-
-    Raises:
-        InputError: if the input is not of either the expected types
-    """
-    root = parse_xml_file(input_file)
-    root = remove_comments_from_element_tree(root)
-    return transform_special_tags_make_localname(root)
-
-
-def transform_special_tags_make_localname(input_tree: etree._Element) -> etree._Element:
-    """
-    This function removes the namespace URIs from the elements' names
-    and transforms the special tags `<annotation>`, `<region>`, `<link>`, `<video-segment>`, `<audio-segment>`
-    to their technically correct form
-    `<resource restype="Annotation">`, `<resource restype="Region">`, `<resource restype="LinkObj">`,
-    `<resource restype="VideoSegment">`, `<resource restype="AudioSegment">`.
-
-    Args:
-        input_tree: unclean tree
-
-    Returns:
-        cleaned tree
-    """
-    tree = transform_into_localnames(input_tree)
-    return _transform_special_tags(tree)
-
-
-def _transform_special_tags(tree: etree._Element) -> etree._Element:
-    tree = deepcopy(tree)
-    for elem in tree.iter():
-        if elem.tag == "annotation":
-            elem.attrib["restype"] = "Annotation"
-            elem.tag = "resource"
-        elif elem.tag == "link":
-            elem.attrib["restype"] = "LinkObj"
-            elem.tag = "resource"
-        elif elem.tag == "region":
-            elem.attrib["restype"] = "Region"
-            elem.tag = "resource"
-        elif elem.tag == "video-segment":
-            elem.attrib["restype"] = "VideoSegment"
-            elem.tag = "resource"
-        elif elem.tag == "audio-segment":
-            elem.attrib["restype"] = "AudioSegment"
-            elem.tag = "resource"
-    return tree
-
-
-def transform_into_localnames(root: etree._Element) -> etree._Element:
-    """
-    This function removes the namespace URIs from the elements' names
-
-    Args:
-        root: unclean tree
-
-    Returns:
-        cleaned tree
-    """
-    tree = deepcopy(root)
-    for elem in tree.iter():
-        elem.tag = etree.QName(elem).localname
-    return tree
-
-
-def remove_comments_from_element_tree(input_tree: etree._Element) -> etree._Element:
-    """
-    This function removes comments and processing instructions.
-    Commented out properties break the XMLProperty constructor.
-
-    Args:
-        input_tree: etree root that will be cleaned
-
-    Returns:
-        clean xml
-    """
-    root = copy.deepcopy(input_tree)
-    for c in root.xpath("//comment()"):
-        c.getparent().remove(c)
-    for c in root.xpath("//processing-instruction()"):
-        c.getparent().remove(c)
-    return root
-
-
-def parse_xml_file(input_file: str | Path) -> etree._Element:
-    """
-    This function parses an XML file and returns an Element Tree
-
-    Args:
-        input_file: path to the input file
-
-    Returns:
-        element tree
-
-    Raises:
-        InputError: if the file contains a syntax error
-    """
-    parser = etree.XMLParser(remove_comments=True, remove_pis=True)
-    try:
-        return etree.parse(source=input_file, parser=parser).getroot()
-    except etree.XMLSyntaxError as err:
-        logger.opt(exception=True).error(f"The XML file contains the following syntax error: {err.msg}")
-        raise InputError(f"The XML file contains the following syntax error: {err.msg}") from None
-
-
-def remove_namespaces_from_xml(data_xml: etree._Element) -> etree._Element:
-    """
-    This function removes all the namespaces from an XML element tree.
-
-    Args:
-        data_xml: xml with namespaces
-
-    Returns:
-        the XMl without the namespaces
-    """
-    xml_no_namespace = copy.deepcopy(data_xml)
-    for elem in xml_no_namespace.iter():
-        if not isinstance(elem, (etree._Comment, etree._ProcessingInstruction)):
-            elem.tag = etree.QName(elem).localname
-    return xml_no_namespace