dsp-tools 9.1.0.post11__py3-none-any.whl → 18.3.0.post13__py3-none-any.whl

dsp_tools/commands/resume_xmlupload/resume_xmlupload.py

@@ -2,21 +2,25 @@ import pickle
 import sys
 
 from loguru import logger
-from termcolor import colored
 
 from dsp_tools.cli.args import ServerCredentials
-from dsp_tools.commands.xmlupload.list_client import ListClient
-from dsp_tools.commands.xmlupload.list_client import ListClientLive
+from dsp_tools.clients.authentication_client_live import AuthenticationClientLive
+from dsp_tools.clients.connection_live import ConnectionLive
+from dsp_tools.clients.legal_info_client import LegalInfoClient
+from dsp_tools.clients.legal_info_client_live import LegalInfoClientLive
+from dsp_tools.clients.project_client import ProjectClient
+from dsp_tools.clients.project_client_live import ProjectClientLive
 from dsp_tools.commands.xmlupload.models.ingest import AssetClient
 from dsp_tools.commands.xmlupload.models.ingest import BulkIngestedAssetClient
 from dsp_tools.commands.xmlupload.models.ingest import DspIngestClientLive
 from dsp_tools.commands.xmlupload.models.upload_clients import UploadClients
 from dsp_tools.commands.xmlupload.models.upload_state import UploadState
-from dsp_tools.commands.xmlupload.project_client import ProjectClient
-from dsp_tools.commands.xmlupload.project_client import ProjectClientLive
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClient
+from dsp_tools.commands.xmlupload.prepare_xml_input.list_client import ListClientLive
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
 from dsp_tools.commands.xmlupload.xmlupload import execute_upload
-from dsp_tools.utils.connection_live import ConnectionLive
+from dsp_tools.utils.ansi_colors import RED
+from dsp_tools.utils.ansi_colors import RESET_TO_DEFAULT
 
 
 def resume_xmlupload(creds: ServerCredentials, skip_first_resource: bool = False) -> bool:
@@ -38,23 +42,19 @@ def resume_xmlupload(creds: ServerCredentials, skip_first_resource: bool = False
 
     _print_and_log(upload_state, server)
 
-    con = ConnectionLive(server)
-    con.login(creds.user, creds.password)
+    auth = AuthenticationClientLive(server, creds.user, creds.password)
+    con = ConnectionLive(server, auth)
 
     ingest_client: AssetClient
     if upload_state.config.media_previously_uploaded:
         ingest_client = BulkIngestedAssetClient()
     else:
-        ingest_client = DspIngestClientLive(
-            dsp_ingest_url=creds.dsp_ingest_url,
-            token=con.get_token(),
-            shortcode=upload_state.config.shortcode,
-            imgdir=".",
-        )
+        ingest_client = DspIngestClientLive(creds.dsp_ingest_url, auth, upload_state.config.shortcode, ".")
 
-    project_client: ProjectClient = ProjectClientLive(con, upload_state.config.shortcode)
-    list_client: ListClient = ListClientLive(con, project_client.get_project_iri())
-    clients = UploadClients(ingest_client, project_client, list_client)
+    project_client: ProjectClient = ProjectClientLive(auth.server, auth)
+    list_client: ListClient = ListClientLive(con, project_client.get_project_iri(upload_state.config.shortcode))
+    legal_info_client: LegalInfoClient = LegalInfoClientLive(server, upload_state.config.shortcode, auth)
+    clients = UploadClients(ingest_client, list_client, legal_info_client)
 
     return execute_upload(clients, upload_state)
 
@@ -77,7 +77,7 @@ def _skip_first_resource(upload_state: UploadState) -> None:
         )
         resp = None
         while resp not in ["y", "n"]:
-            resp = input(colored(msg, color="red"))
+            resp = input(RED + msg + RESET_TO_DEFAULT)
         if resp == "n":
             sys.exit(1)
 

dsp_tools/commands/start_stack.py

@@ -1,3 +1,4 @@
+import contextlib
 import importlib.resources
 import shutil
 import subprocess
@@ -9,11 +10,17 @@ from typing import Optional
 import regex
 import requests
 import yaml
+from jinja2 import Template
 from loguru import logger
 
-from dsp_tools.models.exceptions import UserError
+from dsp_tools.error.exceptions import InputError
+from dsp_tools.error.exceptions import PermanentConnectionError
+from dsp_tools.utils.request_utils import RequestParameters
+from dsp_tools.utils.request_utils import log_request
+from dsp_tools.utils.request_utils import log_response
 
 MAX_FILE_SIZE = 100_000
+MINUTE = 60
 
 
 @dataclass(frozen=True)
@@ -26,7 +33,6 @@ class StackConfiguration:
         enforce_docker_system_prune: if True, prune Docker without asking the user
         suppress_docker_system_prune: if True, don't prune Docker (and don't ask)
         latest_dev_version: if True, start DSP-API from repo's main branch, instead of the latest deployed version
-        api_version_for_validate: if True a fixed API version is taken that has the features necessary for xml-validate
     """
 
     max_file_size: Optional[int] = None
@@ -34,19 +40,23 @@ class StackConfiguration:
     suppress_docker_system_prune: bool = False
     latest_dev_version: bool = False
     upload_test_data: bool = False
-    api_version_for_validate: bool = False
+    custom_host: Optional[str] = None
 
     def __post_init__(self) -> None:
         """
         Validate the input parameters passed by the user.
 
         Raises:
-            UserError: if one of the parameters is invalid
+            InputError: if one of the parameters is invalid
         """
         if self.max_file_size is not None and not 1 <= self.max_file_size <= MAX_FILE_SIZE:
-            raise UserError(f"max_file_size must be between 1 and {MAX_FILE_SIZE}")
+            raise InputError(f"max_file_size must be between 1 and {MAX_FILE_SIZE}")
         if self.enforce_docker_system_prune and self.suppress_docker_system_prune:
-            raise UserError('The arguments "--prune" and "--no-prune" are mutually exclusive')
+            raise InputError('The arguments "--prune" and "--no-prune" are mutually exclusive')
+        if self.custom_host is not None and not regex.match(
+            r"^(((\d{1,3}\.){3}\d{1,3})|((([-\w_~]+\.)*([a-z]){2,})))$", self.custom_host
+        ):
+            raise InputError("Invalid format for custom host. Please, enter an IP or a domain name.")
 
 
 class StackHandler:
@@ -77,28 +87,25 @@ class StackHandler:
         For this reason, we need to know the commit hash of the DSP-API version that is currently deployed,
         so that the files can be retrieved from the correct commit.
 
-        This function reads the commit hash of DSP-API
-        that is configured in start-stack-config.yml,
+        This function reads the version tag in the docker-compose.yml file,
         and constructs the URL prefix necessary to retrieve the files from the DSP-API repository.
 
-        If something goes wrong,
-        the URL prefix falls back to pointing to the main branch of the DSP-API repository.
-
         If the latest development version of DSP-API is started,
         the URL prefix points to the main branch of the DSP-API repository.
 
         Returns:
             URL prefix used to retrieve files from the DSP-API repository
         """
-        url_prefix_base = "https://github.com/dasch-swiss/dsp-api/raw"
+        url_prefix_base = "https://raw.githubusercontent.com/dasch-swiss/dsp-api"
 
         if self.__stack_configuration.latest_dev_version:
             return f"{url_prefix_base}/main/"
 
-        config_file = importlib.resources.files("dsp_tools").joinpath("resources/start-stack/start-stack-config.yml")
-        start_stack_config = yaml.safe_load(config_file.read_bytes())
-        commit_of_used_api_version = start_stack_config["DSP-API commit"]
-        return f"{url_prefix_base}/{commit_of_used_api_version}/"
+        docker_compose_pth = importlib.resources.files("dsp_tools").joinpath("resources/start-stack/docker-compose.yml")
+        docker_compose = yaml.safe_load(docker_compose_pth.read_bytes())
+        tag = docker_compose["services"]["api"]["image"].split(":")[-1]
+
+        return f"{url_prefix_base}/{tag}/"
 
     def _copy_resources_to_home_dir(self) -> None:
         """
@@ -110,6 +117,7 @@ class StackHandler:
         by an earlier run of this method.
         So, this method must always be called, at every run of start-stack.
         """
+        logger.debug("Copying resources to home directory ...")
         docker_path_of_distribution = importlib.resources.files("dsp_tools").joinpath("resources/start-stack")
         for file in docker_path_of_distribution.iterdir():
             with importlib.resources.as_file(file) as f:
@@ -117,8 +125,40 @@ class StackHandler:
             shutil.copy(file_path, self.__docker_path_of_user / file.name)
         if not self.__stack_configuration.latest_dev_version:
             Path(self.__docker_path_of_user / "docker-compose.override.yml").unlink()
-        if not self.__stack_configuration.api_version_for_validate:
-            Path(self.__docker_path_of_user / "docker-compose-validation.yml").unlink()
+
+    def _set_custom_host(self) -> None:
+        """
+        To ensure the frontend can communicate with a backend on a different server, the host in the environments
+        needs to be changed.
+        By design the IRIs match the host of the database.
+
+        This is done by overriding the environment variables in the docker-compose.yml and by replacing the
+        configuration for the frontend.
+        """
+        if self.__stack_configuration.custom_host is not None:
+            logger.debug("Setting custom host...")
+            self.__localhost_url = f"http://{self.__stack_configuration.custom_host}"
+
+            docker_template_path = importlib.resources.files("dsp_tools").joinpath(
+                "resources/start-stack/docker-compose.override-host.j2"
+            )
+            docker_template = Template(docker_template_path.read_text(encoding="utf-8"))
+            docker_template_rendered = docker_template.render(CUSTOM_HOST=self.__stack_configuration.custom_host)
+            Path(self.__docker_path_of_user / "docker-compose.override-host.yml").write_text(
+                docker_template_rendered, encoding="utf-8"
+            )
+
+            dsp_app_config_template_path = importlib.resources.files("dsp_tools").joinpath(
+                "resources/start-stack/dsp-app-config.override-host.j2"
+            )
+            dsp_app_config_template = Template(dsp_app_config_template_path.read_text(encoding="utf-8"))
+            dsp_app_config_rendered = dsp_app_config_template.render(CUSTOM_HOST=self.__stack_configuration.custom_host)
+            Path(self.__docker_path_of_user / "dsp-app-config.json").unlink()
+            Path(self.__docker_path_of_user / "dsp-app-config.json").write_text(
+                dsp_app_config_rendered, encoding="utf-8"
+            )
+        Path(self.__docker_path_of_user / "docker-compose.override-host.j2").unlink()
+        Path(self.__docker_path_of_user / "dsp-app-config.override-host.j2").unlink()
 
     def _get_sipi_docker_config_lua(self) -> None:
         """
@@ -126,14 +166,15 @@ class StackHandler:
         and set the max_file_size parameter if necessary.
 
         Raises:
-            UserError: if max_file_size is set but cannot be injected into sipi.docker-config.lua
+            InputError: if max_file_size is set but cannot be injected into sipi.docker-config.lua
         """
+        logger.debug("Retrieving sipi.docker-config.lua...")
         docker_config_lua_response = requests.get(f"{self.__url_prefix}sipi/config/sipi.docker-config.lua", timeout=30)
         docker_config_lua_text = docker_config_lua_response.text
         if self.__stack_configuration.max_file_size:
             max_post_size_regex = r"max_post_size ?= ?[\'\"]?\d+[MG][\'\"]?"
             if not regex.search(max_post_size_regex, docker_config_lua_text):
-                raise UserError("Unable to set max_file_size. Please try again without this flag.")
+                raise InputError("Unable to set max_file_size. Please try again without this flag.")
             docker_config_lua_text = regex.sub(
                 max_post_size_regex,
                 f"max_post_size = '{self.__stack_configuration.max_file_size}M'",
@@ -147,67 +188,43 @@ class StackHandler:
         Start up the Docker container of the fuseki database.
 
         Raises:
-            UserError: if the database cannot be started
+            InputError: if the database cannot be started
         """
-        cmd = "docker compose up db -d".split()
+        logger.debug("Starting up the fuseki container...")
+        cmd = "docker compose up -d db".split()
         completed_process = subprocess.run(cmd, cwd=self.__docker_path_of_user, check=False)
         if not completed_process or completed_process.returncode != 0:
-            msg = "Cannot start the API: Error while executing 'docker compose up db -d'"
+            msg = "Cannot start the API: Error while executing 'docker compose up -d db'"
             logger.error(f"{msg}. completed_process = '{vars(completed_process)}'")
-            raise UserError(msg)
+            raise InputError(msg)
 
     def _wait_for_fuseki(self) -> None:
         """
         Wait up to 6 minutes, until the fuseki database is up and running.
         This function imitates the behaviour of the script dsp-api/webapi/scripts/wait-for-db.sh.
         """
+        logger.debug("Waiting for the fuseki container to be up and running...")
         for _ in range(6 * 60):
             try:
                 response = requests.get(f"{self.__localhost_url}:3030/$/server", auth=("admin", "test"), timeout=10)
                 if response.ok:
+                    logger.debug("Fuseki is now up and running.")
                     break
             except Exception:  # noqa: BLE001 (blind-except)
                 time.sleep(1)
             time.sleep(1)
 
-    def _create_knora_test_repo(self) -> None:
-        """
-        Inside fuseki, create the "knora-test" repository.
-        This function imitates the behaviour of the script dsp-api/webapi/scripts/fuseki-init-knora-test.sh.
-
-        Raises:
-            UserError: in case of failure
-        """
-        repo_template_response = requests.get(
-            f"{self.__url_prefix}webapi/scripts/fuseki-repository-config.ttl.template",
-            timeout=30,
-        )
-        repo_template = repo_template_response.text
-        repo_template = repo_template.replace("@REPOSITORY@", "knora-test")
-        response = requests.post(
-            f"{self.__localhost_url}:3030/$/datasets",
-            files={"file": ("file.ttl", repo_template, "text/turtle; charset=utf8")},
-            auth=("admin", "test"),
-            timeout=30,
-        )
-        if not response.ok:
-            msg = (
-                "Cannot start DSP-API: Error when creating the 'knora-test' repository. "
-                "Is DSP-API perhaps running already?"
-            )
-            logger.error(f"{msg}. response = {vars(response)}")
-            raise UserError(msg)
-
     def _load_data_into_repo(self) -> None:
         """
         Load some basic ontologies and data into the repository.
         This function imitates the behaviour of the script
-        dsp-api/webapi/target/docker/stage/opt/docker/scripts/fuseki-init-knora-test.sh.
+        dsp-api/webapi/scripts/fuseki-init-knora-test.sh.
 
         Raises:
-            UserError: if one of the graphs cannot be created
+            InputError: if one of the graphs cannot be created
         """
-        graph_prefix = f"{self.__localhost_url}:3030/knora-test/data?graph="
+        logger.debug("Loading data into the 'dsp-repo' repository...")
+        graph_prefix = f"{self.__localhost_url}:3030/dsp-repo/data?graph="
         ttl_files = [
             ("webapi/src/main/resources/knora-ontologies/knora-admin.ttl", "http://www.knora.org/ontology/knora-admin"),
             ("webapi/src/main/resources/knora-ontologies/knora-base.ttl", "http://www.knora.org/ontology/knora-base"),
@@ -224,7 +241,7 @@ class StackHandler:
             if not ttl_response.ok:
                 msg = f"Cannot start DSP-API: Error when retrieving '{self.__url_prefix + ttl_file}'"
                 logger.error(f"{msg}'. response = {vars(ttl_response)}")
-                raise UserError(msg)
+                raise InputError(msg)
             ttl_text = ttl_response.text
             response = requests.post(
                 graph_prefix + graph,
@@ -234,7 +251,7 @@ class StackHandler:
             )
             if not response.ok:
                 logger.error(f"Cannot start DSP-API: Error when creating graph '{graph}'. response = {vars(response)}")
-                raise UserError(f"Cannot start DSP-API: Error when creating graph '{graph}'")
+                raise InputError(f"Cannot start DSP-API: Error when creating graph '{graph}'")
 
     def _create_admin_user(self) -> None:
         """
@@ -242,9 +259,10 @@ class StackHandler:
         The password is the hash for "test".
 
         Raises:
-            UserError: If the user cannot be created.
+            InputError: If the user cannot be created.
         """
-        graph_prefix = f"{self.__localhost_url}:3030/knora-test/data?graph="
+        logger.debug("Creating the default admin user...")
+        graph_prefix = f"{self.__localhost_url}:3030/dsp-repo/data?graph="
         admin_graph = "http://www.knora.org/data/admin"
         admin_user = """
         @prefix xsd:         <http://www.w3.org/2001/XMLSchema#> .
@@ -271,13 +289,12 @@ class StackHandler:
         )
         if not response.ok:
             logger.error(f"Cannot start DSP-API: Error when creating the admin user. response = {vars(response)}")
-            raise UserError("Cannot start DSP-API: Error when creating the admin user.")
+            raise InputError("Cannot start DSP-API: Error when creating the admin user.")
 
     def _initialize_fuseki(self) -> None:
         """
-        Create the "knora-test" repository and load some basic ontologies and data into it.
+        Load some basic ontologies and data into the 'dsp-repo' repository.
         """
-        self._create_knora_test_repo()
         if self.__stack_configuration.upload_test_data:
             self._load_data_into_repo()
         else:
@@ -290,11 +307,13 @@ class StackHandler:
         """
         compose_str = "docker compose -f docker-compose.yml"
         if self.__stack_configuration.latest_dev_version:
+            logger.debug("In order to get the latest dev version, run 'docker compose pull' ...")
             subprocess.run("docker compose pull".split(), cwd=self.__docker_path_of_user, check=True)
             compose_str += " -f docker-compose.override.yml"
-        elif self.__stack_configuration.api_version_for_validate:
-            compose_str += " -f docker-compose-validation.yml"
+        if self.__stack_configuration.custom_host is not None:
+            compose_str += " -f docker-compose.override-host.yml"
         compose_str += " up -d"
+        logger.debug(f"Running '{compose_str}' ...")
         subprocess.run(compose_str.split(), cwd=self.__docker_path_of_user, check=True)
 
     def _wait_for_api(self) -> None:
@@ -302,16 +321,37 @@ class StackHandler:
         Wait until the API is up and running.
         This mimicks the behaviour of the script webapi/scripts/wait-for-api.sh in the DSP-API repository.
         """
-        for _ in range(6 * 60):
+        logger.debug("Waiting for the API to start...")
+        for num_secs in range(6 * 60):
             try:
-                response = requests.get(f"{self.__localhost_url}:3333/health", timeout=1)
-                if not response.ok:
-                    time.sleep(1)
-                    continue
-            except requests.exceptions.RequestException:
-                time.sleep(1)
-                continue
-        print(f"DSP-API is now running on {self.__localhost_url}:3333/ and DSP-APP on {self.__localhost_url}:4200/")
+                params = RequestParameters("GET", f"{self.__localhost_url}:3333/health", timeout=1)
+                log_request(params)
+                response = requests.get(params.url, timeout=params.timeout)
+                log_response(response)
+                if response.ok:
+                    break
+            except requests.exceptions.RequestException as e:
+                logger.debug(f"RequestException while checking API status: {e}")
+            if num_secs > MINUTE / 2 and num_secs % 10 == 0:
+                # There is probably an issue, so we need more logs
+                with contextlib.suppress():
+                    docker_ps_output = subprocess.run(
+                        "docker ps -a".split(), cwd=self.__docker_path_of_user, check=True, capture_output=True
+                    ).stdout.decode("utf-8")
+                    docker_ps_output = "\n\t".join(docker_ps_output.split("\n"))
+                    logger.debug(f"docker ps -a output:\n\t{docker_ps_output}")
+                    docker_logs_output = subprocess.run(
+                        "docker logs start-stack-api-1".split(),
+                        cwd=self.__docker_path_of_user,
+                        check=True,
+                        capture_output=True,
+                    ).stdout.decode("utf-8")
+                    docker_logs_output = "\n\t".join(docker_logs_output.split("\n"))
+                    logger.debug(f"Logs of DSP-API container:\n\t{docker_logs_output}")
+            time.sleep(1)
+        msg = f"DSP-API is now running on {self.__localhost_url}:3333/ and DSP-APP on {self.__localhost_url}:4200/"
+        logger.debug(msg)
+        print(msg)
 
     def _execute_docker_system_prune(self) -> None:
         """
@@ -333,6 +373,7 @@ class StackHandler:
                     "to keep your docker clean and running smoothly. [y/n]"
                 )
         if prune_docker == "y":
+            logger.debug("Running 'docker system prune --volumes -f' ...")
             subprocess.run("docker system prune --volumes -f".split(), cwd=self.__docker_path_of_user, check=False)
 
     def _start_docker_containers(self) -> None:
@@ -356,15 +397,20 @@ class StackHandler:
         After startup, ask user if Docker should be pruned or not.
 
         Raises:
-            UserError: if the stack cannot be started with the parameters passed by the user
+            InputError: if the stack cannot be started with the parameters passed by the user
 
         Returns:
             True if everything went well, False otherwise
         """
-        if subprocess.run("docker stats --no-stream".split(), check=False, capture_output=True).returncode != 0:
-            raise UserError("Docker is not running properly. Please start Docker and try again.")
         self._copy_resources_to_home_dir()
-        self._get_sipi_docker_config_lua()
+        self._set_custom_host()
+        try:
+            self._get_sipi_docker_config_lua()
+        except (requests.ConnectionError, requests.ReadTimeout):
+            raise PermanentConnectionError(
+                "This command requires an internet connection. "
+                "Please ensure that your computer is connected and try again."
+            )
         self._start_docker_containers()
         return True
 
@@ -376,4 +422,7 @@ class StackHandler:
             True if everything went well, False otherwise
         """
         subprocess.run("docker compose down --volumes".split(), cwd=self.__docker_path_of_user, check=True)
+        shutil.rmtree(self.__docker_path_of_user / "sipi", ignore_errors=True)
+        # ignore all errors, because the dir cannot be found if this function is called multiple times,
+        # and because in GitHub CI, python lacks permissions to delete this dir
         return True