ocrd 3.5.1__py3-none-any.whl → 3.7.0__py3-none-any.whl

ocrd_network/runtime_data/__init__.py

@@ -5,10 +5,9 @@ __all__ = [
     "DataNetworkAgent",
     "DataRabbitMQ",
     "DataProcessingWorker",
-    "DataProcessorServer"
 ]
 
 from .deployer import Deployer
 from .hosts import DataHost
-from .network_agents import DataNetworkAgent, DataProcessingWorker, DataProcessorServer
+from .network_agents import DataNetworkAgent, DataProcessingWorker
 from .network_services import DataMongoDB, DataRabbitMQ

ocrd_network/runtime_data/deployer.py

@@ -9,11 +9,10 @@ Each Processing Worker is an instance of an OCR-D processor.
 from __future__ import annotations
 from pathlib import Path
 import psutil
-from time import sleep
-from typing import Dict, List, Union
+from typing import Dict, List
 
 from ocrd import OcrdMetsServer
-from ocrd_utils import config, getLogger, safe_filename
+from ocrd_utils import getLogger
 from ..logging_utils import get_mets_server_logging_file_path
 from ..utils import get_uds_path, is_mets_server_running, stop_mets_server
 from .config_parser import parse_hosts_data, parse_mongodb_data, parse_rabbitmq_data, validate_and_load_config
@@ -34,89 +33,15 @@ class Deployer:
         self.mets_servers_paths: Dict = {}  # {"ws_dir_path": "mets_server_url"}
         self.use_tcp_mets = ps_config.get("use_tcp_mets", False)
 
-    # TODO: Reconsider this.
-    def find_matching_network_agents(
-        self, worker_only: bool = False, server_only: bool = False, docker_only: bool = False,
-        native_only: bool = False, str_names_only: bool = False, unique_only: bool = False, sort: bool = False
-    ) -> Union[List[str], List[object]]:
-        """Finds and returns a list of matching data objects of type:
-        `DataProcessingWorker` and `DataProcessorServer`.
-
-        :py:attr:`worker_only` match only worker network agents (DataProcessingWorker)
-        :py:attr:`server_only` match only server network agents (DataProcessorServer)
-        :py:attr:`docker_only` match only docker network agents (DataProcessingWorker and DataProcessorServer)
-        :py:attr:`native_only` match only native network agents (DataProcessingWorker and DataProcessorServer)
-        :py:attr:`str_names_only` returns the processor_name filed instead of the Data* object
-        :py:attr:`unique_only` remove duplicate names from the matches
-        :py:attr:`sort` sort the result
-
-        `worker_only` and `server_only` are mutually exclusive to each other
-        `docker_only` and `native_only` are mutually exclusive to each other
-        `unique_only` is allowed only together with `str_names_only`
-        """
-
-        if worker_only and server_only:
-            msg = f"Only 'worker_only' or 'server_only' is allowed, not both."
-            self.log.exception(msg)
-            raise ValueError(msg)
-        if docker_only and native_only:
-            msg = f"Only 'docker_only' or 'native_only' is allowed, not both."
-            self.log.exception(msg)
-            raise ValueError(msg)
-        if not str_names_only and unique_only:
-            msg = f"Value 'unique_only' is allowed only together with 'str_names_only'"
-            self.log.exception(msg)
-            raise ValueError(msg)
-        if sort and not str_names_only:
-            msg = f"Value 'sort' is allowed only together with 'str_names_only'"
-            self.log.exception(msg)
-            raise ValueError(msg)
-
-        # Find all matching objects of type DataProcessingWorker or DataProcessorServer
-        matched_objects = []
-        for data_host in self.data_hosts:
-            if not server_only:
-                if not docker_only:
-                    for data_worker in data_host.network_agents_worker_native:
-                        matched_objects.append(data_worker)
-                if not native_only:
-                    for data_worker in data_host.network_agents_worker_docker:
-                        matched_objects.append(data_worker)
-            if not worker_only:
-                if not docker_only:
-                    for data_server in data_host.network_agents_server_native:
-                        matched_objects.append(data_server)
-                if not native_only:
-                    for data_server in data_host.network_agents_server_docker:
-                        matched_objects.append(data_server)
-        if not str_names_only:
-            return matched_objects
-        # Gets only the processor names of the matched objects
-        matched_names = [match.processor_name for match in matched_objects]
-        if not unique_only:
-            return matched_names
-        list_matched = list(dict.fromkeys(matched_names))
-        if not sort:
-            # Removes any duplicate entries from matched names
-            return list_matched
-        list_matched.sort()
-        return list_matched
-
-    def resolve_processor_server_url(self, processor_name) -> str:
-        processor_server_url = ''
-        for data_host in self.data_hosts:
-            processor_server_url = data_host.resolve_processor_server_url(processor_name=processor_name)
-        return processor_server_url
-
-    def deploy_network_agents(self, mongodb_url: str, rabbitmq_url: str) -> None:
-        self.log.debug("Deploying processing workers/processor servers...")
+    def deploy_workers(self, mongodb_url: str, rabbitmq_url: str) -> None:
+        self.log.debug("Deploying processing workers...")
         for host_data in self.data_hosts:
-            host_data.deploy_network_agents(logger=self.log, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
+            host_data.deploy_workers(logger=self.log, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
 
-    def stop_network_agents(self) -> None:
-        self.log.debug("Stopping processing workers/processor servers...")
+    def stop_workers(self) -> None:
+        self.log.debug("Stopping processing workers...")
         for host_data in self.data_hosts:
-            host_data.stop_network_agents(logger=self.log)
+            host_data.stop_workers(logger=self.log)
 
     def deploy_rabbitmq(self) -> str:
         self.data_queue.deploy_rabbitmq(self.log)
@@ -138,7 +63,7 @@ class Deployer:
         If RabbitMQ server is stopped before stopping Processing Workers that may have
         a bad outcome and leave Processing Workers in an unpredictable state.
         """
-        self.stop_network_agents()
+        self.stop_workers()
         self.stop_mongodb()
         self.stop_rabbitmq()
 
@@ -154,7 +79,9 @@ class Deployer:
                 "Removing to avoid any weird behavior before starting the server.")
             Path(mets_server_url).unlink()
         self.log.info(f"Starting UDS mets server: {mets_server_url}")
-        pid = OcrdMetsServer.create_process(mets_server_url=str(mets_server_url), ws_dir_path=str(ws_dir_path), log_file=str(log_file))
+        pid = OcrdMetsServer.create_process(mets_server_url=str(mets_server_url),
+                                            ws_dir_path=str(ws_dir_path),
+                                            log_file=str(log_file))
         self.mets_servers[str(mets_server_url)] = pid
         self.mets_servers_paths[str(ws_dir_path)] = str(mets_server_url)
         return mets_server_url

ocrd_network/runtime_data/hosts.py

@@ -1,9 +1,9 @@
 from logging import Logger
 from time import sleep
-from typing import Dict, List, Union
+from typing import Dict, List
 
 from .connection_clients import create_docker_client, create_ssh_client
-from .network_agents import AgentType, DataNetworkAgent, DataProcessingWorker, DataProcessorServer, DeployType
+from .network_agents import DataProcessingWorker, DeployType
 
 
 class DataHost:
@@ -24,68 +24,39 @@ class DataHost:
         self.ssh_client = None
         self.docker_client = None
 
-        # Time to wait between deploying agents
-        self.wait_between_agent_deploys: float = 0.3
+        # Time to wait between deploying single workers
+        self.wait_between_deploys: float = 0.3
 
-        # Lists of network agents based on their agent and deployment type
-        self.network_agents_worker_native = []
-        self.network_agents_worker_docker = []
-        self.network_agents_server_native = []
-        self.network_agents_server_docker = []
+        # Lists of Processing Workers based on their deployment type
+        self.workers_native = []
+        self.workers_docker = []
 
         if not workers:
             workers = []
         if not servers:
             servers = []
 
-        self.__parse_network_agents_workers(processing_workers=workers)
-        self.__parse_network_agents_servers(processor_servers=servers)
+        self.__parse_workers(processing_workers=workers)
 
-        # Used for caching deployed Processor Servers' ports on the current host
-        # Key: processor_name, Value: list of ports
-        self.processor_servers_ports: dict = {}
+    def __append_workers_to_lists(self, worker_data: DataProcessingWorker) -> None:
+        if worker_data.deploy_type != DeployType.DOCKER and worker_data.deploy_type != DeployType.NATIVE:
+            raise ValueError(f"Processing Worker deploy type is unknown: {worker_data.deploy_type}")
 
-    def __add_deployed_agent_server_port_to_cache(self, processor_name: str, port: int) -> None:
-        if processor_name not in self.processor_servers_ports:
-            self.processor_servers_ports[processor_name] = [port]
-            return
-        self.processor_servers_ports[processor_name] = self.processor_servers_ports[processor_name].append(port)
-
-    def __append_network_agent_to_lists(self, agent_data: DataNetworkAgent) -> None:
-        if agent_data.deploy_type != DeployType.DOCKER and agent_data.deploy_type != DeployType.NATIVE:
-            raise ValueError(f"Network agent deploy type is unknown: {agent_data.deploy_type}")
-        if agent_data.agent_type != AgentType.PROCESSING_WORKER and agent_data.agent_type != AgentType.PROCESSOR_SERVER:
-            raise ValueError(f"Network agent type is unknown: {agent_data.agent_type}")
-
-        if agent_data.deploy_type == DeployType.NATIVE:
+        if worker_data.deploy_type == DeployType.NATIVE:
             self.needs_ssh_connector = True
-            if agent_data.agent_type == AgentType.PROCESSING_WORKER:
-                self.network_agents_worker_native.append(agent_data)
-            if agent_data.agent_type == AgentType.PROCESSOR_SERVER:
-                self.network_agents_server_native.append(agent_data)
-        if agent_data.deploy_type == DeployType.DOCKER:
+            self.workers_native.append(worker_data)
+        if worker_data.deploy_type == DeployType.DOCKER:
             self.needs_docker_connector = True
-            if agent_data.agent_type == AgentType.PROCESSING_WORKER:
-                self.network_agents_worker_docker.append(agent_data)
-            if agent_data.agent_type == AgentType.PROCESSOR_SERVER:
-                self.network_agents_server_docker.append(agent_data)
-
-    def __parse_network_agents_servers(self, processor_servers: List[Dict]):
-        for server in processor_servers:
-            server_data = DataProcessorServer(
-                processor_name=server["name"], deploy_type=server["deploy_type"], host=self.host,
-                port=int(server["port"]), init_by_config=True, pid=None
-            )
-            self.__append_network_agent_to_lists(agent_data=server_data)
+            self.workers_docker.append(worker_data)
 
-    def __parse_network_agents_workers(self, processing_workers: List[Dict]):
+    def __parse_workers(self, processing_workers: List[Dict]):
         for worker in processing_workers:
             worker_data = DataProcessingWorker(
-                processor_name=worker["name"], deploy_type=worker["deploy_type"], host=self.host,
-                init_by_config=True, pid=None
+                processor_name=worker["name"], deploy_type=worker.get("deploy_type", "native"),
+                host=self.host, init_by_config=True, pid=None
             )
             for _ in range(int(worker["number_of_instance"])):
-                self.__append_network_agent_to_lists(agent_data=worker_data)
+                self.__append_workers_to_lists(worker_data=worker_data)
 
     def create_connection_client(self, client_type: str):
         if client_type not in ["docker", "ssh"]:
@@ -97,62 +68,46 @@ class DataHost:
             self.docker_client = create_docker_client(self.host, self.username, self.password, self.keypath)
             return self.docker_client
 
-    def __deploy_network_agent(
-        self, logger: Logger, agent_data: Union[DataProcessorServer, DataProcessingWorker],
+    def __deploy_single_worker(
+        self, logger: Logger, worker_data: DataProcessingWorker,
         mongodb_url: str, rabbitmq_url: str
     ) -> None:
-        deploy_type = agent_data.deploy_type
-        agent_type = agent_data.agent_type
-        name = agent_data.processor_name
-        agent_info = f"network agent: {agent_type}, deploy: {deploy_type}, name: {name}, host: {self.host}"
-        logger.info(f"Deploying {agent_info}")
+        deploy_type = worker_data.deploy_type
+        name = worker_data.processor_name
+        worker_info = f"Processing Worker, deploy: {deploy_type}, name: {name}, host: {self.host}"
+        logger.info(f"Deploying {worker_info}")
 
         connection_client = None
         if deploy_type == DeployType.NATIVE:
-            assert self.ssh_client, f"SSH client connection missing."
+            assert self.ssh_client, "SSH client connection missing."
             connection_client = self.ssh_client
         if deploy_type == DeployType.DOCKER:
-            assert self.docker_client, f"Docker client connection missing."
+            assert self.docker_client, "Docker client connection missing."
             connection_client = self.docker_client
 
-        if agent_type == AgentType.PROCESSING_WORKER:
-            agent_data.deploy_network_agent(logger, connection_client, mongodb_url, rabbitmq_url)
-        if agent_type == AgentType.PROCESSOR_SERVER:
-            agent_data.deploy_network_agent(logger, connection_client, mongodb_url)
+        worker_data.deploy_network_agent(logger, connection_client, mongodb_url, rabbitmq_url)
+        sleep(self.wait_between_deploys)
 
-        sleep(self.wait_between_agent_deploys)
-
-    def __deploy_network_agents_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
+    def __deploy_all_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
         logger.info(f"Deploying processing workers on host: {self.host}")
-        amount_workers = len(self.network_agents_worker_native) + len(self.network_agents_worker_docker)
+        amount_workers = len(self.workers_native) + len(self.workers_docker)
         if not amount_workers:
-            logger.info(f"No processing workers found to be deployed")
-        for data_worker in self.network_agents_worker_native:
-            self.__deploy_network_agent(logger, data_worker, mongodb_url, rabbitmq_url)
-        for data_worker in self.network_agents_worker_docker:
-            self.__deploy_network_agent(logger, data_worker, mongodb_url, rabbitmq_url)
-
-    def __deploy_network_agents_servers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
-        logger.info(f"Deploying processor servers on host: {self.host}")
-        amount_servers = len(self.network_agents_server_native) + len(self.network_agents_server_docker)
-        if not amount_servers:
-            logger.info(f"No processor servers found to be deployed")
-        for data_server in self.network_agents_server_native:
-            self.__deploy_network_agent(logger, data_server, mongodb_url, rabbitmq_url)
-            self.__add_deployed_agent_server_port_to_cache(data_server.processor_name, data_server.port)
-        for data_server in self.network_agents_server_docker:
-            self.__deploy_network_agent(logger, data_server, mongodb_url, rabbitmq_url)
-            self.__add_deployed_agent_server_port_to_cache(data_server.processor_name, data_server.port)
-
-    def deploy_network_agents(self, logger: Logger, mongodb_url: str, rabbitmq_url: str) -> None:
+            logger.info("No processing workers found to be deployed")
+        for data_worker in self.workers_native:
+            self.__deploy_single_worker(logger, data_worker, mongodb_url, rabbitmq_url)
+        for data_worker in self.workers_docker:
+            self.__deploy_single_worker(logger, data_worker, mongodb_url, rabbitmq_url)
+
+    def deploy_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str) -> None:
         if self.needs_ssh_connector and not self.ssh_client:
             logger.debug("Creating missing ssh connector before deploying")
             self.ssh_client = self.create_connection_client(client_type="ssh")
         if self.needs_docker_connector:
             logger.debug("Creating missing docker connector before deploying")
             self.docker_client = self.create_connection_client(client_type="docker")
-        self.__deploy_network_agents_workers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
-        self.__deploy_network_agents_servers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
+
+        self.__deploy_all_workers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
+
         if self.ssh_client:
             self.ssh_client.close()
             self.ssh_client = None
@@ -160,66 +115,42 @@ class DataHost:
             self.docker_client.close()
             self.docker_client = None
 
-    def __stop_network_agent(self, logger: Logger, name: str, deploy_type: DeployType, agent_type: AgentType, pid: str):
-        agent_info = f"network agent: {agent_type}, deploy: {deploy_type}, name: {name}"
+    def __stop_worker(self, logger: Logger, name: str, deploy_type: DeployType, pid: str):
+        worker_info = f"Processing Worker: deploy: {deploy_type}, name: {name}"
         if not pid:
-            logger.warning(f"No pid was passed for {agent_info}")
+            logger.warning(f"No pid was passed for {worker_info}")
             return
-        agent_info += f", pid: {pid}"
-        logger.info(f"Stopping {agent_info}")
+        worker_info += f", pid: {pid}"
+        logger.info(f"Stopping {worker_info}")
         if deploy_type == DeployType.NATIVE:
-            assert self.ssh_client, f"SSH client connection missing"
+            assert self.ssh_client, "SSH client connection missing"
             self.ssh_client.exec_command(f"kill {pid}")
         if deploy_type == DeployType.DOCKER:
-            assert self.docker_client, f"Docker client connection missing"
+            assert self.docker_client, "Docker client connection missing"
             self.docker_client.containers.get(pid).stop()
 
-    def __stop_network_agents_workers(self, logger: Logger):
-        logger.info(f"Stopping processing workers on host: {self.host}")
-        amount_workers = len(self.network_agents_worker_native) + len(self.network_agents_worker_docker)
-        if not amount_workers:
-            logger.warning(f"No active processing workers to be stopped.")
-        for worker in self.network_agents_worker_native:
-            self.__stop_network_agent(logger, worker.processor_name, worker.deploy_type, worker.agent_type, worker.pid)
-        self.network_agents_worker_native = []
-        for worker in self.network_agents_worker_docker:
-            self.__stop_network_agent(logger, worker.processor_name, worker.deploy_type, worker.agent_type, worker.pid)
-        self.network_agents_worker_docker = []
-
-    def __stop_network_agents_servers(self, logger: Logger):
-        logger.info(f"Stopping processor servers on host: {self.host}")
-        amount_servers = len(self.network_agents_server_native) + len(self.network_agents_server_docker)
-        if not amount_servers:
-            logger.warning(f"No active processor servers to be stopped.")
-        for server in self.network_agents_server_native:
-            self.__stop_network_agent(logger, server.processor_name, server.deploy_type, server.agent_type, server.pid)
-        self.network_agents_server_native = []
-        for server in self.network_agents_server_docker:
-            self.__stop_network_agent(logger, server.processor_name, server.deploy_type, server.agent_type, server.pid)
-        self.network_agents_server_docker = []
-
-    def stop_network_agents(self, logger: Logger):
+    def stop_workers(self, logger: Logger):
         if self.needs_ssh_connector and not self.ssh_client:
             logger.debug("Creating missing ssh connector before stopping")
             self.ssh_client = self.create_connection_client(client_type="ssh")
         if self.needs_docker_connector and not self.docker_client:
             logger.debug("Creating missing docker connector before stopping")
             self.docker_client = self.create_connection_client(client_type="docker")
-        self.__stop_network_agents_workers(logger=logger)
-        self.__stop_network_agents_servers(logger=logger)
+
+        logger.info(f"Stopping processing workers on host: {self.host}")
+        amount_workers = len(self.workers_native) + len(self.workers_docker)
+        if not amount_workers:
+            logger.warning("No active processing workers to be stopped.")
+        for worker in self.workers_native:
+            self.__stop_worker(logger, worker.processor_name, worker.deploy_type, worker.pid)
+        self.workers_native = []
+        for worker in self.workers_docker:
+            self.__stop_worker(logger, worker.processor_name, worker.deploy_type, worker.pid)
+        self.workers_docker = []
+
         if self.ssh_client:
             self.ssh_client.close()
             self.ssh_client = None
         if self.docker_client:
             self.docker_client.close()
             self.docker_client = None
-
-    def resolve_processor_server_url(self, processor_name: str) -> str:
-        processor_server_url = ''
-        for data_server in self.network_agents_server_docker:
-            if data_server.processor_name == processor_name:
-                processor_server_url = f"http://{self.host}:{data_server.port}/"
-        for data_server in self.network_agents_server_native:
-            if data_server.processor_name == processor_name:
-                processor_server_url = f"http://{self.host}:{data_server.port}/"
-        return processor_server_url

ocrd_network/runtime_data/network_agents.py

@@ -2,14 +2,15 @@ from logging import Logger
 from typing import Any
 
 from re import search as re_search
-from ..constants import AgentType, DeployType
+from ..constants import DeployType
 
 
 # TODO: Find appropriate replacement for the hack
 def deploy_agent_native_get_pid_hack(logger: Logger, ssh_client, start_cmd: str):
     channel = ssh_client.invoke_shell()
     stdin, stdout = channel.makefile("wb"), channel.makefile("rb")
-    logger.debug(f"Executing command: {start_cmd}")
+    # TODO: set back to debug
+    logger.info(f"Executing command: {start_cmd}")
 
     # TODO: This hack should still be fixed
     #   Note left from @joschrew
@@ -40,14 +41,13 @@ def deploy_agent_docker_template(logger: Logger, docker_client, start_cmd: str):
 
 class DataNetworkAgent:
     def __init__(
-        self, processor_name: str, deploy_type: DeployType, agent_type: AgentType,
+        self, processor_name: str, deploy_type: DeployType,
         host: str, init_by_config: bool, pid: Any = None
     ) -> None:
         self.processor_name = processor_name
         self.deploy_type = deploy_type
         self.host = host
         self.deployed_by_config = init_by_config
-        self.agent_type = agent_type
         # The id is assigned when the agent is deployed
         self.pid = pid
 
@@ -69,42 +69,18 @@ class DataProcessingWorker(DataNetworkAgent):
         self, processor_name: str, deploy_type: DeployType, host: str, init_by_config: bool, pid: Any = None
     ) -> None:
         super().__init__(
-            processor_name=processor_name, host=host, deploy_type=deploy_type, agent_type=AgentType.PROCESSING_WORKER,
+            processor_name=processor_name, host=host, deploy_type=deploy_type,
             init_by_config=init_by_config, pid=pid
         )
 
     def deploy_network_agent(self, logger: Logger, connector_client, database_url: str, queue_url: str):
         if self.deploy_type == DeployType.NATIVE:
-            start_cmd = f"{self.processor_name} {self.agent_type} --database {database_url} --queue {queue_url} &"
+            start_cmd = f"{self.processor_name} --database {database_url} --queue {queue_url} &"
             self.pid = self._start_native_instance(logger, connector_client, start_cmd)
             return self.pid
         if self.deploy_type == DeployType.DOCKER:
             # TODO: add real command to start processing worker in docker here
-            start_cmd = f""
-            self.pid = self._start_docker_instance(logger, connector_client, start_cmd)
-            return self.pid
-        raise RuntimeError(f"Unknown deploy type of {self.__dict__}")
-
-
-class DataProcessorServer(DataNetworkAgent):
-    def __init__(
-        self, processor_name: str, deploy_type: DeployType, host: str, port: int, init_by_config: bool, pid: Any = None
-    ) -> None:
-        super().__init__(
-            processor_name=processor_name, host=host, deploy_type=deploy_type, agent_type=AgentType.PROCESSOR_SERVER,
-            init_by_config=init_by_config, pid=pid
-        )
-        self.port = port
-
-    def deploy_network_agent(self, logger: Logger, connector_client, database_url: str):
-        agent_address = f"{self.host}:{self.port}"
-        if self.deploy_type == DeployType.NATIVE:
-            start_cmd = f"{self.processor_name} {self.agent_type} --address {agent_address} --database {database_url} &"
-            self.pid = self._start_native_instance(logger, connector_client, start_cmd)
-            return self.pid
-        if self.deploy_type == DeployType.DOCKER:
-            # TODO: add real command to start processor server in docker here
-            start_cmd = f""
+            start_cmd = ""
             self.pid = self._start_docker_instance(logger, connector_client, start_cmd)
             return self.pid
         raise RuntimeError(f"Unknown deploy type of {self.__dict__}")

ocrd_network/runtime_data/network_services.py

@@ -129,7 +129,7 @@ class DataRabbitMQ(DataNetworkService):
         rmq_host, rmq_port, rmq_vhost = self.host, int(self.port), self.vhost
         rmq_user, rmq_password = self.cred_username, self.cred_password
         if self.skip_deployment:
-            logger.debug(f"RabbitMQ is managed externally. Skipping deployment.")
+            logger.debug("RabbitMQ is managed externally. Skipping deployment.")
             verify_rabbitmq_available(logger=logger, rabbitmq_address=self.service_url)
             return self.service_url
         if not env:

ocrd_network/server_cache.py

@@ -33,7 +33,7 @@ class CacheLockedPages:
         if not self.locked_pages.get(workspace_key, None):
             self.log.info(f"No entry found in the locked pages cache for workspace key: {workspace_key}")
             return False
-        debug_message = f"Caching the received request due to locked output file grp pages."
+        debug_message = "Caching the received request due to locked output file grp pages."
         for file_group in output_file_grps:
             if file_group in self.locked_pages[workspace_key]:
                 if self.placeholder_all_pages in self.locked_pages[workspace_key][file_group]:

ocrd_network/server_utils.py

@@ -55,7 +55,7 @@ def create_processing_message(logger: Logger, job: DBProcessorJob) -> OcrdProces
         )
         return processing_message
     except ValueError as error:
-        message = f"Failed to create OcrdProcessingMessage from DBProcessorJob"
+        message = "Failed to create OcrdProcessingMessage from DBProcessorJob"
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message, error)
 
 
@@ -124,50 +124,7 @@ async def _get_processor_job_log(logger: Logger, job_id: str) -> FileResponse:
     return FileResponse(path=log_file_path, filename=log_file_path.name)
 
 
-def request_processor_server_tool_json(logger: Logger, processor_server_base_url: str) -> Dict:
-    # Request the ocrd tool json from the Processor Server
-    try:
-        response = requests_get(
-            urljoin(base=processor_server_base_url, url="info"),
-            headers={"Content-Type": "application/json"}
-        )
-    except Exception as error:
-        message = f"Failed to retrieve ocrd tool json from: {processor_server_base_url}"
-        raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message, error)
-    if response.status_code != 200:
-        message = f"Failed to retrieve tool json from: {processor_server_base_url}, code: {response.status_code}"
-        raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message)
-    return response.json()
-
-async def forward_job_to_processor_server(
-    logger: Logger, job_input: PYJobInput, processor_server_base_url: str
-) -> PYJobOutput:
-    try:
-        json_data = dumps(job_input.dict(exclude_unset=True, exclude_none=True))
-    except Exception as error:
-        message = f"Failed to json dump the PYJobInput: {job_input}"
-        raise_http_exception(logger, status.HTTP_500_INTERNAL_SERVER_ERROR, message, error)
-
-    # TODO: The amount of pages should come as a request input
-    # TODO: cf https://github.com/OCR-D/core/pull/1030/files#r1152551161
-    #  currently, use 200 as a default
-    request_timeout = calculate_processing_request_timeout(amount_pages=200, timeout_per_page=20.0)
-
-    # Post a processing job to the Processor Server asynchronously
-    async with AsyncClient(timeout=Timeout(timeout=request_timeout, connect=30.0)) as client:
-        response = await client.post(
-            urljoin(base=processor_server_base_url, url="run"),
-            headers={"Content-Type": "application/json"},
-            json=loads(json_data)
-        )
-    if response.status_code != 202:
-        message = f"Failed to post '{job_input.processor_name}' job to: {processor_server_base_url}"
-        raise_http_exception(logger, status.HTTP_500_INTERNAL_SERVER_ERROR, message)
-    job_output = response.json()
-    return job_output
-
-
-async def get_workflow_content(logger: Logger, workflow_id: str, workflow: Union[UploadFile, None]) -> str:
+async def get_workflow_content(logger: Logger, workflow_id: str, workflow: Union[UploadFile, str, None]) -> str:
     if not workflow and not workflow_id:
         message = "Either 'workflow' must be uploaded as a file or 'workflow_id' must be provided. Both are missing."
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message)
@@ -178,6 +135,9 @@ async def get_workflow_content(logger: Logger, workflow_id: str, workflow: Union
         except ValueError as error:
             message = f"Workflow with id '{workflow_id}' not found"
             raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message, error)
+    if isinstance(workflow, str):
+        with open(workflow) as wf_file:
+            return wf_file.read()
     return await generate_workflow_content(workflow)
 
 
@@ -193,14 +153,14 @@ def parse_workflow_tasks(logger: Logger, workflow_content: str) -> List[Processo
         tasks_list = workflow_content.splitlines()
         return [ProcessorTask.parse(task_str) for task_str in tasks_list if task_str.strip()]
     except ValueError as error:
-        message = f"Failed parsing processing tasks from a workflow."
+        message = "Failed parsing processing tasks from a workflow."
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message, error)
 
 
 def raise_http_exception(logger: Logger, status_code: int, message: str, error: Exception = None) -> None:
     if error:
         message = f"{message} {error}"
-    logger.exception(f"{message}")
+    logger.exception(message)
     raise HTTPException(status_code=status_code, detail=message)
 
 
@@ -214,7 +174,7 @@ def validate_job_input(logger: Logger, processor_name: str, ocrd_tool: dict, job
         )
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message)
     if not ocrd_tool:
-        message = f"Failed parsing processing tasks from a workflow."
+        message = "Failed parsing processing tasks from a workflow."
         raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message)
     try:
         report = ParameterValidator(ocrd_tool).validate(dict(job_input.parameters))
@@ -249,10 +209,10 @@ def validate_first_task_input_file_groups_existence(logger: Logger, mets_path: s
             raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message)
 
 
-def kill_mets_server_zombies(minutes_ago : Optional[int], dry_run : Optional[bool]) -> List[int]:
-    if minutes_ago == None:
+def kill_mets_server_zombies(minutes_ago: Optional[int], dry_run: Optional[bool]) -> List[int]:
+    if minutes_ago is None:
         minutes_ago = 90
-    if dry_run == None:
+    if dry_run is None:
         dry_run = False
 
     now = time()
@@ -271,7 +231,8 @@ def kill_mets_server_zombies(minutes_ago : Optional[int], dry_run : Optional[boo
         if re.match(cmdline_pat, cmdline):
             pid = int(procdir.name)
             ret.append(pid)
-            print(f'METS Server with PID {pid} was created {ctime_ago} minutes ago, more than {minutes_ago}, so killing (cmdline="{cmdline})', file=sys.stderr)
+            print(f'METS Server with PID {pid} was created {ctime_ago} minutes ago, more than {minutes_ago}, '
+                  f'so killing (cmdline="{cmdline})', file=sys.stderr)
             if dry_run:
                 print(f'[dry_run is active] kill {pid}')
             else:

ocrd_network/utils.py

@@ -172,5 +172,6 @@ def stop_mets_server(logger: Logger, mets_server_url: str, ws_dir_path: str) ->
     else:
         ValueError(f"Unexpected protocol type: {protocol}")
 
+
 def get_uds_path(ws_dir_path: str) -> Path:
     return Path(config.OCRD_NETWORK_SOCKETS_ROOT_DIR, f"{safe_filename(ws_dir_path)}.sock")