ocrd 3.5.0__py3-none-any.whl → 3.6.0__py3-none-any.whl

ocrd_network/runtime_data/deployer.py

@@ -9,11 +9,10 @@ Each Processing Worker is an instance of an OCR-D processor.
 from __future__ import annotations
 from pathlib import Path
 import psutil
-from time import sleep
-from typing import Dict, List, Union
+from typing import Dict, List
 
 from ocrd import OcrdMetsServer
-from ocrd_utils import config, getLogger, safe_filename
+from ocrd_utils import getLogger
 from ..logging_utils import get_mets_server_logging_file_path
 from ..utils import get_uds_path, is_mets_server_running, stop_mets_server
 from .config_parser import parse_hosts_data, parse_mongodb_data, parse_rabbitmq_data, validate_and_load_config
@@ -34,89 +33,15 @@ class Deployer:
         self.mets_servers_paths: Dict = {}  # {"ws_dir_path": "mets_server_url"}
         self.use_tcp_mets = ps_config.get("use_tcp_mets", False)
 
-    # TODO: Reconsider this.
-    def find_matching_network_agents(
-        self, worker_only: bool = False, server_only: bool = False, docker_only: bool = False,
-        native_only: bool = False, str_names_only: bool = False, unique_only: bool = False, sort: bool = False
-    ) -> Union[List[str], List[object]]:
-        """Finds and returns a list of matching data objects of type:
-        `DataProcessingWorker` and `DataProcessorServer`.
-
-        :py:attr:`worker_only` match only worker network agents (DataProcessingWorker)
-        :py:attr:`server_only` match only server network agents (DataProcessorServer)
-        :py:attr:`docker_only` match only docker network agents (DataProcessingWorker and DataProcessorServer)
-        :py:attr:`native_only` match only native network agents (DataProcessingWorker and DataProcessorServer)
-        :py:attr:`str_names_only` returns the processor_name filed instead of the Data* object
-        :py:attr:`unique_only` remove duplicate names from the matches
-        :py:attr:`sort` sort the result
-
-        `worker_only` and `server_only` are mutually exclusive to each other
-        `docker_only` and `native_only` are mutually exclusive to each other
-        `unique_only` is allowed only together with `str_names_only`
-        """
-
-        if worker_only and server_only:
-            msg = f"Only 'worker_only' or 'server_only' is allowed, not both."
-            self.log.exception(msg)
-            raise ValueError(msg)
-        if docker_only and native_only:
-            msg = f"Only 'docker_only' or 'native_only' is allowed, not both."
-            self.log.exception(msg)
-            raise ValueError(msg)
-        if not str_names_only and unique_only:
-            msg = f"Value 'unique_only' is allowed only together with 'str_names_only'"
-            self.log.exception(msg)
-            raise ValueError(msg)
-        if sort and not str_names_only:
-            msg = f"Value 'sort' is allowed only together with 'str_names_only'"
-            self.log.exception(msg)
-            raise ValueError(msg)
-
-        # Find all matching objects of type DataProcessingWorker or DataProcessorServer
-        matched_objects = []
-        for data_host in self.data_hosts:
-            if not server_only:
-                if not docker_only:
-                    for data_worker in data_host.network_agents_worker_native:
-                        matched_objects.append(data_worker)
-                if not native_only:
-                    for data_worker in data_host.network_agents_worker_docker:
-                        matched_objects.append(data_worker)
-            if not worker_only:
-                if not docker_only:
-                    for data_server in data_host.network_agents_server_native:
-                        matched_objects.append(data_server)
-                if not native_only:
-                    for data_server in data_host.network_agents_server_docker:
-                        matched_objects.append(data_server)
-        if not str_names_only:
-            return matched_objects
-        # Gets only the processor names of the matched objects
-        matched_names = [match.processor_name for match in matched_objects]
-        if not unique_only:
-            return matched_names
-        list_matched = list(dict.fromkeys(matched_names))
-        if not sort:
-            # Removes any duplicate entries from matched names
-            return list_matched
-        list_matched.sort()
-        return list_matched
-
-    def resolve_processor_server_url(self, processor_name) -> str:
-        processor_server_url = ''
-        for data_host in self.data_hosts:
-            processor_server_url = data_host.resolve_processor_server_url(processor_name=processor_name)
-        return processor_server_url
-
-    def deploy_network_agents(self, mongodb_url: str, rabbitmq_url: str) -> None:
-        self.log.debug("Deploying processing workers/processor servers...")
+    def deploy_workers(self, mongodb_url: str, rabbitmq_url: str) -> None:
+        self.log.debug("Deploying processing workers...")
         for host_data in self.data_hosts:
-            host_data.deploy_network_agents(logger=self.log, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
+            host_data.deploy_workers(logger=self.log, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
 
-    def stop_network_agents(self) -> None:
-        self.log.debug("Stopping processing workers/processor servers...")
+    def stop_workers(self) -> None:
+        self.log.debug("Stopping processing workers...")
         for host_data in self.data_hosts:
-            host_data.stop_network_agents(logger=self.log)
+            host_data.stop_workers(logger=self.log)
 
     def deploy_rabbitmq(self) -> str:
         self.data_queue.deploy_rabbitmq(self.log)
@@ -138,7 +63,7 @@ class Deployer:
         If RabbitMQ server is stopped before stopping Processing Workers that may have
         a bad outcome and leave Processing Workers in an unpredictable state.
         """
-        self.stop_network_agents()
+        self.stop_workers()
         self.stop_mongodb()
         self.stop_rabbitmq()
 
@@ -154,7 +79,9 @@ class Deployer:
                 "Removing to avoid any weird behavior before starting the server.")
             Path(mets_server_url).unlink()
         self.log.info(f"Starting UDS mets server: {mets_server_url}")
-        pid = OcrdMetsServer.create_process(mets_server_url=str(mets_server_url), ws_dir_path=str(ws_dir_path), log_file=str(log_file))
+        pid = OcrdMetsServer.create_process(mets_server_url=str(mets_server_url),
+                                            ws_dir_path=str(ws_dir_path),
+                                            log_file=str(log_file))
         self.mets_servers[str(mets_server_url)] = pid
         self.mets_servers_paths[str(ws_dir_path)] = str(mets_server_url)
         return mets_server_url

ocrd_network/runtime_data/hosts.py

@@ -1,9 +1,9 @@
 from logging import Logger
 from time import sleep
-from typing import Dict, List, Union
+from typing import Dict, List
 
 from .connection_clients import create_docker_client, create_ssh_client
-from .network_agents import AgentType, DataNetworkAgent, DataProcessingWorker, DataProcessorServer, DeployType
+from .network_agents import DataProcessingWorker, DeployType
 
 
 class DataHost:
@@ -24,68 +24,39 @@ class DataHost:
         self.ssh_client = None
         self.docker_client = None
 
-        # Time to wait between deploying agents
-        self.wait_between_agent_deploys: float = 0.3
+        # Time to wait between deploying single workers
+        self.wait_between_deploys: float = 0.3
 
-        # Lists of network agents based on their agent and deployment type
-        self.network_agents_worker_native = []
-        self.network_agents_worker_docker = []
-        self.network_agents_server_native = []
-        self.network_agents_server_docker = []
+        # Lists of Processing Workers based on their deployment type
+        self.workers_native = []
+        self.workers_docker = []
 
         if not workers:
             workers = []
         if not servers:
             servers = []
 
-        self.__parse_network_agents_workers(processing_workers=workers)
-        self.__parse_network_agents_servers(processor_servers=servers)
+        self.__parse_workers(processing_workers=workers)
 
-        # Used for caching deployed Processor Servers' ports on the current host
-        # Key: processor_name, Value: list of ports
-        self.processor_servers_ports: dict = {}
+    def __append_workers_to_lists(self, worker_data: DataProcessingWorker) -> None:
+        if worker_data.deploy_type != DeployType.DOCKER and worker_data.deploy_type != DeployType.NATIVE:
+            raise ValueError(f"Processing Worker deploy type is unknown: {worker_data.deploy_type}")
 
-    def __add_deployed_agent_server_port_to_cache(self, processor_name: str, port: int) -> None:
-        if processor_name not in self.processor_servers_ports:
-            self.processor_servers_ports[processor_name] = [port]
-            return
-        self.processor_servers_ports[processor_name] = self.processor_servers_ports[processor_name].append(port)
-
-    def __append_network_agent_to_lists(self, agent_data: DataNetworkAgent) -> None:
-        if agent_data.deploy_type != DeployType.DOCKER and agent_data.deploy_type != DeployType.NATIVE:
-            raise ValueError(f"Network agent deploy type is unknown: {agent_data.deploy_type}")
-        if agent_data.agent_type != AgentType.PROCESSING_WORKER and agent_data.agent_type != AgentType.PROCESSOR_SERVER:
-            raise ValueError(f"Network agent type is unknown: {agent_data.agent_type}")
-
-        if agent_data.deploy_type == DeployType.NATIVE:
+        if worker_data.deploy_type == DeployType.NATIVE:
             self.needs_ssh_connector = True
-            if agent_data.agent_type == AgentType.PROCESSING_WORKER:
-                self.network_agents_worker_native.append(agent_data)
-            if agent_data.agent_type == AgentType.PROCESSOR_SERVER:
-                self.network_agents_server_native.append(agent_data)
-        if agent_data.deploy_type == DeployType.DOCKER:
+            self.workers_native.append(worker_data)
+        if worker_data.deploy_type == DeployType.DOCKER:
             self.needs_docker_connector = True
-            if agent_data.agent_type == AgentType.PROCESSING_WORKER:
-                self.network_agents_worker_docker.append(agent_data)
-            if agent_data.agent_type == AgentType.PROCESSOR_SERVER:
-                self.network_agents_server_docker.append(agent_data)
-
-    def __parse_network_agents_servers(self, processor_servers: List[Dict]):
-        for server in processor_servers:
-            server_data = DataProcessorServer(
-                processor_name=server["name"], deploy_type=server["deploy_type"], host=self.host,
-                port=int(server["port"]), init_by_config=True, pid=None
-            )
-            self.__append_network_agent_to_lists(agent_data=server_data)
+            self.workers_docker.append(worker_data)
 
-    def __parse_network_agents_workers(self, processing_workers: List[Dict]):
+    def __parse_workers(self, processing_workers: List[Dict]):
         for worker in processing_workers:
             worker_data = DataProcessingWorker(
-                processor_name=worker["name"], deploy_type=worker["deploy_type"], host=self.host,
-                init_by_config=True, pid=None
+                processor_name=worker["name"], deploy_type=worker.get("deploy_type", "native"),
+                host=self.host, init_by_config=True, pid=None
             )
             for _ in range(int(worker["number_of_instance"])):
-                self.__append_network_agent_to_lists(agent_data=worker_data)
+                self.__append_workers_to_lists(worker_data=worker_data)
 
     def create_connection_client(self, client_type: str):
         if client_type not in ["docker", "ssh"]:
@@ -97,62 +68,46 @@ class DataHost:
             self.docker_client = create_docker_client(self.host, self.username, self.password, self.keypath)
             return self.docker_client
 
-    def __deploy_network_agent(
-        self, logger: Logger, agent_data: Union[DataProcessorServer, DataProcessingWorker],
+    def __deploy_single_worker(
+        self, logger: Logger, worker_data: DataProcessingWorker,
         mongodb_url: str, rabbitmq_url: str
     ) -> None:
-        deploy_type = agent_data.deploy_type
-        agent_type = agent_data.agent_type
-        name = agent_data.processor_name
-        agent_info = f"network agent: {agent_type}, deploy: {deploy_type}, name: {name}, host: {self.host}"
-        logger.info(f"Deploying {agent_info}")
+        deploy_type = worker_data.deploy_type
+        name = worker_data.processor_name
+        worker_info = f"Processing Worker, deploy: {deploy_type}, name: {name}, host: {self.host}"
+        logger.info(f"Deploying {worker_info}")
 
         connection_client = None
         if deploy_type == DeployType.NATIVE:
-            assert self.ssh_client, f"SSH client connection missing."
+            assert self.ssh_client, "SSH client connection missing."
             connection_client = self.ssh_client
         if deploy_type == DeployType.DOCKER:
-            assert self.docker_client, f"Docker client connection missing."
+            assert self.docker_client, "Docker client connection missing."
             connection_client = self.docker_client
 
-        if agent_type == AgentType.PROCESSING_WORKER:
-            agent_data.deploy_network_agent(logger, connection_client, mongodb_url, rabbitmq_url)
-        if agent_type == AgentType.PROCESSOR_SERVER:
-            agent_data.deploy_network_agent(logger, connection_client, mongodb_url)
+        worker_data.deploy_network_agent(logger, connection_client, mongodb_url, rabbitmq_url)
+        sleep(self.wait_between_deploys)
 
-        sleep(self.wait_between_agent_deploys)
-
-    def __deploy_network_agents_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
+    def __deploy_all_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
         logger.info(f"Deploying processing workers on host: {self.host}")
-        amount_workers = len(self.network_agents_worker_native) + len(self.network_agents_worker_docker)
+        amount_workers = len(self.workers_native) + len(self.workers_docker)
         if not amount_workers:
-            logger.info(f"No processing workers found to be deployed")
-        for data_worker in self.network_agents_worker_native:
-            self.__deploy_network_agent(logger, data_worker, mongodb_url, rabbitmq_url)
-        for data_worker in self.network_agents_worker_docker:
-            self.__deploy_network_agent(logger, data_worker, mongodb_url, rabbitmq_url)
-
-    def __deploy_network_agents_servers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
-        logger.info(f"Deploying processor servers on host: {self.host}")
-        amount_servers = len(self.network_agents_server_native) + len(self.network_agents_server_docker)
-        if not amount_servers:
-            logger.info(f"No processor servers found to be deployed")
-        for data_server in self.network_agents_server_native:
-            self.__deploy_network_agent(logger, data_server, mongodb_url, rabbitmq_url)
-            self.__add_deployed_agent_server_port_to_cache(data_server.processor_name, data_server.port)
-        for data_server in self.network_agents_server_docker:
-            self.__deploy_network_agent(logger, data_server, mongodb_url, rabbitmq_url)
-            self.__add_deployed_agent_server_port_to_cache(data_server.processor_name, data_server.port)
-
-    def deploy_network_agents(self, logger: Logger, mongodb_url: str, rabbitmq_url: str) -> None:
+            logger.info("No processing workers found to be deployed")
+        for data_worker in self.workers_native:
+            self.__deploy_single_worker(logger, data_worker, mongodb_url, rabbitmq_url)
+        for data_worker in self.workers_docker:
+            self.__deploy_single_worker(logger, data_worker, mongodb_url, rabbitmq_url)
+
+    def deploy_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str) -> None:
         if self.needs_ssh_connector and not self.ssh_client:
             logger.debug("Creating missing ssh connector before deploying")
             self.ssh_client = self.create_connection_client(client_type="ssh")
         if self.needs_docker_connector:
             logger.debug("Creating missing docker connector before deploying")
             self.docker_client = self.create_connection_client(client_type="docker")
-        self.__deploy_network_agents_workers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
-        self.__deploy_network_agents_servers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
+
+        self.__deploy_all_workers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
+
         if self.ssh_client:
             self.ssh_client.close()
             self.ssh_client = None
@@ -160,66 +115,42 @@ class DataHost:
             self.docker_client.close()
             self.docker_client = None
 
-    def __stop_network_agent(self, logger: Logger, name: str, deploy_type: DeployType, agent_type: AgentType, pid: str):
-        agent_info = f"network agent: {agent_type}, deploy: {deploy_type}, name: {name}"
+    def __stop_worker(self, logger: Logger, name: str, deploy_type: DeployType, pid: str):
+        worker_info = f"Processing Worker: deploy: {deploy_type}, name: {name}"
         if not pid:
-            logger.warning(f"No pid was passed for {agent_info}")
+            logger.warning(f"No pid was passed for {worker_info}")
             return
-        agent_info += f", pid: {pid}"
-        logger.info(f"Stopping {agent_info}")
+        worker_info += f", pid: {pid}"
+        logger.info(f"Stopping {worker_info}")
         if deploy_type == DeployType.NATIVE:
-            assert self.ssh_client, f"SSH client connection missing"
+            assert self.ssh_client, "SSH client connection missing"
             self.ssh_client.exec_command(f"kill {pid}")
         if deploy_type == DeployType.DOCKER:
-            assert self.docker_client, f"Docker client connection missing"
+            assert self.docker_client, "Docker client connection missing"
             self.docker_client.containers.get(pid).stop()
 
-    def __stop_network_agents_workers(self, logger: Logger):
-        logger.info(f"Stopping processing workers on host: {self.host}")
-        amount_workers = len(self.network_agents_worker_native) + len(self.network_agents_worker_docker)
-        if not amount_workers:
-            logger.warning(f"No active processing workers to be stopped.")
-        for worker in self.network_agents_worker_native:
-            self.__stop_network_agent(logger, worker.processor_name, worker.deploy_type, worker.agent_type, worker.pid)
-        self.network_agents_worker_native = []
-        for worker in self.network_agents_worker_docker:
-            self.__stop_network_agent(logger, worker.processor_name, worker.deploy_type, worker.agent_type, worker.pid)
-        self.network_agents_worker_docker = []
-
-    def __stop_network_agents_servers(self, logger: Logger):
-        logger.info(f"Stopping processor servers on host: {self.host}")
-        amount_servers = len(self.network_agents_server_native) + len(self.network_agents_server_docker)
-        if not amount_servers:
-            logger.warning(f"No active processor servers to be stopped.")
-        for server in self.network_agents_server_native:
-            self.__stop_network_agent(logger, server.processor_name, server.deploy_type, server.agent_type, server.pid)
-        self.network_agents_server_native = []
-        for server in self.network_agents_server_docker:
-            self.__stop_network_agent(logger, server.processor_name, server.deploy_type, server.agent_type, server.pid)
-        self.network_agents_server_docker = []
-
-    def stop_network_agents(self, logger: Logger):
+    def stop_workers(self, logger: Logger):
         if self.needs_ssh_connector and not self.ssh_client:
             logger.debug("Creating missing ssh connector before stopping")
             self.ssh_client = self.create_connection_client(client_type="ssh")
         if self.needs_docker_connector and not self.docker_client:
             logger.debug("Creating missing docker connector before stopping")
             self.docker_client = self.create_connection_client(client_type="docker")
-        self.__stop_network_agents_workers(logger=logger)
-        self.__stop_network_agents_servers(logger=logger)
+
+        logger.info(f"Stopping processing workers on host: {self.host}")
+        amount_workers = len(self.workers_native) + len(self.workers_docker)
+        if not amount_workers:
+            logger.warning("No active processing workers to be stopped.")
+        for worker in self.workers_native:
+            self.__stop_worker(logger, worker.processor_name, worker.deploy_type, worker.pid)
+        self.workers_native = []
+        for worker in self.workers_docker:
+            self.__stop_worker(logger, worker.processor_name, worker.deploy_type, worker.pid)
+        self.workers_docker = []
+
         if self.ssh_client:
             self.ssh_client.close()
             self.ssh_client = None
         if self.docker_client:
             self.docker_client.close()
             self.docker_client = None
-
-    def resolve_processor_server_url(self, processor_name: str) -> str:
-        processor_server_url = ''
-        for data_server in self.network_agents_server_docker:
-            if data_server.processor_name == processor_name:
-                processor_server_url = f"http://{self.host}:{data_server.port}/"
-        for data_server in self.network_agents_server_native:
-            if data_server.processor_name == processor_name:
-                processor_server_url = f"http://{self.host}:{data_server.port}/"
-        return processor_server_url

ocrd_network/runtime_data/network_agents.py

@@ -2,14 +2,15 @@ from logging import Logger
 from typing import Any
 
 from re import search as re_search
-from ..constants import AgentType, DeployType
+from ..constants import DeployType
 
 
 # TODO: Find appropriate replacement for the hack
 def deploy_agent_native_get_pid_hack(logger: Logger, ssh_client, start_cmd: str):
     channel = ssh_client.invoke_shell()
     stdin, stdout = channel.makefile("wb"), channel.makefile("rb")
-    logger.debug(f"Executing command: {start_cmd}")
+    # TODO: set back to debug
+    logger.info(f"Executing command: {start_cmd}")
 
     # TODO: This hack should still be fixed
     #   Note left from @joschrew
@@ -40,14 +41,13 @@ def deploy_agent_docker_template(logger: Logger, docker_client, start_cmd: str):
 
 class DataNetworkAgent:
     def __init__(
-        self, processor_name: str, deploy_type: DeployType, agent_type: AgentType,
+        self, processor_name: str, deploy_type: DeployType,
         host: str, init_by_config: bool, pid: Any = None
     ) -> None:
         self.processor_name = processor_name
         self.deploy_type = deploy_type
         self.host = host
         self.deployed_by_config = init_by_config
-        self.agent_type = agent_type
         # The id is assigned when the agent is deployed
         self.pid = pid
 
@@ -69,42 +69,18 @@ class DataProcessingWorker(DataNetworkAgent):
         self, processor_name: str, deploy_type: DeployType, host: str, init_by_config: bool, pid: Any = None
     ) -> None:
         super().__init__(
-            processor_name=processor_name, host=host, deploy_type=deploy_type, agent_type=AgentType.PROCESSING_WORKER,
+            processor_name=processor_name, host=host, deploy_type=deploy_type,
             init_by_config=init_by_config, pid=pid
         )
 
     def deploy_network_agent(self, logger: Logger, connector_client, database_url: str, queue_url: str):
         if self.deploy_type == DeployType.NATIVE:
-            start_cmd = f"{self.processor_name} {self.agent_type} --database {database_url} --queue {queue_url} &"
+            start_cmd = f"{self.processor_name} --database {database_url} --queue {queue_url} &"
             self.pid = self._start_native_instance(logger, connector_client, start_cmd)
             return self.pid
         if self.deploy_type == DeployType.DOCKER:
             # TODO: add real command to start processing worker in docker here
-            start_cmd = f""
-            self.pid = self._start_docker_instance(logger, connector_client, start_cmd)
-            return self.pid
-        raise RuntimeError(f"Unknown deploy type of {self.__dict__}")
-
-
-class DataProcessorServer(DataNetworkAgent):
-    def __init__(
-        self, processor_name: str, deploy_type: DeployType, host: str, port: int, init_by_config: bool, pid: Any = None
-    ) -> None:
-        super().__init__(
-            processor_name=processor_name, host=host, deploy_type=deploy_type, agent_type=AgentType.PROCESSOR_SERVER,
-            init_by_config=init_by_config, pid=pid
-        )
-        self.port = port
-
-    def deploy_network_agent(self, logger: Logger, connector_client, database_url: str):
-        agent_address = f"{self.host}:{self.port}"
-        if self.deploy_type == DeployType.NATIVE:
-            start_cmd = f"{self.processor_name} {self.agent_type} --address {agent_address} --database {database_url} &"
-            self.pid = self._start_native_instance(logger, connector_client, start_cmd)
-            return self.pid
-        if self.deploy_type == DeployType.DOCKER:
-            # TODO: add real command to start processor server in docker here
-            start_cmd = f""
+            start_cmd = ""
             self.pid = self._start_docker_instance(logger, connector_client, start_cmd)
             return self.pid
         raise RuntimeError(f"Unknown deploy type of {self.__dict__}")

ocrd_network/runtime_data/network_services.py

@@ -129,7 +129,7 @@ class DataRabbitMQ(DataNetworkService):
         rmq_host, rmq_port, rmq_vhost = self.host, int(self.port), self.vhost
         rmq_user, rmq_password = self.cred_username, self.cred_password
         if self.skip_deployment:
-            logger.debug(f"RabbitMQ is managed externally. Skipping deployment.")
+            logger.debug("RabbitMQ is managed externally. Skipping deployment.")
             verify_rabbitmq_available(logger=logger, rabbitmq_address=self.service_url)
             return self.service_url
         if not env:

ocrd_network/server_cache.py

@@ -33,7 +33,7 @@ class CacheLockedPages:
         if not self.locked_pages.get(workspace_key, None):
             self.log.info(f"No entry found in the locked pages cache for workspace key: {workspace_key}")
             return False
-        debug_message = f"Caching the received request due to locked output file grp pages."
+        debug_message = "Caching the received request due to locked output file grp pages."
         for file_group in output_file_grps:
             if file_group in self.locked_pages[workspace_key]:
                 if self.placeholder_all_pages in self.locked_pages[workspace_key][file_group]:

ocrd_network/server_utils.py

@@ -55,7 +55,7 @@ def create_processing_message(logger: Logger, job: DBProcessorJob) -> OcrdProces
         )
         return processing_message
     except ValueError as error:
-        message = f"Failed to create OcrdProcessingMessage from DBProcessorJob"
+        message = "Failed to create OcrdProcessingMessage from DBProcessorJob"
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message, error)
 
 
@@ -124,50 +124,7 @@ async def _get_processor_job_log(logger: Logger, job_id: str) -> FileResponse:
     return FileResponse(path=log_file_path, filename=log_file_path.name)
 
 
-def request_processor_server_tool_json(logger: Logger, processor_server_base_url: str) -> Dict:
-    # Request the ocrd tool json from the Processor Server
-    try:
-        response = requests_get(
-            urljoin(base=processor_server_base_url, url="info"),
-            headers={"Content-Type": "application/json"}
-        )
-    except Exception as error:
-        message = f"Failed to retrieve ocrd tool json from: {processor_server_base_url}"
-        raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message, error)
-    if response.status_code != 200:
-        message = f"Failed to retrieve tool json from: {processor_server_base_url}, code: {response.status_code}"
-        raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message)
-    return response.json()
-
-async def forward_job_to_processor_server(
-    logger: Logger, job_input: PYJobInput, processor_server_base_url: str
-) -> PYJobOutput:
-    try:
-        json_data = dumps(job_input.dict(exclude_unset=True, exclude_none=True))
-    except Exception as error:
-        message = f"Failed to json dump the PYJobInput: {job_input}"
-        raise_http_exception(logger, status.HTTP_500_INTERNAL_SERVER_ERROR, message, error)
-
-    # TODO: The amount of pages should come as a request input
-    # TODO: cf https://github.com/OCR-D/core/pull/1030/files#r1152551161
-    #  currently, use 200 as a default
-    request_timeout = calculate_processing_request_timeout(amount_pages=200, timeout_per_page=20.0)
-
-    # Post a processing job to the Processor Server asynchronously
-    async with AsyncClient(timeout=Timeout(timeout=request_timeout, connect=30.0)) as client:
-        response = await client.post(
-            urljoin(base=processor_server_base_url, url="run"),
-            headers={"Content-Type": "application/json"},
-            json=loads(json_data)
-        )
-    if response.status_code != 202:
-        message = f"Failed to post '{job_input.processor_name}' job to: {processor_server_base_url}"
-        raise_http_exception(logger, status.HTTP_500_INTERNAL_SERVER_ERROR, message)
-    job_output = response.json()
-    return job_output
-
-
-async def get_workflow_content(logger: Logger, workflow_id: str, workflow: Union[UploadFile, None]) -> str:
+async def get_workflow_content(logger: Logger, workflow_id: str, workflow: Union[UploadFile, str, None]) -> str:
     if not workflow and not workflow_id:
         message = "Either 'workflow' must be uploaded as a file or 'workflow_id' must be provided. Both are missing."
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message)
@@ -178,6 +135,9 @@ async def get_workflow_content(logger: Logger, workflow_id: str, workflow: Union
         except ValueError as error:
             message = f"Workflow with id '{workflow_id}' not found"
             raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message, error)
+    if isinstance(workflow, str):
+        with open(workflow) as wf_file:
+            return wf_file.read()
     return await generate_workflow_content(workflow)
 
 
@@ -193,14 +153,14 @@ def parse_workflow_tasks(logger: Logger, workflow_content: str) -> List[Processo
         tasks_list = workflow_content.splitlines()
         return [ProcessorTask.parse(task_str) for task_str in tasks_list if task_str.strip()]
     except ValueError as error:
-        message = f"Failed parsing processing tasks from a workflow."
+        message = "Failed parsing processing tasks from a workflow."
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message, error)
 
 
 def raise_http_exception(logger: Logger, status_code: int, message: str, error: Exception = None) -> None:
     if error:
         message = f"{message} {error}"
-    logger.exception(f"{message}")
+    logger.exception(message)
     raise HTTPException(status_code=status_code, detail=message)
 
 
@@ -214,7 +174,7 @@ def validate_job_input(logger: Logger, processor_name: str, ocrd_tool: dict, job
         )
         raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message)
     if not ocrd_tool:
-        message = f"Failed parsing processing tasks from a workflow."
+        message = "Failed parsing processing tasks from a workflow."
         raise_http_exception(logger, status.HTTP_404_NOT_FOUND, message)
     try:
         report = ParameterValidator(ocrd_tool).validate(dict(job_input.parameters))
@@ -249,10 +209,10 @@ def validate_first_task_input_file_groups_existence(logger: Logger, mets_path: s
             raise_http_exception(logger, status.HTTP_422_UNPROCESSABLE_ENTITY, message)
 
 
-def kill_mets_server_zombies(minutes_ago : Optional[int], dry_run : Optional[bool]) -> List[int]:
-    if minutes_ago == None:
+def kill_mets_server_zombies(minutes_ago: Optional[int], dry_run: Optional[bool]) -> List[int]:
+    if minutes_ago is None:
         minutes_ago = 90
-    if dry_run == None:
+    if dry_run is None:
         dry_run = False
 
     now = time()
@@ -271,7 +231,8 @@ def kill_mets_server_zombies(minutes_ago : Optional[int], dry_run : Optional[boo
         if re.match(cmdline_pat, cmdline):
             pid = int(procdir.name)
             ret.append(pid)
-            print(f'METS Server with PID {pid} was created {ctime_ago} minutes ago, more than {minutes_ago}, so killing (cmdline="{cmdline})', file=sys.stderr)
+            print(f'METS Server with PID {pid} was created {ctime_ago} minutes ago, more than {minutes_ago}, '
+                  f'so killing (cmdline="{cmdline})', file=sys.stderr)
             if dry_run:
                 print(f'[dry_run is active] kill {pid}')
             else:

ocrd_network/utils.py

@@ -172,5 +172,6 @@ def stop_mets_server(logger: Logger, mets_server_url: str, ws_dir_path: str) ->
     else:
         ValueError(f"Unexpected protocol type: {protocol}")
 
+
 def get_uds_path(ws_dir_path: str) -> Path:
     return Path(config.OCRD_NETWORK_SOCKETS_ROOT_DIR, f"{safe_filename(ws_dir_path)}.sock")

ocrd_utils/__init__.py

@@ -8,11 +8,11 @@ Utility functions and constants usable in various circumstances.
     levels below page (i.e. region, line, word, glyph) between relative coordinates
     w.r.t. a corresponding image and absolute coordinates w.r.t. the top-level image.
     This includes rotation and offset correction, based on affine transformations.
-    (Used by :py:class:`ocrd.workspace.Workspace` methods 
-    :py:meth:`ocrd.workspace.Workspace.image_from_page` and 
+    (Used by :py:class:`ocrd.workspace.Workspace` methods
+    :py:meth:`ocrd.workspace.Workspace.image_from_page` and
     :py:meth:`ocrd.workspace.Workspace.image_from_segment`.)
 
-* :py:func:`rotate_coordinates`, 
+* :py:func:`rotate_coordinates`,
   :py:func:`scale_coordinates`,
   :py:func:`shift_coordinates`,
   :py:func:`transpose_coordinates`,
@@ -23,7 +23,7 @@ Utility functions and constants usable in various circumstances.
     used to pass down the coordinate system along with images (both invariably sharing
     the same operations context) when traversing the element hierarchy top to bottom.
     (Used by :py:class:`ocrd.workspace.Workspace` methods
-    :py:meth:`ocrd.workspace.Workspace.image_from_page` and 
+    :py:meth:`ocrd.workspace.Workspace.image_from_page` and
     :py:meth:`ocrd.workspace.Workspace.image_from_segment`.)
 
 * :py:func:`rotate_image`,