ocrd 3.2.0__py3-none-any.whl → 3.3.0__py3-none-any.whl

ocrd_network/constants.py

@@ -10,8 +10,6 @@ OCRD_ALL_TOOL_JSON = "ocrd-all-tool.json"
 # Used as a placeholder to lock all pages when no page_id is specified
 SERVER_ALL_PAGES_PLACEHOLDER = "all_pages"
 
-# TODO: Make this more configurable
-RESOURCE_MANAGER_SERVER_PORT = 45555
 
 class AgentType(str, Enum):
     PROCESSING_WORKER = "worker"
@@ -51,7 +49,6 @@ class NetworkLoggingDirs(str, Enum):
     PROCESSING_SERVERS = "processing_servers"
     PROCESSING_WORKERS = "processing_workers"
     PROCESSOR_SERVERS = "processor_servers"
-    RESOURCE_MANAGER_SERVERS = "resource_manager_servers"
 
 
 class ServerApiTags(str, Enum):

ocrd_network/logging_utils.py

@@ -50,8 +50,3 @@ def get_processing_worker_logging_file_path(processor_name: str, pid: int) -> Pa
 def get_processor_server_logging_file_path(processor_name: str, pid: int) -> Path:
     log_file: str = f"{AgentType.PROCESSOR_SERVER}.{pid}.{processor_name}.log"
     return Path(get_root_logging_dir(NetworkLoggingDirs.PROCESSOR_SERVERS), log_file)
-
-
-def get_resource_manager_server_logging_file_path(pid: int) -> Path:
-    log_file: str = f"resource_manager_server.{pid}.log"
-    return Path(get_root_logging_dir(NetworkLoggingDirs.RESOURCE_MANAGER_SERVERS), log_file)

ocrd_network/runtime_data/hosts.py

@@ -1,10 +1,9 @@
 from logging import Logger
+from time import sleep
 from typing import Dict, List, Union
 
-from ..constants import RESOURCE_MANAGER_SERVER_PORT
 from .connection_clients import create_docker_client, create_ssh_client
-from .network_agents import (
-    AgentType, DataProcessingWorker, DataProcessorServer, DeployType, deploy_agent_native_get_pid_hack)
+from .network_agents import AgentType, DataNetworkAgent, DataProcessingWorker, DataProcessorServer, DeployType
 
 
 class DataHost:
@@ -12,8 +11,6 @@ class DataHost:
         self, host: str, username: str, password: str, keypath: str, workers: List[Dict], servers: List[Dict]
     ) -> None:
         self.host = host
-        self.resource_manager_port = RESOURCE_MANAGER_SERVER_PORT
-        self.resource_manager_pid = None
         self.username = username
         self.password = password
         self.keypath = keypath
@@ -27,18 +24,22 @@ class DataHost:
         self.ssh_client = None
         self.docker_client = None
 
+        # Time to wait between deploying agents
+        self.wait_between_agent_deploys: float = 0.3
+
         # Lists of network agents based on their agent and deployment type
-        self.network_agents_worker_native: List[DataProcessingWorker] = []
-        self.network_agents_worker_docker: List[DataProcessingWorker] = []
-        self.network_agents_server_native: List[DataProcessorServer] = []
-        self.network_agents_server_docker: List[DataProcessorServer] = []
+        self.network_agents_worker_native = []
+        self.network_agents_worker_docker = []
+        self.network_agents_server_native = []
+        self.network_agents_server_docker = []
 
         if not workers:
             workers = []
         if not servers:
             servers = []
 
-        self.__parse_network_agents(processing_workers=workers, processor_servers=servers)
+        self.__parse_network_agents_workers(processing_workers=workers)
+        self.__parse_network_agents_servers(processor_servers=servers)
 
         # Used for caching deployed Processor Servers' ports on the current host
         # Key: processor_name, Value: list of ports
@@ -50,7 +51,7 @@ class DataHost:
             return
         self.processor_servers_ports[processor_name] = self.processor_servers_ports[processor_name].append(port)
 
-    def __append_network_agent_to_lists(self, agent_data: Union[DataProcessingWorker, DataProcessorServer]) -> None:
+    def __append_network_agent_to_lists(self, agent_data: DataNetworkAgent) -> None:
         if agent_data.deploy_type != DeployType.DOCKER and agent_data.deploy_type != DeployType.NATIVE:
             raise ValueError(f"Network agent deploy type is unknown: {agent_data.deploy_type}")
         if agent_data.agent_type != AgentType.PROCESSING_WORKER and agent_data.agent_type != AgentType.PROCESSOR_SERVER:
@@ -60,16 +61,24 @@ class DataHost:
             self.needs_ssh_connector = True
             if agent_data.agent_type == AgentType.PROCESSING_WORKER:
                 self.network_agents_worker_native.append(agent_data)
-            elif agent_data.agent_type == AgentType.PROCESSOR_SERVER:
+            if agent_data.agent_type == AgentType.PROCESSOR_SERVER:
                 self.network_agents_server_native.append(agent_data)
-        elif agent_data.deploy_type == DeployType.DOCKER:
+        if agent_data.deploy_type == DeployType.DOCKER:
             self.needs_docker_connector = True
             if agent_data.agent_type == AgentType.PROCESSING_WORKER:
                 self.network_agents_worker_docker.append(agent_data)
-            elif agent_data.agent_type == AgentType.PROCESSOR_SERVER:
+            if agent_data.agent_type == AgentType.PROCESSOR_SERVER:
                 self.network_agents_server_docker.append(agent_data)
 
-    def __parse_network_agents(self, processing_workers: List[Dict], processor_servers: List[Dict]):
+    def __parse_network_agents_servers(self, processor_servers: List[Dict]):
+        for server in processor_servers:
+            server_data = DataProcessorServer(
+                processor_name=server["name"], deploy_type=server["deploy_type"], host=self.host,
+                port=int(server["port"]), init_by_config=True, pid=None
+            )
+            self.__append_network_agent_to_lists(agent_data=server_data)
+
+    def __parse_network_agents_workers(self, processing_workers: List[Dict]):
         for worker in processing_workers:
             worker_data = DataProcessingWorker(
                 processor_name=worker["name"], deploy_type=worker["deploy_type"], host=self.host,
@@ -77,12 +86,6 @@ class DataHost:
             )
             for _ in range(int(worker["number_of_instance"])):
                 self.__append_network_agent_to_lists(agent_data=worker_data)
-        for server in processor_servers:
-            server_data = DataProcessorServer(
-                processor_name=server["name"], deploy_type=server["deploy_type"], host=self.host,
-                port=int(server["port"]), init_by_config=True, pid=None
-            )
-            self.__append_network_agent_to_lists(agent_data=server_data)
 
     def create_connection_client(self, client_type: str):
         if client_type not in ["docker", "ssh"]:
@@ -94,38 +97,52 @@ class DataHost:
             self.docker_client = create_docker_client(self.host, self.username, self.password, self.keypath)
             return self.docker_client
 
-    def __deploy_network_agent_resource_manager_server(self, logger: Logger):
-        logger.info(f"Deploying resource manager server on host: {self.host}:{self.resource_manager_port}")
-        start_cmd = f"ocrd network resmgr-server --address {self.host}:{self.resource_manager_port} &"
-        pid = deploy_agent_native_get_pid_hack(logger, self.ssh_client, start_cmd)
-        logger.info(f"Deployed: OCR-D Resource Manager Server [{pid}]: {self.host}:{self.resource_manager_port}")
-        self.resource_manager_pid = pid
+    def __deploy_network_agent(
+        self, logger: Logger, agent_data: Union[DataProcessorServer, DataProcessingWorker],
+        mongodb_url: str, rabbitmq_url: str
+    ) -> None:
+        deploy_type = agent_data.deploy_type
+        agent_type = agent_data.agent_type
+        name = agent_data.processor_name
+        agent_info = f"network agent: {agent_type}, deploy: {deploy_type}, name: {name}, host: {self.host}"
+        logger.info(f"Deploying {agent_info}")
+
+        connection_client = None
+        if deploy_type == DeployType.NATIVE:
+            assert self.ssh_client, f"SSH client connection missing."
+            connection_client = self.ssh_client
+        if deploy_type == DeployType.DOCKER:
+            assert self.docker_client, f"Docker client connection missing."
+            connection_client = self.docker_client
+
+        if agent_type == AgentType.PROCESSING_WORKER:
+            agent_data.deploy_network_agent(logger, connection_client, mongodb_url, rabbitmq_url)
+        if agent_type == AgentType.PROCESSOR_SERVER:
+            agent_data.deploy_network_agent(logger, connection_client, mongodb_url)
+
+        sleep(self.wait_between_agent_deploys)
 
-    def __deploy_network_agents_processing_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
+    def __deploy_network_agents_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
         logger.info(f"Deploying processing workers on host: {self.host}")
         amount_workers = len(self.network_agents_worker_native) + len(self.network_agents_worker_docker)
         if not amount_workers:
             logger.info(f"No processing workers found to be deployed")
         for data_worker in self.network_agents_worker_native:
-            data_worker.deploy_network_agent(logger, self.ssh_client, mongodb_url, rabbitmq_url)
-            logger.info(f"Deployed: {data_worker}")
+            self.__deploy_network_agent(logger, data_worker, mongodb_url, rabbitmq_url)
         for data_worker in self.network_agents_worker_docker:
-            data_worker.deploy_network_agent(logger, self.docker_client, mongodb_url, rabbitmq_url)
-            logger.info(f"Deployed: {data_worker}")
+            self.__deploy_network_agent(logger, data_worker, mongodb_url, rabbitmq_url)
 
-    def __deploy_network_agents_processor_servers(self, logger: Logger, mongodb_url: str):
+    def __deploy_network_agents_servers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
         logger.info(f"Deploying processor servers on host: {self.host}")
         amount_servers = len(self.network_agents_server_native) + len(self.network_agents_server_docker)
         if not amount_servers:
             logger.info(f"No processor servers found to be deployed")
         for data_server in self.network_agents_server_native:
-            data_server.deploy_network_agent(logger, self.ssh_client, mongodb_url)
+            self.__deploy_network_agent(logger, data_server, mongodb_url, rabbitmq_url)
             self.__add_deployed_agent_server_port_to_cache(data_server.processor_name, data_server.port)
-            logger.info(f"Deployed: {data_server}")
         for data_server in self.network_agents_server_docker:
-            data_server.deploy_network_agent(logger, self.docker_client, mongodb_url)
+            self.__deploy_network_agent(logger, data_server, mongodb_url, rabbitmq_url)
             self.__add_deployed_agent_server_port_to_cache(data_server.processor_name, data_server.port)
-            logger.info(f"Deployed: {data_server}")
 
     def deploy_network_agents(self, logger: Logger, mongodb_url: str, rabbitmq_url: str) -> None:
         if self.needs_ssh_connector and not self.ssh_client:
@@ -134,9 +151,8 @@ class DataHost:
         if self.needs_docker_connector:
             logger.debug("Creating missing docker connector before deploying")
             self.docker_client = self.create_connection_client(client_type="docker")
-        self.__deploy_network_agent_resource_manager_server(logger)
-        self.__deploy_network_agents_processing_workers(logger, mongodb_url, rabbitmq_url)
-        self.__deploy_network_agents_processor_servers(logger, mongodb_url)
+        self.__deploy_network_agents_workers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
+        self.__deploy_network_agents_servers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
         if self.ssh_client:
             self.ssh_client.close()
             self.ssh_client = None
@@ -144,12 +160,6 @@ class DataHost:
             self.docker_client.close()
             self.docker_client = None
 
-    def __stop_network_agent_resource_manager_server(self, logger: Logger):
-        logger.info(f"Stopping OCR-D Resource Manager Server [{self.resource_manager_pid}]: "
-                    f"{self.host}:{self.resource_manager_port}")
-        assert self.ssh_client, f"SSH client connection missing"
-        self.ssh_client.exec_command(f"kill {self.resource_manager_pid}")
-
     def __stop_network_agent(self, logger: Logger, name: str, deploy_type: DeployType, agent_type: AgentType, pid: str):
         agent_info = f"network agent: {agent_type}, deploy: {deploy_type}, name: {name}"
         if not pid:
@@ -195,7 +205,6 @@ class DataHost:
         if self.needs_docker_connector and not self.docker_client:
             logger.debug("Creating missing docker connector before stopping")
             self.docker_client = self.create_connection_client(client_type="docker")
-        self.__stop_network_agent_resource_manager_server(logger=logger)
         self.__stop_network_agents_workers(logger=logger)
         self.__stop_network_agents_servers(logger=logger)
         if self.ssh_client:

ocrd_network/runtime_data/network_agents.py

@@ -1,5 +1,4 @@
 from logging import Logger
-from time import sleep
 from typing import Any
 
 from re import search as re_search
@@ -25,8 +24,7 @@ def deploy_agent_native_get_pid_hack(logger: Logger, ssh_client, start_cmd: str)
     output = stdout.read().decode("utf-8")
     stdout.close()
     stdin.close()
-    pid = re_search(r"xyz([0-9]+)xyz", output).group(1)  # type: ignore
-    return pid
+    return re_search(r"xyz([0-9]+)xyz", output).group(1)  # type: ignore
 
 
 # TODO: Implement the actual method that is missing
@@ -53,12 +51,6 @@ class DataNetworkAgent:
         # The id is assigned when the agent is deployed
         self.pid = pid
 
-        # Time to wait between deploying agents
-        self.wait_between_agent_deploys: float = 0.3
-
-    def __str__(self):
-        return f"{self.pid} {self.deploy_type} {self.agent_type} {self.processor_name} on host: {self.host}"
-
     def _start_native_instance(self, logger: Logger, ssh_client, start_cmd: str):
         if self.deploy_type != DeployType.NATIVE:
             raise RuntimeError(f"Mismatch of deploy type when starting network agent: {self.processor_name}")
@@ -84,18 +76,12 @@ class DataProcessingWorker(DataNetworkAgent):
     def deploy_network_agent(self, logger: Logger, connector_client, database_url: str, queue_url: str):
         if self.deploy_type == DeployType.NATIVE:
             start_cmd = f"{self.processor_name} {self.agent_type} --database {database_url} --queue {queue_url} &"
-            assert connector_client, f"SSH client connection missing."
             self.pid = self._start_native_instance(logger, connector_client, start_cmd)
-            sleep(self.wait_between_agent_deploys)
             return self.pid
         if self.deploy_type == DeployType.DOCKER:
             # TODO: add real command to start processing worker in docker here
-            start_cmd = ""
-            assert connector_client, f"Docker client connection missing."
-            if not start_cmd:
-                raise RuntimeError("Missing start command for the Processing Worker in docker mode")
+            start_cmd = f""
             self.pid = self._start_docker_instance(logger, connector_client, start_cmd)
-            sleep(self.wait_between_agent_deploys)
             return self.pid
         raise RuntimeError(f"Unknown deploy type of {self.__dict__}")
 
@@ -110,24 +96,15 @@ class DataProcessorServer(DataNetworkAgent):
         )
         self.port = port
 
-    def __str__(self):
-        return f"{super().__str__}:{self.port}"
-
     def deploy_network_agent(self, logger: Logger, connector_client, database_url: str):
         agent_address = f"{self.host}:{self.port}"
         if self.deploy_type == DeployType.NATIVE:
             start_cmd = f"{self.processor_name} {self.agent_type} --address {agent_address} --database {database_url} &"
-            assert connector_client, f"SSH client connection missing."
             self.pid = self._start_native_instance(logger, connector_client, start_cmd)
-            sleep(self.wait_between_agent_deploys)
             return self.pid
         if self.deploy_type == DeployType.DOCKER:
             # TODO: add real command to start processor server in docker here
-            start_cmd = ""
-            assert connector_client, f"Docker client connection missing."
-            if not start_cmd:
-                raise RuntimeError("Missing start command for the Processor Server in docker mode")
+            start_cmd = f""
             self.pid = self._start_docker_instance(logger, connector_client, start_cmd)
-            sleep(self.wait_between_agent_deploys)
             return self.pid
         raise RuntimeError(f"Unknown deploy type of {self.__dict__}")

ocrd_utils/__init__.py

@@ -118,7 +118,6 @@ from .constants import (
     REGEX_PREFIX,
     REGEX_FILE_ID,
     RESOURCE_LOCATIONS,
-    RESOURCE_TYPES,
     LOG_FORMAT,
     LOG_TIMEFMT,
     VERSION,
@@ -185,7 +184,6 @@ from .os import (
     get_processor_resource_types,
     get_ocrd_tool_json,
     get_moduledir,
-    get_env_locations,
     guess_media_type,
     list_all_resources,
     is_file_in_directory,

ocrd_utils/constants.py

@@ -5,7 +5,6 @@ from .introspect import dist_version
 from re import compile as regex_compile
 
 __all__ = [
-    'DEFAULT_METS_BASENAME',
     'EXT_TO_MIME',
     'LOG_FORMAT',
     'LOG_TIMEFMT',
@@ -15,9 +14,7 @@ __all__ = [
     'PIL_TO_MIME',
     'REGEX_PREFIX',
     'REGEX_FILE_ID',
-    'RESOURCES_DIR_SYSTEM',
     'RESOURCE_LOCATIONS',
-    'RESOURCE_TYPES',
     'VERSION',
 ]
 
@@ -111,8 +108,6 @@ LOG_FORMAT = r'%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s'
 LOG_TIMEFMT = r'%H:%M:%S'
 
 RESOURCE_LOCATIONS = ['data', 'cwd', 'system', 'module']
-RESOURCE_TYPES = ['file', 'directory', 'archive']
-RESOURCES_DIR_SYSTEM = '/usr/local/share/ocrd-resources'
 
 DEFAULT_METS_BASENAME = 'mets.xml'