ocrd 3.6.0__py3-none-any.whl → 3.8.0__py3-none-any.whl

ocrd_network/resource_manager_server.py

@@ -0,0 +1,182 @@
+from datetime import datetime
+from os import getpid
+from shutil import which
+from typing import Any
+from uvicorn import run as uvicorn_run
+from fastapi import APIRouter, FastAPI, HTTPException, status
+
+from ocrd import OcrdResourceManager
+from ocrd_utils import getLogger, get_ocrd_tool_json, initLogging
+from .logging_utils import configure_file_handler_with_formatter, get_resource_manager_server_logging_file_path
+
+
+class ResourceManagerServer(FastAPI):
+    def __init__(self, host: str, port: int) -> None:
+        self.title = f"OCR-D Resource Manager Server"
+        super().__init__(
+            title=self.title,
+            on_startup=[self.on_startup],
+            on_shutdown=[self.on_shutdown],
+            description=self.title
+        )
+        initLogging()
+        self.log = getLogger("ocrd_network.resource_manager_server")
+        log_file = get_resource_manager_server_logging_file_path(pid=getpid())
+        configure_file_handler_with_formatter(self.log, log_file=log_file, mode="a")
+
+        self.resmgr_instance = OcrdResourceManager()
+
+        self.hostname = host
+        self.port = port
+
+        self.add_api_routes()
+
+    def start(self):
+        uvicorn_run(self, host=self.hostname, port=int(self.port))
+
+    async def on_startup(self):
+        self.log.info(f"Starting {self.title}")
+        pass
+
+    async def on_shutdown(self) -> None:
+        pass
+
+    def add_api_routes(self):
+        base_router = APIRouter()
+        base_router.add_api_route(
+            path="/",
+            endpoint=self.home_page,
+            methods=["GET"],
+            status_code=status.HTTP_200_OK,
+            summary="Get information about the OCR-D Resource Manager Server"
+        )
+        base_router.add_api_route(
+            path="/list_available",
+            endpoint=self.list_available_resources,
+            methods=["GET"],
+            status_code=status.HTTP_200_OK,
+            summary=""
+        )
+        base_router.add_api_route(
+            path="/list_installed",
+            endpoint=self.list_installed_resources,
+            methods=["GET"],
+            status_code=status.HTTP_200_OK,
+            summary=""
+        )
+        base_router.add_api_route(
+            path="/download",
+            endpoint=self.download_resource,
+            methods=["GET"],
+            status_code=status.HTTP_200_OK,
+            summary=""
+        )
+        self.include_router(base_router)
+
+    async def home_page(self):
+        message = f"The home page of the {self.title}"
+        json_message = {
+            "message": message,
+            "time": datetime.now().strftime("%Y-%m-%d %H:%M")
+        }
+        return json_message
+
+    async def list_available_resources(
+        self,
+        executable: Any = "ocrd-dummy",
+        dynamic: bool = True,
+        name: Any = None,
+        database: Any = None,
+        url: Any = None
+    ):
+        if executable == '*':
+            message = f"'*' is not an acceptable executable name! Try with a specific executable."
+            self.log.error(message)
+            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=message)
+        result = self.resmgr_instance.list_available(executable, dynamic, name, database, url)
+        json_message = {
+            "result": result
+        }
+        return json_message
+
+    async def list_installed_resources(self, executable: Any = None):
+        if executable == '*':
+            message = f"'*' is not an acceptable executable name! Try with a specific executable."
+            self.log.error(message)
+            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=message)
+        result = self.resmgr_instance.list_available(executable)
+        json_message = {
+            "result": result
+        }
+        return json_message
+
+    async def download_resource(
+        self,
+        executable: str = "ocrd-dummy",
+        name: Any = None,
+        location: Any = None,
+        any_url: str = '',
+        no_dynamic: bool = False,
+        resource_type: str = 'file',
+        path_in_archive: str = '.',
+        allow_uninstalled: bool = True,
+        overwrite: bool = True
+    ):
+        resmgr = OcrdResourceManager()
+        response = []
+        if executable == '*':
+            message = f"'*' is not an acceptable executable name! Try with a specific executable."
+            self.log.error(message)
+            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=message)
+        if name == '*':
+            name = None
+        if executable and not which(executable):
+            if not allow_uninstalled:
+                message = (f"Executable '{executable}' is not installed. To download resources anyway, "
+                           f"use the -a/--allow-uninstalled flag")
+                self.log.error(message)
+                raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=message)
+            else:
+                message = f"Executable '{executable}' is not installed, but downloading resources anyway."
+                self.log.info(message)
+                response.append(message)
+        reslist = resmgr.list_available(executable=executable, dynamic=not no_dynamic, name=name)
+        if not any(r[1] for r in reslist):
+            message = f"No resources {name} found in registry for executable {executable}"
+            self.log.info(message)
+            response.append(message)
+            if executable and name:
+                reslist = [(executable, [{
+                    'url': any_url or '???',
+                    'name': name,
+                    'type': resource_type,
+                    'path_in_archive': path_in_archive}]
+                )]
+        for this_executable, this_reslist in reslist:
+            resource_locations = get_ocrd_tool_json(this_executable)['resource_locations']
+            if not location:
+                location = resource_locations[0]
+            elif location not in resource_locations:
+                response.append(
+                    f"The selected --location {location} is not in the {this_executable}'s resource search path, "
+                    f"refusing to install to invalid location. Instead installing to: {resource_locations[0]}")
+            res_dest_dir = resmgr.build_resource_dest_dir(location=location, executable=this_executable)
+            for res_dict in this_reslist:
+                try:
+                    fpath = resmgr.handle_resource(
+                        res_dict=res_dict,
+                        executable=this_executable,
+                        dest_dir=res_dest_dir,
+                        any_url=any_url,
+                        overwrite=overwrite,
+                        resource_type=resource_type,
+                        path_in_archive=path_in_archive
+                    )
+                    if not fpath:
+                        continue
+                except FileExistsError as exc:
+                    response.append(str(exc))
+                usage = res_dict.get('parameter_usage', 'as-is')
+                response.append(f"Use in parameters as '{resmgr.parameter_usage(res_dict['name'], usage)}'")
+        json_message = { "result": response }
+        return json_message

ocrd_network/runtime_data/connection_clients.py

@@ -36,7 +36,7 @@ class CustomDockerClient(DockerClient):
                 raise ValueError("Both 'password' and 'keypath' provided - one must be provided")
         if ("password" not in kwargs) and ("keypath" not in kwargs):
             raise ValueError("Missing 'password' or 'keypath' - one must be provided")
-        self.api = APIClient(base_url=f"ssh://{host}", use_ssh_client=True, version="1.41")
+        self.api = APIClient(base_url=f"ssh://{host}", use_ssh_client=True, version="auto")
         self.api.mount(
             prefix="http+docker://ssh", adapter=self.CustomSshHttpAdapter(base_url=f"ssh://{user}@{host}:22", **kwargs)
         )

ocrd_network/runtime_data/hosts.py

@@ -1,9 +1,13 @@
 from logging import Logger
-from time import sleep
-from typing import Dict, List
+from typing import Dict, List, Optional
 
-from .connection_clients import create_docker_client, create_ssh_client
-from .network_agents import DataProcessingWorker, DeployType
+from docker import APIClient
+from paramiko import SSHClient
+
+from ..constants import RESOURCE_MANAGER_SERVER_PORT
+from .connection_clients import CustomDockerClient, create_docker_client, create_ssh_client
+from .network_agents import (
+    DataProcessingWorker, DeployType, deploy_agent_native_get_pid_hack)
 
 
 class DataHost:
@@ -11,6 +15,8 @@ class DataHost:
         self, host: str, username: str, password: str, keypath: str, workers: List[Dict], servers: List[Dict]
     ) -> None:
         self.host = host
+        self.resource_manager_port = RESOURCE_MANAGER_SERVER_PORT
+        self.resource_manager_pid = None
         self.username = username
         self.password = password
         self.keypath = keypath
@@ -22,14 +28,11 @@ class DataHost:
 
         # Connection clients, ssh for native deployment, docker for docker deployment
         self.ssh_client = None
-        self.docker_client = None
-
-        # Time to wait between deploying single workers
-        self.wait_between_deploys: float = 0.3
+        self.docker_client: Optional[CustomDockerClient] = None
 
-        # Lists of Processing Workers based on their deployment type
-        self.workers_native = []
-        self.workers_docker = []
+        # Lists of network agents based on their agent and deployment type
+        self.workers_native: List[DataProcessingWorker] = []
+        self.workers_docker: List[DataProcessingWorker] = []
 
         if not workers:
             workers = []
@@ -68,6 +71,13 @@ class DataHost:
             self.docker_client = create_docker_client(self.host, self.username, self.password, self.keypath)
             return self.docker_client
 
+    def __deploy_network_agent_resource_manager_server(self, logger: Logger):
+        logger.info(f"Deploying resource manager server on host: {self.host}:{self.resource_manager_port}")
+        start_cmd = f"ocrd network resmgr-server --address {self.host}:{self.resource_manager_port} &"
+        pid = deploy_agent_native_get_pid_hack(logger, self.ssh_client, start_cmd)
+        logger.info(f"Deployed: OCR-D Resource Manager Server [{pid}]: {self.host}:{self.resource_manager_port}")
+        self.resource_manager_pid = pid
+
     def __deploy_single_worker(
         self, logger: Logger, worker_data: DataProcessingWorker,
         mongodb_url: str, rabbitmq_url: str
@@ -86,7 +96,6 @@ class DataHost:
             connection_client = self.docker_client
 
         worker_data.deploy_network_agent(logger, connection_client, mongodb_url, rabbitmq_url)
-        sleep(self.wait_between_deploys)
 
     def __deploy_all_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str):
         logger.info(f"Deploying processing workers on host: {self.host}")
@@ -95,17 +104,24 @@ class DataHost:
             logger.info("No processing workers found to be deployed")
         for data_worker in self.workers_native:
             self.__deploy_single_worker(logger, data_worker, mongodb_url, rabbitmq_url)
+            logger.info(f"Deployed: {data_worker}")
         for data_worker in self.workers_docker:
             self.__deploy_single_worker(logger, data_worker, mongodb_url, rabbitmq_url)
+            logger.info(f"Deployed: {data_worker}")
 
     def deploy_workers(self, logger: Logger, mongodb_url: str, rabbitmq_url: str) -> None:
         if self.needs_ssh_connector and not self.ssh_client:
             logger.debug("Creating missing ssh connector before deploying")
-            self.ssh_client = self.create_connection_client(client_type="ssh")
+            client = self.create_connection_client(client_type="ssh")
+            assert isinstance(client, SSHClient)
+            self.ssh_client = client
         if self.needs_docker_connector:
             logger.debug("Creating missing docker connector before deploying")
-            self.docker_client = self.create_connection_client(client_type="docker")
+            client = self.create_connection_client(client_type="docker")
+            assert isinstance(client, CustomDockerClient)
+            self.docker_client = client
 
+        self.__deploy_network_agent_resource_manager_server(logger)
         self.__deploy_all_workers(logger=logger, mongodb_url=mongodb_url, rabbitmq_url=rabbitmq_url)
 
         if self.ssh_client:
@@ -115,6 +131,12 @@ class DataHost:
             self.docker_client.close()
             self.docker_client = None
 
+    def __stop_network_agent_resource_manager_server(self, logger: Logger):
+        logger.info(f"Stopping OCR-D Resource Manager Server [{self.resource_manager_pid}]: "
+                    f"{self.host}:{self.resource_manager_port}")
+        assert self.ssh_client, "SSH client connection missing"
+        self.ssh_client.exec_command(f"kill {self.resource_manager_pid}")
+
     def __stop_worker(self, logger: Logger, name: str, deploy_type: DeployType, pid: str):
         worker_info = f"Processing Worker: deploy: {deploy_type}, name: {name}"
         if not pid:
@@ -132,10 +154,15 @@ class DataHost:
     def stop_workers(self, logger: Logger):
         if self.needs_ssh_connector and not self.ssh_client:
             logger.debug("Creating missing ssh connector before stopping")
-            self.ssh_client = self.create_connection_client(client_type="ssh")
+            client = self.create_connection_client(client_type="ssh")
+            assert isinstance(client, SSHClient)
+            self.ssh_client = client
         if self.needs_docker_connector and not self.docker_client:
             logger.debug("Creating missing docker connector before stopping")
-            self.docker_client = self.create_connection_client(client_type="docker")
+            client = self.create_connection_client(client_type="docker")
+            assert isinstance(client, CustomDockerClient)
+            self.docker_client = client
+        self.__stop_network_agent_resource_manager_server(logger=logger)
 
         logger.info(f"Stopping processing workers on host: {self.host}")
         amount_workers = len(self.workers_native) + len(self.workers_docker)

ocrd_network/runtime_data/network_agents.py

@@ -1,4 +1,5 @@
 from logging import Logger
+from time import sleep
 from typing import Any
 
 from re import search as re_search
@@ -25,7 +26,8 @@ def deploy_agent_native_get_pid_hack(logger: Logger, ssh_client, start_cmd: str)
     output = stdout.read().decode("utf-8")
     stdout.close()
     stdin.close()
-    return re_search(r"xyz([0-9]+)xyz", output).group(1)  # type: ignore
+    pid = re_search(r"xyz([0-9]+)xyz", output).group(1)  # type: ignore
+    return pid
 
 
 # TODO: Implement the actual method that is missing
@@ -51,6 +53,12 @@ class DataNetworkAgent:
         # The id is assigned when the agent is deployed
         self.pid = pid
 
+        # Time to wait between deploying agents
+        self.wait_between_agent_deploys: float = 0.3
+
+    def __str__(self):
+        return f"{self.pid} {self.deploy_type} {self.processor_name} on host: {self.host}"
+
     def _start_native_instance(self, logger: Logger, ssh_client, start_cmd: str):
         if self.deploy_type != DeployType.NATIVE:
             raise RuntimeError(f"Mismatch of deploy type when starting network agent: {self.processor_name}")
@@ -76,11 +84,17 @@ class DataProcessingWorker(DataNetworkAgent):
     def deploy_network_agent(self, logger: Logger, connector_client, database_url: str, queue_url: str):
         if self.deploy_type == DeployType.NATIVE:
             start_cmd = f"{self.processor_name} --database {database_url} --queue {queue_url} &"
+            assert connector_client, f"SSH client connection missing."
             self.pid = self._start_native_instance(logger, connector_client, start_cmd)
+            sleep(self.wait_between_agent_deploys)
             return self.pid
         if self.deploy_type == DeployType.DOCKER:
             # TODO: add real command to start processing worker in docker here
             start_cmd = ""
+            assert connector_client, f"Docker client connection missing."
+            if not start_cmd:
+                raise RuntimeError("Missing start command for the Processing Worker in docker mode")
             self.pid = self._start_docker_instance(logger, connector_client, start_cmd)
+            sleep(self.wait_between_agent_deploys)
             return self.pid
         raise RuntimeError(f"Unknown deploy type of {self.__dict__}")

ocrd_utils/__init__.py

@@ -70,7 +70,8 @@ Utility functions and constants usable in various circumstances.
 
     filesystem-related utilities
 
-* :py:func:`is_string`,
+* :py:func:`is_git_url`,
+  :py:func:`is_string`,
   :py:func:`membername`,
   :py:func:`concat_padded`,
   :py:func:`nth_url_segment`,
@@ -118,6 +119,7 @@ from .constants import (
     REGEX_PREFIX,
     REGEX_FILE_ID,
     RESOURCE_LOCATIONS,
+    RESOURCE_TYPES,
     LOG_FORMAT,
     LOG_TIMEFMT,
     VERSION,
@@ -184,9 +186,11 @@ from .os import (
     get_processor_resource_types,
     get_ocrd_tool_json,
     get_moduledir,
+    get_env_locations,
     guess_media_type,
     list_all_resources,
     is_file_in_directory,
+    is_git_url,
     list_resource_candidates,
     atomic_write,
     pushd_popd,

ocrd_utils/constants.py

@@ -5,6 +5,7 @@ from .introspect import dist_version
 from re import compile as regex_compile
 
 __all__ = [
+    'DEFAULT_METS_BASENAME',
     'EXT_TO_MIME',
     'LOG_FORMAT',
     'LOG_TIMEFMT',
@@ -14,7 +15,9 @@ __all__ = [
     'PIL_TO_MIME',
     'REGEX_PREFIX',
     'REGEX_FILE_ID',
+    'RESOURCES_DIR_SYSTEM',
     'RESOURCE_LOCATIONS',
+    'RESOURCE_TYPES',
     'VERSION',
 ]
 
@@ -108,6 +111,8 @@ LOG_FORMAT = r'%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s'
 LOG_TIMEFMT = r'%H:%M:%S'
 
 RESOURCE_LOCATIONS = ['data', 'cwd', 'system', 'module']
+RESOURCE_TYPES = ['file', 'directory', 'archive']
+RESOURCES_DIR_SYSTEM = '/usr/local/share/ocrd-resources'
 
 DEFAULT_METS_BASENAME = 'mets.xml'
 

ocrd_utils/logging.py

@@ -75,6 +75,9 @@ _ocrdLevel2pythonLevel = {
 
 
 def tf_disable_interactive_logs():
+    """
+    Disable the interactive logging of tf/keras and set the log level to error or higher
+    """
     try:
         from os import environ  # pylint: disable=import-outside-toplevel
         # This env variable must be set before importing from Keras