ocrd 3.5.1__py3-none-any.whl → 3.7.0__py3-none-any.whl

ocrd/decorators/__init__.py

@@ -30,7 +30,7 @@ def ocrd_cli_wrap_processor(
     working_dir=None,
     dump_json=False,
     dump_module_dir=False,
-    help=False, # pylint: disable=redefined-builtin
+    help=False,  # pylint: disable=redefined-builtin
     profile=False,
     profile_file=None,
     version=False,
@@ -41,7 +41,6 @@ def ocrd_cli_wrap_processor(
     list_resources=False,
     # ocrd_network params start #
     subcommand=None,
-    address=None,
     queue=None,
     log_filename=None,
     database=None,
@@ -88,9 +87,8 @@ def ocrd_cli_wrap_processor(
     if list_resources:
         processor.list_resources()
         sys.exit()
-    if subcommand or address or queue or database:
-        # Used for checking/starting network agents for the WebAPI architecture
-        check_and_run_network_agent(processorClass, subcommand, address, database, queue)
+    if subcommand == "worker" or queue or database:
+        check_and_run_processing_worker(processorClass, database, queue)
 
     if 'parameter' in kwargs:
         # Disambiguate parameter file/literal, and resolve file
@@ -110,10 +108,10 @@ def ocrd_cli_wrap_processor(
     if not kwargs.get('input_file_grp', None):
         raise ValueError('-I/--input-file-grp is required')
     if 'output_file_grp' not in kwargs:
-        raise ValueError('-O/--output-file-grp is required') # actually, it may be None
+        raise ValueError('-O/--output-file-grp is required')  # actually, it may be None
     resolver = Resolver()
     working_dir, mets, _, mets_server_url = \
-            resolver.resolve_mets_arguments(working_dir, mets, None, mets_server_url)
+        resolver.resolve_mets_arguments(working_dir, mets, None, mets_server_url)
     workspace = resolver.workspace_from_url(mets, working_dir, mets_server_url=mets_server_url)
     page_id = kwargs.get('page_id')
     if debug:
@@ -122,7 +120,10 @@ def ocrd_cli_wrap_processor(
         config.OCRD_EXISTING_OUTPUT = 'ABORT'
     if overwrite:
         config.OCRD_EXISTING_OUTPUT = 'OVERWRITE'
-    report = WorkspaceValidator.check_file_grp(workspace, kwargs['input_file_grp'], '' if overwrite else kwargs['output_file_grp'], page_id)
+    report = WorkspaceValidator.check_file_grp(workspace,
+                                               kwargs['input_file_grp'],
+                                               '' if overwrite else kwargs['output_file_grp'],
+                                               page_id)
     if not report.is_valid:
         raise Exception("Invalid input/output file grps:\n\t%s" % '\n\t'.join(report.errors))
     # Set up profiling behavior from environment variables/flags
@@ -138,6 +139,7 @@ def ocrd_cli_wrap_processor(
         print("Profiling...")
         pr = cProfile.Profile()
         pr.enable()
+
         def goexit():
             pr.disable()
             print("Profiling completed")
@@ -146,6 +148,7 @@ def ocrd_cli_wrap_processor(
             s = io.StringIO()
             pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats()
             print(s.getvalue())
+
         atexit.register(goexit)
     if log_filename:
         log_ctx = redirect_stderr_and_stdout_to_file(log_filename)
@@ -155,53 +158,26 @@ def ocrd_cli_wrap_processor(
         run_processor(processorClass, mets_url=mets, workspace=workspace, **kwargs)
 
 
-def check_and_run_network_agent(ProcessorClass, subcommand: str, address: str, database: str, queue: str):
-    """
+def check_and_run_processing_worker(ProcessorClass, database: str, queue: str):
+    """ Check/start Processing Worker for the WebAPI architecture
     """
-    from ocrd_network import ProcessingWorker, ProcessorServer, AgentType
-    SUBCOMMANDS = [AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER]
-
-    if not subcommand:
-        raise ValueError(f"Subcommand options --address --queue and --database are only valid for subcommands: {SUBCOMMANDS}")
-    if subcommand not in SUBCOMMANDS:
-        raise ValueError(f"SUBCOMMAND can only be one of {SUBCOMMANDS}")
+    from ocrd_network import ProcessingWorker
 
     if not database:
-        raise ValueError(f"Option '--database' is invalid for subcommand {subcommand}")
-
-    if subcommand == AgentType.PROCESSOR_SERVER:
-        if not address:
-            raise ValueError(f"Option '--address' required for subcommand {subcommand}")
-        if queue:
-            raise ValueError(f"Option '--queue' invalid for subcommand {subcommand}")
-    if subcommand == AgentType.PROCESSING_WORKER:
-        if address:
-            raise ValueError(f"Option '--address' invalid for subcommand {subcommand}")
-        if not queue:
-            raise ValueError(f"Option '--queue' required for subcommand {subcommand}")
+        raise ValueError("Option '--database' is required for the Processing Worker")
+    if not queue:
+        raise ValueError("Option '--queue' is required for the Processing Worker")
 
     processor = ProcessorClass(workspace=None)
-    if subcommand == AgentType.PROCESSING_WORKER:
-        processing_worker = ProcessingWorker(
-            rabbitmq_addr=queue,
-            mongodb_addr=database,
-            processor_name=processor.ocrd_tool['executable'],
-            ocrd_tool=processor.ocrd_tool,
-            processor_class=ProcessorClass,
-        )
-        # The RMQConsumer is initialized and a connection to the RabbitMQ is performed
-        processing_worker.connect_consumer()
-        # Start consuming from the queue with name `processor_name`
-        processing_worker.start_consuming()
-    elif subcommand == AgentType.PROCESSOR_SERVER:
-        # TODO: Better validate that inside the ProcessorServer itself
-        host, port = address.split(':')
-        processor_server = ProcessorServer(
-            mongodb_addr=database,
-            processor_name=processor.ocrd_tool['executable'],
-            processor_class=ProcessorClass,
-        )
-        processor_server.run_server(host=host, port=int(port))
-    else:
-        raise ValueError(f"Unknown network agent type, must be one of: {SUBCOMMANDS}")
+    processing_worker = ProcessingWorker(
+        rabbitmq_addr=queue,
+        mongodb_addr=database,
+        processor_name=processor.ocrd_tool['executable'],
+        ocrd_tool=processor.ocrd_tool,
+        processor_class=ProcessorClass,
+    )
+    # The RMQConsumer is initialized and a connection to the RabbitMQ is performed
+    processing_worker.connect_consumer()
+    # Start consuming from the queue with name `processor_name`
+    processing_worker.start_consuming()
     sys.exit(0)

ocrd/decorators/loglevel_option.py

@@ -1,14 +1,17 @@
 import click
 from ocrd_utils.logging import setOverrideLogLevel
 
+
 __all__ = ['ocrd_loglevel']
 
+
 def _setOverrideLogLevel(ctx, param, value):    # pylint: disable=unused-argument
     if value is None:   # Explicitly test for None because logging.DEBUG == 0
         return
     setOverrideLogLevel(value)
     return value
 
+
 loglevel_option = click.option('-l', '--log-level', help="Log level",
                                type=click.Choice([
                                    'OFF', 'ERROR', 'WARN',
@@ -16,6 +19,7 @@ loglevel_option = click.option('-l', '--log-level', help="Log level",
                                ]),
                                default=None, callback=_setOverrideLogLevel)
 
+
 def ocrd_loglevel(f):
     """
     Add an option '--log-level' to set the log level.

ocrd/decorators/mets_find_options.py

@@ -1,7 +1,8 @@
 from click import option
 
+
 def mets_find_options(f):
-    for opt in  [
+    for opt in [
         option('-G', '--file-grp', help="fileGrp USE", metavar='FILTER'),
         option('-m', '--mimetype', help="Media type to look for", metavar='FILTER'),
         option('-g', '--page-id', help="Page ID", metavar='FILTER'),

ocrd/decorators/ocrd_cli_options.py

@@ -1,12 +1,10 @@
 import click
-from click import option, Path, group, command, argument
+from click import option, Path, argument
 from ocrd_utils import DEFAULT_METS_BASENAME
-from ocrd_network import AgentType
 from .parameter_option import parameter_option, parameter_override_option
 from .loglevel_option import loglevel_option
 from ocrd_network import (
     DatabaseParamType,
-    ServerAddressParamType,
     QueueServerParamType
 )
 
@@ -40,7 +38,6 @@ def ocrd_cli_options(f):
         parameter_override_option,
         loglevel_option,
         option('--log-filename', default=None),
-        option('--address', type=ServerAddressParamType()),
         option('--queue', type=QueueServerParamType()),
         option('--database', type=DatabaseParamType()),
         option('-R', '--resolve-resource'),
@@ -50,13 +47,12 @@ def ocrd_cli_options(f):
         option('-D', '--dump-module-dir', is_flag=True, default=False),
         option('-h', '--help', is_flag=True, default=False),
         option('-V', '--version', is_flag=True, default=False),
-        # Subcommand, only used for 'worker'/'server'. Cannot be handled in
+        # Subcommand, only used for 'worker'. Cannot be handled in
         # click because processors use the @command decorator and even if they
         # were using `group`, you cannot combine have a command with
         # subcommands. So we have to work around that by creating a
         # pseudo-subcommand handled in ocrd_cli_wrap_processor
-        argument('subcommand', nargs=1, required=False,
-                 type=click.Choice(list(map(str, AgentType)))),
+        argument('subcommand', nargs=1, required=False, type=click.Choice(["worker"])),
     ]
     for param in params:
         param(f)

ocrd/decorators/parameter_option.py

@@ -7,17 +7,18 @@ def _handle_param_option(ctx, param, value):
     from ocrd_utils import parse_json_string_or_file
     return parse_json_string_or_file(*list(value))
 
+
 parameter_option = option('-p', '--parameter',
-                                help="Parameters, either JSON string or path to JSON file",
-                                multiple=True,
-                                default=[],
-                                # now handled in ocrd_cli_wrap_processor to resolve processor preset files
-                                # callback=_handle_param_option
-                                callback=lambda ctx, param, kv: list(kv))
+                          help="Parameters, either JSON string or path to JSON file",
+                          multiple=True,
+                          default=[],
+                          # now handled in ocrd_cli_wrap_processor to resolve processor preset files
+                          # callback=_handle_param_option
+                          callback=lambda ctx, param, kv: list(kv))
 
 parameter_override_option = option('-P', '--parameter-override',
-                                help="Parameter override",
-                                nargs=2,
-                                multiple=True,
-                                callback=lambda ctx, param, kv: kv)
-                                # callback=lambda ctx, param, kv: {kv[0]: kv[1]})
+                                   help="Parameter override",
+                                   nargs=2,
+                                   multiple=True,
+                                   # callback=lambda ctx, param, kv: {kv[0]: kv[1]})
+                                   callback=lambda ctx, param, kv: kv)

ocrd/mets_server.py

@@ -3,12 +3,12 @@
 """
 import os
 import re
-from os import _exit, chmod
+from os import chmod
 import signal
 from typing import Dict, Optional, Union, List, Tuple
 from time import sleep
 from pathlib import Path
-from subprocess import Popen, run as subprocess_run
+from subprocess import Popen
 from urllib.parse import urlparse
 import socket
 import atexit
@@ -258,12 +258,12 @@ class ClientSideOcrdMets:
 
     def add_agent(self, **kwargs):
         if not self.multiplexing_mode:
-            return self.session.request("POST", f"{self.url}/agent", json=OcrdAgentModel.create(**kwargs).dict())
+            return self.session.request("POST", f"{self.url}/agent", json=OcrdAgentModel.create(**kwargs).model_dump())
         else:
             self.session.request(
                 "POST",
                 self.url,
-                json=MpxReq.add_agent(self.ws_dir_path, OcrdAgentModel.create(**kwargs).dict())
+                json=MpxReq.add_agent(self.ws_dir_path, OcrdAgentModel.create(**kwargs).model_dump())
             ).json()
             return OcrdAgentModel.create(**kwargs)
 
@@ -305,7 +305,7 @@ class ClientSideOcrdMets:
             mimetype=mimetype, url=url, local_filename=local_filename
         )
         # add force+ignore
-        kwargs = {**kwargs, **data.dict()}
+        kwargs = {**kwargs, **data.model_dump()}
 
         if not self.multiplexing_mode:
             r = self.session.request("POST", f"{self.url}/file", data=kwargs)
@@ -424,7 +424,7 @@ class OcrdMetsServer:
         # Wait for the mets server to start
         sleep(2)
         if sub_process.poll():
-            raise RuntimeError(f"Mets server starting failed. See {log_file} for errors")
+            raise RuntimeError(f"Starting METS Server failed. See {log_file} for errors")
         return sub_process.pid
 
     @staticmethod
@@ -433,12 +433,12 @@ class OcrdMetsServer:
         sleep(3)
         try:
             os.kill(mets_server_pid, signal.SIGKILL)
-        except ProcessLookupError as e:
+        except ProcessLookupError:
             pass
 
     def shutdown(self):
         pid = os.getpid()
-        self.log.info(f"Shutdown method of mets server[{pid}] invoked, sending SIGTERM signal.")
+        self.log.info(f"Shutdown method of METS Server[{pid}] invoked, sending SIGTERM signal.")
         os.kill(pid, signal.SIGTERM)
         if self.is_uds:
             if Path(self.url).exists():
@@ -446,7 +446,7 @@ class OcrdMetsServer:
                 Path(self.url).unlink()
 
     def startup(self):
-        self.log.info(f"Configuring the Mets Server")
+        self.log.info("Configuring the METS Server")
 
         workspace = self.workspace
 
@@ -516,10 +516,6 @@ class OcrdMetsServer:
             self.log.debug(f"GET /physical_pages -> {response}")
             return response
 
-        @app.get(path='/physical_pages', response_model=OcrdPageListModel)
-        async def physical_pages():
-            return {'physical_pages': workspace.mets.physical_pages}
-
         @app.get(path='/file_groups', response_model=OcrdFileGroupListModel)
         async def file_groups():
             response = {'file_groups': workspace.mets.file_groups}
@@ -534,7 +530,7 @@ class OcrdMetsServer:
 
         @app.post(path='/agent', response_model=OcrdAgentModel)
         async def add_agent(agent: OcrdAgentModel):
-            kwargs = agent.dict()
+            kwargs = agent.model_dump()
             kwargs['_type'] = kwargs.pop('type')
             workspace.mets.add_agent(**kwargs)
             response = agent
@@ -579,7 +575,7 @@ class OcrdMetsServer:
                 local_filename=local_filename
             )
             # Add to workspace
-            kwargs = file_resource.dict()
+            kwargs = file_resource.model_dump()
             workspace.add_file(**kwargs, force=force)
             response = file_resource
             self.log.debug(f"POST /file -> {response.__dict__}")