ocrd 3.0.0a2__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

ocrd/processor/base.py

@@ -9,6 +9,7 @@ __all__ = [
     'run_processor'
 ]
 
+from functools import cached_property
 from os.path import exists, join
 from shutil import copyfileobj
 import json
@@ -20,36 +21,48 @@ import sys
 import inspect
 import tarfile
 import io
-from warnings import warn
+import weakref
+from frozendict import frozendict
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+
+from click import wrap_text
 from deprecated import deprecated
 from requests import HTTPError
 
-from ocrd.workspace import Workspace
+from ..workspace import Workspace
+from ..mets_server import ClientSideOcrdMets
 from ocrd_models.ocrd_file import OcrdFileType
-from ocrd.processor.ocrd_page_result import OcrdPageResult
+from .ocrd_page_result import OcrdPageResult
 from ocrd_utils import (
     VERSION as OCRD_VERSION,
     MIMETYPE_PAGE,
     MIME_TO_EXT,
     config,
     getLogger,
-    initLogging,
     list_resource_candidates,
     pushd_popd,
     list_all_resources,
     get_processor_resource_types,
     resource_filename,
-    resource_string,
+    parse_json_file_with_comments,
     make_file_id,
     deprecation_warning
 )
 from ocrd_validators import ParameterValidator
-from ocrd_models.ocrd_page import MetadataItemType, LabelType, LabelsType, OcrdPage, to_xml
+from ocrd_models.ocrd_page import (
+    PageType,
+    AlternativeImageType,
+    MetadataItemType,
+    LabelType,
+    LabelsType,
+    OcrdPage,
+    to_xml,
+)
 from ocrd_modelfactory import page_from_file
 from ocrd_validators.ocrd_tool_validator import OcrdToolValidator
 
 # XXX imports must remain for backwards-compatibility
-from .helpers import run_cli, run_processor, generate_processor_help # pylint: disable=unused-import
+from .helpers import run_cli, run_processor # pylint: disable=unused-import
 
 
 class ResourceNotFoundError(FileNotFoundError):
@@ -94,54 +107,163 @@ class MissingInputFile(ValueError):
 
 class Processor():
     """
-    A processor is a tool that implements the uniform OCR-D command-line interface
-    for run-time data processing. That is, it executes a single workflow step,
-    or a combination of workflow steps, on the workspace (represented by local METS).
-    It reads input files for all or requested physical pages of the input fileGrp(s),
-    and writes output files for them into the output fileGrp(s). It may take 
-    a number of optional or mandatory parameters.
+    A processor is a tool that implements the uniform OCR-D
+    `command-line interface for run-time data processing <https://ocr-d.de/en/spec/cli>`_.
+
+    That is, it executes a single workflow step, or a combination of workflow steps,
+    on the workspace (represented by local METS). It reads input files for all or selected
+    physical pages of the input fileGrp(s), computes additional annotation, and writes output
+    files for them into the output fileGrp(s). It may take a number of optional or mandatory
+    parameters.
+    """
+
+    max_instances : int = -1
+    """
+    maximum number of cached instances (ignored if negative), to be applied on top of
+    :py:data:`~ocrd_utils.config.OCRD_MAX_PROCESSOR_CACHE` (i.e. whatever is smaller).
+
+    (Override this if you know how many instances fit into memory - GPU / CPU RAM - at once.)
+    """
+
+    max_workers : int = -1
+    """
+    maximum number of processor threads for page-parallel processing (ignored if negative),
+    to be applied on top of :py:data:`~ocrd_utils.config.OCRD_MAX_PARALLEL_PAGES` (i.e.
+    whatever is smaller).
+
+    (Override this if you know how many pages fit into processing units - GPU shaders / CPU cores
+    - at once, or if your class is not thread-safe.)
+    """
+
+    max_page_seconds : int = -1
+    """
+    maximum number of seconds may be spent processing a single page (ignored if negative),
+    to be applied on top of :py:data:`~ocrd_utils.config.OCRD_PROCESSING_PAGE_TIMEOUT`
+    (i.e. whatever is smaller).
+
+    (Override this if you know how costly this processor may be, irrespective of image size
+    or complexity of the page.)
     """
 
     @property
+    def metadata_filename(self) -> str:
+        """
+        Relative location of the ``ocrd-tool.json`` file inside the package.
+
+        Used by :py:data:`metadata_location`.
+
+        (Override if ``ocrd-tool.json`` is not in the root of the module,
+        e.g. ``namespace/ocrd-tool.json`` or ``data/ocrd-tool.json``).
+        """
+        return 'ocrd-tool.json'
+
+    @cached_property
+    def metadata_location(self) -> Path:
+        """
+        Absolute path of the ``ocrd-tool.json`` file as distributed with the package.
+
+        Used by :py:data:`metadata_rawdict`.
+
+        (Override if ``ocrd-tool.json`` is not distributed with the Python package.)
+        """
+        # XXX HACK
+        module_tokens = self.__module__.split('.')
+        if module_tokens[0] == 'src':
+            module_tokens.pop(0)
+        return resource_filename(module_tokens[0], self.metadata_filename)
+
+    @cached_property
+    def metadata_rawdict(self) -> dict:
+        """
+        Raw (unvalidated, unexpanded) ``ocrd-tool.json`` dict contents of the package.
+
+        Used by :py:data:`metadata`.
+
+        (Override if ``ocrd-tool.json`` is not in a file.)
+        """
+        return parse_json_file_with_comments(self.metadata_location)
+
+    @cached_property
     def metadata(self) -> dict:
-        """the ocrd-tool.json dict of the package"""
-        if hasattr(self, '_metadata'):
-            return self._metadata
-        self._metadata = json.loads(resource_string(self.__module__.split('.')[0], 'ocrd-tool.json'))
-        report = OcrdToolValidator.validate(self._metadata)
+        """
+        The ``ocrd-tool.json`` dict contents of the package, according to the OCR-D
+        `spec <https://ocr-d.de/en/spec/ocrd_tool>`_ for processor tools.
+
+        After deserialisation, it also gets validated against the
+        `schema <https://ocr-d.de/en/spec/ocrd_tool#definition>`_ with all defaults
+        expanded.
+
+        Used by :py:data:`ocrd_tool` and :py:data:`version`.
+
+        (Override if you want to provide metadata programmatically instead of a
+        JSON file.)
+        """
+        metadata = self.metadata_rawdict
+        report = OcrdToolValidator.validate(metadata)
         if not report.is_valid:
-            # FIXME: remove when bertsky/core#10 is merged
-            self.logger = getLogger(f'ocrd.processor.{self.__class__.__name__}')
-            self.logger.error(f"The ocrd-tool.json of this processor is {'problematic' if not report.errors else 'invalid'}:\n{report.to_xml()}.\nPlease open an issue at {self._metadata['git_url']}.")
-        return self._metadata
+            self.logger.error(f"The ocrd-tool.json of this processor is {'problematic' if not report.errors else 'invalid'}:\n"
+                              f"{report.to_xml()}.\nPlease open an issue at {metadata.get('git_url', 'the website')}.")
+        return metadata
 
-    @property
+    @cached_property
     def version(self) -> str:
-        """the version of the package"""
-        if hasattr(self, '_version'):
-            return self._version
-        self._version = self.metadata['version']
-        return self._version
+        """
+        The program version of the package.
+        Usually the ``version`` part of :py:data:`metadata`.
 
-    @property
+        (Override if you do not want to use :py:data:`metadata` lookup
+        mechanism.)
+        """
+        return self.metadata['version']
+
+    @cached_property
     def executable(self) -> str:
-        """the executable name of this processor tool"""
-        if hasattr(self, '_executable'):
-            return self._executable
-        self._executable = os.path.basename(inspect.stack()[-1].filename)
-        return self._executable
+        """
+        The executable name of this processor tool. Taken from the runtime
+        filename.
 
-    @property
+        Used by :py:data:`ocrd_tool` for lookup in :py:data:`metadata`.
+
+        (Override if your entry-point name deviates from the ``executable``
+        name, or the processor gets instantiated from another runtime.)
+        """
+        return os.path.basename(inspect.stack()[-1].filename)
+
+    @cached_property
     def ocrd_tool(self) -> dict:
-        """the ocrd-tool.json dict of this processor tool"""
-        if hasattr(self, '_ocrd_tool'):
-            return self._ocrd_tool
-        self._ocrd_tool = self.metadata['tools'][self.executable]
-        return self._ocrd_tool
+        """
+        The ``ocrd-tool.json`` dict contents of this processor tool.
+        Usually the :py:data:`executable` key of the ``tools`` part
+        of :py:data:`metadata`.
+
+        (Override if you do not want to use :py:data:`metadata` lookup
+        mechanism.)
+        """
+        return self.metadata['tools'][self.executable]
+
+    @property
+    def parameter(self) -> Optional[dict]:
+        """the runtime parameter dict to be used by this processor"""
+        if hasattr(self, '_parameter'):
+            return self._parameter
+        return None
+
+    @parameter.setter
+    def parameter(self, parameter : dict) -> None:
+        if self.parameter is not None:
+            self.shutdown()
+        parameterValidator = ParameterValidator(self.ocrd_tool)
+        report = parameterValidator.validate(parameter)
+        if not report.is_valid:
+            raise ValueError(f'Invalid parameters:\n{report.to_xml()}')
+        # make parameter dict read-only
+        self._parameter = frozendict(parameter)
+        # (re-)run setup to load models etc
+        self.setup()
 
     def __init__(
             self,
-            # FIXME: deprecate in favor of process_workspace(workspace)
+            # FIXME: remove in favor of process_workspace(workspace)
             workspace : Optional[Workspace],
             ocrd_tool=None,
             parameter=None,
@@ -179,12 +301,12 @@ class Processor():
         if ocrd_tool is not None:
             deprecation_warning("Passing 'ocrd_tool' as keyword argument to Processor is deprecated - "
                                 "use or override metadata/executable/ocrd-tool properties instead")
-            self._ocrd_tool = ocrd_tool
-            self._executable = ocrd_tool['executable']
+            self.ocrd_tool = ocrd_tool
+            self.executable = ocrd_tool['executable']
         if version is not None:
             deprecation_warning("Passing 'version' as keyword argument to Processor is deprecated - "
                                 "use or override metadata/version properties instead")
-            self._version = version
+            self.version = version
         if workspace is not None:
             deprecation_warning("Passing a workspace argument other than 'None' to Processor "
                                 "is deprecated - pass as argument to process_workspace instead")
@@ -204,19 +326,14 @@ class Processor():
                                 "is deprecated - pass as argument to process_workspace instead")
             self.page_id = page_id or None
         self.download = download_files
-        if parameter is None:
-            parameter = {}
-        parameterValidator = ParameterValidator(self.ocrd_tool)
-
-        report = parameterValidator.validate(parameter)
-        if not report.is_valid:
-            raise ValueError("Invalid parameters %s" % report.errors)
-        self.parameter = parameter
-        # NOTE: this is the logger to be used by processor implementations,
-        # `processor.base` default implementations should use
-        # :py:attr:`self._base_logger`
+        #: The logger to be used by processor implementations.
+        # `ocrd.processor.base` internals should use :py:attr:`self._base_logger`
         self.logger = getLogger(f'ocrd.processor.{self.__class__.__name__}')
         self._base_logger = getLogger('ocrd.processor.base')
+        if parameter is not None:
+            self.parameter = parameter
+        # ensure that shutdown gets called at destruction
+        self._finalizer = weakref.finalize(self, self.shutdown)
         # workaround for deprecated#72 (@deprecated decorator does not work for subclasses):
         setattr(self, 'process',
                 deprecated(version='3.0', reason='process() should be replaced with process_page() and process_workspace()')(getattr(self, 'process')))
@@ -254,14 +371,10 @@ class Processor():
                     assert len(grps) >= minimum, msg % (len(grps), str(spec))
                 if maximum > 0:
                     assert len(grps) <= maximum, msg % (len(grps), str(spec))
-        # FIXME: maybe we should enforce the cardinality properties to be specified or apply default=1 here
-        # (but we already have ocrd-tool validation, and these first need to be adopted by implementors)
-        if 'input_file_grp_cardinality' in self.ocrd_tool:
-            assert_file_grp_cardinality(input_file_grps, self.ocrd_tool['input_file_grp_cardinality'],
-                                        "Unexpected number of input file groups %d vs %s")
-        if 'output_file_grp_cardinality' in self.ocrd_tool:
-            assert_file_grp_cardinality(output_file_grps, self.ocrd_tool['output_file_grp_cardinality'],
-                                        "Unexpected number of output file groups %d vs %s")
+        assert_file_grp_cardinality(input_file_grps, self.ocrd_tool['input_file_grp_cardinality'],
+                                    "Unexpected number of input file groups %d vs %s")
+        assert_file_grp_cardinality(output_file_grps, self.ocrd_tool['output_file_grp_cardinality'],
+                                    "Unexpected number of output file groups %d vs %s")
         for input_file_grp in input_file_grps:
             assert input_file_grp in self.workspace.mets.file_groups
         # keep this for backwards compatibility:
@@ -272,14 +385,12 @@ class Processor():
         Print :py:attr:`ocrd_tool` on stdout.
         """
         print(json.dumps(self.ocrd_tool, indent=True))
-        return
 
     def dump_module_dir(self):
         """
         Print :py:attr:`moduledir` on stdout.
         """
         print(self.moduledir)
-        return
 
     def list_resources(self):
         """
@@ -287,7 +398,6 @@ class Processor():
         """
         for res in self.list_all_resources():
             print(res)
-        return
 
     def setup(self) -> None:
         """
@@ -299,6 +409,16 @@ class Processor():
         """
         pass
 
+    def shutdown(self) -> None:
+        """
+        Bring down the processor after data processing,
+        after to changing back from the workspace directory but
+        before exiting (or setting up with different parameters).
+
+        (Override this to unload models from memory etc.)
+        """
+        pass
+
     @deprecated(version='3.0', reason='process() should be replaced with process_page_pcgts() or process_page_file() or process_workspace()')
     def process(self) -> None:
         """
@@ -330,7 +450,29 @@ class Processor():
             self.workspace = workspace
             self.verify()
             try:
-                # FIXME: add page parallelization by running multiprocessing.Pool (#322)
+                nr_succeeded = 0
+                nr_skipped = 0
+                nr_copied = 0
+
+                # set up multithreading
+                if self.max_workers <= 0:
+                    max_workers = max(0, config.OCRD_MAX_PARALLEL_PAGES)
+                else:
+                    max_workers = max(0, min(config.OCRD_MAX_PARALLEL_PAGES, self.max_workers))
+                if max_workers > 1:
+                    assert isinstance(workspace.mets, ClientSideOcrdMets), \
+                        "OCRD_MAX_PARALLEL_PAGES>1 requires also using --mets-server-url"
+                if self.max_page_seconds <= 0:
+                    max_seconds = max(0, config.OCRD_PROCESSING_PAGE_TIMEOUT)
+                else:
+                    max_seconds = max(0, min(config.OCRD_PROCESSING_PAGE_TIMEOUT, self.max_page_seconds))
+                executor = ThreadPoolExecutor(
+                    max_workers=max_workers or 1,
+                    thread_name_prefix=f"pagetask.{workspace.mets.unique_identifier}"
+                )
+                self._base_logger.debug("started executor %s", str(executor))
+                tasks = {}
+
                 for input_file_tuple in self.zip_input_files(on_error='abort', require_first=False):
                     input_files : List[Optional[OcrdFileType]] = [None] * len(input_file_tuple)
                     page_id = next(input_file.pageId
@@ -349,35 +491,55 @@ class Processor():
                         except (ValueError, FileNotFoundError, HTTPError) as e:
                             self._base_logger.error(repr(e))
                             self._base_logger.warning(f"failed downloading file {input_file} for page {page_id}")
+                    # process page
+                    tasks[executor.submit(self.process_page_file, *input_files)] = (page_id, input_files)
+                self._base_logger.debug("submitted %d processing tasks", len(tasks))
+
+                for task in tasks:
+                    # wait for results, handle errors
+                    page_id, input_files = tasks[task]
                     # FIXME: differentiate error cases in various ways:
                     # - ResourceNotFoundError → use ResourceManager to download (once), then retry
                     # - transient (I/O or OOM) error → maybe sleep, retry
                     # - persistent (data) error → skip / dummy / raise
                     try:
-                        self.process_page_file(*input_files)
-                    except Exception as err:
-                        # we have to be broad here, but want to exclude NotImplementedError
-                        if isinstance(err, NotImplementedError):
+                        self._base_logger.debug("waiting for output of task %s (page %s) max_seconds=%d", task, page_id, max_seconds)
+                        task.result(timeout=max_seconds or None)
+                        nr_succeeded += 1
+                    # exclude NotImplementedError, so we can try process() below
+                    except NotImplementedError:
+                        raise
+                    # handle input failures separately
+                    except FileExistsError as err:
+                        if config.OCRD_EXISTING_OUTPUT == 'ABORT':
                             raise err
-                        if isinstance(err, FileExistsError):
-                            if config.OCRD_EXISTING_OUTPUT == 'ABORT':
-                                raise err
-                            if config.OCRD_EXISTING_OUTPUT == 'SKIP':
-                                continue
-                            if config.OCRD_EXISTING_OUTPUT == 'OVERWRITE':
-                                # too late here, must not happen
-                                raise Exception(f"got {err} despite OCRD_EXISTING_OUTPUT==OVERWRITE")
-                        # FIXME: re-usable/actionable logging
-                        self._base_logger.exception(f"Failure on page {page_id}: {err}")
+                        if config.OCRD_EXISTING_OUTPUT == 'SKIP':
+                            continue
+                        if config.OCRD_EXISTING_OUTPUT == 'OVERWRITE':
+                            # too late here, must not happen
+                            raise Exception(f"got {err} despite OCRD_EXISTING_OUTPUT==OVERWRITE")
+                    # broad coverage of output failures (including TimeoutError)
+                    except (Exception, TimeoutError) as err:
+                        # FIXME: add re-usable/actionable logging
+                        self._base_logger.error(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
                         if config.OCRD_MISSING_OUTPUT == 'ABORT':
                             raise err
                         if config.OCRD_MISSING_OUTPUT == 'SKIP':
+                            nr_skipped += 1
                             continue
                         if config.OCRD_MISSING_OUTPUT == 'COPY':
                             self._copy_page_file(input_files[0])
+                            nr_copied += 1
                         else:
                             desc = config.describe('OCRD_MISSING_OUTPUT', wrap_text=False, indent_text=False)
                             raise ValueError(f"unknown configuration value {config.OCRD_MISSING_OUTPUT} - {desc}")
+
+                if nr_skipped > 0 and nr_succeeded / nr_skipped < config.OCRD_MAX_MISSING_OUTPUTS:
+                    raise Exception(f"too many failures with skipped output ({nr_skipped})")
+                if nr_copied > 0 and nr_succeeded / nr_copied < config.OCRD_MAX_MISSING_OUTPUTS:
+                    raise Exception(f"too many failures with fallback output ({nr_skipped})")
+                executor.shutdown()
+
             except NotImplementedError:
                 # fall back to deprecated method
                 self.process()
@@ -401,13 +563,14 @@ class Processor():
         output_file_id = make_file_id(input_file, self.output_file_grp)
         input_pcgts.set_pcGtsId(output_file_id)
         self.add_metadata(input_pcgts)
-        self.workspace.add_file(file_id=output_file_id,
-                                file_grp=self.output_file_grp,
-                                page_id=input_file.pageId,
-                                local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
-                                mimetype=MIMETYPE_PAGE,
-                                content=to_xml(input_pcgts),
-                                force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
+        self.workspace.add_file(
+            file_id=output_file_id,
+            file_grp=self.output_file_grp,
+            page_id=input_file.pageId,
+            local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
+            mimetype=MIMETYPE_PAGE,
+            content=to_xml(input_pcgts),
+            force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
         )
 
     def process_page_file(self, *input_files : Optional[OcrdFileType]) -> None:
@@ -439,7 +602,18 @@ class Processor():
         for image_result in result.images:
             image_file_id = f'{output_file_id}_{image_result.file_id_suffix}'
             image_file_path = join(self.output_file_grp, f'{image_file_id}.png')
-            image_result.alternative_image.set_filename(image_file_path)
+            if isinstance(image_result.alternative_image, PageType):
+                # special case: not an alternative image, but replacing the original image
+                # (this is needed by certain processors when the original's coordinate system
+                #  cannot or must not be kept)
+                image_result.alternative_image.set_imageFilename(image_file_path)
+                image_result.alternative_image.set_imageWidth(image_result.pil.width)
+                image_result.alternative_image.set_imageHeight(image_result.pil.height)
+            elif isinstance(image_result.alternative_image, AlternativeImageType):
+                image_result.alternative_image.set_filename(image_file_path)
+            else:
+                raise ValueError(f"process_page_pcgts returned an OcrdPageResultImage of unknown type "
+                                 f"{type(image_result.alternative_image)}")
             self.workspace.save_image_file(
                 image_result.pil,
                 image_file_id,
@@ -450,13 +624,14 @@ class Processor():
             )
         result.pcgts.set_pcGtsId(output_file_id)
         self.add_metadata(result.pcgts)
-        self.workspace.add_file(file_id=output_file_id,
-                                file_grp=self.output_file_grp,
-                                page_id=page_id,
-                                local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
-                                mimetype=MIMETYPE_PAGE,
-                                content=to_xml(result.pcgts),
-                                force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
+        self.workspace.add_file(
+            file_id=output_file_id,
+            file_grp=self.output_file_grp,
+            page_id=page_id,
+            local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
+            mimetype=MIMETYPE_PAGE,
+            content=to_xml(result.pcgts),
+            force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
         )
 
     def process_page_pcgts(self, *input_pcgts : Optional[OcrdPage], page_id : Optional[str] = None) -> OcrdPageResult:
@@ -668,7 +843,7 @@ class Processor():
         # can actually be much more costly than traversing the ltree.
         # This might depend on the number of pages vs number of fileGrps.
 
-        pages = dict()
+        pages = {}
         for i, ifg in enumerate(ifgs):
             files_ = sorted(self.workspace.mets.find_all_files(
                     pageId=self.page_id, fileGrp=ifg, mimetype=mimetype),
@@ -723,7 +898,7 @@ class Processor():
         if self.page_id and not any(pages):
             self._base_logger.critical(f"Could not find any files for selected pageId {self.page_id}.\n"
                                        f"compare '{self.page_id}' with the output of 'orcd workspace list-page'.")
-        ifts = list()
+        ifts = []
         for page, ifiles in pages.items():
             for i, ifg in enumerate(ifgs):
                 if not ifiles[i]:
@@ -738,3 +913,151 @@ class Processor():
             if ifiles[0] or not require_first:
                 ifts.append(tuple(ifiles))
         return ifts
+
+def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
+    """Generate a string describing the full CLI of this processor including params.
+
+    Args:
+         ocrd_tool (dict): this processor's ``tools`` section of the module's ``ocrd-tool.json``
+         processor_instance (object, optional): the processor implementation
+             (for adding any module/class/function docstrings)
+        subcommand (string): 'worker' or 'server'
+    """
+    doc_help = ''
+    if processor_instance:
+        module = inspect.getmodule(processor_instance)
+        if module and module.__doc__:
+            doc_help += '\n' + inspect.cleandoc(module.__doc__) + '\n'
+        if processor_instance.__doc__:
+            doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + '\n'
+        # Try to find the most concrete docstring among the various methods that an implementation
+        # could overload, first serving.
+        # In doing so, compare with Processor to avoid a glitch in the way py>=3.5 inherits docstrings.
+        # (They are supposed to only repeat information inspect.getdoc, rather than inherit __doc__ itself.)
+        for method in ['process_page_pcgts', 'process_page_file', 'process_workspace', 'process']:
+            instance_method = getattr(processor_instance, method)
+            superclass_method = getattr(Processor, method)
+            if instance_method.__doc__ and instance_method.__doc__ != superclass_method.__doc__:
+                doc_help += '\n' + inspect.cleandoc(instance_method.__doc__) + '\n'
+                break
+        if doc_help:
+            doc_help = '\n\n' + wrap_text(doc_help, width=72,
+                                          initial_indent='  > ',
+                                          subsequent_indent='  > ',
+                                          preserve_paragraphs=True)
+    subcommands = '''\
+    worker      Start a processing worker rather than do local processing
+    server      Start a processor server rather than do local processing
+'''
+
+    processing_worker_options = '''\
+  --queue                         The RabbitMQ server address in format
+                                  "amqp://{user}:{pass}@{host}:{port}/{vhost}"
+                                  [amqp://admin:admin@localhost:5672]
+  --database                      The MongoDB server address in format
+                                  "mongodb://{host}:{port}"
+                                  [mongodb://localhost:27018]
+  --log-filename                  Filename to redirect STDOUT/STDERR to,
+                                  if specified.
+'''
+
+    processing_server_options = '''\
+  --address                       The Processor server address in format
+                                  "{host}:{port}"
+  --database                      The MongoDB server address in format
+                                  "mongodb://{host}:{port}"
+                                  [mongodb://localhost:27018]
+'''
+
+    processing_options = '''\
+  -m, --mets URL-PATH             URL or file path of METS to process [./mets.xml]
+  -w, --working-dir PATH          Working directory of local workspace [dirname(URL-PATH)]
+  -I, --input-file-grp USE        File group(s) used as input
+  -O, --output-file-grp USE       File group(s) used as output
+  -g, --page-id ID                Physical page ID(s) to process instead of full document []
+  --overwrite                     Remove existing output pages/images
+                                  (with "--page-id", remove only those).
+                                  Short-hand for OCRD_EXISTING_OUTPUT=OVERWRITE
+  --debug                         Abort on any errors with full stack trace.
+                                  Short-hand for OCRD_MISSING_OUTPUT=ABORT
+  --profile                       Enable profiling
+  --profile-file PROF-PATH        Write cProfile stats to PROF-PATH. Implies "--profile"
+  -p, --parameter JSON-PATH       Parameters, either verbatim JSON string
+                                  or JSON file path
+  -P, --param-override KEY VAL    Override a single JSON object key-value pair,
+                                  taking precedence over --parameter
+  -U, --mets-server-url URL       URL of a METS Server for parallel incremental access to METS
+                                  If URL starts with http:// start an HTTP server there,
+                                  otherwise URL is a path to an on-demand-created unix socket
+  -l, --log-level [OFF|ERROR|WARN|INFO|DEBUG|TRACE]
+                                  Override log level globally [INFO]
+  --log-filename LOG-PATH         File to redirect stderr logging to (overriding ocrd_logging.conf).
+'''
+
+    information_options = '''\
+  -C, --show-resource RESNAME     Dump the content of processor resource RESNAME
+  -L, --list-resources            List names of processor resources
+  -J, --dump-json                 Dump tool description as JSON
+  -D, --dump-module-dir           Show the 'module' resource location path for this processor
+  -h, --help                      Show this message
+  -V, --version                   Show version
+'''
+
+    parameter_help = ''
+    if 'parameters' not in ocrd_tool or not ocrd_tool['parameters']:
+        parameter_help = '  NONE\n'
+    else:
+        def wrap(s):
+            return wrap_text(s, initial_indent=' '*3,
+                             subsequent_indent=' '*4,
+                             width=72, preserve_paragraphs=True)
+        for param_name, param in ocrd_tool['parameters'].items():
+            parameter_help += wrap('"%s" [%s%s]' % (
+                param_name,
+                param['type'],
+                ' - REQUIRED' if 'required' in param and param['required'] else
+                ' - %s' % json.dumps(param['default']) if 'default' in param else ''))
+            parameter_help += '\n ' + wrap(param['description'])
+            if 'enum' in param:
+                parameter_help += '\n ' + wrap('Possible values: %s' % json.dumps(param['enum']))
+            parameter_help += "\n"
+
+    if not subcommand:
+        return f'''\
+Usage: {ocrd_tool['executable']} [worker|server] [OPTIONS]
+
+  {ocrd_tool['description']}{doc_help}
+
+Subcommands:
+{subcommands}
+Options for processing:
+{processing_options}
+Options for information:
+{information_options}
+Parameters:
+{parameter_help}
+'''
+    elif subcommand == 'worker':
+        return f'''\
+Usage: {ocrd_tool['executable']} worker [OPTIONS]
+
+  Run {ocrd_tool['executable']} as a processing worker.
+
+  {ocrd_tool['description']}{doc_help}
+
+Options:
+{processing_worker_options}
+'''
+    elif subcommand == 'server':
+        return f'''\
+Usage: {ocrd_tool['executable']} server [OPTIONS]
+
+  Run {ocrd_tool['executable']} as a processor sever.
+
+  {ocrd_tool['description']}{doc_help}
+
+Options:
+{processing_server_options}
+'''
+    else:
+        pass