ocrd 3.0.0b1__py3-none-any.whl → 3.0.0b3__py3-none-any.whl

ocrd/cli/__init__.py

@@ -61,11 +61,11 @@ Variables:
 \b
 {config.describe('OCRD_DOWNLOAD_INPUT')}
 \b
-{config.describe('OCRD_MISSING_INPUT')}
+{config.describe('OCRD_MISSING_INPUT', wrap_text=False)}
 \b
-{config.describe('OCRD_MISSING_OUTPUT')}
+{config.describe('OCRD_MISSING_OUTPUT', wrap_text=False)}
 \b
-{config.describe('OCRD_EXISTING_OUTPUT')}
+{config.describe('OCRD_EXISTING_OUTPUT', wrap_text=False)}
 \b
 {config.describe('OCRD_METS_CACHING')}
 \b

ocrd/decorators/ocrd_cli_options.py

@@ -39,6 +39,7 @@ def ocrd_cli_options(f):
         parameter_option,
         parameter_override_option,
         loglevel_option,
+        option('--log-filename', default=None),
         option('--address', type=ServerAddressParamType()),
         option('--queue', type=QueueServerParamType()),
         option('--database', type=DatabaseParamType()),
@@ -48,7 +49,6 @@ def ocrd_cli_options(f):
         option('-D', '--dump-module-dir', is_flag=True, default=False),
         option('-h', '--help', is_flag=True, default=False),
         option('-V', '--version', is_flag=True, default=False),
-        option('--log-filename', default=None),
         # Subcommand, only used for 'worker'/'server'. Cannot be handled in
         # click because processors use the @command decorator and even if they
         # were using `group`, you cannot combine have a command with

ocrd/lib.bash

@@ -156,6 +156,7 @@ ocrd__parse_argv () {
     while [[ "${1:-}" = -* ]];do
         case "$1" in
             -l|--log-level) ocrd__argv[log_level]=$2 ; shift ;;
+            --log-filename) exec 2> "$2" ; shift ;;
             -h|--help|--usage) ocrd__usage; exit ;;
             -J|--dump-json) ocrd__dumpjson; exit ;;
             -D|--dump-module-dir) echo $(dirname "$OCRD_TOOL_JSON"); exit ;;

ocrd/mets_server.py

@@ -120,7 +120,7 @@ class ClientSideOcrdMets:
 
     def __init__(self, url, workspace_path: Optional[str] = None):
         self.protocol = "tcp" if url.startswith("http://") else "uds"
-        self.log = getLogger(f"ocrd.mets_client[{url}]")
+        self.log = getLogger(f"ocrd.models.ocrd_mets.client.{url}")
         self.url = url if self.protocol == "tcp" else f'http+unix://{url.replace("/", "%2F")}'
         self.ws_dir_path = workspace_path if workspace_path else None
 

ocrd/processor/__init__.py

@@ -3,6 +3,7 @@ from .base import (
     ResourceNotFoundError,
     NonUniqueInputFile,
     MissingInputFile,
+    generate_processor_help,
 )
 from .ocrd_page_result import (
     OcrdPageResult,
@@ -11,5 +12,4 @@ from .ocrd_page_result import (
 from .helpers import (
     run_cli,
     run_processor,
-    generate_processor_help
 )

ocrd/processor/base.py

@@ -23,12 +23,16 @@ import tarfile
 import io
 import weakref
 from frozendict import frozendict
+from concurrent.futures import ThreadPoolExecutor, TimeoutError
+
+from click import wrap_text
 from deprecated import deprecated
 from requests import HTTPError
 
-from ocrd.workspace import Workspace
+from ..workspace import Workspace
+from ..mets_server import ClientSideOcrdMets
 from ocrd_models.ocrd_file import OcrdFileType
-from ocrd.processor.ocrd_page_result import OcrdPageResult
+from .ocrd_page_result import OcrdPageResult
 from ocrd_utils import (
     VERSION as OCRD_VERSION,
     MIMETYPE_PAGE,
@@ -58,7 +62,7 @@ from ocrd_modelfactory import page_from_file
 from ocrd_validators.ocrd_tool_validator import OcrdToolValidator
 
 # XXX imports must remain for backwards-compatibility
-from .helpers import run_cli, run_processor, generate_processor_help # pylint: disable=unused-import
+from .helpers import run_cli, run_processor # pylint: disable=unused-import
 
 
 class ResourceNotFoundError(FileNotFoundError):
@@ -118,7 +122,27 @@ class Processor():
     maximum number of cached instances (ignored if negative), to be applied on top of
     :py:data:`~ocrd_utils.config.OCRD_MAX_PROCESSOR_CACHE` (i.e. whatever is smaller).
 
-    (Override this if you know how many instances fit into memory at once.)
+    (Override this if you know how many instances fit into memory - GPU / CPU RAM - at once.)
+    """
+
+    max_workers : int = -1
+    """
+    maximum number of processor threads for page-parallel processing (ignored if negative),
+    to be applied on top of :py:data:`~ocrd_utils.config.OCRD_MAX_PARALLEL_PAGES` (i.e.
+    whatever is smaller).
+
+    (Override this if you know how many pages fit into processing units - GPU shaders / CPU cores
+    - at once, or if your class is not thread-safe.)
+    """
+
+    max_page_seconds : int = -1
+    """
+    maximum number of seconds may be spent processing a single page (ignored if negative),
+    to be applied on top of :py:data:`~ocrd_utils.config.OCRD_PROCESSING_PAGE_TIMEOUT`
+    (i.e. whatever is smaller).
+
+    (Override this if you know how costly this processor may be, irrespective of image size
+    or complexity of the page.)
     """
 
     @property
@@ -142,7 +166,11 @@ class Processor():
 
         (Override if ``ocrd-tool.json`` is not distributed with the Python package.)
         """
-        return resource_filename(self.__module__.split('.')[0], self.metadata_filename)
+        # XXX HACK
+        module_tokens = self.__module__.split('.')
+        if module_tokens[0] == 'src':
+            module_tokens.pop(0)
+        return resource_filename(module_tokens[0], self.metadata_filename)
 
     @cached_property
     def metadata_rawdict(self) -> dict:
@@ -273,12 +301,12 @@ class Processor():
         if ocrd_tool is not None:
             deprecation_warning("Passing 'ocrd_tool' as keyword argument to Processor is deprecated - "
                                 "use or override metadata/executable/ocrd-tool properties instead")
-            self._ocrd_tool = ocrd_tool
-            self._executable = ocrd_tool['executable']
+            self.ocrd_tool = ocrd_tool
+            self.executable = ocrd_tool['executable']
         if version is not None:
             deprecation_warning("Passing 'version' as keyword argument to Processor is deprecated - "
                                 "use or override metadata/version properties instead")
-            self._version = version
+            self.version = version
         if workspace is not None:
             deprecation_warning("Passing a workspace argument other than 'None' to Processor "
                                 "is deprecated - pass as argument to process_workspace instead")
@@ -422,13 +450,35 @@ class Processor():
             self.workspace = workspace
             self.verify()
             try:
-                # FIXME: add page parallelization by running multiprocessing.Pool (#322)
+                nr_succeeded = 0
+                nr_skipped = 0
+                nr_copied = 0
+
+                # set up multithreading
+                if self.max_workers <= 0:
+                    max_workers = max(0, config.OCRD_MAX_PARALLEL_PAGES)
+                else:
+                    max_workers = max(0, min(config.OCRD_MAX_PARALLEL_PAGES, self.max_workers))
+                if max_workers > 1:
+                    assert isinstance(workspace.mets, ClientSideOcrdMets), \
+                        "OCRD_MAX_PARALLEL_PAGES>1 requires also using --mets-server-url"
+                if self.max_page_seconds <= 0:
+                    max_seconds = max(0, config.OCRD_PROCESSING_PAGE_TIMEOUT)
+                else:
+                    max_seconds = max(0, min(config.OCRD_PROCESSING_PAGE_TIMEOUT, self.max_page_seconds))
+                executor = ThreadPoolExecutor(
+                    max_workers=max_workers or 1,
+                    thread_name_prefix=f"pagetask.{workspace.mets.unique_identifier}"
+                )
+                self._base_logger.debug("started executor %s with %d workers", str(executor), max_workers or 1)
+                tasks = {}
+
                 for input_file_tuple in self.zip_input_files(on_error='abort', require_first=False):
                     input_files : List[Optional[OcrdFileType]] = [None] * len(input_file_tuple)
                     page_id = next(input_file.pageId
                                    for input_file in input_file_tuple
                                    if input_file)
-                    self._base_logger.info(f"processing page {page_id}")
+                    self._base_logger.info(f"preparing page {page_id}")
                     for i, input_file in enumerate(input_file_tuple):
                         if input_file is None:
                             # file/page not found in this file grp
@@ -441,35 +491,56 @@ class Processor():
                         except (ValueError, FileNotFoundError, HTTPError) as e:
                             self._base_logger.error(repr(e))
                             self._base_logger.warning(f"failed downloading file {input_file} for page {page_id}")
+                    # process page
+                    tasks[executor.submit(self.process_page_file, *input_files)] = (page_id, input_files)
+                self._base_logger.debug("submitted %d processing tasks", len(tasks))
+
+                for task in tasks:
+                    # wait for results, handle errors
+                    page_id, input_files = tasks[task]
                     # FIXME: differentiate error cases in various ways:
                     # - ResourceNotFoundError → use ResourceManager to download (once), then retry
                     # - transient (I/O or OOM) error → maybe sleep, retry
                     # - persistent (data) error → skip / dummy / raise
                     try:
-                        self.process_page_file(*input_files)
-                    except Exception as err:
-                        # we have to be broad here, but want to exclude NotImplementedError
-                        if isinstance(err, NotImplementedError):
+                        self._base_logger.debug("waiting for output of task %s (page %s) max_seconds=%d", task, page_id, max_seconds)
+                        task.result(timeout=max_seconds or None)
+                        nr_succeeded += 1
+                    # exclude NotImplementedError, so we can try process() below
+                    except NotImplementedError:
+                        raise
+                    # handle input failures separately
+                    except FileExistsError as err:
+                        if config.OCRD_EXISTING_OUTPUT == 'ABORT':
                             raise err
-                        if isinstance(err, FileExistsError):
-                            if config.OCRD_EXISTING_OUTPUT == 'ABORT':
-                                raise err
-                            if config.OCRD_EXISTING_OUTPUT == 'SKIP':
-                                continue
-                            if config.OCRD_EXISTING_OUTPUT == 'OVERWRITE':
-                                # too late here, must not happen
-                                raise Exception(f"got {err} despite OCRD_EXISTING_OUTPUT==OVERWRITE")
-                        # FIXME: re-usable/actionable logging
-                        self._base_logger.exception(f"Failure on page {page_id}: {err}")
+                        if config.OCRD_EXISTING_OUTPUT == 'SKIP':
+                            continue
+                        if config.OCRD_EXISTING_OUTPUT == 'OVERWRITE':
+                            # too late here, must not happen
+                            raise Exception(f"got {err} despite OCRD_EXISTING_OUTPUT==OVERWRITE")
+                    # broad coverage of output failures (including TimeoutError)
+                    except (Exception, TimeoutError) as err:
+                        # FIXME: add re-usable/actionable logging
                         if config.OCRD_MISSING_OUTPUT == 'ABORT':
+                            self._base_logger.error(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
                             raise err
+                        self._base_logger.exception(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
                         if config.OCRD_MISSING_OUTPUT == 'SKIP':
+                            nr_skipped += 1
                             continue
                         if config.OCRD_MISSING_OUTPUT == 'COPY':
                             self._copy_page_file(input_files[0])
+                            nr_copied += 1
                         else:
                             desc = config.describe('OCRD_MISSING_OUTPUT', wrap_text=False, indent_text=False)
                             raise ValueError(f"unknown configuration value {config.OCRD_MISSING_OUTPUT} - {desc}")
+
+                if nr_skipped > 0 and nr_succeeded / nr_skipped < config.OCRD_MAX_MISSING_OUTPUTS:
+                    raise Exception(f"too many failures with skipped output ({nr_skipped})")
+                if nr_copied > 0 and nr_succeeded / nr_copied < config.OCRD_MAX_MISSING_OUTPUTS:
+                    raise Exception(f"too many failures with fallback output ({nr_skipped})")
+                executor.shutdown()
+
             except NotImplementedError:
                 # fall back to deprecated method
                 self.process()
@@ -493,13 +564,14 @@ class Processor():
         output_file_id = make_file_id(input_file, self.output_file_grp)
         input_pcgts.set_pcGtsId(output_file_id)
         self.add_metadata(input_pcgts)
-        self.workspace.add_file(file_id=output_file_id,
-                                file_grp=self.output_file_grp,
-                                page_id=input_file.pageId,
-                                local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
-                                mimetype=MIMETYPE_PAGE,
-                                content=to_xml(input_pcgts),
-                                force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
+        self.workspace.add_file(
+            file_id=output_file_id,
+            file_grp=self.output_file_grp,
+            page_id=input_file.pageId,
+            local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
+            mimetype=MIMETYPE_PAGE,
+            content=to_xml(input_pcgts),
+            force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
         )
 
     def process_page_file(self, *input_files : Optional[OcrdFileType]) -> None:
@@ -516,6 +588,7 @@ class Processor():
         input_pcgts : List[Optional[OcrdPage]] = [None] * len(input_files)
         assert isinstance(input_files[0], get_args(OcrdFileType))
         page_id = input_files[0].pageId
+        self._base_logger.info("processing page %s", page_id)
         for i, input_file in enumerate(input_files):
             assert isinstance(input_file, get_args(OcrdFileType))
             self._base_logger.debug(f"parsing file {input_file.ID} for page {page_id}")
@@ -532,6 +605,9 @@ class Processor():
             image_file_id = f'{output_file_id}_{image_result.file_id_suffix}'
             image_file_path = join(self.output_file_grp, f'{image_file_id}.png')
             if isinstance(image_result.alternative_image, PageType):
+                # special case: not an alternative image, but replacing the original image
+                # (this is needed by certain processors when the original's coordinate system
+                #  cannot or must not be kept)
                 image_result.alternative_image.set_imageFilename(image_file_path)
                 image_result.alternative_image.set_imageWidth(image_result.pil.width)
                 image_result.alternative_image.set_imageHeight(image_result.pil.height)
@@ -550,13 +626,14 @@ class Processor():
             )
         result.pcgts.set_pcGtsId(output_file_id)
         self.add_metadata(result.pcgts)
-        self.workspace.add_file(file_id=output_file_id,
-                                file_grp=self.output_file_grp,
-                                page_id=page_id,
-                                local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
-                                mimetype=MIMETYPE_PAGE,
-                                content=to_xml(result.pcgts),
-                                force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
+        self.workspace.add_file(
+            file_id=output_file_id,
+            file_grp=self.output_file_grp,
+            page_id=page_id,
+            local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
+            mimetype=MIMETYPE_PAGE,
+            content=to_xml(result.pcgts),
+            force=config.OCRD_EXISTING_OUTPUT == 'OVERWRITE',
         )
 
     def process_page_pcgts(self, *input_pcgts : Optional[OcrdPage], page_id : Optional[str] = None) -> OcrdPageResult:
@@ -838,3 +915,151 @@ class Processor():
             if ifiles[0] or not require_first:
                 ifts.append(tuple(ifiles))
         return ifts
+
+def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
+    """Generate a string describing the full CLI of this processor including params.
+
+    Args:
+         ocrd_tool (dict): this processor's ``tools`` section of the module's ``ocrd-tool.json``
+         processor_instance (object, optional): the processor implementation
+             (for adding any module/class/function docstrings)
+        subcommand (string): 'worker' or 'server'
+    """
+    doc_help = ''
+    if processor_instance:
+        module = inspect.getmodule(processor_instance)
+        if module and module.__doc__:
+            doc_help += '\n' + inspect.cleandoc(module.__doc__) + '\n'
+        if processor_instance.__doc__:
+            doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + '\n'
+        # Try to find the most concrete docstring among the various methods that an implementation
+        # could overload, first serving.
+        # In doing so, compare with Processor to avoid a glitch in the way py>=3.5 inherits docstrings.
+        # (They are supposed to only repeat information inspect.getdoc, rather than inherit __doc__ itself.)
+        for method in ['process_page_pcgts', 'process_page_file', 'process_workspace', 'process']:
+            instance_method = getattr(processor_instance, method)
+            superclass_method = getattr(Processor, method)
+            if instance_method.__doc__ and instance_method.__doc__ != superclass_method.__doc__:
+                doc_help += '\n' + inspect.cleandoc(instance_method.__doc__) + '\n'
+                break
+        if doc_help:
+            doc_help = '\n\n' + wrap_text(doc_help, width=72,
+                                          initial_indent='  > ',
+                                          subsequent_indent='  > ',
+                                          preserve_paragraphs=True)
+    subcommands = '''\
+    worker      Start a processing worker rather than do local processing
+    server      Start a processor server rather than do local processing
+'''
+
+    processing_worker_options = '''\
+  --queue                         The RabbitMQ server address in format
+                                  "amqp://{user}:{pass}@{host}:{port}/{vhost}"
+                                  [amqp://admin:admin@localhost:5672]
+  --database                      The MongoDB server address in format
+                                  "mongodb://{host}:{port}"
+                                  [mongodb://localhost:27018]
+  --log-filename                  Filename to redirect STDOUT/STDERR to,
+                                  if specified.
+'''
+
+    processing_server_options = '''\
+  --address                       The Processor server address in format
+                                  "{host}:{port}"
+  --database                      The MongoDB server address in format
+                                  "mongodb://{host}:{port}"
+                                  [mongodb://localhost:27018]
+'''
+
+    processing_options = '''\
+  -m, --mets URL-PATH             URL or file path of METS to process [./mets.xml]
+  -w, --working-dir PATH          Working directory of local workspace [dirname(URL-PATH)]
+  -I, --input-file-grp USE        File group(s) used as input
+  -O, --output-file-grp USE       File group(s) used as output
+  -g, --page-id ID                Physical page ID(s) to process instead of full document []
+  --overwrite                     Remove existing output pages/images
+                                  (with "--page-id", remove only those).
+                                  Short-hand for OCRD_EXISTING_OUTPUT=OVERWRITE
+  --debug                         Abort on any errors with full stack trace.
+                                  Short-hand for OCRD_MISSING_OUTPUT=ABORT
+  --profile                       Enable profiling
+  --profile-file PROF-PATH        Write cProfile stats to PROF-PATH. Implies "--profile"
+  -p, --parameter JSON-PATH       Parameters, either verbatim JSON string
+                                  or JSON file path
+  -P, --param-override KEY VAL    Override a single JSON object key-value pair,
+                                  taking precedence over --parameter
+  -U, --mets-server-url URL       URL of a METS Server for parallel incremental access to METS
+                                  If URL starts with http:// start an HTTP server there,
+                                  otherwise URL is a path to an on-demand-created unix socket
+  -l, --log-level [OFF|ERROR|WARN|INFO|DEBUG|TRACE]
+                                  Override log level globally [INFO]
+  --log-filename LOG-PATH         File to redirect stderr logging to (overriding ocrd_logging.conf).
+'''
+
+    information_options = '''\
+  -C, --show-resource RESNAME     Dump the content of processor resource RESNAME
+  -L, --list-resources            List names of processor resources
+  -J, --dump-json                 Dump tool description as JSON
+  -D, --dump-module-dir           Show the 'module' resource location path for this processor
+  -h, --help                      Show this message
+  -V, --version                   Show version
+'''
+
+    parameter_help = ''
+    if 'parameters' not in ocrd_tool or not ocrd_tool['parameters']:
+        parameter_help = '  NONE\n'
+    else:
+        def wrap(s):
+            return wrap_text(s, initial_indent=' '*3,
+                             subsequent_indent=' '*4,
+                             width=72, preserve_paragraphs=True)
+        for param_name, param in ocrd_tool['parameters'].items():
+            parameter_help += wrap('"%s" [%s%s]' % (
+                param_name,
+                param['type'],
+                ' - REQUIRED' if 'required' in param and param['required'] else
+                ' - %s' % json.dumps(param['default']) if 'default' in param else ''))
+            parameter_help += '\n ' + wrap(param['description'])
+            if 'enum' in param:
+                parameter_help += '\n ' + wrap('Possible values: %s' % json.dumps(param['enum']))
+            parameter_help += "\n"
+
+    if not subcommand:
+        return f'''\
+Usage: {ocrd_tool['executable']} [worker|server] [OPTIONS]
+
+  {ocrd_tool['description']}{doc_help}
+
+Subcommands:
+{subcommands}
+Options for processing:
+{processing_options}
+Options for information:
+{information_options}
+Parameters:
+{parameter_help}
+'''
+    elif subcommand == 'worker':
+        return f'''\
+Usage: {ocrd_tool['executable']} worker [OPTIONS]
+
+  Run {ocrd_tool['executable']} as a processing worker.
+
+  {ocrd_tool['description']}{doc_help}
+
+Options:
+{processing_worker_options}
+'''
+    elif subcommand == 'server':
+        return f'''\
+Usage: {ocrd_tool['executable']} server [OPTIONS]
+
+  Run {ocrd_tool['executable']} as a processor sever.
+
+  {ocrd_tool['description']}{doc_help}
+
+Options:
+{processing_server_options}
+'''
+    else:
+        pass

ocrd/processor/helpers.py

@@ -8,13 +8,11 @@ import inspect
 from subprocess import run
 from typing import List, Optional
 
-from click import wrap_text
-from ocrd.workspace import Workspace
+from ..workspace import Workspace
 from ocrd_utils import freeze_args, getLogger, config, setOverrideLogLevel, getLevelName, sparkline
 
 
 __all__ = [
-    'generate_processor_help',
     'run_cli',
     'run_processor'
 ]
@@ -213,147 +211,6 @@ def run_cli(
     return result.returncode
 
 
-def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
-    """Generate a string describing the full CLI of this processor including params.
-
-    Args:
-         ocrd_tool (dict): this processor's ``tools`` section of the module's ``ocrd-tool.json``
-         processor_instance (object, optional): the processor implementation
-             (for adding any module/class/function docstrings)
-        subcommand (string): 'worker' or 'server'
-    """
-    doc_help = ''
-    if processor_instance:
-        module = inspect.getmodule(processor_instance)
-        if module and module.__doc__:
-            doc_help += '\n' + inspect.cleandoc(module.__doc__) + '\n'
-        if processor_instance.__doc__:
-            doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + '\n'
-        if processor_instance.process_workspace.__doc__:
-            doc_help += '\n' + inspect.cleandoc(processor_instance.process_workspace.__doc__) + '\n'
-        if processor_instance.process.__doc__:
-            doc_help += '\n' + inspect.cleandoc(processor_instance.process.__doc__) + '\n'
-        if doc_help:
-            doc_help = '\n\n' + wrap_text(doc_help, width=72,
-                                          initial_indent='  > ',
-                                          subsequent_indent='  > ',
-                                          preserve_paragraphs=True)
-    subcommands = '''\
-    worker      Start a processing worker rather than do local processing
-    server      Start a processor server rather than do local processing
-'''
-
-    processing_worker_options = '''\
-  --queue                         The RabbitMQ server address in format
-                                  "amqp://{user}:{pass}@{host}:{port}/{vhost}"
-                                  [amqp://admin:admin@localhost:5672]
-  --database                      The MongoDB server address in format
-                                  "mongodb://{host}:{port}"
-                                  [mongodb://localhost:27018]
-  --log-filename                  Filename to redirect STDOUT/STDERR to,
-                                  if specified.
-'''
-
-    processing_server_options = '''\
-  --address                       The Processor server address in format
-                                  "{host}:{port}"
-  --database                      The MongoDB server address in format
-                                  "mongodb://{host}:{port}"
-                                  [mongodb://localhost:27018]
-'''
-
-    processing_options = '''\
-  -m, --mets URL-PATH             URL or file path of METS to process [./mets.xml]
-  -w, --working-dir PATH          Working directory of local workspace [dirname(URL-PATH)]
-  -I, --input-file-grp USE        File group(s) used as input
-  -O, --output-file-grp USE       File group(s) used as output
-  -g, --page-id ID                Physical page ID(s) to process instead of full document []
-  --overwrite                     Remove existing output pages/images
-                                  (with "--page-id", remove only those).
-                                  Short-hand for OCRD_EXISTING_OUTPUT=OVERWRITE
-  --debug                         Abort on any errors with full stack trace.
-                                  Short-hand for OCRD_MISSING_OUTPUT=ABORT
-  --profile                       Enable profiling
-  --profile-file PROF-PATH        Write cProfile stats to PROF-PATH. Implies "--profile"
-  -p, --parameter JSON-PATH       Parameters, either verbatim JSON string
-                                  or JSON file path
-  -P, --param-override KEY VAL    Override a single JSON object key-value pair,
-                                  taking precedence over --parameter
-  -U, --mets-server-url URL       URL of a METS Server for parallel incremental access to METS
-                                  If URL starts with http:// start an HTTP server there,
-                                  otherwise URL is a path to an on-demand-created unix socket
-  -l, --log-level [OFF|ERROR|WARN|INFO|DEBUG|TRACE]
-                                  Override log level globally [INFO]
-'''
-
-    information_options = '''\
-  -C, --show-resource RESNAME     Dump the content of processor resource RESNAME
-  -L, --list-resources            List names of processor resources
-  -J, --dump-json                 Dump tool description as JSON
-  -D, --dump-module-dir           Show the 'module' resource location path for this processor
-  -h, --help                      Show this message
-  -V, --version                   Show version
-'''
-
-    parameter_help = ''
-    if 'parameters' not in ocrd_tool or not ocrd_tool['parameters']:
-        parameter_help = '  NONE\n'
-    else:
-        def wrap(s):
-            return wrap_text(s, initial_indent=' '*3,
-                             subsequent_indent=' '*4,
-                             width=72, preserve_paragraphs=True)
-        for param_name, param in ocrd_tool['parameters'].items():
-            parameter_help += wrap('"%s" [%s%s]' % (
-                param_name,
-                param['type'],
-                ' - REQUIRED' if 'required' in param and param['required'] else
-                ' - %s' % json.dumps(param['default']) if 'default' in param else ''))
-            parameter_help += '\n ' + wrap(param['description'])
-            if 'enum' in param:
-                parameter_help += '\n ' + wrap('Possible values: %s' % json.dumps(param['enum']))
-            parameter_help += "\n"
-
-    if not subcommand:
-        return f'''\
-Usage: {ocrd_tool['executable']} [worker|server] [OPTIONS]
-
-  {ocrd_tool['description']}{doc_help}
-
-Subcommands:
-{subcommands}
-Options for processing:
-{processing_options}
-Options for information:
-{information_options}
-Parameters:
-{parameter_help}
-'''
-    elif subcommand == 'worker':
-        return f'''\
-Usage: {ocrd_tool['executable']} worker [OPTIONS]
-
-  Run {ocrd_tool['executable']} as a processing worker.
-
-  {ocrd_tool['description']}{doc_help}
-
-Options:
-{processing_worker_options}
-'''
-    elif subcommand == 'server':
-        return f'''\
-Usage: {ocrd_tool['executable']} server [OPTIONS]
-
-  Run {ocrd_tool['executable']} as a processor sever.
-
-  {ocrd_tool['description']}{doc_help}
-
-Options:
-{processing_server_options}
-'''
-    else:
-        pass
-
 
 # not decorated here but at runtime (on first use)
 #@freeze_args

ocrd/workspace.py

@@ -95,8 +95,8 @@ class Workspace():
             if self.is_remote:
                 mets = ClientSideOcrdMets(mets_server_url, self.directory)
                 if mets.workspace_path != self.directory:
-                    raise ValueError(f"METS server {mets_server_url} workspace directory {mets.workspace_path} differs "
-                            f"from local workspace directory {self.directory}. These are not the same workspaces.")
+                    raise ValueError(f"METS server {mets_server_url} workspace directory '{mets.workspace_path}' differs "
+                            f"from local workspace directory '{self.directory}'. These are not the same workspaces.")
             else:
                 mets = OcrdMets(filename=self.mets_target)
         self.mets = mets

{ocrd-3.0.0b1.dist-info → ocrd-3.0.0b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0b1
+Version: 3.0.0b3
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -94,17 +94,12 @@ complete stack of OCR-D-related software.
 
 The easiest way to install is via `pip`:
 
-```sh
-pip install ocrd
+    pip install ocrd
 
-# or just the functionality you need, e.g.
-
-pip install ocrd_modelfactory
-```
 
 All Python software released by [OCR-D](https://github.com/OCR-D) requires Python 3.8 or higher.
 
-**NOTE** Some OCR-D-Tools (or even test cases) _might_ reveal an unintended behavior if you have specific environment modifications, like:
+> **NOTE** Some OCR-D tools (or even test cases) _might_ reveal an unintended behavior if you have specific environment modifications, like:
 * using a custom build of [ImageMagick](https://github.com/ImageMagick/ImageMagick), whose format delegates are different from what OCR-D supposes
 * custom Python logging configurations in your personal account
 
@@ -129,7 +124,6 @@ Almost all behaviour of the OCR-D/core software is configured via CLI options an
 
 Some parts of the software are configured via environment variables:
 
-* `OCRD_METS_CACHING`: If set to `true`, access to the METS file is cached, speeding in-memory search and modification.
 * `OCRD_PROFILE`: This variable configures the built-in CPU and memory profiling. If empty, no profiling is done. Otherwise expected to contain any of the following tokens:
   * `CPU`: Enable CPU profiling of processor runs
   * `RSS`: Enable RSS memory profiling
@@ -142,18 +136,46 @@ Some parts of the software are configured via environment variables:
 * `XDG_CONFIG_HOME`: Directory to look for `./ocrd/resources.yml` (i.e. `ocrd resmgr` user database) – defaults to `$HOME/.config`.
 * `XDG_DATA_HOME`: Directory to look for `./ocrd-resources/*` (i.e. `ocrd resmgr` data location) – defaults to `$HOME/.local/share`.
 
-* `OCRD_DOWNLOAD_RETRIES`: Number of times to retry failed attempts for downloads of workspace files.
+* `OCRD_DOWNLOAD_RETRIES`: Number of times to retry failed attempts for downloads of resources or workspace files.
 * `OCRD_DOWNLOAD_TIMEOUT`: Timeout in seconds for connecting or reading (comma-separated) when downloading.
 
+* `OCRD_MISSING_INPUT`: How to deal with missing input files (for some fileGrp/pageId) during processing:
+  * `SKIP`: ignore and proceed with next page's input
+  * `ABORT`: throw `MissingInputFile` exception
+
+* `OCRD_MISSING_OUTPUT`: How to deal with missing output files (for some fileGrp/pageId) during processing:
+  * `SKIP`: ignore and proceed processing next page
+  * `COPY`: fall back to copying input PAGE to output fileGrp for page
+  * `ABORT`: re-throw whatever caused processing to fail
+
+* `OCRD_MAX_MISSING_OUTPUTS`: Maximal rate of skipped/fallback pages among all processed pages before aborting (decimal fraction, ignored if negative).
+
+* `OCRD_EXISTING_OUTPUT`: How to deal with already existing output files (for some fileGrp/pageId) during processing:
+  * `SKIP`: ignore and proceed processing next page
+  * `OVERWRITE`: force writing result to output fileGrp for page
+  * `ABORT`: re-throw `FileExistsError` exception
+
+
 * `OCRD_METS_CACHING`: Whether to enable in-memory storage of OcrdMets data structures for speedup during processing or workspace operations.
 
 * `OCRD_MAX_PROCESSOR_CACHE`: Maximum number of processor instances (for each set of parameters) to be kept in memory (including loaded models) for processing workers or processor servers.
 
+* `OCRD_MAX_PARALLEL_PAGES`: Maximum number of processor threads for page-parallel processing (within each Processor's selected page range, independent of the number of Processing Workers or Processor Servers). If set `>1`, then a METS Server must be used for METS synchronisation.
+
+* `OCRD_PROCESSING_PAGE_TIMEOUT`: Timeout in seconds for processing a single page. If set >0, when exceeded, the same as OCRD_MISSING_OUTPUT applies.
+
 * `OCRD_NETWORK_SERVER_ADDR_PROCESSING`: Default address of Processing Server to connect to (for `ocrd network client processing`).
 * `OCRD_NETWORK_SERVER_ADDR_WORKFLOW`: Default address of Workflow Server to connect to (for `ocrd network client workflow`).
 * `OCRD_NETWORK_SERVER_ADDR_WORKSPACE`: Default address of Workspace Server to connect to (for `ocrd network client workspace`).
 * `OCRD_NETWORK_RABBITMQ_CLIENT_CONNECT_ATTEMPTS`: Number of attempts for a worker to create its queue. Helpful if the rabbitmq-server needs time to be fully started.
 
+* `OCRD_NETWORK_CLIENT_POLLING_SLEEP`: How many seconds to sleep before trying `ocrd network client` again.
+* `OCRD_NETWORK_CLIENT_POLLING_TIMEOUT`: Timeout for a blocking `ocrd network client` (in seconds).
+
+* `OCRD_NETWORK_SOCKETS_ROOT_DIR`: The root directory where all mets server related socket files are created.
+* `OCRD_NETWORK_LOGS_ROOT_DIR`: The root directory where all ocrd_network related file logs are stored.
+
+
 
 ## Packages
 

{ocrd-3.0.0b1.dist-info → ocrd-3.0.0b3.dist-info}/RECORD

@@ -1,16 +1,16 @@
 ocrd/__init__.py,sha256=ZswMVmlqFhAEIzMR3my6IKPq9XLH21aDPC_m_8Jh4dA,1076
 ocrd/constants.py,sha256=6dn3mG54WqHsKInmLZp4kJjNqqPtBoFoSuLUuRbOps0,740
-ocrd/lib.bash,sha256=7amCf_0mwE3tkz-e_HxA30aQnL1x2O4BF_8dE9fbqJg,10692
-ocrd/mets_server.py,sha256=M7hZOvOEbCfx5jCjeZ5Uv4CgviVZjGV15N8SN1ATpNY,20149
+ocrd/lib.bash,sha256=dlOQd36OVsibwEPXg7i-e_JmIc-3uH-SIwGcE8OJ3Cc,10744
+ocrd/mets_server.py,sha256=SrCnCmWBuyEauHrNF0jLZO_Wi8WoPCMWxhec2wE5Y3w,20160
 ocrd/ocrd-all-tool.json,sha256=9bX2VYnUwhTAzAvKaoT77BFzbgBGgyIt7qBqARpwWNc,586
 ocrd/resolver.py,sha256=Ba9ALQbTXz6_mla4VqN9tAfHoj6aKuNJAU4tIDnjcHE,14952
 ocrd/resource_list.yml,sha256=82-PiqkZnka1kTj3MQqNn4wXWKHHtoFchsQuetWuqFs,2633
 ocrd/resource_manager.py,sha256=8BMVKJq8J56hugi8vtGn9Ffuk7oRkbs197aG74aKbCY,16733
 ocrd/task_sequence.py,sha256=spiaUQaMM7M8WdBDoQGmLuTPm7tOugYXD6rcJ2UXzxw,6991
-ocrd/workspace.py,sha256=t5xveWhSLj5cbsuVkOqT6VZEQ9DRCxucT7FUvTNFnDA,65604
+ocrd/workspace.py,sha256=4s0qscEosS7rQ0jfn1qJeT9B3eC31YippAX-RUjXghA,65608
 ocrd/workspace_backup.py,sha256=iab_JjZ_mMP-G8NIUk4PZmfpNlQuGRoqc3NbTSSew1w,3621
 ocrd/workspace_bagger.py,sha256=yU8H3xR5WmQKvgQewac71ie-DUWcfLnMS01D55zsEHQ,11971
-ocrd/cli/__init__.py,sha256=XyYcbIuajaS2YM6HEWD4dfitdAzn111AWIaFPsTHoKQ,2621
+ocrd/cli/__init__.py,sha256=lNR6wMf7JhQ8Jf33tUkowJr0mB3423OMY0_6dkMRLvU,2672
 ocrd/cli/bashlib.py,sha256=XGcO-MmYM3xJBRkSCLEZcGs0hqbw2GR8oyijJPtKnYM,5888
 ocrd/cli/log.py,sha256=6_FrVmTKIIVNUaNLkuOJx8pvPhensHMuayJ0PA7T-XA,1562
 ocrd/cli/network.py,sha256=oWBHFEURxfUdb_t-F4svP_ri7o5mqBoNQnLZLbsZLTA,602
@@ -23,11 +23,11 @@ ocrd/cli/zip.py,sha256=MMJLw3OXWiJVfVtrdJcBkbB8vA1IzSautluazZRuCQ0,5910
 ocrd/decorators/__init__.py,sha256=IJlA1XcdVBO6Hxm9rNDya7QYcqeWcaXXuLtGjfjcen8,7596
 ocrd/decorators/loglevel_option.py,sha256=tgipROEu3t4hkwWvFssd80k2SbTBwBIC4WNE6Gc-XAg,798
 ocrd/decorators/mets_find_options.py,sha256=d4oATKMP6bFQHNqOK6nLqgUiWF2FYdkPvzkTVRMYpKo,635
-ocrd/decorators/ocrd_cli_options.py,sha256=-bpYOyHod3pMmbooyw_dIOqRnhCoZbdmE8vrnWbQRNA,2621
+ocrd/decorators/ocrd_cli_options.py,sha256=4pcBLAFPSpYZLj6r9Yj1GZOQl4r_RWU00pyA4mHwFQk,2621
 ocrd/decorators/parameter_option.py,sha256=n8hYw7XVTd3i3tvpK8F1Jx_CqRp6EGF9qJVH95yj92Q,1076
-ocrd/processor/__init__.py,sha256=m2lYRvJogmt2Xb4UgqjCXWkfOtqpKW3h8OEWDnblwTQ,273
-ocrd/processor/base.py,sha256=EvzR-uN5fDH1jKRWnm4CpCVpqge4Z7aprzW2PeXMqxU,38521
-ocrd/processor/helpers.py,sha256=7TmhKRrgi-T_q8dJX6DIVEVZ3qmNABJEcz3S9RXSUjI,16484
+ocrd/processor/__init__.py,sha256=39ymNwYRdc-b_OJzzKmWCvo2ga3KdsGSYDHE1Hzkn_w,274
+ocrd/processor/base.py,sha256=341APZGx6zCbuxgX-XTkKhPfeQkqblykmC9zSMPH3ss,48843
+ocrd/processor/helpers.py,sha256=Lp9zbHYCLpT3GnPzl-p7UCSFU5Nx99gYEYXwW04v0RI,10157
 ocrd/processor/ocrd_page_result.py,sha256=AazEmnWyPEN47TxXVg0WUQpgFNV_mlIiExwwycUj0nQ,490
 ocrd/processor/builtin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ocrd/processor/builtin/dummy_processor.py,sha256=6ORike_59wb_UUivhA6Iw_Ldg0AaMzX37F7qL9R8S_A,3704
@@ -87,12 +87,12 @@ ocrd_network/runtime_data/hosts.py,sha256=ml19ptzH4TFofyJR-Qp_Mn3sZUFbWoNe__rRXZ
 ocrd_network/runtime_data/network_agents.py,sha256=5p_zKLqECBIHLw-Ya6eKcKSZcUM4ESiipEIphVxHBEA,5192
 ocrd_network/runtime_data/network_services.py,sha256=xrPpFUU_Pa-XzGe2FEt5RmO17xqykIUmTr_9g6S7XSs,7892
 ocrd_utils/__init__.py,sha256=U_zAQJwxg_aJ4CR84CKMNAUP6Cob8Er8Ikj42JmnUKo,5977
-ocrd_utils/config.py,sha256=Zs7XXjfWa1oQ1tvnu6DR1WmfafohMPU9KR-Qx3qoC74,9818
+ocrd_utils/config.py,sha256=Rkqv5wWEmlDDD0l1IWo9TPgn5ppPnHPRH9FfkMST29E,11117
 ocrd_utils/constants.py,sha256=ImbG1d8t2MW3uuFi-mN6aY90Zn74liAKZBKlfuKN86w,3278
 ocrd_utils/deprecate.py,sha256=4i50sZsA3Eevqn5D-SL5yGf9KEZfGCV4A5Anzn1GRMs,1026
 ocrd_utils/image.py,sha256=zNNX1cnRy6yvrxx8mnYQiqWraAh5-i4a1AOfCCg4SmI,24781
 ocrd_utils/introspect.py,sha256=gfBlmeEFuRmRUSgdSK0jOxRpYqDRXl2IAE6gv2MZ6as,1977
-ocrd_utils/logging.py,sha256=Mw49E_JX3lGNi2TEOK3qXInQf5E2zg-Vbse5uFgxO9M,8156
+ocrd_utils/logging.py,sha256=5_-5T5OWSYicNk8SQyjVqdRj2bVl-gDK1Th-C7oW_HE,8248
 ocrd_utils/ocrd_logging.conf,sha256=kl9x9JS1d8h8F0QZabvrjZtW1iApIaChvkImYafKO5g,3623
 ocrd_utils/os.py,sha256=acRRdDBI8L6BK0Mf773yKEzwdpZSFRBJEKB2crL4EjU,9865
 ocrd_utils/str.py,sha256=JIhsyWphqJuxJAzhRQJUqlZ44AGOeObEPJMFhfWhfhQ,10084
@@ -118,9 +118,9 @@ ocrd_validators/xlink.xsd,sha256=8fW7YAMWXN2PbB_MMvj9H5ZeFoEBDzuYBtlGC8_6ijw,318
 ocrd_validators/xsd_mets_validator.py,sha256=4GWfLyqkmca0x7osDuXuExYuM0HWVrKoqn0S35sFhHU,467
 ocrd_validators/xsd_page_validator.py,sha256=BNz_9u-Ek4UCeyZu3KxSQoolfW9lvuaSR9nIu1XXxeE,467
 ocrd_validators/xsd_validator.py,sha256=6HrVAf6SzCvfUIuQdIzz9bOq4V-zhyii9yrUPoK2Uvo,2094
-ocrd-3.0.0b1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ocrd-3.0.0b1.dist-info/METADATA,sha256=NKl_vrDFcwK3ayhp9_fe_2gAPESgGghDz3QQcdJRSRw,8785
-ocrd-3.0.0b1.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-ocrd-3.0.0b1.dist-info/entry_points.txt,sha256=tV_gAdO8cbnOjS0GmKfJKbN60xBAV2DQRX6hEjleSjE,94
-ocrd-3.0.0b1.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
-ocrd-3.0.0b1.dist-info/RECORD,,
+ocrd-3.0.0b3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ocrd-3.0.0b3.dist-info/METADATA,sha256=WZhPkJV0F8A5k-0IVK8HZ5zGWVWwYSa6FuDlpkuh4Xc,10397
+ocrd-3.0.0b3.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+ocrd-3.0.0b3.dist-info/entry_points.txt,sha256=tV_gAdO8cbnOjS0GmKfJKbN60xBAV2DQRX6hEjleSjE,94
+ocrd-3.0.0b3.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
+ocrd-3.0.0b3.dist-info/RECORD,,

{ocrd-3.0.0b1.dist-info → ocrd-3.0.0b3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (74.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ocrd_utils/config.py

@@ -78,14 +78,26 @@ class OcrdEnvConfig():
             raise ValueError(f"Unregistered env variable {name}")
         return self._variables[name].has_default
 
+    def reset_defaults(self):
+        for name in self._variables:
+            try:
+                # we cannot use hasattr, because that delegates to getattr,
+                # which we override and provide defaults for (which of course
+                # cannot be removed)
+                if self.__getattribute__(name):
+                    delattr(self, name)
+            except AttributeError:
+                pass
+
     def describe(self, name, *args, **kwargs):
         if not name in self._variables:
             raise ValueError(f"Unregistered env variable {name}")
         return self._variables[name].describe(*args, **kwargs)
 
     def __getattr__(self, name):
+        # will be called if name is not accessible (has not been added directly yet)
         if not name in self._variables:
-            raise ValueError(f"Unregistered env variable {name}")
+            raise AttributeError(f"Unregistered env variable {name}")
         var_obj = self._variables[name]
         try:
             raw_value = self.raw_value(name)
@@ -120,6 +132,16 @@ config.add('OCRD_MAX_PROCESSOR_CACHE',
     parser=int,
     default=(True, 128))
 
+config.add('OCRD_MAX_PARALLEL_PAGES',
+    description="Maximum number of processor threads for page-parallel processing (within each Processor's selected page range, independent of the number of Processing Workers or Processor Servers). If set >1, then a METS Server must be used for METS synchronisation.",
+    parser=int,
+    default=(True, 1))
+
+config.add('OCRD_PROCESSING_PAGE_TIMEOUT',
+    description="Timeout in seconds for processing a single page. If set >0, when exceeded, the same as OCRD_MISSING_OUTPUT applies.",
+    parser=int,
+    default=(True, 0))
+
 config.add("OCRD_PROFILE",
     description="""\
 Whether to enable gathering runtime statistics
@@ -184,6 +206,11 @@ How to deal with missing output files (for some fileGrp/pageId) during processin
     validator=lambda val: val in ['SKIP', 'COPY', 'ABORT'],
     parser=str)
 
+config.add("OCRD_MAX_MISSING_OUTPUTS",
+    description="Maximal rate of skipped/fallback pages among all processed pages before aborting (decimal fraction, ignored if negative).",
+    default=(True, 0.1),
+    parser=float)
+
 config.add("OCRD_EXISTING_OUTPUT",
     description="""\
 How to deal with already existing output files (for some fileGrp/pageId) during processing:

ocrd_utils/logging.py

@@ -210,11 +210,13 @@ def disableLogging(silent=not config.OCRD_LOGGING_DEBUG):
     # logging.basicConfig(level=logging.CRITICAL)
     # logging.disable(logging.ERROR)
     # remove all handlers for the ocrd logger
-    for logger_name in ROOT_OCRD_LOGGERS:
+    for logger_name in ROOT_OCRD_LOGGERS + ['']:
         for handler in logging.getLogger(logger_name).handlers[:]:
             logging.getLogger(logger_name).removeHandler(handler)
     for logger_name in LOGGING_DEFAULTS:
         logging.getLogger(logger_name).setLevel(logging.NOTSET)
+    # Python default log level is WARNING
+    logging.root.setLevel(logging.WARNING)
 
 # Initializing stream handlers at module level
 # would cause message output in all runtime contexts,