ocrd 3.0.0b5__tar.gz → 3.0.0b7__tar.gz

{ocrd-3.0.0b5/src/ocrd.egg-info → ocrd-3.0.0b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0b5
+Version: 3.0.0b7
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -26,6 +26,7 @@ Requires-Dist: httpx>=0.22.0
 Requires-Dist: importlib_metadata; python_version < "3.8"
 Requires-Dist: importlib_resources; python_version < "3.10"
 Requires-Dist: jsonschema>=4
+Requires-Dist: loky
 Requires-Dist: lxml
 Requires-Dist: memory-profiler>=0.58.0
 Requires-Dist: numpy

ocrd-3.0.0b7/VERSION

@@ -0,0 +1 @@
+3.0.0b7

{ocrd-3.0.0b5 → ocrd-3.0.0b7}/requirements.txt

@@ -13,6 +13,7 @@ httpx>=0.22.0
 importlib_metadata ; python_version < '3.8'
 importlib_resources ; python_version < '3.10'
 jsonschema>=4
+loky
 lxml
 memory-profiler >= 0.58.0
 # XXX explicitly do not restrict the numpy version because different

{ocrd-3.0.0b5 → ocrd-3.0.0b7}/src/ocrd/cli/workspace.py

@@ -149,7 +149,8 @@ def workspace_clone(ctx, clobber_mets, download, file_grp, file_id, page_id, mim
         LOG.warning(DeprecationWarning("Use 'ocrd workspace --directory DIR clone' instead of argument 'WORKSPACE_DIR' ('%s')" % workspace_dir))
         ctx.directory = workspace_dir
 
-    assert not ctx.mets_server_url
+    assert not ctx.mets_server_url, \
+        f"clone cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
     workspace = ctx.resolver.workspace_from_url(
         mets_url,
         dst_dir=ctx.directory,
@@ -185,7 +186,8 @@ def workspace_init(ctx, clobber_mets, directory):
     if directory:
         LOG.warning(DeprecationWarning("Use 'ocrd workspace --directory DIR init' instead of argument 'DIRECTORY' ('%s')" % directory))
         ctx.directory = directory
-    assert not ctx.mets_server_url
+    assert not ctx.mets_server_url, \
+        f"init cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
     workspace = ctx.resolver.workspace_from_nothing(
         directory=ctx.directory,
         mets_basename=ctx.mets_basename,
@@ -506,6 +508,8 @@ def workspace_remove_file(ctx, id, force, keep_file):  # pylint: disable=redefin
     (If any ``ID`` starts with ``//``, then its remainder
      will be interpreted as a regular expression.)
     """
+    assert not ctx.mets_server_url, \
+        f"remove cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
     workspace = ctx.workspace()
     for i in id:
         workspace.remove_file(i, force=force, keep_file=keep_file)
@@ -524,6 +528,8 @@ def rename_group(ctx, old, new):
     """
     Rename fileGrp (USE attribute ``NEW`` to ``OLD``).
     """
+    assert not ctx.mets_server_url, \
+        f"rename-group cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
     workspace = ctx.workspace()
     workspace.rename_file_group(old, new)
     workspace.save_mets()
@@ -545,6 +551,8 @@ def remove_group(ctx, group, recursive, force, keep_files):
     (If any ``GROUP`` starts with ``//``, then its remainder
      will be interpreted as a regular expression.)
     """
+    assert not ctx.mets_server_url, \
+        f"remove-group cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
     workspace = ctx.workspace()
     for g in group:
         workspace.remove_file_group(g, recursive=recursive, force=force, keep_files=keep_files)
@@ -567,6 +575,8 @@ def prune_files(ctx, file_grp, mimetype, page_id, file_id):
     (If any ``FILTER`` starts with ``//``, then its remainder
      will be interpreted as a regular expression.)
     """
+    assert not ctx.mets_server_url, \
+        f"prune-files cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
     workspace = ctx.workspace()
     with pushd_popd(workspace.directory):
         for f in workspace.find_files(
@@ -673,19 +683,15 @@ def list_pages(ctx, output_field, output_format, chunk_number, chunk_index, page
      will be interpreted as a regular expression.)
     """
     workspace = ctx.workspace()
-    find_kwargs = {}
-    if page_id_range and 'ID' in output_field:
-        find_kwargs['pageId'] = page_id_range
-    page_ids = sorted({x.pageId for x in workspace.mets.find_files(**find_kwargs) if x.pageId})
     ret = []
-
-    if output_field == ['ID']:
-        ret = [[x] for x in page_ids]
-    else:
-        for i, page_div in enumerate(workspace.mets.get_physical_pages(for_pageIds=','.join(page_ids), return_divs=True)):
+    if page_id_range or list(output_field) != ['ID']:
+        for i, page_div in enumerate(workspace.mets.get_physical_pages(for_pageIds=page_id_range, return_divs=True)):
             ret.append([])
             for k in output_field:
                 ret[i].append(page_div.get(k, 'None'))
+    else:
+        for page_id in workspace.mets.physical_pages:
+            ret.append([page_id])
 
     if numeric_range:
         start, end = map(int, numeric_range.split('..'))
@@ -762,6 +768,8 @@ def update_page(ctx, attr_value_pairs, order, orderlabel, contentids, page_id):
     if contentids:
         update_kwargs['CONTENTIDS'] = contentids
     try:
+        assert not ctx.mets_server_url, \
+            f"update-page cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
         workspace = ctx.workspace()
         workspace.mets.update_physical_page_attributes(page_id, **update_kwargs)
         workspace.save_mets()
@@ -800,6 +808,8 @@ def merge(ctx, overwrite, force, copy_files, filegrp_mapping, fileid_mapping, pa
     mets_path = Path(mets_path)
     if filegrp_mapping:
         filegrp_mapping = loads(filegrp_mapping)
+    assert not ctx.mets_server_url, \
+        f"merge cannot be performed with METS Server - stop server, rerun without -U {ctx.mets_server_url}"
     workspace = ctx.workspace()
     other_workspace = Workspace(ctx.resolver, directory=str(mets_path.parent), mets_basename=str(mets_path.name))
     workspace.merge(

{ocrd-3.0.0b5 → ocrd-3.0.0b7}/src/ocrd/decorators/__init__.py

@@ -13,7 +13,6 @@ from ocrd_utils import (
     redirect_stderr_and_stdout_to_file,
 )
 from ocrd_validators import WorkspaceValidator
-from ocrd_network import ProcessingWorker, ProcessorServer, AgentType
 
 from ..resolver import Resolver
 from ..processor.base import ResourceNotFoundError, run_processor
@@ -23,8 +22,6 @@ from .parameter_option import parameter_option, parameter_override_option
 from .ocrd_cli_options import ocrd_cli_options
 from .mets_find_options import mets_find_options
 
-SUBCOMMANDS = [AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER]
-
 
 def ocrd_cli_wrap_processor(
     processorClass,
@@ -88,11 +85,9 @@ def ocrd_cli_wrap_processor(
     if list_resources:
         processor.list_resources()
         sys.exit()
-    if subcommand:
+    if subcommand or address or queue or database:
         # Used for checking/starting network agents for the WebAPI architecture
         check_and_run_network_agent(processorClass, subcommand, address, database, queue)
-    elif address or queue or database:
-        raise ValueError(f"Subcommand options --address --queue and --database are only valid for subcommands: {SUBCOMMANDS}")
 
     # from here: single-run processing context
     initLogging()
@@ -162,6 +157,11 @@ def ocrd_cli_wrap_processor(
 def check_and_run_network_agent(ProcessorClass, subcommand: str, address: str, database: str, queue: str):
     """
     """
+    from ocrd_network import ProcessingWorker, ProcessorServer, AgentType
+    SUBCOMMANDS = [AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER]
+
+    if not subcommand:
+        raise ValueError(f"Subcommand options --address --queue and --database are only valid for subcommands: {SUBCOMMANDS}")
     if subcommand not in SUBCOMMANDS:
         raise ValueError(f"SUBCOMMAND can only be one of {SUBCOMMANDS}")
 

{ocrd-3.0.0b5 → ocrd-3.0.0b7}/src/ocrd/processor/base.py

@@ -16,14 +16,21 @@ import json
 import os
 from os import getcwd
 from pathlib import Path
-from typing import Any, List, Optional, Union, get_args
+from typing import Any, Dict, List, Optional, Tuple, Union, get_args
 import sys
+import logging
+import logging.handlers
 import inspect
 import tarfile
 import io
-import weakref
+from collections import defaultdict
 from frozendict import frozendict
-from concurrent.futures import ThreadPoolExecutor, TimeoutError
+# concurrent.futures is buggy in py38,
+# this is where the fixes came from:
+from loky import Future, ProcessPoolExecutor
+import multiprocessing as mp
+from threading import Timer
+from _thread import interrupt_main
 
 from click import wrap_text
 from deprecated import deprecated
@@ -105,6 +112,31 @@ class MissingInputFile(ValueError):
                         f"and pageId {pageId} under mimetype {mimetype or 'PAGE+image(s)'}")
         super().__init__(self.message)
 
+class DummyFuture:
+    """
+    Mimics some of `concurrent.futures.Future` but runs immediately.
+    """
+    def __init__(self, fn, *args, **kwargs):
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+    def result(self):
+        return self.fn(*self.args, **self.kwargs)
+class DummyExecutor:
+    """
+    Mimics some of `concurrent.futures.ProcessPoolExecutor` but runs
+    everything immediately in this process.
+    """
+    def __init__(self, initializer=None, initargs=(), **kwargs):
+        initializer(*initargs)
+    def shutdown(self, **kwargs):
+        pass
+    def submit(self, fn, *args, **kwargs) -> DummyFuture:
+        return DummyFuture(fn, *args, **kwargs)
+
+TFuture = Union[DummyFuture, Future]
+TExecutor = Union[DummyExecutor, ProcessPoolExecutor]
+
 class Processor():
     """
     A processor is a tool that implements the uniform OCR-D
@@ -127,12 +159,12 @@ class Processor():
 
     max_workers : int = -1
     """
-    maximum number of processor threads for page-parallel processing (ignored if negative),
+    maximum number of processor forks for page-parallel processing (ignored if negative),
     to be applied on top of :py:data:`~ocrd_utils.config.OCRD_MAX_PARALLEL_PAGES` (i.e.
     whatever is smaller).
 
     (Override this if you know how many pages fit into processing units - GPU shaders / CPU cores
-    - at once, or if your class is not thread-safe.)
+    - at once, or if your class already creates threads prior to forking, e.g. during ``setup``.)
     """
 
     max_page_seconds : int = -1
@@ -335,12 +367,14 @@ class Processor():
         self._base_logger = getLogger('ocrd.processor.base')
         if parameter is not None:
             self.parameter = parameter
-        # ensure that shutdown gets called at destruction
-        self._finalizer = weakref.finalize(self, self.shutdown)
         # workaround for deprecated#72 (@deprecated decorator does not work for subclasses):
         setattr(self, 'process',
                 deprecated(version='3.0', reason='process() should be replaced with process_page_pcgts() or process_page_file() or process_workspace()')(getattr(self, 'process')))
 
+    def __del__(self):
+        self._base_logger.debug("shutting down")
+        self.shutdown()
+
     def show_help(self, subcommand=None):
         """
         Print a usage description including the standard CLI and all of this processor's ocrd-tool
@@ -456,6 +490,9 @@ class Processor():
         for the given :py:data:`page_id` (or all pages)
         under the given :py:data:`parameter`.
 
+        Delegates to :py:meth:`.process_workspace_submit_tasks`
+        and :py:meth:`.process_workspace_handle_tasks`.
+
         (This will iterate over pages and files, calling
         :py:meth:`.process_page_file` and handling exceptions.
         It should be overridden by subclasses to handle cases
@@ -465,11 +502,7 @@ class Processor():
             self.workspace = workspace
             self.verify()
             try:
-                nr_succeeded = 0
-                nr_skipped = 0
-                nr_copied = 0
-
-                # set up multithreading
+                # set up multitasking
                 max_workers = max(0, config.OCRD_MAX_PARALLEL_PAGES)
                 if self.max_workers > 0 and self.max_workers < config.OCRD_MAX_PARALLEL_PAGES:
                     self._base_logger.info("limiting number of threads from %d to %d", max_workers, self.max_workers)
@@ -481,80 +514,34 @@ class Processor():
                 if self.max_page_seconds > 0 and self.max_page_seconds < config.OCRD_PROCESSING_PAGE_TIMEOUT:
                     self._base_logger.info("limiting page timeout from %d to %d sec", max_seconds, self.max_page_seconds)
                     max_seconds = self.max_page_seconds
-                executor = ThreadPoolExecutor(
+
+                if max_workers > 1:
+                    executor_cls = ProcessPoolExecutor
+                    log_queue = mp.Queue()
+                    # forward messages from log queue (in subprocesses) to all root handlers
+                    log_listener = logging.handlers.QueueListener(log_queue, *logging.root.handlers, respect_handler_level=True)
+                else:
+                    executor_cls = DummyExecutor
+                    log_queue = None
+                    log_listener = None
+                executor = executor_cls(
                     max_workers=max_workers or 1,
-                    thread_name_prefix=f"pagetask.{workspace.mets.unique_identifier}"
+                    # only forking method avoids pickling
+                    context=mp.get_context('fork'),
+                    # share processor instance as global to avoid pickling
+                    initializer=_page_worker_set_ctxt,
+                    initargs=(self, log_queue),
                 )
-                self._base_logger.debug("started executor %s with %d workers", str(executor), max_workers or 1)
-                tasks = {}
-
-                for input_file_tuple in self.zip_input_files(on_error='abort', require_first=False):
-                    input_files : List[Optional[OcrdFileType]] = [None] * len(input_file_tuple)
-                    page_id = next(input_file.pageId
-                                   for input_file in input_file_tuple
-                                   if input_file)
-                    self._base_logger.info(f"preparing page {page_id}")
-                    for i, input_file in enumerate(input_file_tuple):
-                        if input_file is None:
-                            # file/page not found in this file grp
-                            continue
-                        input_files[i] = input_file
-                        if not self.download:
-                            continue
-                        try:
-                            input_files[i] = self.workspace.download_file(input_file)
-                        except (ValueError, FileNotFoundError, HTTPError) as e:
-                            self._base_logger.error(repr(e))
-                            self._base_logger.warning(f"failed downloading file {input_file} for page {page_id}")
-                    # process page
-                    tasks[executor.submit(self.process_page_file, *input_files)] = (page_id, input_files)
-                self._base_logger.debug("submitted %d processing tasks", len(tasks))
-
-                for task in tasks:
-                    # wait for results, handle errors
-                    page_id, input_files = tasks[task]
-                    # FIXME: differentiate error cases in various ways:
-                    # - ResourceNotFoundError → use ResourceManager to download (once), then retry
-                    # - transient (I/O or OOM) error → maybe sleep, retry
-                    # - persistent (data) error → skip / dummy / raise
-                    try:
-                        self._base_logger.debug("waiting for output of task %s (page %s) max_seconds=%d", task, page_id, max_seconds)
-                        task.result(timeout=max_seconds or None)
-                        nr_succeeded += 1
-                    # exclude NotImplementedError, so we can try process() below
-                    except NotImplementedError:
-                        raise
-                    # handle input failures separately
-                    except FileExistsError as err:
-                        if config.OCRD_EXISTING_OUTPUT == 'ABORT':
-                            raise err
-                        if config.OCRD_EXISTING_OUTPUT == 'SKIP':
-                            continue
-                        if config.OCRD_EXISTING_OUTPUT == 'OVERWRITE':
-                            # too late here, must not happen
-                            raise Exception(f"got {err} despite OCRD_EXISTING_OUTPUT==OVERWRITE")
-                    # broad coverage of output failures (including TimeoutError)
-                    except (Exception, TimeoutError) as err:
-                        # FIXME: add re-usable/actionable logging
-                        if config.OCRD_MISSING_OUTPUT == 'ABORT':
-                            self._base_logger.error(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
-                            raise err
-                        self._base_logger.exception(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
-                        if config.OCRD_MISSING_OUTPUT == 'SKIP':
-                            nr_skipped += 1
-                            continue
-                        if config.OCRD_MISSING_OUTPUT == 'COPY':
-                            self._copy_page_file(input_files[0])
-                            nr_copied += 1
-                        else:
-                            desc = config.describe('OCRD_MISSING_OUTPUT', wrap_text=False, indent_text=False)
-                            raise ValueError(f"unknown configuration value {config.OCRD_MISSING_OUTPUT} - {desc}")
-
-                if nr_skipped > 0 and nr_succeeded / nr_skipped < config.OCRD_MAX_MISSING_OUTPUTS:
-                    raise Exception(f"too many failures with skipped output ({nr_skipped})")
-                if nr_copied > 0 and nr_succeeded / nr_copied < config.OCRD_MAX_MISSING_OUTPUTS:
-                    raise Exception(f"too many failures with fallback output ({nr_skipped})")
-                executor.shutdown()
+                if max_workers > 1:
+                    log_listener.start()
+                try:
+                    self._base_logger.debug("started executor %s with %d workers", str(executor), max_workers or 1)
+                    tasks = self.process_workspace_submit_tasks(executor, max_seconds)
+                    stats = self.process_workspace_handle_tasks(tasks)
+                finally:
+                    executor.shutdown(kill_workers=True, wait=False)
+                    if max_workers > 1:
+                        log_listener.stop()
 
             except NotImplementedError:
                 # fall back to deprecated method
@@ -564,6 +551,190 @@ class Processor():
                     # suppress the NotImplementedError context
                     raise err from None
 
+    def process_workspace_submit_tasks(self, executor : TExecutor, max_seconds : int) -> Dict[TFuture, Tuple[str, List[Optional[OcrdFileType]]]]:
+        """
+        Look up all input files of the given ``workspace``
+        from the given :py:data:`input_file_grp`
+        for the given :py:data:`page_id` (or all pages),
+        and schedules calling :py:meth:`.process_page_file`
+        on them for each page via `executor` (enforcing
+        a per-page time limit of `max_seconds`).
+
+        When running with `OCRD_MAX_PARALLEL_PAGES>1` and
+        the workspace via METS Server, the executor will fork
+        this many worker parallel subprocesses each processing
+        one page at a time. (Interprocess communication is
+        done via task and result queues.)
+
+        Otherwise, tasks are run sequentially in the
+        current process.
+
+        Delegates to :py:meth:`.zip_input_files` to get 
+        the input files for each page, and then calls
+        :py:meth:`.process_workspace_submit_page_task`.
+
+        Returns a dict mapping the per-page tasks
+        (i.e. futures submitted to the executor)
+        to their corresponding pageId and input files.
+        """
+        tasks = {}
+        for input_file_tuple in self.zip_input_files(on_error='abort', require_first=False):
+            task, page_id, input_files = self.process_workspace_submit_page_task(executor, max_seconds, input_file_tuple)
+            tasks[task] = (page_id, input_files)
+        self._base_logger.debug("submitted %d processing tasks", len(tasks))
+        return tasks
+
+    def process_workspace_submit_page_task(self, executor : TExecutor, max_seconds : int, input_file_tuple : List[Optional[OcrdFileType]]) -> Tuple[TFuture, str, List[Optional[OcrdFileType]]]:
+        """
+        Ensure all input files for a single page are
+        downloaded to the workspace, then schedule
+        :py:meth:`.process_process_file` to be run on
+        them via `executor` (enforcing a per-page time
+        limit of `max_seconds`).
+
+        Delegates to :py:meth:`.process_page_file`
+        (wrapped in :py:func:`_page_worker` to share
+        the processor instance across forked processes).
+
+        \b
+        Returns a tuple of:
+        - the scheduled future object,
+        - the corresponding pageId,
+        - the corresponding input files.
+        """
+        input_files : List[Optional[OcrdFileType]] = [None] * len(input_file_tuple)
+        page_id = next(input_file.pageId
+                       for input_file in input_file_tuple
+                       if input_file)
+        self._base_logger.info(f"preparing page {page_id}")
+        for i, input_file in enumerate(input_file_tuple):
+            if input_file is None:
+                # file/page not found in this file grp
+                continue
+            input_files[i] = input_file
+            if not self.download:
+                continue
+            try:
+                input_files[i] = self.workspace.download_file(input_file)
+            except (ValueError, FileNotFoundError, HTTPError) as e:
+                self._base_logger.error(repr(e))
+                self._base_logger.warning(f"failed downloading file {input_file} for page {page_id}")
+        # process page
+        #executor.submit(self.process_page_file, *input_files)
+        return executor.submit(_page_worker, max_seconds, *input_files), page_id, input_files
+
+    def process_workspace_handle_tasks(self, tasks : Dict[TFuture, Tuple[str, List[Optional[OcrdFileType]]]]) -> Tuple[int, int, Dict[str, int], int]:
+        """
+        Look up scheduled per-page futures one by one,
+        handle errors (exceptions) and gather results.
+
+        \b
+        Enforces policies configured by the following
+        environment variables:
+        - `OCRD_EXISTING_OUTPUT` (abort/skip/overwrite)
+        - `OCRD_MISSING_OUTPUT` (abort/skip/fallback-copy)
+        - `OCRD_MAX_MISSING_OUTPUTS` (abort after all).
+
+        \b
+        Returns a tuple of:
+        - the number of successfully processed pages
+        - the number of failed (i.e. skipped or copied) pages
+        - a dict of the type and corresponding number of exceptions seen
+        - the number of total requested pages (i.e. success+fail+existing).
+
+        Delegates to :py:meth:`.process_workspace_handle_page_task`
+        for each page.
+        """
+        # aggregate info for logging:
+        nr_succeeded = 0
+        nr_failed = 0
+        nr_errors = defaultdict(int) # count causes
+        if config.OCRD_MISSING_OUTPUT == 'SKIP':
+            reason = "skipped"
+        elif config.OCRD_MISSING_OUTPUT == 'COPY':
+            reason = "fallback-copied"
+        for task in tasks:
+            # wait for results, handle errors
+            page_id, input_files = tasks[task]
+            result = self.process_workspace_handle_page_task(page_id, input_files, task)
+            if isinstance(result, Exception):
+                nr_errors[result.__class__.__name__] += 1
+                nr_failed += 1
+                # FIXME: this is just prospective, because len(tasks)==nr_failed+nr_succeeded is not guaranteed
+                if config.OCRD_MAX_MISSING_OUTPUTS > 0 and nr_failed / len(tasks) > config.OCRD_MAX_MISSING_OUTPUTS:
+                    # already irredeemably many failures, stop short
+                    nr_errors = dict(nr_errors)
+                    raise Exception(f"too many failures with {reason} output ({nr_failed} of {nr_failed+nr_succeeded}, {str(nr_errors)})")
+            elif result:
+                nr_succeeded += 1
+            # else skipped - already exists
+        nr_errors = dict(nr_errors)
+        if nr_failed > 0:
+            nr_all = nr_succeeded + nr_failed
+            if config.OCRD_MAX_MISSING_OUTPUTS > 0 and nr_failed / nr_all > config.OCRD_MAX_MISSING_OUTPUTS:
+                raise Exception(f"too many failures with {reason} output ({nr_failed} of {nr_all}, {str(nr_errors)})")
+            self._base_logger.warning("%s %d of %d pages due to %s", reason, nr_failed, nr_all, str(nr_errors))
+        return nr_succeeded, nr_failed, nr_errors, len(tasks)
+
+    def process_workspace_handle_page_task(self, page_id : str, input_files : List[Optional[OcrdFileType]], task : TFuture) -> Union[bool, Exception]:
+        """
+        \b
+        Await a single page result and handle errors (exceptions), 
+        enforcing policies configured by the following
+        environment variables:
+        - `OCRD_EXISTING_OUTPUT` (abort/skip/overwrite)
+        - `OCRD_MISSING_OUTPUT` (abort/skip/fallback-copy)
+        - `OCRD_MAX_MISSING_OUTPUTS` (abort after all).
+
+        \b
+        Returns
+        - true in case of success
+        - false in case the output already exists
+        - the exception in case of failure
+        """
+        # FIXME: differentiate error cases in various ways:
+        # - ResourceNotFoundError → use ResourceManager to download (once), then retry
+        # - transient (I/O or OOM) error → maybe sleep, retry
+        # - persistent (data) error → skip / dummy / raise
+        try:
+            self._base_logger.debug("waiting for output of task %s (page %s)", task, page_id)
+            # timeout kwarg on future is useless: it only raises TimeoutError here,
+            # but does not stop the running process/thread, and executor itself
+            # offers nothing to that effect:
+            # task.result(timeout=max_seconds or None)
+            # so we instead applied the timeout within the worker function
+            task.result()
+            return True
+        except NotImplementedError:
+            # exclude NotImplementedError, so we can try process() below
+            raise
+        # handle input failures separately
+        except FileExistsError as err:
+            if config.OCRD_EXISTING_OUTPUT == 'ABORT':
+                raise err
+            if config.OCRD_EXISTING_OUTPUT == 'SKIP':
+                return False
+            if config.OCRD_EXISTING_OUTPUT == 'OVERWRITE':
+                # too late here, must not happen
+                raise Exception(f"got {err} despite OCRD_EXISTING_OUTPUT==OVERWRITE")
+        except KeyboardInterrupt:
+            raise
+        # broad coverage of output failures (including TimeoutError)
+        except Exception as err:
+            # FIXME: add re-usable/actionable logging
+            if config.OCRD_MISSING_OUTPUT == 'ABORT':
+                self._base_logger.error(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
+                raise err
+            self._base_logger.exception(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
+            if config.OCRD_MISSING_OUTPUT == 'SKIP':
+                pass
+            elif config.OCRD_MISSING_OUTPUT == 'COPY':
+                self._copy_page_file(input_files[0])
+            else:
+                desc = config.describe('OCRD_MISSING_OUTPUT', wrap_text=False, indent_text=False)
+                raise ValueError(f"unknown configuration value {config.OCRD_MISSING_OUTPUT} - {desc}")
+            return err
+
     def _copy_page_file(self, input_file : OcrdFileType) -> None:
         """
         Copy the given ``input_file`` of the :py:data:`workspace`,
@@ -618,6 +789,12 @@ class Processor():
                 # not PAGE and not an image to generate PAGE for
                 self._base_logger.error(f"non-PAGE input for page {page_id}: {err}")
         output_file_id = make_file_id(input_files[0], self.output_file_grp)
+        output_file = next(self.workspace.mets.find_files(ID=output_file_id), None)
+        if output_file and config.OCRD_EXISTING_OUTPUT != 'OVERWRITE':
+            # short-cut avoiding useless computation:
+            raise FileExistsError(
+                f"A file with ID=={output_file_id} already exists {output_file} and neither force nor ignore are set"
+            )
         result = self.process_page_pcgts(*input_pcgts, page_id=page_id)
         for image_result in result.images:
             image_file_id = f'{output_file_id}_{image_result.file_id_suffix}'
@@ -934,6 +1111,47 @@ class Processor():
                 ifts.append(tuple(ifiles))
         return ifts
 
+_page_worker_processor = None
+"""
+This global binding for the processor is required to avoid
+squeezing the processor through a mp.Queue (which is impossible
+due to unpicklable attributes like .workspace.mets._tree anyway)
+when calling Processor.process_page_file as page worker processes
+in Processor.process_workspace. Forking allows inheriting global
+objects, and with the METS Server we do not mutate the local
+processor instance anyway.
+"""
+def _page_worker_set_ctxt(processor, log_queue):
+    """
+    Overwrites `ocrd.processor.base._page_worker_processor` instance
+    for sharing with subprocesses in ProcessPoolExecutor initializer.
+    """
+    global _page_worker_processor
+    _page_worker_processor = processor
+    if log_queue:
+        # replace all log handlers with just one queue handler
+        logging.root.handlers = [logging.handlers.QueueHandler(log_queue)]
+
+def _page_worker(timeout, *input_files):
+    """
+    Wraps a `Processor.process_page_file` call as payload (call target)
+    of the ProcessPoolExecutor workers, but also enforces the given timeout.
+    """
+    page_id = next((file.pageId for file in input_files
+                    if hasattr(file, 'pageId')), "")
+    if timeout > 0:
+        timer = Timer(timeout, interrupt_main)
+        timer.start()
+    try:
+        _page_worker_processor.process_page_file(*input_files)
+        _page_worker_processor.logger.debug("page worker completed for page %s", page_id)
+    except KeyboardInterrupt:
+        _page_worker_processor.logger.debug("page worker timed out for page %s", page_id)
+        raise TimeoutError()
+    finally:
+        if timeout > 0:
+            timer.cancel()
+
 def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
     """Generate a string describing the full CLI of this processor including params.