ocrd 3.0.0b6__py3-none-any.whl → 3.0.0b7__py3-none-any.whl

ocrd/processor/base.py

@@ -18,10 +18,11 @@ from os import getcwd
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union, get_args
 import sys
+import logging
+import logging.handlers
 import inspect
 import tarfile
 import io
-import weakref
 from collections import defaultdict
 from frozendict import frozendict
 # concurrent.futures is buggy in py38,
@@ -158,12 +159,12 @@ class Processor():
 
     max_workers : int = -1
     """
-    maximum number of processor threads for page-parallel processing (ignored if negative),
+    maximum number of processor forks for page-parallel processing (ignored if negative),
     to be applied on top of :py:data:`~ocrd_utils.config.OCRD_MAX_PARALLEL_PAGES` (i.e.
     whatever is smaller).
 
     (Override this if you know how many pages fit into processing units - GPU shaders / CPU cores
-    - at once, or if your class is not thread-safe.)
+    - at once, or if your class already creates threads prior to forking, e.g. during ``setup``.)
     """
 
     max_page_seconds : int = -1
@@ -366,12 +367,14 @@ class Processor():
         self._base_logger = getLogger('ocrd.processor.base')
         if parameter is not None:
             self.parameter = parameter
-        # ensure that shutdown gets called at destruction
-        self._finalizer = weakref.finalize(self, self.shutdown)
         # workaround for deprecated#72 (@deprecated decorator does not work for subclasses):
         setattr(self, 'process',
                 deprecated(version='3.0', reason='process() should be replaced with process_page_pcgts() or process_page_file() or process_workspace()')(getattr(self, 'process')))
 
+    def __del__(self):
+        self._base_logger.debug("shutting down")
+        self.shutdown()
+
     def show_help(self, subcommand=None):
         """
         Print a usage description including the standard CLI and all of this processor's ocrd-tool
@@ -514,22 +517,31 @@ class Processor():
 
                 if max_workers > 1:
                     executor_cls = ProcessPoolExecutor
+                    log_queue = mp.Queue()
+                    # forward messages from log queue (in subprocesses) to all root handlers
+                    log_listener = logging.handlers.QueueListener(log_queue, *logging.root.handlers, respect_handler_level=True)
                 else:
                     executor_cls = DummyExecutor
+                    log_queue = None
+                    log_listener = None
                 executor = executor_cls(
                     max_workers=max_workers or 1,
                     # only forking method avoids pickling
                     context=mp.get_context('fork'),
                     # share processor instance as global to avoid pickling
                     initializer=_page_worker_set_ctxt,
-                    initargs=(self,),
+                    initargs=(self, log_queue),
                 )
+                if max_workers > 1:
+                    log_listener.start()
                 try:
                     self._base_logger.debug("started executor %s with %d workers", str(executor), max_workers or 1)
                     tasks = self.process_workspace_submit_tasks(executor, max_seconds)
                     stats = self.process_workspace_handle_tasks(tasks)
                 finally:
                     executor.shutdown(kill_workers=True, wait=False)
+                    if max_workers > 1:
+                        log_listener.stop()
 
             except NotImplementedError:
                 # fall back to deprecated method
@@ -1109,13 +1121,16 @@ in Processor.process_workspace. Forking allows inheriting global
 objects, and with the METS Server we do not mutate the local
 processor instance anyway.
 """
-def _page_worker_set_ctxt(processor):
+def _page_worker_set_ctxt(processor, log_queue):
     """
     Overwrites `ocrd.processor.base._page_worker_processor` instance
     for sharing with subprocesses in ProcessPoolExecutor initializer.
     """
     global _page_worker_processor
     _page_worker_processor = processor
+    if log_queue:
+        # replace all log handlers with just one queue handler
+        logging.root.handlers = [logging.handlers.QueueHandler(log_queue)]
 
 def _page_worker(timeout, *input_files):
     """

ocrd/processor/concurrent.py

@@ -0,0 +1,909 @@
+# copied from CPython@aac89b54c5ee03c4d64fbdfbb6ea3001e26aa83a
+# (which fixes hanging shutdown() and a memory leak on exceptions)
+# but reverting b61b818d916942aad1f8f3e33181801c4a1ed14b and
+# 90549676e063c2c818cfc14213d3adb7edcc2bd5 here
+# so it can be used standalone in Python 3.8.
+
+# Copyright 2009 Brian Quinlan. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Implements ProcessPoolExecutor.
+
+The following diagram and text describe the data-flow through the system:
+
+|======================= In-process =====================|== Out-of-process ==|
+
++----------+     +----------+       +--------+     +-----------+    +---------+
+|          |  => | Work Ids |       |        |     | Call Q    |    | Process |
+|          |     +----------+       |        |     +-----------+    |  Pool   |
+|          |     | ...      |       |        |     | ...       |    +---------+
+|          |     | 6        |    => |        |  => | 5, call() | => |         |
+|          |     | 7        |       |        |     | ...       |    |         |
+| Process  |     | ...      |       | Local  |     +-----------+    | Process |
+|  Pool    |     +----------+       | Worker |                      |  #1..n  |
+| Executor |                        | Thread |                      |         |
+|          |     +----------- +     |        |     +-----------+    |         |
+|          | <=> | Work Items | <=> |        | <=  | Result Q  | <= |         |
+|          |     +------------+     |        |     +-----------+    |         |
+|          |     | 6: call()  |     |        |     | ...       |    |         |
+|          |     |    future  |     |        |     | 4, result |    |         |
+|          |     | ...        |     |        |     | 3, except |    |         |
++----------+     +------------+     +--------+     +-----------+    +---------+
+
+Executor.submit() called:
+- creates a uniquely numbered _WorkItem and adds it to the "Work Items" dict
+- adds the id of the _WorkItem to the "Work Ids" queue
+
+Local worker thread:
+- reads work ids from the "Work Ids" queue and looks up the corresponding
+  WorkItem from the "Work Items" dict: if the work item has been cancelled then
+  it is simply removed from the dict, otherwise it is repackaged as a
+  _CallItem and put in the "Call Q". New _CallItems are put in the "Call Q"
+  until "Call Q" is full. NOTE: the size of the "Call Q" is kept small because
+  calls placed in the "Call Q" can no longer be cancelled with Future.cancel().
+- reads _ResultItems from "Result Q", updates the future stored in the
+  "Work Items" dict and deletes the dict entry
+
+Process #1..n:
+- reads _CallItems from "Call Q", executes the calls, and puts the resulting
+  _ResultItems in "Result Q"
+"""
+
+__author__ = 'Brian Quinlan (brian@sweetapp.com)'
+
+import atexit
+import os
+from concurrent.futures import _base
+import queue
+import multiprocessing as mp
+# This import is required to load the multiprocessing.connection submodule
+# so that it can be accessed later as `mp.connection`
+import multiprocessing.connection
+from multiprocessing.queues import Queue
+import threading
+import weakref
+from functools import partial
+import itertools
+import sys
+from traceback import format_exception
+
+# Workers are created as daemon threads and processes. This is done to allow the
+# interpreter to exit when there are still idle processes in a
+# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However,
+# allowing workers to die with the interpreter has two undesirable properties:
+#   - The workers would still be running during interpreter shutdown,
+#     meaning that they would fail in unpredictable ways.
+#   - The workers could be killed while evaluating a work item, which could
+#     be bad if the callable being evaluated has external side-effects e.g.
+#     writing to a file.
+#
+# To work around this problem, an exit handler is installed which tells the
+# workers to exit when their work queues are empty and then waits until the
+# threads/processes finish.
+
+_threads_wakeups = weakref.WeakKeyDictionary()
+_global_shutdown = False
+
+# Workers are created as daemon threads. This is done to allow the interpreter
+# to exit when there are still idle threads in a ThreadPoolExecutor's thread
+# pool (i.e. shutdown() was not called). However, allowing workers to die with
+# the interpreter has two undesirable properties:
+#   - The workers would still be running during interpreter shutdown,
+#     meaning that they would fail in unpredictable ways.
+#   - The workers could be killed while evaluating a work item, which could
+#     be bad if the callable being evaluated has external side-effects e.g.
+#     writing to a file.
+#
+# To work around this problem, an exit handler is installed which tells the
+# workers to exit when their work queues are empty and then waits until the
+# threads finish.
+
+class _ThreadWakeup:
+    def __init__(self):
+        self._closed = False
+        self._reader, self._writer = mp.Pipe(duplex=False)
+
+    def close(self):
+        # Please note that we do not take the shutdown lock when
+        # calling clear() (to avoid deadlocking) so this method can
+        # only be called safely from the same thread as all calls to
+        # clear() even if you hold the shutdown lock. Otherwise we
+        # might try to read from the closed pipe.
+        if not self._closed:
+            self._closed = True
+            self._writer.close()
+            self._reader.close()
+
+    def wakeup(self):
+        if not self._closed:
+            self._writer.send_bytes(b"")
+
+    def clear(self):
+        if not self._closed:
+            while self._reader.poll():
+                self._reader.recv_bytes()
+
+# Workers are created as daemon threads and processes. This is done to allow the
+# interpreter to exit when there are still idle processes in a
+# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However,
+# allowing workers to die with the interpreter has two undesirable properties:
+#   - The workers would still be running during interpreter shutdown,
+#     meaning that they would fail in unpredictable ways.
+#   - The workers could be killed while evaluating a work item, which could
+#     be bad if the callable being evaluated has external side-effects e.g.
+#     writing to a file.
+#
+# To work around this problem, an exit handler is installed which tells the
+# workers to exit when their work queues are empty and then waits until the
+# threads/processes finish.
+
+def _python_exit():
+    global _global_shutdown
+    _global_shutdown = True
+    items = list(_threads_wakeups.items())
+    for _, thread_wakeup in items:
+        # call not protected by ProcessPoolExecutor._shutdown_lock
+        thread_wakeup.wakeup()
+    for t, _ in items:
+        t.join()
+
+# Controls how many more calls than processes will be queued in the call queue.
+# A smaller number will mean that processes spend more time idle waiting for
+# work while a larger number will make Future.cancel() succeed less frequently
+# (Futures in the call queue cannot be cancelled).
+EXTRA_QUEUED_CALLS = 1
+
+
+# On Windows, WaitForMultipleObjects is used to wait for processes to finish.
+# It can wait on, at most, 63 objects. There is an overhead of two objects:
+# - the result queue reader
+# - the thread wakeup reader
+_MAX_WINDOWS_WORKERS = 63 - 2
+
+# Hack to embed stringification of remote traceback in local traceback
+
+class _RemoteTraceback(Exception):
+    def __init__(self, tb):
+        self.tb = tb
+    def __str__(self):
+        return self.tb
+
+class _ExceptionWithTraceback:
+    def __init__(self, exc, tb):
+        tb = ''.join(format_exception(type(exc), exc, tb))
+        self.exc = exc
+        # Traceback object needs to be garbage-collected as its frames
+        # contain references to all the objects in the exception scope
+        self.exc.__traceback__ = None
+        self.tb = '\n"""\n%s"""' % tb
+    def __reduce__(self):
+        return _rebuild_exc, (self.exc, self.tb)
+
+def _rebuild_exc(exc, tb):
+    exc.__cause__ = _RemoteTraceback(tb)
+    return exc
+
+class _WorkItem(object):
+    def __init__(self, future, fn, args, kwargs):
+        self.future = future
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+
+class _ResultItem(object):
+    def __init__(self, work_id, exception=None, result=None, exit_pid=None):
+        self.work_id = work_id
+        self.exception = exception
+        self.result = result
+        self.exit_pid = exit_pid
+
+class _CallItem(object):
+    def __init__(self, work_id, fn, args, kwargs):
+        self.work_id = work_id
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+
+
+class _SafeQueue(Queue):
+    """Safe Queue set exception to the future object linked to a job"""
+    def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock,
+                 thread_wakeup):
+        self.pending_work_items = pending_work_items
+        self.shutdown_lock = shutdown_lock
+        self.thread_wakeup = thread_wakeup
+        super().__init__(max_size, ctx=ctx)
+
+    def _on_queue_feeder_error(self, e, obj):
+        if isinstance(obj, _CallItem):
+            tb = format_exception(type(e), e, e.__traceback__)
+            e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb)))
+            work_item = self.pending_work_items.pop(obj.work_id, None)
+            with self.shutdown_lock:
+                self.thread_wakeup.wakeup()
+            # work_item can be None if another process terminated. In this
+            # case, the executor_manager_thread fails all work_items
+            # with BrokenProcessPool
+            if work_item is not None:
+                work_item.future.set_exception(e)
+        else:
+            super()._on_queue_feeder_error(e, obj)
+
+
+def _process_chunk(fn, chunk):
+    """ Processes a chunk of an iterable passed to map.
+
+    Runs the function passed to map() on a chunk of the
+    iterable passed to map.
+
+    This function is run in a separate process.
+
+    """
+    return [fn(*args) for args in chunk]
+
+
+def _sendback_result(result_queue, work_id, result=None, exception=None,
+                     exit_pid=None):
+    """Safely send back the given result or exception"""
+    try:
+        result_queue.put(_ResultItem(work_id, result=result,
+                                     exception=exception, exit_pid=exit_pid))
+    except BaseException as e:
+        exc = _ExceptionWithTraceback(e, e.__traceback__)
+        result_queue.put(_ResultItem(work_id, exception=exc,
+                                     exit_pid=exit_pid))
+
+
+def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None):
+    """Evaluates calls from call_queue and places the results in result_queue.
+
+    This worker is run in a separate process.
+
+    Args:
+        call_queue: A ctx.Queue of _CallItems that will be read and
+            evaluated by the worker.
+        result_queue: A ctx.Queue of _ResultItems that will written
+            to by the worker.
+        initializer: A callable initializer, or None
+        initargs: A tuple of args for the initializer
+    """
+    if initializer is not None:
+        try:
+            initializer(*initargs)
+        except BaseException:
+            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
+            # The parent will notice that the process stopped and
+            # mark the pool broken
+            return
+    num_tasks = 0
+    exit_pid = None
+    while True:
+        call_item = call_queue.get(block=True)
+        if call_item is None:
+            # Wake up queue management thread
+            result_queue.put(os.getpid())
+            return
+
+        if max_tasks is not None:
+            num_tasks += 1
+            if num_tasks >= max_tasks:
+                exit_pid = os.getpid()
+
+        try:
+            r = call_item.fn(*call_item.args, **call_item.kwargs)
+        except BaseException as e:
+            exc = _ExceptionWithTraceback(e, e.__traceback__)
+            _sendback_result(result_queue, call_item.work_id, exception=exc,
+                             exit_pid=exit_pid)
+        else:
+            _sendback_result(result_queue, call_item.work_id, result=r,
+                             exit_pid=exit_pid)
+            del r
+
+        # Liberate the resource as soon as possible, to avoid holding onto
+        # open files or shared memory that is not needed anymore
+        del call_item
+
+        if exit_pid is not None:
+            return
+
+
+class _ExecutorManagerThread(threading.Thread):
+    """Manages the communication between this process and the worker processes.
+
+    The manager is run in a local thread.
+
+    Args:
+        executor: A reference to the ProcessPoolExecutor that owns
+            this thread. A weakref will be own by the manager as well as
+            references to internal objects used to introspect the state of
+            the executor.
+    """
+
+    def __init__(self, executor):
+        # Store references to necessary internals of the executor.
+
+        # A _ThreadWakeup to allow waking up the queue_manager_thread from the
+        # main Thread and avoid deadlocks caused by permanently locked queues.
+        self.thread_wakeup = executor._executor_manager_thread_wakeup
+        self.shutdown_lock = executor._shutdown_lock
+
+        # A weakref.ref to the ProcessPoolExecutor that owns this thread. Used
+        # to determine if the ProcessPoolExecutor has been garbage collected
+        # and that the manager can exit.
+        # When the executor gets garbage collected, the weakref callback
+        # will wake up the queue management thread so that it can terminate
+        # if there is no pending work item.
+        def weakref_cb(_,
+                       thread_wakeup=self.thread_wakeup,
+                       shutdown_lock=self.shutdown_lock,
+                       mp_util_debug=mp.util.debug):
+            mp_util_debug('Executor collected: triggering callback for'
+                          ' QueueManager wakeup')
+            with shutdown_lock:
+                thread_wakeup.wakeup()
+
+        self.executor_reference = weakref.ref(executor, weakref_cb)
+
+        # A list of the ctx.Process instances used as workers.
+        self.processes = executor._processes
+
+        # A ctx.Queue that will be filled with _CallItems derived from
+        # _WorkItems for processing by the process workers.
+        self.call_queue = executor._call_queue
+
+        # A ctx.SimpleQueue of _ResultItems generated by the process workers.
+        self.result_queue = executor._result_queue
+
+        # A queue.Queue of work ids e.g. Queue([5, 6, ...]).
+        self.work_ids_queue = executor._work_ids
+
+        # Maximum number of tasks a worker process can execute before
+        # exiting safely
+        self.max_tasks_per_child = executor._max_tasks_per_child
+
+        # A dict mapping work ids to _WorkItems e.g.
+        #     {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
+        self.pending_work_items = executor._pending_work_items
+
+        # Set this thread to be daemonized
+        super().__init__()
+        self.daemon = True
+
+    def run(self):
+        # Main loop for the executor manager thread.
+
+        while True:
+            # gh-109047: During Python finalization, self.call_queue.put()
+            # creation of a thread can fail with RuntimeError.
+            try:
+                self.add_call_item_to_queue()
+            except BaseException as exc:
+                cause = format_exception(exc)
+                self.terminate_broken(cause)
+                return
+
+            result_item, is_broken, cause = self.wait_result_broken_or_wakeup()
+
+            if is_broken:
+                self.terminate_broken(cause)
+                return
+            if result_item is not None:
+                self.process_result_item(result_item)
+
+                process_exited = result_item.exit_pid is not None
+                if process_exited:
+                    p = self.processes.pop(result_item.exit_pid)
+                    p.join()
+
+                # Delete reference to result_item to avoid keeping references
+                # while waiting on new results.
+                del result_item
+
+                if executor := self.executor_reference():
+                    if process_exited:
+                        with self.shutdown_lock:
+                            executor._adjust_process_count()
+                    else:
+                        executor._idle_worker_semaphore.release()
+                    del executor
+
+            if self.is_shutting_down():
+                self.flag_executor_shutting_down()
+
+                # When only canceled futures remain in pending_work_items, our
+                # next call to wait_result_broken_or_wakeup would hang forever.
+                # This makes sure we have some running futures or none at all.
+                self.add_call_item_to_queue()
+
+                # Since no new work items can be added, it is safe to shutdown
+                # this thread if there are no pending work items.
+                if not self.pending_work_items:
+                    self.join_executor_internals()
+                    return
+
+    def add_call_item_to_queue(self):
+        # Fills call_queue with _WorkItems from pending_work_items.
+        # This function never blocks.
+        while True:
+            if self.call_queue.full():
+                return
+            try:
+                work_id = self.work_ids_queue.get(block=False)
+            except queue.Empty:
+                return
+            else:
+                work_item = self.pending_work_items[work_id]
+
+                if work_item.future.set_running_or_notify_cancel():
+                    self.call_queue.put(_CallItem(work_id,
+                                                  work_item.fn,
+                                                  work_item.args,
+                                                  work_item.kwargs),
+                                        block=True)
+                else:
+                    del self.pending_work_items[work_id]
+                    continue
+
+    def wait_result_broken_or_wakeup(self):
+        # Wait for a result to be ready in the result_queue while checking
+        # that all worker processes are still running, or for a wake up
+        # signal send. The wake up signals come either from new tasks being
+        # submitted, from the executor being shutdown/gc-ed, or from the
+        # shutdown of the python interpreter.
+        result_reader = self.result_queue._reader
+        assert not self.thread_wakeup._closed
+        wakeup_reader = self.thread_wakeup._reader
+        readers = [result_reader, wakeup_reader]
+        worker_sentinels = [p.sentinel for p in list(self.processes.values())]
+        ready = mp.connection.wait(readers + worker_sentinels)
+
+        cause = None
+        is_broken = True
+        result_item = None
+        if result_reader in ready:
+            try:
+                result_item = result_reader.recv()
+                is_broken = False
+            except BaseException as exc:
+                cause = format_exception(exc)
+
+        elif wakeup_reader in ready:
+            is_broken = False
+
+        # No need to hold the _shutdown_lock here because:
+        # 1. we're the only thread to use the wakeup reader
+        # 2. we're also the only thread to call thread_wakeup.close()
+        # 3. we want to avoid a possible deadlock when both reader and writer
+        #    would block (gh-105829)
+        self.thread_wakeup.clear()
+
+        return result_item, is_broken, cause
+
+    def process_result_item(self, result_item):
+        # Process the received a result_item. This can be either the PID of a
+        # worker that exited gracefully or a _ResultItem
+
+        # Received a _ResultItem so mark the future as completed.
+        work_item = self.pending_work_items.pop(result_item.work_id, None)
+        # work_item can be None if another process terminated (see above)
+        if work_item is not None:
+            if result_item.exception:
+                work_item.future.set_exception(result_item.exception)
+            else:
+                work_item.future.set_result(result_item.result)
+
+    def is_shutting_down(self):
+        # Check whether we should start shutting down the executor.
+        executor = self.executor_reference()
+        # No more work items can be added if:
+        #   - The interpreter is shutting down OR
+        #   - The executor that owns this worker has been collected OR
+        #   - The executor that owns this worker has been shutdown.
+        return (_global_shutdown or executor is None
+                or executor._shutdown_thread)
+
+    def _terminate_broken(self, cause):
+        # Terminate the executor because it is in a broken state. The cause
+        # argument can be used to display more information on the error that
+        # lead the executor into becoming broken.
+
+        # Mark the process pool broken so that submits fail right now.
+        executor = self.executor_reference()
+        if executor is not None:
+            executor._broken = ('A child process terminated '
+                                'abruptly, the process pool is not '
+                                'usable anymore')
+            executor._shutdown_thread = True
+            executor = None
+
+        # All pending tasks are to be marked failed with the following
+        # BrokenProcessPool error
+        bpe = BrokenProcessPool("A process in the process pool was "
+                                "terminated abruptly while the future was "
+                                "running or pending.")
+        if cause is not None:
+            bpe.__cause__ = _RemoteTraceback(
+                f"\n'''\n{''.join(cause)}'''")
+
+        # Mark pending tasks as failed.
+        for work_id, work_item in self.pending_work_items.items():
+            try:
+                work_item.future.set_exception(bpe)
+            except _base.InvalidStateError:
+                # set_exception() fails if the future is cancelled: ignore it.
+                # Trying to check if the future is cancelled before calling
+                # set_exception() would leave a race condition if the future is
+                # cancelled between the check and set_exception().
+                pass
+            # Delete references to object. See issue16284
+            del work_item
+        self.pending_work_items.clear()
+
+        # Terminate remaining workers forcibly: the queues or their
+        # locks may be in a dirty state and block forever.
+        for p in self.processes.values():
+            p.terminate()
+
+        self.call_queue._terminate_broken()
+
+        # clean up resources
+        self._join_executor_internals(broken=True)
+
+    def terminate_broken(self, cause):
+        with self.shutdown_lock:
+            self._terminate_broken(cause)
+
+    def flag_executor_shutting_down(self):
+        # Flag the executor as shutting down and cancel remaining tasks if
+        # requested as early as possible if it is not gc-ed yet.
+        executor = self.executor_reference()
+        if executor is not None:
+            executor._shutdown_thread = True
+            # Cancel pending work items if requested.
+            if executor._cancel_pending_futures:
+                # Cancel all pending futures and update pending_work_items
+                # to only have futures that are currently running.
+                new_pending_work_items = {}
+                for work_id, work_item in self.pending_work_items.items():
+                    if not work_item.future.cancel():
+                        new_pending_work_items[work_id] = work_item
+                self.pending_work_items = new_pending_work_items
+                # Drain work_ids_queue since we no longer need to
+                # add items to the call queue.
+                while True:
+                    try:
+                        self.work_ids_queue.get_nowait()
+                    except queue.Empty:
+                        break
+                # Make sure we do this only once to not waste time looping
+                # on running processes over and over.
+                executor._cancel_pending_futures = False
+
+    def shutdown_workers(self):
+        n_children_to_stop = self.get_n_children_alive()
+        n_sentinels_sent = 0
+        # Send the right number of sentinels, to make sure all children are
+        # properly terminated.
+        while (n_sentinels_sent < n_children_to_stop
+                and self.get_n_children_alive() > 0):
+            for i in range(n_children_to_stop - n_sentinels_sent):
+                try:
+                    self.call_queue.put_nowait(None)
+                    n_sentinels_sent += 1
+                except queue.Full:
+                    break
+
+    def join_executor_internals(self):
+        with self.shutdown_lock:
+            self._join_executor_internals()
+
+    def _join_executor_internals(self, broken=False):
+        # If broken, call_queue was closed and so can no longer be used.
+        if not broken:
+            self.shutdown_workers()
+
+        # Release the queue's resources as soon as possible.
+        self.call_queue.close()
+        self.call_queue.join_thread()
+        self.thread_wakeup.close()
+
+        # If .join() is not called on the created processes then
+        # some ctx.Queue methods may deadlock on Mac OS X.
+        for p in self.processes.values():
+            if broken:
+                p.terminate()
+            p.join()
+
+    def get_n_children_alive(self):
+        # This is an upper bound on the number of children alive.
+        return sum(p.is_alive() for p in self.processes.values())
+
+
+_system_limits_checked = False
+_system_limited = None
+
+
+def _check_system_limits():
+    global _system_limits_checked, _system_limited
+    if _system_limits_checked:
+        if _system_limited:
+            raise NotImplementedError(_system_limited)
+    _system_limits_checked = True
+    try:
+        import multiprocessing.synchronize  # noqa: F401
+    except ImportError:
+        _system_limited = (
+            "This Python build lacks multiprocessing.synchronize, usually due "
+            "to named semaphores being unavailable on this platform."
+        )
+        raise NotImplementedError(_system_limited)
+    try:
+        nsems_max = os.sysconf("SC_SEM_NSEMS_MAX")
+    except (AttributeError, ValueError):
+        # sysconf not available or setting not available
+        return
+    if nsems_max == -1:
+        # indetermined limit, assume that limit is determined
+        # by available memory only
+        return
+    if nsems_max >= 256:
+        # minimum number of semaphores available
+        # according to POSIX
+        return
+    _system_limited = ("system provides too few semaphores (%d"
+                       " available, 256 necessary)" % nsems_max)
+    raise NotImplementedError(_system_limited)
+
+
+def _chain_from_iterable_of_lists(iterable):
+    """
+    Specialized implementation of itertools.chain.from_iterable.
+    Each item in *iterable* should be a list.  This function is
+    careful not to keep references to yielded objects.
+    """
+    for element in iterable:
+        element.reverse()
+        while element:
+            yield element.pop()
+
+
+class BrokenProcessPool(_base.BrokenExecutor):
+    """
+    Raised when a process in a ProcessPoolExecutor terminated abruptly
+    while a future was in the running state.
+    """
+
+
+class ProcessPoolExecutor(_base.Executor):
+    def __init__(self, max_workers=None, mp_context=None,
+                 initializer=None, initargs=(), *, max_tasks_per_child=None):
+        """Initializes a new ProcessPoolExecutor instance.
+
+        Args:
+            max_workers: The maximum number of processes that can be used to
+                execute the given calls. If None or not given then as many
+                worker processes will be created as the machine has processors.
+            mp_context: A multiprocessing context to launch the workers created
+                using the multiprocessing.get_context('start method') API. This
+                object should provide SimpleQueue, Queue and Process.
+            initializer: A callable used to initialize worker processes.
+            initargs: A tuple of arguments to pass to the initializer.
+            max_tasks_per_child: The maximum number of tasks a worker process
+                can complete before it will exit and be replaced with a fresh
+                worker process. The default of None means worker process will
+                live as long as the executor. Requires a non-'fork' mp_context
+                start method. When given, we default to using 'spawn' if no
+                mp_context is supplied.
+        """
+        _check_system_limits()
+
+        if max_workers is None:
+            self._max_workers = os.process_cpu_count() or 1
+            if sys.platform == 'win32':
+                self._max_workers = min(_MAX_WINDOWS_WORKERS,
+                                        self._max_workers)
+        else:
+            if max_workers <= 0:
+                raise ValueError("max_workers must be greater than 0")
+            elif (sys.platform == 'win32' and
+                max_workers > _MAX_WINDOWS_WORKERS):
+                raise ValueError(
+                    f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
+
+            self._max_workers = max_workers
+
+        if mp_context is None:
+            if max_tasks_per_child is not None:
+                mp_context = mp.get_context("spawn")
+            else:
+                mp_context = mp.get_context()
+        self._mp_context = mp_context
+
+        # https://github.com/python/cpython/issues/90622
+        self._safe_to_dynamically_spawn_children = (
+                self._mp_context.get_start_method(allow_none=False) != "fork")
+
+        if initializer is not None and not callable(initializer):
+            raise TypeError("initializer must be a callable")
+        self._initializer = initializer
+        self._initargs = initargs
+
+        if max_tasks_per_child is not None:
+            if not isinstance(max_tasks_per_child, int):
+                raise TypeError("max_tasks_per_child must be an integer")
+            elif max_tasks_per_child <= 0:
+                raise ValueError("max_tasks_per_child must be >= 1")
+            if self._mp_context.get_start_method(allow_none=False) == "fork":
+                # https://github.com/python/cpython/issues/90622
+                raise ValueError("max_tasks_per_child is incompatible with"
+                                 " the 'fork' multiprocessing start method;"
+                                 " supply a different mp_context.")
+        self._max_tasks_per_child = max_tasks_per_child
+
+        # Management thread
+        self._executor_manager_thread = None
+
+        # Map of pids to processes
+        self._processes = {}
+
+        # Shutdown is a two-step process.
+        self._shutdown_thread = False
+        self._shutdown_lock = threading.Lock()
+        self._idle_worker_semaphore = threading.Semaphore(0)
+        self._broken = False
+        self._queue_count = 0
+        self._pending_work_items = {}
+        self._cancel_pending_futures = False
+
+        # _ThreadWakeup is a communication channel used to interrupt the wait
+        # of the main loop of executor_manager_thread from another thread (e.g.
+        # when calling executor.submit or executor.shutdown). We do not use the
+        # _result_queue to send wakeup signals to the executor_manager_thread
+        # as it could result in a deadlock if a worker process dies with the
+        # _result_queue write lock still acquired.
+        #
+        # _shutdown_lock must be locked to access _ThreadWakeup.close() and
+        # .wakeup(). Care must also be taken to not call clear or close from
+        # more than one thread since _ThreadWakeup.clear() is not protected by
+        # the _shutdown_lock
+        self._executor_manager_thread_wakeup = _ThreadWakeup()
+
+        # Create communication channels for the executor
+        # Make the call queue slightly larger than the number of processes to
+        # prevent the worker processes from idling. But don't make it too big
+        # because futures in the call queue cannot be cancelled.
+        queue_size = self._max_workers + EXTRA_QUEUED_CALLS
+        self._call_queue = _SafeQueue(
+            max_size=queue_size, ctx=self._mp_context,
+            pending_work_items=self._pending_work_items,
+            shutdown_lock=self._shutdown_lock,
+            thread_wakeup=self._executor_manager_thread_wakeup)
+        # Killed worker processes can produce spurious "broken pipe"
+        # tracebacks in the queue's own worker thread. But we detect killed
+        # processes anyway, so silence the tracebacks.
+        self._call_queue._ignore_epipe = True
+        self._result_queue = mp_context.SimpleQueue()
+        self._work_ids = queue.Queue()
+
+    def _start_executor_manager_thread(self):
+        if self._executor_manager_thread is None:
+            # Start the processes so that their sentinels are known.
+            if not self._safe_to_dynamically_spawn_children:  # ie, using fork.
+                self._launch_processes()
+            self._executor_manager_thread = _ExecutorManagerThread(self)
+            self._executor_manager_thread.start()
+            _threads_wakeups[self._executor_manager_thread] = \
+                self._executor_manager_thread_wakeup
+
+    def _adjust_process_count(self):
+        # if there's an idle process, we don't need to spawn a new one.
+        if self._idle_worker_semaphore.acquire(blocking=False):
+            return
+
+        process_count = len(self._processes)
+        if process_count < self._max_workers:
+            # Assertion disabled as this codepath is also used to replace a
+            # worker that unexpectedly dies, even when using the 'fork' start
+            # method. That means there is still a potential deadlock bug. If a
+            # 'fork' mp_context worker dies, we'll be forking a new one when
+            # we know a thread is running (self._executor_manager_thread).
+            #assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622'
+            self._spawn_process()
+
+    def _launch_processes(self):
+        # https://github.com/python/cpython/issues/90622
+        assert not self._executor_manager_thread, (
+                'Processes cannot be fork()ed after the thread has started, '
+                'deadlock in the child processes could result.')
+        for _ in range(len(self._processes), self._max_workers):
+            self._spawn_process()
+
+    def _spawn_process(self):
+        p = self._mp_context.Process(
+            target=_process_worker,
+            args=(self._call_queue,
+                  self._result_queue,
+                  self._initializer,
+                  self._initargs,
+                  self._max_tasks_per_child))
+        p.start()
+        self._processes[p.pid] = p
+
+    def submit(self, fn, /, *args, **kwargs):
+        with self._shutdown_lock:
+            if self._broken:
+                raise BrokenProcessPool(self._broken)
+            if self._shutdown_thread:
+                raise RuntimeError('cannot schedule new futures after shutdown')
+            if _global_shutdown:
+                raise RuntimeError('cannot schedule new futures after '
+                                   'interpreter shutdown')
+
+            f = _base.Future()
+            w = _WorkItem(f, fn, args, kwargs)
+
+            self._pending_work_items[self._queue_count] = w
+            self._work_ids.put(self._queue_count)
+            self._queue_count += 1
+            # Wake up queue management thread
+            self._executor_manager_thread_wakeup.wakeup()
+
+            if self._safe_to_dynamically_spawn_children:
+                self._adjust_process_count()
+            self._start_executor_manager_thread()
+            return f
+    submit.__doc__ = _base.Executor.submit.__doc__
+
+    def map(self, fn, *iterables, timeout=None, chunksize=1):
+        """Returns an iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+            chunksize: If greater than one, the iterables will be chopped into
+                chunks of size chunksize and submitted to the process pool.
+                If set to one, the items in the list will be sent one at a time.
+
+        Returns:
+            An iterator equivalent to: map(func, *iterables) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If fn(*args) raises for any values.
+        """
+        if chunksize < 1:
+            raise ValueError("chunksize must be >= 1.")
+
+        results = super().map(partial(_process_chunk, fn),
+                              itertools.batched(zip(*iterables), chunksize),
+                              timeout=timeout)
+        return _chain_from_iterable_of_lists(results)
+
+    def shutdown(self, wait=True, *, cancel_futures=False):
+        with self._shutdown_lock:
+            self._cancel_pending_futures = cancel_futures
+            self._shutdown_thread = True
+            if self._executor_manager_thread_wakeup is not None:
+                # Wake up queue management thread
+                self._executor_manager_thread_wakeup.wakeup()
+
+        if self._executor_manager_thread is not None and wait:
+            self._executor_manager_thread.join()
+        # To reduce the risk of opening too many files, remove references to
+        # objects that use file descriptors.
+        self._executor_manager_thread = None
+        self._call_queue = None
+        if self._result_queue is not None and wait:
+            self._result_queue.close()
+        self._result_queue = None
+        self._processes = None
+        self._executor_manager_thread_wakeup = None
+
+    shutdown.__doc__ = _base.Executor.shutdown.__doc__
+
+atexit.register(_python_exit)

{ocrd-3.0.0b6.dist-info → ocrd-3.0.0b7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0b6
+Version: 3.0.0b7
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0

{ocrd-3.0.0b6.dist-info → ocrd-3.0.0b7.dist-info}/RECORD

@@ -26,7 +26,8 @@ ocrd/decorators/mets_find_options.py,sha256=d4oATKMP6bFQHNqOK6nLqgUiWF2FYdkPvzkT
 ocrd/decorators/ocrd_cli_options.py,sha256=hr2EugwAY_-GJ7F7g77Od9o9eAqhfLBHSpfmCql2OCU,2665
 ocrd/decorators/parameter_option.py,sha256=n8hYw7XVTd3i3tvpK8F1Jx_CqRp6EGF9qJVH95yj92Q,1076
 ocrd/processor/__init__.py,sha256=39ymNwYRdc-b_OJzzKmWCvo2ga3KdsGSYDHE1Hzkn_w,274
-ocrd/processor/base.py,sha256=n3eJWmh8jFFrwZrjV_pBoL8uV-VgFy9BGOBJtKKEaH8,58432
+ocrd/processor/base.py,sha256=8kFrYLd1cSHBaBolHjXdIVHwmV8muKgBCBrAYw7kWTQ,59154
+ocrd/processor/concurrent.py,sha256=IMMBFGDGqfpgm7Rp6J_dnXVckyBKntwARwcKDnxadHQ,38217
 ocrd/processor/helpers.py,sha256=8ngrqAJ01BSoSJNsIoK_YfA8QdryM5y0MqaqA9f7ELM,10483
 ocrd/processor/ocrd_page_result.py,sha256=eDkpyVHcpaBzTHXiGrcNk9PP9Xr-XZru2w_uoX_ZeNA,510
 ocrd/processor/builtin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -92,8 +93,8 @@ ocrd_utils/constants.py,sha256=ImbG1d8t2MW3uuFi-mN6aY90Zn74liAKZBKlfuKN86w,3278
 ocrd_utils/deprecate.py,sha256=4i50sZsA3Eevqn5D-SL5yGf9KEZfGCV4A5Anzn1GRMs,1026
 ocrd_utils/image.py,sha256=zNNX1cnRy6yvrxx8mnYQiqWraAh5-i4a1AOfCCg4SmI,24781
 ocrd_utils/introspect.py,sha256=gfBlmeEFuRmRUSgdSK0jOxRpYqDRXl2IAE6gv2MZ6as,1977
-ocrd_utils/logging.py,sha256=hC5_v0NDOeD_sVSFey8bmJM6bmhxgKgSHPU4EEuWP9Y,8386
-ocrd_utils/ocrd_logging.conf,sha256=kl9x9JS1d8h8F0QZabvrjZtW1iApIaChvkImYafKO5g,3623
+ocrd_utils/logging.py,sha256=sHNfih9kBfvKsHdLqMK_ew9Pld1GsRyYlrZHIySujnw,7313
+ocrd_utils/ocrd_logging.conf,sha256=JlWmA_5vg6HnjPGjTC4mA5vFHqmnEinwllSTiOw5CCo,3473
 ocrd_utils/os.py,sha256=acRRdDBI8L6BK0Mf773yKEzwdpZSFRBJEKB2crL4EjU,9865
 ocrd_utils/str.py,sha256=cRgqYILDGOAqWr0qrCrV52I3y4wvpwDVtnBGEUjXNS4,10116
 ocrd_validators/__init__.py,sha256=ZFc-UqRVBk9o1YesZFmr9lOepttNJ_NKx1Zdb7g_YsU,972
@@ -118,9 +119,9 @@ ocrd_validators/xlink.xsd,sha256=8fW7YAMWXN2PbB_MMvj9H5ZeFoEBDzuYBtlGC8_6ijw,318
 ocrd_validators/xsd_mets_validator.py,sha256=4GWfLyqkmca0x7osDuXuExYuM0HWVrKoqn0S35sFhHU,467
 ocrd_validators/xsd_page_validator.py,sha256=BNz_9u-Ek4UCeyZu3KxSQoolfW9lvuaSR9nIu1XXxeE,467
 ocrd_validators/xsd_validator.py,sha256=6HrVAf6SzCvfUIuQdIzz9bOq4V-zhyii9yrUPoK2Uvo,2094
-ocrd-3.0.0b6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ocrd-3.0.0b6.dist-info/METADATA,sha256=Hvoh6U5USfeXBfTBrUIIoYWEUkj020L6pKRp81HW-lQ,10417
-ocrd-3.0.0b6.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-ocrd-3.0.0b6.dist-info/entry_points.txt,sha256=tV_gAdO8cbnOjS0GmKfJKbN60xBAV2DQRX6hEjleSjE,94
-ocrd-3.0.0b6.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
-ocrd-3.0.0b6.dist-info/RECORD,,
+ocrd-3.0.0b7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ocrd-3.0.0b7.dist-info/METADATA,sha256=lc6oZVb9SPl97qvspPigal4yb_1DBBRZwE3GQUTAK_o,10417
+ocrd-3.0.0b7.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+ocrd-3.0.0b7.dist-info/entry_points.txt,sha256=tV_gAdO8cbnOjS0GmKfJKbN60xBAV2DQRX6hEjleSjE,94
+ocrd-3.0.0b7.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
+ocrd-3.0.0b7.dist-info/RECORD,,

ocrd_utils/logging.py

@@ -46,14 +46,8 @@ __all__ = [
     'setOverrideLogLevel',
 ]
 
-# These are the loggers we add handlers to
-ROOT_OCRD_LOGGERS = [
-    '',
-    'ocrd',
-    'ocrd_network'
-]
-
 LOGGING_DEFAULTS = {
+    '': logging.WARNING,
     'ocrd': logging.INFO,
     'ocrd_network': logging.INFO,
     # 'ocrd.resolver': logging.INFO,
@@ -114,18 +108,15 @@ def setOverrideLogLevel(lvl, silent=not config.OCRD_LOGGING_DEBUG):
         lvl (string): Log level name.
         silent (boolean): Whether to log the override call
     """
-    if not _initialized_flag:
-        initLogging(silent=silent)
-    ocrd_logger = logging.getLogger('ocrd')
-
-    if lvl is None:
-        if not silent:
-            print('[LOGGING] Reset log level override', file=sys.stderr)
-        ocrd_logger.setLevel(logging.NOTSET)
-    else:
-        if not silent:
-            print(f'[LOGGING] Overriding ocrd log level to {lvl}', file=sys.stderr)
-        ocrd_logger.setLevel(lvl)
+    if lvl is not None:
+        lvl = getLevelName(lvl)
+        if not _initialized_flag:
+            initLogging(silent=silent)
+        # affect all configured loggers
+        for logger_name in logging.root.manager.loggerDict:
+            if not silent:
+                print(f'[LOGGING] Overriding {logger_name} log level to {lvl}', file=sys.stderr)
+            logging.getLogger(logger_name).setLevel(lvl)
 
 def get_logging_config_files():
     """
@@ -159,20 +150,11 @@ def initLogging(builtin_only=False, force_reinit=False, silent=not config.OCRD_L
         - silent (bool): Whether to log logging behavior by printing to stderr
     """
     global _initialized_flag
-    if _initialized_flag and not force_reinit:
-        return
-    # disableLogging()
-
-    # https://docs.python.org/3/library/logging.html#logging.disable
-    # If logging.disable(logging.NOTSET) is called, it effectively removes this
-    # overriding level, so that logging output again depends on the effective
-    # levels of individual loggers.
-    logging.disable(logging.NOTSET)
-
-    # remove all handlers for the ocrd root loggers
-    for logger_name in ROOT_OCRD_LOGGERS:
-        for handler in logging.getLogger(logger_name).handlers[:]:
-            logging.getLogger(logger_name).removeHandler(handler)
+    if _initialized_flag:
+        if force_reinit:
+            disableLogging(silent=silent)
+        else:
+            return
 
     config_file = None
     if not builtin_only:
@@ -191,11 +173,8 @@ def initLogging(builtin_only=False, force_reinit=False, silent=not config.OCRD_L
         ocrd_handler = logging.StreamHandler(stream=sys.stderr)
         ocrd_handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT, datefmt=LOG_TIMEFMT))
         ocrd_handler.setLevel(logging.DEBUG)
-        for logger_name in ROOT_OCRD_LOGGERS:
-            logger = logging.getLogger(logger_name)
-            logger.addHandler(ocrd_handler)
-            if logger_name:
-                logger.propagate = False # avoid duplication (from root handler)
+        root_logger = logging.getLogger('')
+        root_logger.addHandler(ocrd_handler)
         for logger_name, logger_level in LOGGING_DEFAULTS.items():
             logging.getLogger(logger_name).setLevel(logger_level)
     _initialized_flag = True
@@ -211,24 +190,16 @@ def disableLogging(silent=not config.OCRD_LOGGING_DEBUG):
     if _initialized_flag and not silent:
         print("[LOGGING] Disabling logging", file=sys.stderr)
     _initialized_flag = False
-    # logging.basicConfig(level=logging.CRITICAL)
-    # logging.disable(logging.ERROR)
-    # remove all handlers for the ocrd logger
-    for logger_name in ROOT_OCRD_LOGGERS:
-        for handler in logging.getLogger(logger_name).handlers[:]:
-            logging.getLogger(logger_name).removeHandler(handler)
-    for logger_name in LOGGING_DEFAULTS:
-        logging.getLogger(logger_name).setLevel(logging.NOTSET)
+    # remove all handlers we might have added (via initLogging on builtin or file config)
+    for logger_name in logging.root.manager.loggerDict:
+        if not silent:
+            print(f'[LOGGING] Resetting {logger_name} log level and handlers')
+        logger = logging.getLogger(logger_name)
+        logger.setLevel(logging.NOTSET)
+        for handler in logger.handlers[:]:
+            logger.removeHandler(handler)
+    for handler in logging.root.handlers[:]:
+        logging.root.removeHandler(handler)
     # Python default log level is WARNING
     logging.root.setLevel(logging.WARNING)
 
-# Initializing stream handlers at module level
-# would cause message output in all runtime contexts,
-# including those which are already run for std output
-# (--dump-json, --version, ocrd-tool, bashlib etc).
-# So this needs to be an opt-in from the CLIs/decorators:
-#initLogging()
-# Also, we even have to block log output for libraries
-# (like matplotlib/tensorflow) which set up logging
-# themselves already:
-disableLogging()

ocrd_utils/ocrd_logging.conf

@@ -34,7 +34,7 @@ keys=defaultFormatter,detailedFormatter
 # default logger "root" using consoleHandler
 #
 [logger_root]
-level=INFO
+level=WARNING
 handlers=consoleHandler,fileHandler
 
 
@@ -56,22 +56,22 @@ handlers=consoleHandler,fileHandler
 # ocrd loggers
 [logger_ocrd]
 level=INFO
-handlers=consoleHandler,fileHandler
+handlers=
 qualname=ocrd
-propagate=0
 
 [logger_ocrd_network]
 level=INFO
-handlers=consoleHandler,processingServerHandler
+#handlers=consoleHandler,processingServerHandler
+handlers=processingServerHandler
 qualname=ocrd_network
-propagate=0
+#propagate=0
 
 #
 # logger tensorflow
 #
 [logger_ocrd_tensorflow]
 level=ERROR
-handlers=consoleHandler
+handlers=
 qualname=tensorflow
 
 #
@@ -79,7 +79,7 @@ qualname=tensorflow
 #
 [logger_ocrd_shapely_geos]
 level=ERROR
-handlers=consoleHandler
+handlers=
 qualname=shapely.geos
 
 
@@ -88,7 +88,7 @@ qualname=shapely.geos
 #
 [logger_ocrd_PIL]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=PIL
 
 #
@@ -96,34 +96,32 @@ qualname=PIL
 #
 [logger_paramiko]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=paramiko
-propagate=0
 
 [logger_paramiko_transport]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=paramiko.transport
-propagate=0
 
 #
 # uvicorn loggers
 #
 [logger_uvicorn]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=uvicorn
 [logger_uvicorn_access]
 level=WARN
-handlers=consoleHandler
+handlers=
 qualname=uvicorn.access
 [logger_uvicorn_error]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=uvicorn.error
 [logger_multipart]
 level=INFO
-handlers=consoleHandler
+handlers=
 qualname=multipart