ocrd 3.0.0b6__tar.gz → 3.0.0b7__tar.gz

{ocrd-3.0.0b6/src/ocrd.egg-info → ocrd-3.0.0b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0b6
+Version: 3.0.0b7
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0

ocrd-3.0.0b7/VERSION

@@ -0,0 +1 @@
+3.0.0b7

{ocrd-3.0.0b6 → ocrd-3.0.0b7}/src/ocrd/processor/base.py

@@ -18,10 +18,11 @@ from os import getcwd
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union, get_args
 import sys
+import logging
+import logging.handlers
 import inspect
 import tarfile
 import io
-import weakref
 from collections import defaultdict
 from frozendict import frozendict
 # concurrent.futures is buggy in py38,
@@ -158,12 +159,12 @@ class Processor():
 
     max_workers : int = -1
     """
-    maximum number of processor threads for page-parallel processing (ignored if negative),
+    maximum number of processor forks for page-parallel processing (ignored if negative),
     to be applied on top of :py:data:`~ocrd_utils.config.OCRD_MAX_PARALLEL_PAGES` (i.e.
     whatever is smaller).
 
     (Override this if you know how many pages fit into processing units - GPU shaders / CPU cores
-    - at once, or if your class is not thread-safe.)
+    - at once, or if your class already creates threads prior to forking, e.g. during ``setup``.)
     """
 
     max_page_seconds : int = -1
@@ -366,12 +367,14 @@ class Processor():
         self._base_logger = getLogger('ocrd.processor.base')
         if parameter is not None:
             self.parameter = parameter
-        # ensure that shutdown gets called at destruction
-        self._finalizer = weakref.finalize(self, self.shutdown)
         # workaround for deprecated#72 (@deprecated decorator does not work for subclasses):
         setattr(self, 'process',
                 deprecated(version='3.0', reason='process() should be replaced with process_page_pcgts() or process_page_file() or process_workspace()')(getattr(self, 'process')))
 
+    def __del__(self):
+        self._base_logger.debug("shutting down")
+        self.shutdown()
+
     def show_help(self, subcommand=None):
         """
         Print a usage description including the standard CLI and all of this processor's ocrd-tool
@@ -514,22 +517,31 @@ class Processor():
 
                 if max_workers > 1:
                     executor_cls = ProcessPoolExecutor
+                    log_queue = mp.Queue()
+                    # forward messages from log queue (in subprocesses) to all root handlers
+                    log_listener = logging.handlers.QueueListener(log_queue, *logging.root.handlers, respect_handler_level=True)
                 else:
                     executor_cls = DummyExecutor
+                    log_queue = None
+                    log_listener = None
                 executor = executor_cls(
                     max_workers=max_workers or 1,
                     # only forking method avoids pickling
                     context=mp.get_context('fork'),
                     # share processor instance as global to avoid pickling
                     initializer=_page_worker_set_ctxt,
-                    initargs=(self,),
+                    initargs=(self, log_queue),
                 )
+                if max_workers > 1:
+                    log_listener.start()
                 try:
                     self._base_logger.debug("started executor %s with %d workers", str(executor), max_workers or 1)
                     tasks = self.process_workspace_submit_tasks(executor, max_seconds)
                     stats = self.process_workspace_handle_tasks(tasks)
                 finally:
                     executor.shutdown(kill_workers=True, wait=False)
+                    if max_workers > 1:
+                        log_listener.stop()
 
             except NotImplementedError:
                 # fall back to deprecated method
@@ -1109,13 +1121,16 @@ in Processor.process_workspace. Forking allows inheriting global
 objects, and with the METS Server we do not mutate the local
 processor instance anyway.
 """
-def _page_worker_set_ctxt(processor):
+def _page_worker_set_ctxt(processor, log_queue):
     """
     Overwrites `ocrd.processor.base._page_worker_processor` instance
     for sharing with subprocesses in ProcessPoolExecutor initializer.
     """
     global _page_worker_processor
     _page_worker_processor = processor
+    if log_queue:
+        # replace all log handlers with just one queue handler
+        logging.root.handlers = [logging.handlers.QueueHandler(log_queue)]
 
 def _page_worker(timeout, *input_files):
     """