ocrd 3.9.1__tar.gz → 3.10.0__tar.gz

{ocrd-3.9.1/src/ocrd.egg-info → ocrd-3.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: ocrd
-Version: 3.9.1
+Version: 3.10.0
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -12,9 +12,10 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: atomicwrites>=1.3.0
-Requires-Dist: beanie~=1.7
+Requires-Dist: beanie>=2.0.1
 Requires-Dist: click>=7
 Requires-Dist: cryptography<43.0.0
+Requires-Dist: cysignals
 Requires-Dist: Deprecated==1.2.0
 Requires-Dist: docker>=7.1.0
 Requires-Dist: elementpath
@@ -28,7 +29,7 @@ Requires-Dist: httpx>=0.22.0
 Requires-Dist: importlib_metadata; python_version < "3.8"
 Requires-Dist: importlib_resources; python_version < "3.10"
 Requires-Dist: jsonschema>=4
-Requires-Dist: loky
+Requires-Dist: pebble
 Requires-Dist: lxml
 Requires-Dist: memory-profiler>=0.58.0
 Requires-Dist: numpy
@@ -39,6 +40,7 @@ Requires-Dist: paramiko
 Requires-Dist: pika>=1.2.0
 Requires-Dist: Pillow>=7.2.0
 Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pymongo>=4.15.5
 Requires-Dist: python-magic
 Requires-Dist: python-multipart
 Requires-Dist: pyyaml
@@ -48,6 +50,7 @@ Requires-Dist: shapely<2.0.2; python_version < "3.9"
 Requires-Dist: shapely>=2; python_version >= "3.9"
 Requires-Dist: uvicorn
 Requires-Dist: uvicorn>=0.17.6
+Dynamic: license-file
 
 # OCR-D/core
 

ocrd-3.10.0/VERSION

@@ -0,0 +1 @@
+3.10.0

{ocrd-3.9.1 → ocrd-3.10.0}/requirements.txt

@@ -1,7 +1,8 @@
 atomicwrites >= 1.3.0
-beanie~=1.7
+beanie>=2.0.1
 click >=7
 cryptography < 43.0.0
+cysignals
 Deprecated == 1.2.0
 docker>=7.1.0
 elementpath
@@ -15,7 +16,7 @@ httpx>=0.22.0
 importlib_metadata ; python_version < '3.8'
 importlib_resources ; python_version < '3.10'
 jsonschema>=4
-loky
+pebble
 lxml
 memory-profiler >= 0.58.0
 # XXX explicitly do not restrict the numpy version because different
@@ -28,6 +29,7 @@ paramiko
 pika>=1.2.0
 Pillow >= 7.2.0
 pydantic >= 2.0.0
+pymongo >= 4.15.5
 python-magic
 python-multipart
 pyyaml

{ocrd-3.9.1 → ocrd-3.10.0}/src/ocrd/processor/base.py

@@ -25,12 +25,13 @@ import tarfile
 import io
 from collections import defaultdict
 from frozendict import frozendict
-# concurrent.futures is buggy in py38,
-# this is where the fixes came from:
-from loky import Future, ProcessPoolExecutor
+# concurrent.futures cannot timeout-kill workers
+from pebble import ProcessFuture as Future, ProcessPool as ProcessPoolExecutor
+from concurrent.futures import TimeoutError
 import multiprocessing as mp
-from multiprocessing.pool import ThreadPool
+import threading
 
+from cysignals import alarm
 from click import wrap_text
 from deprecated import deprecated
 from requests import HTTPError
@@ -113,6 +114,13 @@ class MissingInputFile(ValueError):
         super().__init__(self.message)
 
 
+class IncompleteProcessorImplementation(NotImplementedError):
+    """
+    An exception signifying the Processor subclass is incomplete,
+    because either :py:meth:`Processor.process_page_pcgts()` or
+    :py:meth:`Processor.process()` was not overridden.
+    """
+
 class DummyFuture:
     """
     Mimics some of `concurrent.futures.Future` but runs immediately.
@@ -134,12 +142,18 @@ class DummyExecutor:
     def __init__(self, initializer=None, initargs=(), **kwargs):
         initializer(*initargs)
 
-    def shutdown(self, **kwargs):
+    def stop(self):
         # allow gc to catch processor instance (unless cached)
         _page_worker_set_ctxt(None, None)
 
-    def submit(self, fn, *args, **kwargs) -> DummyFuture:
-        return DummyFuture(fn, *args, **kwargs)
+    def join(self, **kwargs):
+        pass
+
+    def schedule(self, fn, args=None, kwargs=None, timeout=None) -> DummyFuture:
+        args = args or []
+        kwargs = kwargs or {}
+        timeout = timeout or 0
+        return DummyFuture(fn, *args, **kwargs, timeout=timeout)
 
 
 TFuture = Union[DummyFuture, Future]
@@ -490,7 +504,7 @@ class Processor():
         (This contains the main functionality and needs to be
         overridden by subclasses.)
         """
-        raise NotImplementedError()
+        raise IncompleteProcessorImplementation()
 
     def process_workspace(self, workspace: Workspace) -> None:
         """
@@ -525,7 +539,7 @@ class Processor():
                     self._base_logger.info("limiting page timeout from %d to %d sec", max_seconds, self.max_page_seconds)
                     max_seconds = self.max_page_seconds
 
-                if max_workers > 1:
+                if isinstance(workspace.mets, ClientSideOcrdMets):
                     executor_cls = ProcessPoolExecutor
                     log_queue = mp.get_context('fork').Queue()
                 else:
@@ -539,7 +553,8 @@ class Processor():
                     initializer=_page_worker_set_ctxt,
                     initargs=(self, log_queue),
                 )
-                if max_workers > 1:
+                if isinstance(workspace.mets, ClientSideOcrdMets):
+                    assert executor.active # ensure pre-forking
                     # forward messages from log queue (in subprocesses) to all root handlers
                     log_listener = logging.handlers.QueueListener(log_queue, *logging.root.handlers,
                                                                   respect_handler_level=True)
@@ -550,21 +565,22 @@ class Processor():
                     tasks = self.process_workspace_submit_tasks(executor, max_seconds)
                     stats = self.process_workspace_handle_tasks(tasks)
                 finally:
-                    executor.shutdown(kill_workers=True, wait=False)
+                    executor.stop()
+                    executor.join(timeout=3.0) # raises TimeoutError
                     self._base_logger.debug("stopped executor %s after %d tasks", str(executor), len(tasks) if tasks else -1)
-                    if max_workers > 1:
+                    if isinstance(workspace.mets, ClientSideOcrdMets):
                         # can cause deadlock:
                         #log_listener.stop()
                         # not much better:
                         #log_listener.enqueue_sentinel()
                         pass
 
-            except NotImplementedError:
+            except IncompleteProcessorImplementation:
                 # fall back to deprecated method
                 try:
                     self.process()
                 except Exception as err:
-                    # suppress the NotImplementedError context
+                    # suppress the IncompleteProcessorImplementation context
                     raise err from None
 
     def process_workspace_submit_tasks(self, executor: TExecutor, max_seconds: int) -> Dict[
@@ -640,7 +656,7 @@ class Processor():
                 self._base_logger.warning(f"failed downloading file {input_file} for page {page_id}")
         # process page
         #executor.submit(self.process_page_file, *input_files)
-        return executor.submit(_page_worker, max_seconds, *input_files), page_id, input_files
+        return executor.schedule(_page_worker, args=input_files, timeout=max_seconds), page_id, input_files
 
     def process_workspace_handle_tasks(self, tasks: Dict[TFuture, Tuple[str, List[Optional[OcrdFileType]]]]) -> Tuple[
             int, int, Dict[str, int], int]:
@@ -726,11 +742,12 @@ class Processor():
             # but does not stop the running process/thread, and executor itself
             # offers nothing to that effect:
             # task.result(timeout=max_seconds or None)
-            # so we instead applied the timeout within the worker function
+            # so we instead passed the timeout to the submit (schedule) function
             task.result()
+            self._base_logger.debug("page worker completed for page %s", page_id)
             return True
-        except NotImplementedError:
-            # exclude NotImplementedError, so we can try process() below
+        except IncompleteProcessorImplementation:
+            # pass this through, so we can try process() below
             raise
         # handle input failures separately
         except FileExistsError as err:
@@ -744,7 +761,9 @@ class Processor():
         except KeyboardInterrupt:
             raise
         # broad coverage of output failures (including TimeoutError)
-        except Exception as err:
+        except (Exception, TimeoutError) as err:
+            if isinstance(err, TimeoutError):
+                self._base_logger.debug("page worker timed out for page %s", page_id)
             # FIXME: add re-usable/actionable logging
             if config.OCRD_MISSING_OUTPUT == 'ABORT':
                 self._base_logger.error(f"Failure on page {page_id}: {str(err) or err.__class__.__name__}")
@@ -896,7 +915,7 @@ class Processor():
         (This contains the main functionality and must be overridden by subclasses,
         unless it does not get called by some overriden :py:meth:`.process_page_file`.)
         """
-        raise NotImplementedError()
+        raise IncompleteProcessorImplementation()
 
     def add_metadata(self, pcgts: OcrdPage) -> None:
         """
@@ -1182,22 +1201,33 @@ def _page_worker_set_ctxt(processor, log_queue):
         logging.root.handlers = [logging.handlers.QueueHandler(log_queue)]
 
 
-def _page_worker(timeout, *input_files):
+def _page_worker(*input_files, timeout=0):
     """
     Wraps a `Processor.process_page_file` call as payload (call target)
-    of the ProcessPoolExecutor workers, but also enforces the given timeout.
+    of the ProcessPoolExecutor workers.
     """
+    #_page_worker_processor.process_page_file(*input_files)
     page_id = next((file.pageId for file in input_files
                     if hasattr(file, 'pageId')), "")
-    pool = ThreadPool(processes=1)
+    if timeout:
+        if threading.current_thread() is not threading.main_thread():
+            # does not work outside of main thread
+            # (because the exception/interrupt goes there):
+            raise ValueError("cannot apply page worker timeout outside main thread")
+        # based on setitimer() / SIGALRM - only available on Unix+Cygwin
+        # (but we need to interrupt even when in syscalls
+        #  and cannot rely on Timer threads, because
+        #  processor implementations might not work with threads),
+        alarm.alarm(timeout)
     try:
-        #_page_worker_processor.process_page_file(*input_files)
-        async_result = pool.apply_async(_page_worker_processor.process_page_file, input_files)
-        async_result.get(timeout or None)
+        _page_worker_processor.process_page_file(*input_files)
         _page_worker_processor.logger.debug("page worker completed for page %s", page_id)
-    except mp.TimeoutError:
+    except alarm.AlarmInterrupt:
         _page_worker_processor.logger.debug("page worker timed out for page %s", page_id)
-        raise
+        raise TimeoutError
+    finally:
+        if timeout:
+            alarm.cancel_alarm()
 
 
 def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):

{ocrd-3.9.1 → ocrd-3.10.0/src/ocrd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: ocrd
-Version: 3.9.1
+Version: 3.10.0
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -12,9 +12,10 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: atomicwrites>=1.3.0
-Requires-Dist: beanie~=1.7
+Requires-Dist: beanie>=2.0.1
 Requires-Dist: click>=7
 Requires-Dist: cryptography<43.0.0
+Requires-Dist: cysignals
 Requires-Dist: Deprecated==1.2.0
 Requires-Dist: docker>=7.1.0
 Requires-Dist: elementpath
@@ -28,7 +29,7 @@ Requires-Dist: httpx>=0.22.0
 Requires-Dist: importlib_metadata; python_version < "3.8"
 Requires-Dist: importlib_resources; python_version < "3.10"
 Requires-Dist: jsonschema>=4
-Requires-Dist: loky
+Requires-Dist: pebble
 Requires-Dist: lxml
 Requires-Dist: memory-profiler>=0.58.0
 Requires-Dist: numpy
@@ -39,6 +40,7 @@ Requires-Dist: paramiko
 Requires-Dist: pika>=1.2.0
 Requires-Dist: Pillow>=7.2.0
 Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pymongo>=4.15.5
 Requires-Dist: python-magic
 Requires-Dist: python-multipart
 Requires-Dist: pyyaml
@@ -48,6 +50,7 @@ Requires-Dist: shapely<2.0.2; python_version < "3.9"
 Requires-Dist: shapely>=2; python_version >= "3.9"
 Requires-Dist: uvicorn
 Requires-Dist: uvicorn>=0.17.6
+Dynamic: license-file
 
 # OCR-D/core
 

{ocrd-3.9.1 → ocrd-3.10.0}/src/ocrd.egg-info/requires.txt

@@ -1,7 +1,8 @@
 atomicwrites>=1.3.0
-beanie~=1.7
+beanie>=2.0.1
 click>=7
 cryptography<43.0.0
+cysignals
 Deprecated==1.2.0
 docker>=7.1.0
 elementpath
@@ -13,7 +14,7 @@ gitpython
 gdown
 httpx>=0.22.0
 jsonschema>=4
-loky
+pebble
 lxml
 memory-profiler>=0.58.0
 numpy
@@ -24,6 +25,7 @@ paramiko
 pika>=1.2.0
 Pillow>=7.2.0
 pydantic>=2.0.0
+pymongo>=4.15.5
 python-magic
 python-multipart
 pyyaml

{ocrd-3.9.1 → ocrd-3.10.0}/src/ocrd_network/database.py

@@ -14,9 +14,8 @@ database (runs in docker) currently has no volume set.
 """
 from beanie import init_beanie
 from beanie.operators import In
-from motor.motor_asyncio import AsyncIOMotorClient
 from pathlib import Path
-from pymongo import MongoClient, uri_parser as mongo_uri_parser
+from pymongo import AsyncMongoClient, MongoClient, uri_parser as mongo_uri_parser
 from re import sub as re_sub
 from typing import List
 from uuid import uuid4
@@ -26,7 +25,7 @@ from .utils import call_sync
 
 
 async def initiate_database(db_url: str, db_name: str = 'ocrd'):
-    client = AsyncIOMotorClient(db_url)
+    client = AsyncMongoClient(db_url)
     await init_beanie(
         database=client.get_default_database(default=db_name),
         document_models=[DBProcessorJob, DBWorkflowJob, DBWorkspace, DBWorkflowScript]

{ocrd-3.9.1 → ocrd-3.10.0}/src/ocrd_validators/page.xsd

@@ -1321,9 +1321,16 @@
 		<annotation>
 			<documentation>
 			Definition of the reading order within the page.
-			To express a reading order between elements
-			they have to be included in an OrderedGroup.
-			Groups may contain further groups.
+			To express a reading order between regions,
+			they have to be referenced by an OrderedGroup.
+			To express a non-sequential relationship (while still
+			referencing them as part of the overall structural tree),
+			they have to be referenced by an UnorderedGroup.
+			Groups may contain further groups, just as regions may
+			contain further regions.
+			Regions may be referenced only once by any ReadingOrder
+			element. (That is, the @regionRef define an injective
+			mapping from the ReadingOrder tree to the Region tree.)
 			</documentation>
 		</annotation>
 		<choice minOccurs="1" maxOccurs="1">
@@ -1336,26 +1343,26 @@
 			</annotation>
 		</attribute>
 	</complexType>
-	<complexType name="RegionRefIndexedType">
+	<complexType name="RegionRefType">
 		<annotation>
-			<documentation>Numbered region</documentation>
+			<documentation>Region reference in an UnorderedGroup(Indexed)</documentation>
 		</annotation>
-		<attribute name="index" type="int" use="required">
-			<annotation>
-				<documentation>Position (order number) of this item within the current hierarchy level.</documentation>
-			</annotation>
-		</attribute>
 		<attribute name="regionRef" type="IDREF" use="required"/>
 	</complexType>
-	<complexType name="OrderedGroupIndexedType">
+	<complexType name="OrderedGroupType">
 		<annotation>
 			<documentation>
-			Indexed group containing ordered elements
+			Group containing index-ordered elements within an UnorderedGroup(Indexed)
+			Elements must be sorted ascending strictly monotonically according to their
+			@index, regardless of their type. (That is, the index of each two consecutive
+			OrderedGroupIndexed, UnorderedGroupIndexed or RegionRefIndexed elements
+			must increase by at least one.)
 			</documentation>
 		</annotation>
 		<sequence>
 			<element name="UserDefined" type="pc:UserDefinedType"
-				 minOccurs="0" maxOccurs="1"/>
+				 minOccurs="0" maxOccurs="1">
+			</element>
 			<element name="Labels" type="pc:LabelsType"
 				 minOccurs="0" maxOccurs="unbounded">
 				<annotation>
@@ -1378,23 +1385,15 @@
 				</documentation>
 			</annotation>
 		</attribute>
-		<attribute name="index" type="int" use="required">
-			<annotation>
-				<documentation>
-				Position (order number) of this item within the
-				current hierarchy level.
-				</documentation>
-			</annotation>
-		</attribute>
 		<attribute name="caption" type="string"/>
 		<attribute name="type" type="pc:GroupTypeSimpleType"/>
 		<attribute name="continuation" type="boolean">
 			<annotation>
 				<documentation>
-				Is this group a continuation of another group (from
-				previous column or page, for example)?
+				Is this group a continuation of another group
+				(from previous column or page, for example)?
 				</documentation>
-			</annotation>
+        		</annotation>
 		</attribute>
 		<attribute name="custom" type="string">
 			<annotation>
@@ -1403,10 +1402,13 @@
 		</attribute>
 		<attribute name="comments" type="string"/>
 	</complexType>
-	<complexType name="UnorderedGroupIndexedType">
+	<complexType name="UnorderedGroupType">
 		<annotation>
 			<documentation>
-			Indexed group containing unordered elements
+			Group containing unordered elements within an UnorderedGroup(Indexed)
+			Elements need not be sorted, and may be mixed by type. (That is,
+			the sequence of	OrderedGroup, UnorderedGroup or RegionRef elements
+			may be ordered arbitrarily.)
 			</documentation>
 		</annotation>
 		<sequence>
@@ -1435,14 +1437,6 @@
 				</documentation>
 			</annotation>
 		</attribute>
-		<attribute name="index" type="int" use="required">
-			<annotation>
-				<documentation>
-				Position (order number) of this item within the
-				current hierarchy level.
-				</documentation>
-			</annotation>
-		</attribute>
 		<attribute name="caption" type="string"/>
 		<attribute name="type" type="pc:GroupTypeSimpleType"/>
 		<attribute name="continuation" type="boolean">
@@ -1460,19 +1454,30 @@
 		</attribute>
 		<attribute name="comments" type="string"/>
 	</complexType>
-	<complexType name="RegionRefType">
+	<complexType name="RegionRefIndexedType">
+		<annotation>
+			<documentation>Region reference within an OrderedGroup(Indexed)</documentation>
+		</annotation>
+		<attribute name="index" type="int" use="required">
+			<annotation>
+				<documentation>Position (order number) of this item within the current hierarchy level.</documentation>
+			</annotation>
+		</attribute>
 		<attribute name="regionRef" type="IDREF" use="required"/>
 	</complexType>
-	<complexType name="OrderedGroupType">
+	<complexType name="OrderedGroupIndexedType">
 		<annotation>
 			<documentation>
-			Numbered group (contains ordered elements)
+			Group containing index-ordered elements within an OrderedGroup(Indexed).
+			Elements must be sorted ascending strictly monotonically according to their
+			@index, regardless of their type. (That is, the index of each two consecutive
+			OrderedGroupIndexed, UnorderedGroupIndexed or RegionRefIndexed elements
+			must increase by at least one.)
 			</documentation>
 		</annotation>
 		<sequence>
 			<element name="UserDefined" type="pc:UserDefinedType"
-				 minOccurs="0" maxOccurs="1">
-			</element>
+				 minOccurs="0" maxOccurs="1"/>
 			<element name="Labels" type="pc:LabelsType"
 				 minOccurs="0" maxOccurs="unbounded">
 				<annotation>
@@ -1495,15 +1500,23 @@
 				</documentation>
 			</annotation>
 		</attribute>
+		<attribute name="index" type="int" use="required">
+			<annotation>
+				<documentation>
+				Position (order number) of this item within the
+				current hierarchy level.
+				</documentation>
+			</annotation>
+		</attribute>
 		<attribute name="caption" type="string"/>
 		<attribute name="type" type="pc:GroupTypeSimpleType"/>
 		<attribute name="continuation" type="boolean">
 			<annotation>
 				<documentation>
-				Is this group a continuation of another group
-				(from previous column or page, for example)?
+				Is this group a continuation of another group (from
+				previous column or page, for example)?
 				</documentation>
-        		</annotation>
+			</annotation>
 		</attribute>
 		<attribute name="custom" type="string">
 			<annotation>
@@ -1512,10 +1525,13 @@
 		</attribute>
 		<attribute name="comments" type="string"/>
 	</complexType>
-	<complexType name="UnorderedGroupType">
+	<complexType name="UnorderedGroupIndexedType">
 		<annotation>
 			<documentation>
-			Numbered group (contains unordered elements)
+			Group containing unordered elements within an OrderedGroup(Indexed)
+			Elements need not be sorted, and may be mixed by type. (That is,
+			the sequence of	OrderedGroup, UnorderedGroup or RegionRef elements
+			may be ordered arbitrarily.)
 			</documentation>
 		</annotation>
 		<sequence>
@@ -1544,6 +1560,14 @@
 				</documentation>
 			</annotation>
 		</attribute>
+		<attribute name="index" type="int" use="required">
+			<annotation>
+				<documentation>
+				Position (order number) of this item within the
+				current hierarchy level.
+				</documentation>
+			</annotation>
+		</attribute>
 		<attribute name="caption" type="string"/>
 		<attribute name="type" type="pc:GroupTypeSimpleType"/>
 		<attribute name="continuation" type="boolean">
@@ -2325,6 +2349,7 @@
 				<element name="TableRegion" type="pc:TableRegionType"/>
 				<element name="ChartRegion" type="pc:ChartRegionType"/>
 				<element name="SeparatorRegion" type="pc:SeparatorRegionType"/>
+				<element name="MapRegion" type="pc:MapRegionType"/>
 				<element name="MathsRegion" type="pc:MathsRegionType"/>
 				<element name="ChemRegion" type="pc:ChemRegionType"/>
 				<element name="MusicRegion" type="pc:MusicRegionType"/>

ocrd-3.9.1/VERSION

@@ -1 +0,0 @@
-3.9.1