ocrd 3.4.0__tar.gz → 3.5.0__tar.gz

{ocrd-3.4.0/src/ocrd.egg-info → ocrd-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.4.0
+Version: 3.5.0
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -21,7 +21,7 @@ Requires-Dist: elementpath
 Requires-Dist: fastapi>=0.78.0
 Requires-Dist: filetype
 Requires-Dist: Flask
-Requires-Dist: frozendict>=2.3.4
+Requires-Dist: frozendict>=2.4.0
 Requires-Dist: gdown
 Requires-Dist: httpx>=0.22.0
 Requires-Dist: importlib_metadata; python_version < "3.8"

ocrd-3.5.0/VERSION

@@ -0,0 +1 @@
+3.5.0

{ocrd-3.4.0 → ocrd-3.5.0}/requirements.txt

@@ -8,7 +8,7 @@ elementpath
 fastapi>=0.78.0
 filetype
 Flask
-frozendict>=2.3.4
+frozendict>=2.4.0
 gdown
 httpx>=0.22.0
 importlib_metadata ; python_version < '3.8'

{ocrd-3.4.0 → ocrd-3.5.0}/src/ocrd/cli/__init__.py

@@ -67,6 +67,12 @@ Variables:
 \b
 {config.describe('OCRD_EXISTING_OUTPUT', wrap_text=False)}
 \b
+{config.describe('OCRD_MAX_MISSING_OUTPUTS')}
+\b
+{config.describe('OCRD_MAX_PARALLEL_PAGES')}
+\b
+{config.describe('OCRD_PROCESSING_PAGE_TIMEOUT')}
+\b
 {config.describe('OCRD_METS_CACHING')}
 \b
 {config.describe('OCRD_MAX_PROCESSOR_CACHE')}

{ocrd-3.4.0 → ocrd-3.5.0}/src/ocrd/decorators/ocrd_cli_options.py

@@ -56,7 +56,7 @@ def ocrd_cli_options(f):
         # subcommands. So we have to work around that by creating a
         # pseudo-subcommand handled in ocrd_cli_wrap_processor
         argument('subcommand', nargs=1, required=False,
-                 type=click.Choice([AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER])),
+                 type=click.Choice(list(map(str, AgentType)))),
     ]
     for param in params:
         param(f)

{ocrd-3.4.0 → ocrd-3.5.0}/src/ocrd/processor/base.py

@@ -29,8 +29,7 @@ from frozendict import frozendict
 # this is where the fixes came from:
 from loky import Future, ProcessPoolExecutor
 import multiprocessing as mp
-from threading import Timer
-from _thread import interrupt_main
+from multiprocessing.pool import ThreadPool
 
 from click import wrap_text
 from deprecated import deprecated
@@ -783,11 +782,16 @@ class Processor():
         page_id = input_files[input_pos].pageId
         self._base_logger.info("processing page %s", page_id)
         for i, input_file in enumerate(input_files):
+            grp = self.input_file_grp.split(',')[i]
             if input_file is None:
-                grp = self.input_file_grp.split(',')[i]
                 self._base_logger.debug(f"ignoring missing file for input fileGrp {grp} for page {page_id}")
                 continue
             assert isinstance(input_file, get_args(OcrdFileType))
+            if not input_file.local_filename:
+                self._base_logger.error(f'No local file exists for page {page_id} in file group {grp}')
+                if config.OCRD_MISSING_INPUT == 'ABORT':
+                    raise MissingInputFile(grp, page_id, input_file.mimetype)
+                continue
             self._base_logger.debug(f"parsing file {input_file.ID} for page {page_id}")
             try:
                 page_ = page_from_file(input_file)
@@ -796,6 +800,9 @@ class Processor():
             except ValueError as err:
                 # not PAGE and not an image to generate PAGE for
                 self._base_logger.error(f"non-PAGE input for page {page_id}: {err}")
+        if not any(input_pcgts):
+            self._base_logger.warning(f'skipping page {page_id}')
+            return
         output_file_id = make_file_id(input_files[input_pos], self.output_file_grp)
         if input_files[input_pos].fileGrp == self.output_file_grp:
             # input=output fileGrp: re-use ID exactly
@@ -1107,7 +1114,11 @@ class Processor():
             self._base_logger.critical(f"Could not find any files for selected pageId {self.page_id}.\n"
                                        f"compare '{self.page_id}' with the output of 'orcd workspace list-page'.")
         ifts = []
-        for page, ifiles in pages.items():
+        # use physical page order
+        for page in self.workspace.mets.physical_pages:
+            if page not in pages:
+                continue
+            ifiles = pages[page]
             for i, ifg in enumerate(ifgs):
                 if not ifiles[i]:
                     # could be from non-unique with on_error=skip or from true gap
@@ -1150,18 +1161,15 @@ def _page_worker(timeout, *input_files):
     """
     page_id = next((file.pageId for file in input_files
                     if hasattr(file, 'pageId')), "")
-    if timeout > 0:
-        timer = Timer(timeout, interrupt_main)
-        timer.start()
+    pool = ThreadPool(processes=1)
     try:
-        _page_worker_processor.process_page_file(*input_files)
+        #_page_worker_processor.process_page_file(*input_files)
+        async_result = pool.apply_async(_page_worker_processor.process_page_file, input_files)
+        async_result.get(timeout or None)
         _page_worker_processor.logger.debug("page worker completed for page %s", page_id)
-    except KeyboardInterrupt:
+    except mp.TimeoutError:
         _page_worker_processor.logger.debug("page worker timed out for page %s", page_id)
-        raise TimeoutError()
-    finally:
-        if timeout > 0:
-            timer.cancel()
+        raise
 
 def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
     """Generate a string describing the full CLI of this processor including params.

{ocrd-3.4.0 → ocrd-3.5.0}/src/ocrd/workspace.py

@@ -777,16 +777,14 @@ class Workspace():
             raise Exception('Found no AlternativeImage that satisfies all requirements ' +
                             'filename="%s" in page "%s"' % (
                                 filename, page_id))
-        if not all(feature in page_coords['features']
-                   for feature in feature_selector.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'selector="%s" in page "%s"' % (
-                                feature_selector, page_id))
-        if any(feature in page_coords['features']
-               for feature in feature_filter.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'filter="%s" in page "%s"' % (
-                                feature_filter, page_id))
+        if (not all(feature in page_coords['features']
+                    for feature in feature_selector.split(',') if feature) or
+            any(feature in page_coords['features']
+                for feature in feature_filter.split(',') if feature)):
+            raise Exception('Found no AlternativeImage that satisfies all requirements' +
+                            ' selector="%s"' % feature_selector +
+                            ' filter="%s"' % feature_filter +
+                            ' in page "%s"' % page_id)
         # ensure DPI will be set in image meta-data again
         if 'DPI' in page_coords:
             dpi = page_coords['DPI']
@@ -1038,16 +1036,14 @@ class Workspace():
             raise Exception('Found no AlternativeImage that satisfies all requirements ' +
                             'filename="%s" in segment "%s"' % (
                                 filename, segment.id))
-        if not all(feature in segment_coords['features']
-                   for feature in feature_selector.split(',') if feature):
+        if (not all(feature in segment_coords['features']
+                    for feature in feature_selector.split(',') if feature) or
+            any(feature in segment_coords['features']
+                for feature in feature_filter.split(',') if feature)):
             raise Exception('Found no AlternativeImage that satisfies all requirements' +
-                            'selector="%s" in segment "%s"' % (
-                                feature_selector, segment.id))
-        if any(feature in segment_coords['features']
-               for feature in feature_filter.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'filter="%s" in segment "%s"' % (
-                                feature_filter, segment.id))
+                            ' selector="%s"' % feature_selector +
+                            ' filter="%s"' % feature_filter +
+                            ' in segment "%s"' % segment.id)
         # ensure DPI will be set in image meta-data again
         if 'DPI' in segment_coords:
             dpi = segment_coords['DPI']

{ocrd-3.4.0 → ocrd-3.5.0/src/ocrd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.4.0
+Version: 3.5.0
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -21,7 +21,7 @@ Requires-Dist: elementpath
 Requires-Dist: fastapi>=0.78.0
 Requires-Dist: filetype
 Requires-Dist: Flask
-Requires-Dist: frozendict>=2.3.4
+Requires-Dist: frozendict>=2.4.0
 Requires-Dist: gdown
 Requires-Dist: httpx>=0.22.0
 Requires-Dist: importlib_metadata; python_version < "3.8"

{ocrd-3.4.0 → ocrd-3.5.0}/src/ocrd.egg-info/requires.txt

@@ -8,7 +8,7 @@ elementpath
 fastapi>=0.78.0
 filetype
 Flask
-frozendict>=2.3.4
+frozendict>=2.4.0
 gdown
 httpx>=0.22.0
 jsonschema>=4

ocrd-3.5.0/src/ocrd_models/constants.py

@@ -0,0 +1,205 @@
+"""
+Constants for ocrd_models.
+"""
+from re import Pattern
+from enum import Enum, auto
+from dataclasses import dataclass, field
+from abc import ABC, abstractmethod
+from typing import Any, List, Optional, Union
+from ocrd_utils import resource_string
+
+__all__ = [
+    'IDENTIFIER_PRIORITY',
+    'METS_XML_EMPTY',
+    'NAMESPACES',
+    'TAG_METS_AGENT',
+    'TAG_METS_DIV',
+    'TAG_METS_FILE',
+    'TAG_METS_FILEGRP',
+    'TAG_METS_FILESEC',
+    'TAG_METS_FPTR',
+    'TAG_METS_FLOCAT',
+    'TAG_METS_METSHDR',
+    'TAG_METS_NAME',
+    'TAG_METS_NOTE',
+    'TAG_METS_STRUCTMAP',
+    'TAG_MODS_IDENTIFIER',
+    'TAG_PAGE_ALTERNATIVEIMAGE',
+    'TAG_PAGE_COORDS',
+    'TAG_PAGE_READINGORDER',
+    'TAG_PAGE_REGIONREFINDEXED',
+    'TAG_PAGE_TEXTLINE',
+    'TAG_PAGE_TEXTEQUIV',
+    'TAG_PAGE_TEXTREGION',
+    'METS_PAGE_DIV_ATTRIBUTE',
+    'METS_STRUCT_DIV_ATTRIBUTE',
+    'METS_DIV_ATTRIBUTE_ATOM_PATTERN',
+    'METS_DIV_ATTRIBUTE_RANGE_PATTERN',
+    'METS_DIV_ATTRIBUTE_REGEX_PATTERN',
+    'PAGE_REGION_TYPES',
+    'PAGE_ALTIMG_FEATURES',
+]
+
+
+IDENTIFIER_PRIORITY = ['purl', 'urn', 'doi', 'url']
+
+METS_XML_EMPTY = resource_string(__package__, 'mets-empty.xml')
+
+NAMESPACES = {
+    'mets': "http://www.loc.gov/METS/",
+    'mods': "http://www.loc.gov/mods/v3",
+    'xlink': "http://www.w3.org/1999/xlink",
+    'page': "http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15",
+    'xsl': 'http://www.w3.org/1999/XSL/Transform#',
+    'ocrd': 'https://ocr-d.de',
+}
+
+TAG_METS_AGENT            = '{%s}agent' % NAMESPACES['mets']
+TAG_METS_DIV              = '{%s}div' % NAMESPACES['mets']
+TAG_METS_FILE             = '{%s}file' % NAMESPACES['mets']
+TAG_METS_FILEGRP          = '{%s}fileGrp' % NAMESPACES['mets']
+TAG_METS_FILESEC          = '{%s}fileSec' % NAMESPACES['mets']
+TAG_METS_FPTR             = '{%s}fptr' % NAMESPACES['mets']
+TAG_METS_FLOCAT           = '{%s}FLocat' % NAMESPACES['mets']
+TAG_METS_METSHDR          = '{%s}metsHdr' % NAMESPACES['mets']
+TAG_METS_NAME             = '{%s}name' % NAMESPACES['mets']
+TAG_METS_NOTE             = '{%s}note' % NAMESPACES['mets']
+TAG_METS_STRUCTMAP        = '{%s}structMap' % NAMESPACES['mets']
+
+TAG_MODS_IDENTIFIER       = '{%s}identifier' % NAMESPACES['mods']
+
+TAG_PAGE_ALTERNATIVEIMAGE = '{%s}AlternativeImage' % NAMESPACES['page']
+TAG_PAGE_COORDS           = '{%s}Coords' % NAMESPACES['page']
+TAG_PAGE_READINGORDER     = '{%s}ReadingOrder' % NAMESPACES['page']
+TAG_PAGE_REGIONREFINDEXED = '{%s}RegionRefIndexed' % NAMESPACES['page']
+TAG_PAGE_TEXTLINE         = '{%s}TextLine' % NAMESPACES['page']
+TAG_PAGE_TEXTEQUIV        = '{%s}TextEquiv' % NAMESPACES['page']
+TAG_PAGE_TEXTREGION       = '{%s}TextRegion' % NAMESPACES['page']
+
+PAGE_REGION_TYPES = [
+    'Advert', 'Chart', 'Chem', 'Custom', 'Graphic', 'Image',
+    'LineDrawing', 'Map', 'Maths', 'Music', 'Noise',
+    'Separator', 'Table', 'Text', 'Unknown'
+]
+
+PAGE_ALTIMG_FEATURES = [
+    'binarized',
+    'grayscale_normalized',
+    'despeckled',
+    'cropped',
+    'deskewed',
+    'rotated-90',
+    'rotated-180',
+    'rotated-270',
+    'dewarped',
+    'clipped',
+]
+
+
+class METS_PAGE_DIV_ATTRIBUTE(Enum):
+    """page selection attributes of PHYSICAL mets:structMap//mets:div"""
+    ID = auto()
+    ORDER = auto()
+    ORDERLABEL = auto()
+    LABEL = auto()
+    CONTENTIDS = auto()
+
+    @classmethod
+    def names(cls):
+        return [x.name for x in cls]
+    @classmethod
+    def type_prefix(cls):
+        """disambiguation prefix to use for all subtypes"""
+        return "physical:"
+    def prefix(self):
+        """disambiguation prefix to use for this attribute type"""
+        return self.type_prefix() + self.name.lower() + ":"
+
+class METS_STRUCT_DIV_ATTRIBUTE(Enum):
+    """page selection attributes of LOGICAL mets:structMap//mets:div"""
+    ID = auto()
+    DMDID = auto()
+    TYPE = auto()
+    LABEL = auto()
+
+    @classmethod
+    def names(cls):
+        return [x.name for x in cls]
+    @classmethod
+    def type_prefix(cls):
+        """disambiguation prefix to use for all subtypes"""
+        return "logical:"
+    def prefix(self):
+        """disambiguation prefix to use for this attribute type"""
+        return self.type_prefix() + self.name.lower() + ":"
+
+@dataclass
+class METS_DIV_ATTRIBUTE_PATTERN(ABC):
+    """page selection pattern (abstract supertype)"""
+
+    expr: Any
+    """pattern value to match a mets:div against"""
+    attr: List[Union[METS_PAGE_DIV_ATTRIBUTE, METS_STRUCT_DIV_ATTRIBUTE]] = field(
+        default_factory=lambda: list(METS_PAGE_DIV_ATTRIBUTE) + list(METS_STRUCT_DIV_ATTRIBUTE))
+    """attribute type(s) to match a mets:div for
+    (pre-disambiguated with prefix syntax, or filled upon first match)
+    """
+    has_matched: bool = field(init=False, default=False)
+    """whether this pattern has already been matched"""
+
+    def attr_prefix(self):
+        """attribute type disambiguation prefix corresponding to the current state of disambiguation"""
+        if self.attr == list(METS_PAGE_DIV_ATTRIBUTE) + list(METS_STRUCT_DIV_ATTRIBUTE):
+            return ""
+        if self.attr == list(METS_PAGE_DIV_ATTRIBUTE):
+            return METS_PAGE_DIV_ATTRIBUTE.type_prefix()
+        if self.attr == list(METS_STRUCT_DIV_ATTRIBUTE):
+            return METS_STRUCT_DIV_ATTRIBUTE.type_prefix()
+        assert len(self.attr) == 1, "unexpected type ambiguity: %s" % repr(self.attr)
+        return self.attr[0].prefix()
+
+    @abstractmethod
+    def _matches(self, input) -> bool:
+        return
+    def matches(self, input) -> bool:
+        """does the selection pattern match on the given attribute value?"""
+        if (matched := self._matches(input)):
+            self.has_matched = True
+        return matched
+
+@dataclass
+class METS_DIV_ATTRIBUTE_ATOM_PATTERN(METS_DIV_ATTRIBUTE_PATTERN):
+    """page selection pattern for literal (single value) matching"""
+
+    expr: str
+    def __repr__(self):
+        return "%s%s" % (self.attr_prefix(), self.expr)
+    def _matches(self, input):
+        return input == self.expr
+
+@dataclass
+class METS_DIV_ATTRIBUTE_RANGE_PATTERN(METS_DIV_ATTRIBUTE_PATTERN):
+    """page selection pattern for interval (list expansion) matching"""
+
+    expr: List[str]
+    start: str = field(init=False)
+    """first value of the range after expansion, before matching-exhausting"""
+    stop: str = field(init=False)
+    """last value of the range after expansion, before matching-exhausting"""
+    def __post_init__(self):
+        self.start = self.expr[0]
+        self.stop = self.expr[-1]
+    def __repr__(self):
+        return "%s%s..%s" % (self.attr_prefix(), self.start, self.stop)
+    def _matches(self, input):
+        return input in self.expr
+
+@dataclass
+class METS_DIV_ATTRIBUTE_REGEX_PATTERN(METS_DIV_ATTRIBUTE_PATTERN):
+    """page selection pattern for regular expression matching"""
+
+    expr: Pattern
+    def __repr__(self):
+        return "%s//%s" % (self.attr_prefix(), self.expr.pattern)
+    def _matches(self, input):
+        return bool(self.expr.fullmatch(input))