ocrd 3.4.0__py3-none-any.whl → 3.5.0__py3-none-any.whl

ocrd/cli/__init__.py

@@ -67,6 +67,12 @@ Variables:
 \b
 {config.describe('OCRD_EXISTING_OUTPUT', wrap_text=False)}
 \b
+{config.describe('OCRD_MAX_MISSING_OUTPUTS')}
+\b
+{config.describe('OCRD_MAX_PARALLEL_PAGES')}
+\b
+{config.describe('OCRD_PROCESSING_PAGE_TIMEOUT')}
+\b
 {config.describe('OCRD_METS_CACHING')}
 \b
 {config.describe('OCRD_MAX_PROCESSOR_CACHE')}

ocrd/decorators/ocrd_cli_options.py

@@ -56,7 +56,7 @@ def ocrd_cli_options(f):
         # subcommands. So we have to work around that by creating a
         # pseudo-subcommand handled in ocrd_cli_wrap_processor
         argument('subcommand', nargs=1, required=False,
-                 type=click.Choice([AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER])),
+                 type=click.Choice(list(map(str, AgentType)))),
     ]
     for param in params:
         param(f)

ocrd/processor/base.py

@@ -29,8 +29,7 @@ from frozendict import frozendict
 # this is where the fixes came from:
 from loky import Future, ProcessPoolExecutor
 import multiprocessing as mp
-from threading import Timer
-from _thread import interrupt_main
+from multiprocessing.pool import ThreadPool
 
 from click import wrap_text
 from deprecated import deprecated
@@ -783,11 +782,16 @@ class Processor():
         page_id = input_files[input_pos].pageId
         self._base_logger.info("processing page %s", page_id)
         for i, input_file in enumerate(input_files):
+            grp = self.input_file_grp.split(',')[i]
             if input_file is None:
-                grp = self.input_file_grp.split(',')[i]
                 self._base_logger.debug(f"ignoring missing file for input fileGrp {grp} for page {page_id}")
                 continue
             assert isinstance(input_file, get_args(OcrdFileType))
+            if not input_file.local_filename:
+                self._base_logger.error(f'No local file exists for page {page_id} in file group {grp}')
+                if config.OCRD_MISSING_INPUT == 'ABORT':
+                    raise MissingInputFile(grp, page_id, input_file.mimetype)
+                continue
             self._base_logger.debug(f"parsing file {input_file.ID} for page {page_id}")
             try:
                 page_ = page_from_file(input_file)
@@ -796,6 +800,9 @@ class Processor():
             except ValueError as err:
                 # not PAGE and not an image to generate PAGE for
                 self._base_logger.error(f"non-PAGE input for page {page_id}: {err}")
+        if not any(input_pcgts):
+            self._base_logger.warning(f'skipping page {page_id}')
+            return
         output_file_id = make_file_id(input_files[input_pos], self.output_file_grp)
         if input_files[input_pos].fileGrp == self.output_file_grp:
             # input=output fileGrp: re-use ID exactly
@@ -1107,7 +1114,11 @@ class Processor():
             self._base_logger.critical(f"Could not find any files for selected pageId {self.page_id}.\n"
                                        f"compare '{self.page_id}' with the output of 'orcd workspace list-page'.")
         ifts = []
-        for page, ifiles in pages.items():
+        # use physical page order
+        for page in self.workspace.mets.physical_pages:
+            if page not in pages:
+                continue
+            ifiles = pages[page]
             for i, ifg in enumerate(ifgs):
                 if not ifiles[i]:
                     # could be from non-unique with on_error=skip or from true gap
@@ -1150,18 +1161,15 @@ def _page_worker(timeout, *input_files):
     """
     page_id = next((file.pageId for file in input_files
                     if hasattr(file, 'pageId')), "")
-    if timeout > 0:
-        timer = Timer(timeout, interrupt_main)
-        timer.start()
+    pool = ThreadPool(processes=1)
     try:
-        _page_worker_processor.process_page_file(*input_files)
+        #_page_worker_processor.process_page_file(*input_files)
+        async_result = pool.apply_async(_page_worker_processor.process_page_file, input_files)
+        async_result.get(timeout or None)
         _page_worker_processor.logger.debug("page worker completed for page %s", page_id)
-    except KeyboardInterrupt:
+    except mp.TimeoutError:
         _page_worker_processor.logger.debug("page worker timed out for page %s", page_id)
-        raise TimeoutError()
-    finally:
-        if timeout > 0:
-            timer.cancel()
+        raise
 
 def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
     """Generate a string describing the full CLI of this processor including params.

ocrd/workspace.py

@@ -777,16 +777,14 @@ class Workspace():
             raise Exception('Found no AlternativeImage that satisfies all requirements ' +
                             'filename="%s" in page "%s"' % (
                                 filename, page_id))
-        if not all(feature in page_coords['features']
-                   for feature in feature_selector.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'selector="%s" in page "%s"' % (
-                                feature_selector, page_id))
-        if any(feature in page_coords['features']
-               for feature in feature_filter.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'filter="%s" in page "%s"' % (
-                                feature_filter, page_id))
+        if (not all(feature in page_coords['features']
+                    for feature in feature_selector.split(',') if feature) or
+            any(feature in page_coords['features']
+                for feature in feature_filter.split(',') if feature)):
+            raise Exception('Found no AlternativeImage that satisfies all requirements' +
+                            ' selector="%s"' % feature_selector +
+                            ' filter="%s"' % feature_filter +
+                            ' in page "%s"' % page_id)
         # ensure DPI will be set in image meta-data again
         if 'DPI' in page_coords:
             dpi = page_coords['DPI']
@@ -1038,16 +1036,14 @@ class Workspace():
             raise Exception('Found no AlternativeImage that satisfies all requirements ' +
                             'filename="%s" in segment "%s"' % (
                                 filename, segment.id))
-        if not all(feature in segment_coords['features']
-                   for feature in feature_selector.split(',') if feature):
+        if (not all(feature in segment_coords['features']
+                    for feature in feature_selector.split(',') if feature) or
+            any(feature in segment_coords['features']
+                for feature in feature_filter.split(',') if feature)):
             raise Exception('Found no AlternativeImage that satisfies all requirements' +
-                            'selector="%s" in segment "%s"' % (
-                                feature_selector, segment.id))
-        if any(feature in segment_coords['features']
-               for feature in feature_filter.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'filter="%s" in segment "%s"' % (
-                                feature_filter, segment.id))
+                            ' selector="%s"' % feature_selector +
+                            ' filter="%s"' % feature_filter +
+                            ' in segment "%s"' % segment.id)
         # ensure DPI will be set in image meta-data again
         if 'DPI' in segment_coords:
             dpi = segment_coords['DPI']

{ocrd-3.4.0.dist-info → ocrd-3.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.4.0
+Version: 3.5.0
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -21,7 +21,7 @@ Requires-Dist: elementpath
 Requires-Dist: fastapi>=0.78.0
 Requires-Dist: filetype
 Requires-Dist: Flask
-Requires-Dist: frozendict>=2.3.4
+Requires-Dist: frozendict>=2.4.0
 Requires-Dist: gdown
 Requires-Dist: httpx>=0.22.0
 Requires-Dist: jsonschema>=4

{ocrd-3.4.0.dist-info → ocrd-3.5.0.dist-info}/RECORD

@@ -7,10 +7,10 @@ ocrd/resolver.py,sha256=A7BrZlUGrfJye-etaEuT-fdJFgvQcCxWovjufT-WmRY,15119
 ocrd/resource_list.yml,sha256=82-PiqkZnka1kTj3MQqNn4wXWKHHtoFchsQuetWuqFs,2633
 ocrd/resource_manager.py,sha256=kIWDoKxWH4IJE1gcoTcCRQjYjieCqiQclyuyF6Y9b8A,16813
 ocrd/task_sequence.py,sha256=spiaUQaMM7M8WdBDoQGmLuTPm7tOugYXD6rcJ2UXzxw,6991
-ocrd/workspace.py,sha256=pv9Ug1BPI8Uy8kgZ2tAskqKhgMcUx1ACg1QHqAgMbcM,65201
+ocrd/workspace.py,sha256=eLuGSJtOh3y2miKgcF8219YH1RkAaEi-qwXHarz8O8k,64916
 ocrd/workspace_backup.py,sha256=iab_JjZ_mMP-G8NIUk4PZmfpNlQuGRoqc3NbTSSew1w,3621
 ocrd/workspace_bagger.py,sha256=yU8H3xR5WmQKvgQewac71ie-DUWcfLnMS01D55zsEHQ,11971
-ocrd/cli/__init__.py,sha256=-BiwIakeCkWx0Jd2yX9_ahfdV4VKz_5yqGEJ_2zKakQ,2734
+ocrd/cli/__init__.py,sha256=LpQb8ne1nzAq2j52lGWDTZlBCmrLwUsz17PTwJkWNcU,2884
 ocrd/cli/bashlib.py,sha256=ypFBM3-IULz_IEBx0Y04eGt9VbQWwEWm4ujm9g_hPWY,6009
 ocrd/cli/log.py,sha256=6_FrVmTKIIVNUaNLkuOJx8pvPhensHMuayJ0PA7T-XA,1562
 ocrd/cli/network.py,sha256=oWBHFEURxfUdb_t-F4svP_ri7o5mqBoNQnLZLbsZLTA,602
@@ -23,10 +23,10 @@ ocrd/cli/zip.py,sha256=MMJLw3OXWiJVfVtrdJcBkbB8vA1IzSautluazZRuCQ0,5910
 ocrd/decorators/__init__.py,sha256=n2Lb1WLXGlvPrhNTSGZYRqugpa__MZSWV546EmQnTtc,7678
 ocrd/decorators/loglevel_option.py,sha256=tgipROEu3t4hkwWvFssd80k2SbTBwBIC4WNE6Gc-XAg,798
 ocrd/decorators/mets_find_options.py,sha256=d4oATKMP6bFQHNqOK6nLqgUiWF2FYdkPvzkTVRMYpKo,635
-ocrd/decorators/ocrd_cli_options.py,sha256=lIvtE8re1VmpHm45u71ltE0QJS8nyd28HhLC7zGSvlo,2691
+ocrd/decorators/ocrd_cli_options.py,sha256=psS7u42mXTOWIXQd9kcrgW7kDnFURHbmZ0946aqBz3A,2659
 ocrd/decorators/parameter_option.py,sha256=n8hYw7XVTd3i3tvpK8F1Jx_CqRp6EGF9qJVH95yj92Q,1076
 ocrd/processor/__init__.py,sha256=39ymNwYRdc-b_OJzzKmWCvo2ga3KdsGSYDHE1Hzkn_w,274
-ocrd/processor/base.py,sha256=eQzrwPcGxyo0y7jlAF2g5jOabB-jq_LuJfTn87Qxjwk,59929
+ocrd/processor/base.py,sha256=_h0V5FevEPLb1q0zGtShuKXRj_tOWhD0M7_ufn34MPc,60476
 ocrd/processor/helpers.py,sha256=WFdC5zeB8F7T0FkpJwfTqWsSPNRtBCBUmFLgixw-rYs,10999
 ocrd/processor/ocrd_page_result.py,sha256=eDkpyVHcpaBzTHXiGrcNk9PP9Xr-XZru2w_uoX_ZeNA,510
 ocrd/processor/builtin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -36,12 +36,12 @@ ocrd/processor/builtin/dummy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 ocrd/processor/builtin/dummy/ocrd-tool.json,sha256=NgMAXN1AQpGk4Ss73ThDY4QyFPKhj54qcrdeCGwTb10,2339
 ocrd_modelfactory/__init__.py,sha256=NyJT1uSvmeEwibRFOkh0AEoVnYfP0mzxU--pP23B-TQ,4404
 ocrd_models/__init__.py,sha256=A0aj0mOraNb-xfiUueACdoaqISnp0qH-F49nTJg2vCs,380
-ocrd_models/constants.py,sha256=kvvAAro_1YOTRWwFgbrGEFeDZ8_u0S624Y3icNNk4Oo,2987
+ocrd_models/constants.py,sha256=z5XAFMgz3pttMJOHVzTWNZr3ZqMjonVIDmXk3GQTJ30,6954
 ocrd_models/mets-empty.xml,sha256=dFixfbxSXrgjZx9BfdIKWHX-khNmp7dNYaFe2qQSwCY,1203
 ocrd_models/ocrd_agent.py,sha256=E9OtDhz9UfKb6ou2qvsuCL9NlO1V6zMb0s8nVq8dVos,5609
 ocrd_models/ocrd_exif.py,sha256=wRSprHxCy9LCXw41Fi9kp-CbFc5NFX9ZFIFNszB41qk,4585
 ocrd_models/ocrd_file.py,sha256=7lyHezuNnl2FEYV1lV35-QTCrgYAL-3wO2ulFUNq2Ak,9717
-ocrd_models/ocrd_mets.py,sha256=cRBEnvRNmRUhcrKouqygTQqYA-XdgyA-FNLwt86V0vg,42878
+ocrd_models/ocrd_mets.py,sha256=FHZnztf1cfWim_sAtTVFXt2ZuQx2HVDTQ1xIobIVIeQ,50540
 ocrd_models/ocrd_page.py,sha256=TTCnvpKGyZx1dqH8LnDiVVVPjU6emWGVLO_4o9rQHtw,6233
 ocrd_models/ocrd_page_generateds.py,sha256=IWoN3V-v3C4JgyPaFh9OQC87ob__wUP1Q6ELBxhLA1w,841794
 ocrd_models/ocrd_xml_base.py,sha256=OW57mXLlwm1nH8CNefvXmwLRws9KL9zSrb-3vH--mX8,1641
@@ -51,7 +51,7 @@ ocrd_models/xpath_functions.py,sha256=AwR8tHf56-mmIksnw_GeOQ760sxNHqK92T7z9OfsEE
 ocrd_network/__init__.py,sha256=gMejC614J5PPGgXDKBiQS0jt-Jx8qOrLbWH7zt8x8Gs,374
 ocrd_network/client.py,sha256=rzqtamZ8krRRy-QTO-AeWH8Lr3HhRiQe2R1-Lovd40g,3020
 ocrd_network/client_utils.py,sha256=VVZMNBgGznh41exZ78S48X3DDwHcWTuOq-LNdxjRvak,5002
-ocrd_network/constants.py,sha256=IeNtcU6dqonDE3Zw83_61auhS8X2b8wsjAxYg1zvK-M,1902
+ocrd_network/constants.py,sha256=AAcE6zZQNcNp2oqPD6oIgoVLSs4IHTkg8AS92WCQ6Xo,1968
 ocrd_network/database.py,sha256=fcft7vdRDoR7vmPL1xNYTIeOg5DwRPcggwYDYxLy5ik,10706
 ocrd_network/logging_utils.py,sha256=ijWpM8B943Jx6F0NeK3ggni0198UYjM5NCkYpARLk_E,2472
 ocrd_network/param_validators.py,sha256=Jl1VwiPPKJ50k-xEHLdvW-1QDOkJHCiMz4k9Ipqm-Uc,1489
@@ -96,7 +96,7 @@ ocrd_utils/image.py,sha256=zNNX1cnRy6yvrxx8mnYQiqWraAh5-i4a1AOfCCg4SmI,24781
 ocrd_utils/introspect.py,sha256=gfBlmeEFuRmRUSgdSK0jOxRpYqDRXl2IAE6gv2MZ6as,1977
 ocrd_utils/logging.py,sha256=XYTL7DxUvdX4V56jhAYH6PkhjMFOmaa0kf_XkhSTTe0,7816
 ocrd_utils/ocrd_logging.conf,sha256=JlWmA_5vg6HnjPGjTC4mA5vFHqmnEinwllSTiOw5CCo,3473
-ocrd_utils/os.py,sha256=acRRdDBI8L6BK0Mf773yKEzwdpZSFRBJEKB2crL4EjU,9865
+ocrd_utils/os.py,sha256=tMjikpVXJ8sCgYBOrgjgT3vlR2Pok39nSKysYc6mUQ4,9863
 ocrd_utils/str.py,sha256=cRgqYILDGOAqWr0qrCrV52I3y4wvpwDVtnBGEUjXNS4,10116
 ocrd_validators/__init__.py,sha256=ZFc-UqRVBk9o1YesZFmr9lOepttNJ_NKx1Zdb7g_YsU,972
 ocrd_validators/bagit-profile.yml,sha256=sdQJlSi7TOn1E9WYMOZ1shewJ-i_nPaKmsAFkh28TGY,1011
@@ -120,9 +120,9 @@ ocrd_validators/xlink.xsd,sha256=8fW7YAMWXN2PbB_MMvj9H5ZeFoEBDzuYBtlGC8_6ijw,318
 ocrd_validators/xsd_mets_validator.py,sha256=4GWfLyqkmca0x7osDuXuExYuM0HWVrKoqn0S35sFhHU,467
 ocrd_validators/xsd_page_validator.py,sha256=BNz_9u-Ek4UCeyZu3KxSQoolfW9lvuaSR9nIu1XXxeE,467
 ocrd_validators/xsd_validator.py,sha256=6HrVAf6SzCvfUIuQdIzz9bOq4V-zhyii9yrUPoK2Uvo,2094
-ocrd-3.4.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ocrd-3.4.0.dist-info/METADATA,sha256=ykbQAx7XiG5cpWXfFTsRrSiZl2SMKNVL2qqCazc2BJk,10442
-ocrd-3.4.0.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-ocrd-3.4.0.dist-info/entry_points.txt,sha256=4hcJ2LkK_OlIabHnKgFit35Ap7b5Lz1Gb4hzkxV0Kiw,152
-ocrd-3.4.0.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
-ocrd-3.4.0.dist-info/RECORD,,
+ocrd-3.5.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ocrd-3.5.0.dist-info/METADATA,sha256=t6X-RzqcEpiZDXRaRR0kSfU6hgu4fAa5qLpGVD4Lhjs,10442
+ocrd-3.5.0.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+ocrd-3.5.0.dist-info/entry_points.txt,sha256=4hcJ2LkK_OlIabHnKgFit35Ap7b5Lz1Gb4hzkxV0Kiw,152
+ocrd-3.5.0.dist-info/top_level.txt,sha256=pUgiN42t4KXC5rvpi6V8atza31XP4SCznXpXlVlvomM,75
+ocrd-3.5.0.dist-info/RECORD,,

ocrd_models/constants.py

@@ -1,7 +1,11 @@
 """
 Constants for ocrd_models.
 """
+from re import Pattern
 from enum import Enum, auto
+from dataclasses import dataclass, field
+from abc import ABC, abstractmethod
+from typing import Any, List, Optional, Union
 from ocrd_utils import resource_string
 
 __all__ = [
@@ -28,6 +32,10 @@ __all__ = [
     'TAG_PAGE_TEXTEQUIV',
     'TAG_PAGE_TEXTREGION',
     'METS_PAGE_DIV_ATTRIBUTE',
+    'METS_STRUCT_DIV_ATTRIBUTE',
+    'METS_DIV_ATTRIBUTE_ATOM_PATTERN',
+    'METS_DIV_ATTRIBUTE_RANGE_PATTERN',
+    'METS_DIV_ATTRIBUTE_REGEX_PATTERN',
     'PAGE_REGION_TYPES',
     'PAGE_ALTIMG_FEATURES',
 ]
@@ -89,6 +97,7 @@ PAGE_ALTIMG_FEATURES = [
 
 
 class METS_PAGE_DIV_ATTRIBUTE(Enum):
+    """page selection attributes of PHYSICAL mets:structMap//mets:div"""
     ID = auto()
     ORDER = auto()
     ORDERLABEL = auto()
@@ -98,3 +107,99 @@ class METS_PAGE_DIV_ATTRIBUTE(Enum):
     @classmethod
     def names(cls):
         return [x.name for x in cls]
+    @classmethod
+    def type_prefix(cls):
+        """disambiguation prefix to use for all subtypes"""
+        return "physical:"
+    def prefix(self):
+        """disambiguation prefix to use for this attribute type"""
+        return self.type_prefix() + self.name.lower() + ":"
+
+class METS_STRUCT_DIV_ATTRIBUTE(Enum):
+    """page selection attributes of LOGICAL mets:structMap//mets:div"""
+    ID = auto()
+    DMDID = auto()
+    TYPE = auto()
+    LABEL = auto()
+
+    @classmethod
+    def names(cls):
+        return [x.name for x in cls]
+    @classmethod
+    def type_prefix(cls):
+        """disambiguation prefix to use for all subtypes"""
+        return "logical:"
+    def prefix(self):
+        """disambiguation prefix to use for this attribute type"""
+        return self.type_prefix() + self.name.lower() + ":"
+
+@dataclass
+class METS_DIV_ATTRIBUTE_PATTERN(ABC):
+    """page selection pattern (abstract supertype)"""
+
+    expr: Any
+    """pattern value to match a mets:div against"""
+    attr: List[Union[METS_PAGE_DIV_ATTRIBUTE, METS_STRUCT_DIV_ATTRIBUTE]] = field(
+        default_factory=lambda: list(METS_PAGE_DIV_ATTRIBUTE) + list(METS_STRUCT_DIV_ATTRIBUTE))
+    """attribute type(s) to match a mets:div for
+    (pre-disambiguated with prefix syntax, or filled upon first match)
+    """
+    has_matched: bool = field(init=False, default=False)
+    """whether this pattern has already been matched"""
+
+    def attr_prefix(self):
+        """attribute type disambiguation prefix corresponding to the current state of disambiguation"""
+        if self.attr == list(METS_PAGE_DIV_ATTRIBUTE) + list(METS_STRUCT_DIV_ATTRIBUTE):
+            return ""
+        if self.attr == list(METS_PAGE_DIV_ATTRIBUTE):
+            return METS_PAGE_DIV_ATTRIBUTE.type_prefix()
+        if self.attr == list(METS_STRUCT_DIV_ATTRIBUTE):
+            return METS_STRUCT_DIV_ATTRIBUTE.type_prefix()
+        assert len(self.attr) == 1, "unexpected type ambiguity: %s" % repr(self.attr)
+        return self.attr[0].prefix()
+
+    @abstractmethod
+    def _matches(self, input) -> bool:
+        return
+    def matches(self, input) -> bool:
+        """does the selection pattern match on the given attribute value?"""
+        if (matched := self._matches(input)):
+            self.has_matched = True
+        return matched
+
+@dataclass
+class METS_DIV_ATTRIBUTE_ATOM_PATTERN(METS_DIV_ATTRIBUTE_PATTERN):
+    """page selection pattern for literal (single value) matching"""
+
+    expr: str
+    def __repr__(self):
+        return "%s%s" % (self.attr_prefix(), self.expr)
+    def _matches(self, input):
+        return input == self.expr
+
+@dataclass
+class METS_DIV_ATTRIBUTE_RANGE_PATTERN(METS_DIV_ATTRIBUTE_PATTERN):
+    """page selection pattern for interval (list expansion) matching"""
+
+    expr: List[str]
+    start: str = field(init=False)
+    """first value of the range after expansion, before matching-exhausting"""
+    stop: str = field(init=False)
+    """last value of the range after expansion, before matching-exhausting"""
+    def __post_init__(self):
+        self.start = self.expr[0]
+        self.stop = self.expr[-1]
+    def __repr__(self):
+        return "%s%s..%s" % (self.attr_prefix(), self.start, self.stop)
+    def _matches(self, input):
+        return input in self.expr
+
+@dataclass
+class METS_DIV_ATTRIBUTE_REGEX_PATTERN(METS_DIV_ATTRIBUTE_PATTERN):
+    """page selection pattern for regular expression matching"""
+
+    expr: Pattern
+    def __repr__(self):
+        return "%s//%s" % (self.attr_prefix(), self.expr.pattern)
+    def _matches(self, input):
+        return bool(self.expr.fullmatch(input))

ocrd_models/ocrd_mets.py

@@ -29,7 +29,12 @@ from .constants import (
     IDENTIFIER_PRIORITY,
     TAG_MODS_IDENTIFIER,
     METS_XML_EMPTY,
-    METS_PAGE_DIV_ATTRIBUTE
+    METS_PAGE_DIV_ATTRIBUTE,
+    METS_STRUCT_DIV_ATTRIBUTE,
+    METS_DIV_ATTRIBUTE_PATTERN,
+    METS_DIV_ATTRIBUTE_ATOM_PATTERN,
+    METS_DIV_ATTRIBUTE_RANGE_PATTERN,
+    METS_DIV_ATTRIBUTE_REGEX_PATTERN,
 )
 
 from .ocrd_xml_base import OcrdXmlDocument, ET      # type: ignore
@@ -43,9 +48,11 @@ class OcrdMets(OcrdXmlDocument):
     API to a single METS file
     """
     _cache_flag : bool
-    # Cache for the pages (mets:div)
-    # The dictionary's Key: 'div.ID'
-    # The dictionary's Value: a 'div' object at some memory location
+    # Cache for the physical pages (mets:div) - two nested dictionaries
+    # The outer dictionary's key: attribute type
+    # The outer dictionary's value: inner dictionary
+    # The inner dictionary's key: attribute value (str)
+    # The inner dictionary's value: a 'div' object at some memory location
     _page_cache : Dict[METS_PAGE_DIV_ATTRIBUTE, Dict[str, ET._Element]]
     # Cache for the files (mets:file) - two nested dictionaries
     # The outer dictionary's Key: 'fileGrp.USE'
@@ -59,6 +66,12 @@ class OcrdMets(OcrdXmlDocument):
     # The inner dictionary's Key: 'fptr.FILEID'
     # The inner dictionary's Value: a 'fptr' object at some memory location
     _fptr_cache : Dict[str, Dict[str, ET._Element]]
+    # Cache for the logical structural divs (mets:div) - two nested dictionaries
+    # The outer dictionary's key: attribute type
+    # The outer dictionary's value: inner dictionary
+    # The inner dictionary's key: attribute value (str)
+    # The inner dictionary's value: a list of corresponding physical div.ID
+    _struct_cache : Dict[METS_STRUCT_DIV_ATTRIBUTE, Dict[str, List[str]]]
 
     @staticmethod
     def empty_mets(now : Optional[str] = None, cache_flag : bool = False):
@@ -111,7 +124,6 @@ class OcrdMets(OcrdXmlDocument):
             return
 
         log = getLogger('ocrd.models.ocrd_mets._fill_caches-files')
-
         for el_fileGrp in el_fileSec.findall('mets:fileGrp', NS):
             fileGrp_use = el_fileGrp.get('USE')
 
@@ -124,10 +136,10 @@ class OcrdMets(OcrdXmlDocument):
                 # log.info("File added to the cache: %s" % file_id)
 
         # Fill with pages
+        log = getLogger('ocrd.models.ocrd_mets._fill_caches-pages')
         el_div_list = tree_root.findall(".//mets:div[@TYPE='page']", NS)
         if len(el_div_list) == 0:
             return
-        log = getLogger('ocrd.models.ocrd_mets._fill_caches-pages')
 
         for el_div in el_div_list:
             div_id = el_div.get('ID')
@@ -148,11 +160,30 @@ class OcrdMets(OcrdXmlDocument):
         # log.info("Len of page_cache: %s" % len(self._page_cache[METS_PAGE_DIV_ATTRIBUTE.ID]))
         # log.info("Len of fptr_cache: %s" % len(self._fptr_cache))
 
+        # Fill with logical divs
+        log = getLogger('ocrd.models.ocrd_mets._fill_caches-structs')
+        el_struct_list = tree_root.findall("mets:structMap[@TYPE='LOGICAL']//mets:div", NS)
+        el_smlink_list = tree_root.findall("mets:structLink/mets:smLink", NS)
+        if len(el_struct_list) == 0 or len(el_smlink_list) == 0:
+            return
+        smlink_map = {}
+        for link in el_smlink_list:
+            link_log = link.get('{%s}from' % NS['xlink'])
+            link_phy = link.get('{%s}to' % NS['xlink'])
+            smlink_map.setdefault(link_log, list()).append(link_phy)
+        for el_div in el_struct_list:
+            for attr in METS_STRUCT_DIV_ATTRIBUTE:
+                val = self._struct_cache[attr].setdefault(str(el_div.get(attr.name)), list())
+                val.extend(smlink_map.get(el_div.get('ID'), []))
+
+        # log.info("Len of struct_cache: %s" % len(self._struct_cache[METS_STRUCT_DIV_ATTRIBUTE.ID]))
+
     def _initialize_caches(self) -> None:
         self._file_cache = {}
         # NOTE we can only guarantee uniqueness for @ID and @ORDER
         self._page_cache = {k : {} for k in METS_PAGE_DIV_ATTRIBUTE}
         self._fptr_cache = {}
+        self._struct_cache = {k : {} for k in METS_STRUCT_DIV_ATTRIBUTE}
 
     def _refresh_caches(self) -> None:
         if self._cache_flag:
@@ -253,12 +284,20 @@ class OcrdMets(OcrdXmlDocument):
         :py:attr:`url` and :py:attr:`mimetype` parameters can each be either a
         literal string, or a regular expression if the string starts with
         ``//`` (double slash).
+
         If it is a regex, the leading ``//`` is removed and candidates are matched
         against the regex with `re.fullmatch`. If it is a literal string, comparison
         is done with string equality.
-        The :py:attr:`pageId` parameter supports the numeric range operator ``..``. For
-        example, to find all files in pages ``PHYS_0001`` to ``PHYS_0003``,
-        ``PHYS_0001..PHYS_0003`` will be expanded to ``PHYS_0001,PHYS_0002,PHYS_0003``.
+
+        The :py:attr:`pageId` parameter also supports comma-separated lists, as well
+        as the numeric range operator ``..`` and the negation operator ``~``.
+
+        For example, to find all files in pages ``PHYS_0001`` to ``PHYS_0003``, the
+        both expressions ``PHYS_0001..PHYS_0003`` and ``PHYS_0001,PHYS_0002,PHYS_0003``
+        will be expanded to the same 3 pages. To find all files above that subrange,
+        both expressions ``~PHYS_0001..PHYS_0003`` and ``~PHYS_0001,~PHYS_0002,~PHYS_0003``
+        will be expanded to ``PHYS_0004`` and upwards.
+
         Keyword Args:
             ID (string) : ``@ID`` of the ``mets:file``
             fileGrp (string) : ``@USE`` of the ``mets:fileGrp`` to list files of
@@ -609,101 +648,73 @@ class OcrdMets(OcrdXmlDocument):
 
             return self.physical_pages
 
-        # log = getLogger('ocrd.models.ocrd_mets.get_physical_pages')
+        log = getLogger('ocrd.models.ocrd_mets.get_physical_pages')
         if for_pageIds is not None:
-            ret = []
             page_attr_patterns = []
-            page_attr_patterns_raw = re.split(r',', for_pageIds)
-            for pageId_token in page_attr_patterns_raw:
+            page_attr_antipatterns = []
+            for pageId_token in re.split(r',', for_pageIds):
+                pageId_token_raw = pageId_token
+                # prefix for disambiguation of attribute?
+                attr = list(METS_PAGE_DIV_ATTRIBUTE) + list(METS_STRUCT_DIV_ATTRIBUTE)
+                for attr_type in [METS_STRUCT_DIV_ATTRIBUTE, METS_PAGE_DIV_ATTRIBUTE]:
+                    if pageId_token.startswith(attr_type.type_prefix()):
+                        for attr_val in list(attr_type):
+                            if pageId_token.startswith(attr_val.prefix()):
+                                # disambiguated to e.g. "logical:label:"
+                                attr = [attr_val]
+                                pageId_token = pageId_token[len(attr_val.prefix()):]
+                                break
+                        if len(attr) > 1:
+                            # just "logical:" or "physical:"
+                            attr = list(attr_type)
+                            pageId_token = pageId_token[len(attr_type.type_prefix()):]
+                        break
+                if not pageId_token:
+                    raise ValueError("invalid pageId syntax '%s': empty after type prefix" % pageId_token_raw)
+                # negation prefix
+                if pageId_token.startswith('~'):
+                    page_attr_xpatterns = page_attr_antipatterns
+                    pageId_token = pageId_token[1:]
+                else:
+                    page_attr_xpatterns = page_attr_patterns
+                if not pageId_token:
+                    raise ValueError("invalid pageId syntax '%s': empty after negator prefix" % pageId_token_raw)
+                # operator prefix
                 if pageId_token.startswith(REGEX_PREFIX):
-                    page_attr_patterns.append((None, re.compile(pageId_token[REGEX_PREFIX_LEN:])))
+                    pageId_token = pageId_token[REGEX_PREFIX_LEN:]
+                    if not pageId_token:
+                        raise ValueError("invalid pageId syntax '%s': empty after regex prefix" % pageId_token_raw)
+                    val_expr = re.compile(pageId_token)
+                    page_attr_xpatterns.append(
+                        METS_DIV_ATTRIBUTE_REGEX_PATTERN(val_expr, attr))
                 elif '..' in pageId_token:
-                    val_range = generate_range(*pageId_token.split('..', 1))
-                    page_attr_patterns.append(val_range)
+                    try:
+                        val_range = generate_range(*pageId_token.split('..', 1))
+                    except ValueError as e:
+                        raise ValueError("invalid pageId syntax '%s': %s" % (pageId_token_raw, str(e))) from None
+                    page_attr_xpatterns.append(
+                        METS_DIV_ATTRIBUTE_RANGE_PATTERN(val_range, attr))
                 else:
-                    page_attr_patterns.append(pageId_token)
-            if not page_attr_patterns:
+                    if not pageId_token:
+                        raise ValueError("invalid pageId syntax '%s': empty" % pageId_token_raw)
+                    page_attr_xpatterns.append(
+                        METS_DIV_ATTRIBUTE_ATOM_PATTERN(pageId_token, attr))
+                log.debug("parsed pattern '%s' to %s", pageId_token_raw, page_attr_xpatterns[-1])
+            if not page_attr_patterns and not page_attr_antipatterns:
                 return []
-            range_patterns_first_last = [(x[0], x[-1]) if isinstance(x, list) else None for x in page_attr_patterns]
-            page_attr_patterns_copy = list(page_attr_patterns)
-            if self._cache_flag:
-                for pat in page_attr_patterns:
-                    try:
-                        attr : METS_PAGE_DIV_ATTRIBUTE
-                        if isinstance(pat, str):
-                            attr = next(a for a in list(METS_PAGE_DIV_ATTRIBUTE) if pat in self._page_cache[a])
-                            cache_keys = [pat]
-                        elif isinstance(pat, list):
-                            attr = next(a for a in list(METS_PAGE_DIV_ATTRIBUTE) if any(x in self._page_cache[a] for x in pat))
-                            cache_keys = [v for v in pat if v in self._page_cache[attr]]
-                            for k in cache_keys:
-                                pat.remove(k)
-                        elif isinstance(pat, tuple):
-                            _, re_pat = pat
-                            attr = next(a for a in list(METS_PAGE_DIV_ATTRIBUTE) for v in self._page_cache[a] if re_pat.fullmatch(v))
-                            cache_keys = [v for v in self._page_cache[attr] if re_pat.fullmatch(v)]
-                        else:
-                            raise ValueError
-                        if return_divs:
-                            ret += [self._page_cache[attr][v] for v in cache_keys]
-                        else:
-                            ret += [self._page_cache[attr][v].get('ID') for v in cache_keys]
-                    except StopIteration:
-                        raise ValueError(f"{pat} matches none of the keys of any of the _page_caches.")
+            if page_attr_patterns:
+                divs = self.get_physical_page_patterns(page_attr_patterns)
             else:
-                page_attr_patterns_matched = []
-                for page in self._tree.getroot().xpath(
-                        'mets:structMap[@TYPE="PHYSICAL"]/mets:div[@TYPE="physSequence"]/mets:div[@TYPE="page"]',
-                        namespaces=NS):
-                    patterns_exhausted = []
-                    for pat_idx, pat in enumerate(page_attr_patterns):
-                        try:
-                            if isinstance(pat, str):
-                                attr = next(a for a in list(METS_PAGE_DIV_ATTRIBUTE) if pat == page.get(a.name))
-                                ret.append(page if return_divs else page.get('ID'))
-                                patterns_exhausted.append(pat)
-                            elif isinstance(pat, list):
-                                if not isinstance(pat[0], METS_PAGE_DIV_ATTRIBUTE):
-                                    pat.insert(0, next(a for a in list(METS_PAGE_DIV_ATTRIBUTE) if any(x == page.get(a.name) for x in pat)))
-                                attr_val = page.get(pat[0].name)
-                                if attr_val in pat:
-                                    pat.remove(attr_val)
-                                    ret.append(page if return_divs else page.get('ID'))
-                                if len(pat) == 1:
-                                    patterns_exhausted.append(pat)
-                            elif isinstance(pat, tuple):
-                                attr, re_pat = pat
-                                if not attr:
-                                    attr = next(a for a in list(METS_PAGE_DIV_ATTRIBUTE) if re_pat.fullmatch(page.get(a.name) or ''))
-                                    page_attr_patterns[pat_idx] = (attr, re_pat)
-                                if re_pat.fullmatch(page.get(attr.name) or ''):
-                                    ret.append(page if return_divs else page.get('ID'))
-                            else:
-                                raise ValueError
-                            page_attr_patterns_matched.append(pat)
-                        except StopIteration:
-                            continue
-                    for p in patterns_exhausted:
-                        page_attr_patterns.remove(p)
-                unmatched = [x for x in page_attr_patterns_copy if x not in page_attr_patterns_matched]
-                if unmatched:
-                    raise ValueError(f"Patterns {unmatched} match none of the pages")
-
-            ranges_without_start_match = []
-            ranges_without_last_match = []
-            for idx, pat in enumerate(page_attr_patterns_copy):
-                if isinstance(pat, list):
-                    start, last = range_patterns_first_last[idx]
-                    if start in pat:
-                        print(pat, start, last)
-                        ranges_without_start_match.append(page_attr_patterns_raw[idx])
-                    # if last in pat:
-                    #     ranges_without_last_match.append(page_attr_patterns_raw[idx])
-            if ranges_without_start_match:
-                raise ValueError(f"Start of range patterns {ranges_without_start_match} not matched - invalid range")
-            # if ranges_without_last_match:
-            #     raise ValueError(f"End of range patterns {ranges_without_last_match} not matched - invalid range")
-            return ret
+                all_pages = [METS_DIV_ATTRIBUTE_REGEX_PATTERN(
+                    re.compile(".*"), [METS_PAGE_DIV_ATTRIBUTE.ID])]
+                divs = self.get_physical_page_patterns(all_pages)
+            if page_attr_antipatterns:
+                antidivs = self.get_physical_page_patterns(page_attr_antipatterns)
+                divs = [div for div in divs if div not in antidivs]
+            if return_divs:
+                return divs
+            else:
+                return [div.get('ID') for div in divs]
 
         if for_fileIds == []:
             return []
@@ -731,6 +742,129 @@ class OcrdMets(OcrdXmlDocument):
                             ret[index] = page.get('ID')
         return ret
 
+    def get_physical_page_patterns(self, page_attr_patterns: List[METS_DIV_ATTRIBUTE_PATTERN]) -> List[ET._Element]:
+        log = getLogger('ocrd.models.ocrd_mets.get_physical_pages')
+        ret = []
+        page_attr_patterns_copy = list(page_attr_patterns)
+        if self._cache_flag:
+            for pat in page_attr_patterns:
+                for attr in pat.attr:
+                    if isinstance(attr, METS_PAGE_DIV_ATTRIBUTE):
+                        cache = self._page_cache[attr]
+                    else:
+                        cache = self._struct_cache[attr]
+                    if (isinstance(pat, METS_DIV_ATTRIBUTE_RANGE_PATTERN) and
+                        # @TYPE makes no sense in range expressions
+                        # @LABEL makes no sense in range expressions
+                        attr in [METS_STRUCT_DIV_ATTRIBUTE.TYPE,
+                                 METS_STRUCT_DIV_ATTRIBUTE.LABEL]):
+                        continue
+                    if cache_keys := [v for v in cache if pat.matches(v)]:
+                        if isinstance(attr, METS_PAGE_DIV_ATTRIBUTE):
+                            ret += [cache[v] for v in cache_keys]
+                            log.debug('physical matches for %s: %s', pat, str(cache_keys))
+                        else:
+                            for v in cache_keys:
+                                ret += [self._page_cache[METS_PAGE_DIV_ATTRIBUTE.ID][p]
+                                        for p in cache[v]]
+                            log.debug('logical matches for %s: %s', pat, str(cache_keys))
+                        if isinstance(pat, METS_DIV_ATTRIBUTE_RANGE_PATTERN):
+                            # remove matches for final range check
+                            for v in cache_keys:
+                                pat.expr.remove(v)
+                        break
+                if not cache_keys:
+                    raise ValueError(f"{pat} matches none of the keys of any of the _page_caches and _struct_caches.")
+        else:
+            # cache logical structmap:
+            el_struct_list = self._tree.getroot().findall("mets:structMap[@TYPE='LOGICAL']//mets:div", NS)
+            el_smlink_list = self._tree.getroot().findall("mets:structLink/mets:smLink", NS)
+            smlink_map = {}
+            for link in el_smlink_list:
+                link_log = link.get('{%s}from' % NS['xlink'])
+                link_phy = link.get('{%s}to' % NS['xlink'])
+                smlink_map.setdefault(link_log, list()).append(link_phy)
+            struct_cache = {k: {} for k in METS_STRUCT_DIV_ATTRIBUTE}
+            for el_div in el_struct_list:
+                for attr in METS_STRUCT_DIV_ATTRIBUTE:
+                    if not el_div.get(attr.name):
+                        # avoid mapping None indiscriminately
+                        continue
+                    val = struct_cache[attr].setdefault(str(el_div.get(attr.name)), list())
+                    val.extend(smlink_map.get(el_div.get('ID'), []))
+            log.debug("found %d smLink entries for %d logical divs", len(el_smlink_list), len(el_struct_list))
+            page_attr_patterns_matched = []
+            for page in self._tree.getroot().xpath(
+                    'mets:structMap[@TYPE="PHYSICAL"]/mets:div[@TYPE="physSequence"]/mets:div[@TYPE="page"]',
+                    namespaces=NS):
+                patterns_exhausted = []
+                for pat in page_attr_patterns:
+                    for attr in pat.attr:
+                        if isinstance(attr, METS_PAGE_DIV_ATTRIBUTE):
+                            cache = [page.get(attr.name) or '']
+                        else:
+                            cache = struct_cache[attr]
+                        if (isinstance(pat, METS_DIV_ATTRIBUTE_RANGE_PATTERN) and
+                            # @TYPE makes no sense in range expressions
+                            # @LABEL makes no sense in range expressions
+                            attr in [METS_STRUCT_DIV_ATTRIBUTE.TYPE,
+                                     METS_STRUCT_DIV_ATTRIBUTE.LABEL]):
+                            continue
+                        if cache_keys := [v for v in cache if pat.matches(v)]:
+                            pat.attr = [attr] # disambiguate next
+                            if isinstance(attr, METS_PAGE_DIV_ATTRIBUTE):
+                                ret.append(page)
+                                log.debug('physical match for %s on page %s', pat, page.get('ID'))
+                                if isinstance(pat, METS_DIV_ATTRIBUTE_ATOM_PATTERN):
+                                    patterns_exhausted.append(pat)
+                                elif isinstance(pat, METS_DIV_ATTRIBUTE_RANGE_PATTERN):
+                                    # remove for efficiency and final range check
+                                    pat.expr.remove(cache_keys[0])
+                                    if not pat.expr:
+                                        patterns_exhausted.append(pat)
+                            elif cache_key := next((v for v in cache_keys
+                                                    if page.get('ID') in cache[v]), None):
+                                ret.append(page)
+                                log.debug('logical match for %s on page %s', pat, page.get('ID'))
+                                cache[cache_key].remove(page.get('ID'))
+                                # remove for efficiency and final range check
+                                if not cache[cache_key]:
+                                    if isinstance(pat, METS_DIV_ATTRIBUTE_ATOM_PATTERN):
+                                        patterns_exhausted.append(pat)
+                                    elif isinstance(pat, METS_DIV_ATTRIBUTE_RANGE_PATTERN):
+                                        pat.expr.remove(cache_key)
+                                        if not pat.expr:
+                                            patterns_exhausted.append(pat)
+                            break # no more attributes for this pattern
+                    # keep matching in order to exhaust and consume pattern list
+                    #if page in ret:
+                    #    break # no more patterns for this page
+                for p in patterns_exhausted:
+                    page_attr_patterns.remove(p)
+            unmatched = [pat for pat in page_attr_patterns_copy
+                         if not pat.has_matched]
+            if unmatched:
+                raise ValueError(f"Patterns {unmatched} match none of the pages")
+
+        ranges_without_start_match = []
+        ranges_without_stop_match = []
+        for pat in page_attr_patterns_copy:
+            if isinstance(pat, METS_DIV_ATTRIBUTE_RANGE_PATTERN):
+                # range expression, expanded to pattern list
+                # list items get consumed (pat.expr.remove) when matched,
+                # exhausted patterns also get consumed (page_attr_patterns.remove)
+                # (but top-level list copy references the same list objects)
+                if pat.start in pat.expr:
+                    log.debug((pat, pat.expr))
+                    ranges_without_start_match.append(pat)
+                # if pat.stop in pat.expr:
+                #     ranges_without_stop_match.append(pat)
+        if ranges_without_start_match:
+            raise ValueError(f"Start of range patterns {ranges_without_start_match} not matched - invalid range")
+        # if ranges_without_stop_match:
+        #     raise ValueError(f"End of range patterns {ranges_without_stop_match} not matched - invalid range")
+        return ret
+
     def set_physical_page_for_file(self, pageId : str, ocrd_file : OcrdFile, 
                                    order : Optional[str] = None, orderlabel : Optional[str] = None) -> None:
         """

ocrd_network/constants.py

@@ -11,12 +11,16 @@ OCRD_ALL_TOOL_JSON = "ocrd-all-tool.json"
 SERVER_ALL_PAGES_PLACEHOLDER = "all_pages"
 
 
-class AgentType(str, Enum):
+class StrEnum(str, Enum):
+    def __str__(self):
+        return self.value
+
+class AgentType(StrEnum):
     PROCESSING_WORKER = "worker"
     PROCESSOR_SERVER = "server"
 
 
-class DeployType(str, Enum):
+class DeployType(StrEnum):
     # Deployed by the Processing Server config file
     DOCKER = "docker"
     NATIVE = "native"
@@ -26,7 +30,7 @@ class DeployType(str, Enum):
 
 
 # TODO: Make the states uppercase
-class JobState(str, Enum):
+class JobState(StrEnum):
     # The processing job is cached inside the Processing Server requests cache
     cached = "CACHED"
     # The processing job was cancelled due to failed dependencies
@@ -43,7 +47,7 @@ class JobState(str, Enum):
     unset = "UNSET"
 
 
-class NetworkLoggingDirs(str, Enum):
+class NetworkLoggingDirs(StrEnum):
     METS_SERVERS = "mets_servers"
     PROCESSING_JOBS = "processing_jobs"
     PROCESSING_SERVERS = "processing_servers"
@@ -51,7 +55,7 @@ class NetworkLoggingDirs(str, Enum):
     PROCESSOR_SERVERS = "processor_servers"
 
 
-class ServerApiTags(str, Enum):
+class ServerApiTags(StrEnum):
     ADMIN = "admin"
     DISCOVERY = "discovery"
     PROCESSING = "processing"

ocrd_utils/os.py

@@ -254,7 +254,7 @@ def guess_media_type(input_file : str, fallback : str = None, application_xml :
     if mimetype is None:
         mimetype = EXT_TO_MIME.get(''.join(Path(input_file).suffixes), fallback)
     if mimetype is None:
-        raise ValueError("Could not determine MIME type of input_file must")
+        raise ValueError(f"Could not determine MIME type of {input_file}")
     if mimetype == 'application/xml':
         mimetype = application_xml
     return mimetype