ocrd 3.3.2__tar.gz → 3.4.1__tar.gz

{ocrd-3.3.2/src/ocrd.egg-info → ocrd-3.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.3.2
+Version: 3.4.1
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0

ocrd-3.4.1/VERSION

@@ -0,0 +1 @@
+3.4.1

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd/cli/__init__.py

@@ -67,6 +67,12 @@ Variables:
 \b
 {config.describe('OCRD_EXISTING_OUTPUT', wrap_text=False)}
 \b
+{config.describe('OCRD_MAX_MISSING_OUTPUTS')}
+\b
+{config.describe('OCRD_MAX_PARALLEL_PAGES')}
+\b
+{config.describe('OCRD_PROCESSING_PAGE_TIMEOUT')}
+\b
 {config.describe('OCRD_METS_CACHING')}
 \b
 {config.describe('OCRD_MAX_PROCESSOR_CACHE')}

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd/cli/ocrd_tool.py

@@ -18,7 +18,8 @@ from ocrd.processor import Processor
 from ocrd_utils import (
     set_json_key_value_overrides,
     parse_json_string_or_file,
-    parse_json_string_with_comments as loads
+    parse_json_string_with_comments as loads,
+    get_moduledir
 )
 from ocrd_validators import ParameterValidator, OcrdToolValidator
 
@@ -104,6 +105,13 @@ def ocrd_tool_list(ctx):
 def ocrd_tool_dump(ctx):
     print(dumps(ctx.json['tools'], indent=True))
 
+@ocrd_tool_cli.command('dump-module-dirs', help="Dump module directory of each tool")
+@pass_ocrd_tool
+def ocrd_tool_dump_module_dirs(ctx):
+    print(dumps({tool_name: get_moduledir(tool_name)
+                 for tool_name in ctx.json['tools']},
+                indent=True))
+
 # ----------------------------------------------------------------------
 # ocrd ocrd-tool tool
 # ----------------------------------------------------------------------

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd/decorators/ocrd_cli_options.py

@@ -56,7 +56,7 @@ def ocrd_cli_options(f):
         # subcommands. So we have to work around that by creating a
         # pseudo-subcommand handled in ocrd_cli_wrap_processor
         argument('subcommand', nargs=1, required=False,
-                 type=click.Choice([AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER])),
+                 type=click.Choice(list(map(str, AgentType)))),
     ]
     for param in params:
         param(f)

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd/mets_server.py

@@ -46,7 +46,7 @@ class OcrdFileModel(BaseModel):
     ):
         return OcrdFileModel(
             file_grp=file_grp, file_id=file_id, page_id=page_id, mimetype=mimetype, url=url,
-            local_filename=str(local_filename)
+            local_filename=str(local_filename) if local_filename else None
         )
 
 
@@ -314,7 +314,7 @@ class ClientSideOcrdMets:
         else:
             r = self.session.request("POST", self.url, json=MpxReq.add_file(self.ws_dir_path, kwargs))
             if not r.ok:
-                raise RuntimeError(f"Failed to add file ({str(data)}): {r.json()[errors]}")
+                raise RuntimeError(f"Failed to add file ({str(data)}): {r.json()}")
 
         return ClientSideOcrdFile(
             None, fileGrp=file_grp,

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd/processor/base.py

@@ -29,8 +29,7 @@ from frozendict import frozendict
 # this is where the fixes came from:
 from loky import Future, ProcessPoolExecutor
 import multiprocessing as mp
-from threading import Timer
-from _thread import interrupt_main
+from multiprocessing.pool import ThreadPool
 
 from click import wrap_text
 from deprecated import deprecated
@@ -783,11 +782,16 @@ class Processor():
         page_id = input_files[input_pos].pageId
         self._base_logger.info("processing page %s", page_id)
         for i, input_file in enumerate(input_files):
+            grp = self.input_file_grp.split(',')[i]
             if input_file is None:
-                grp = self.input_file_grp.split(',')[i]
                 self._base_logger.debug(f"ignoring missing file for input fileGrp {grp} for page {page_id}")
                 continue
             assert isinstance(input_file, get_args(OcrdFileType))
+            if not input_file.local_filename:
+                self._base_logger.error(f'No local file exists for page {page_id} in file group {grp}')
+                if config.OCRD_MISSING_INPUT == 'ABORT':
+                    raise MissingInputFile(grp, page_id, input_file.mimetype)
+                continue
             self._base_logger.debug(f"parsing file {input_file.ID} for page {page_id}")
             try:
                 page_ = page_from_file(input_file)
@@ -796,6 +800,9 @@ class Processor():
             except ValueError as err:
                 # not PAGE and not an image to generate PAGE for
                 self._base_logger.error(f"non-PAGE input for page {page_id}: {err}")
+        if not any(input_pcgts):
+            self._base_logger.warning(f'skipping page {page_id}')
+            return
         output_file_id = make_file_id(input_files[input_pos], self.output_file_grp)
         if input_files[input_pos].fileGrp == self.output_file_grp:
             # input=output fileGrp: re-use ID exactly
@@ -1107,7 +1114,11 @@ class Processor():
             self._base_logger.critical(f"Could not find any files for selected pageId {self.page_id}.\n"
                                        f"compare '{self.page_id}' with the output of 'orcd workspace list-page'.")
         ifts = []
-        for page, ifiles in pages.items():
+        # use physical page order
+        for page in self.workspace.mets.physical_pages:
+            if page not in pages:
+                continue
+            ifiles = pages[page]
             for i, ifg in enumerate(ifgs):
                 if not ifiles[i]:
                     # could be from non-unique with on_error=skip or from true gap
@@ -1150,18 +1161,15 @@ def _page_worker(timeout, *input_files):
     """
     page_id = next((file.pageId for file in input_files
                     if hasattr(file, 'pageId')), "")
-    if timeout > 0:
-        timer = Timer(timeout, interrupt_main)
-        timer.start()
+    pool = ThreadPool(processes=1)
     try:
-        _page_worker_processor.process_page_file(*input_files)
+        #_page_worker_processor.process_page_file(*input_files)
+        async_result = pool.apply_async(_page_worker_processor.process_page_file, input_files)
+        async_result.get(timeout or None)
         _page_worker_processor.logger.debug("page worker completed for page %s", page_id)
-    except KeyboardInterrupt:
+    except mp.TimeoutError:
         _page_worker_processor.logger.debug("page worker timed out for page %s", page_id)
-        raise TimeoutError()
-    finally:
-        if timeout > 0:
-            timer.cancel()
+        raise
 
 def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
     """Generate a string describing the full CLI of this processor including params.

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd/workspace.py

@@ -5,7 +5,7 @@ from shutil import copyfileobj
 from re import sub
 from tempfile import NamedTemporaryFile
 from contextlib import contextmanager
-from typing import Optional, Union
+from typing import Optional, Union, Callable
 
 from cv2 import COLOR_GRAY2BGR, COLOR_RGB2BGR, cvtColor
 from PIL import Image
@@ -457,6 +457,20 @@ class Workspace():
             with atomic_write(self.mets_target) as f:
                 f.write(self.mets.to_xml(xmllint=True).decode('utf-8'))
 
+    def _apply_mets_file(self, filename_or_url: str, fun: Callable):
+        if not filename_or_url:
+            # avoid "finding" just any file
+            raise ValueError("requires non-empty filename or URL")
+        with pushd_popd(self.directory):
+            if Path(filename_or_url).exists():
+                return fun(filename_or_url)
+            if image_file := next(self.mets.find_files(local_filename=str(filename_or_url)), None):
+                return fun(image_file.local_filename)
+            if image_file := next(self.mets.find_files(url=str(filename_or_url)), None):
+                return fun(self.download_file(image_file).local_filename)
+            with download_temporary_file(filename_or_url) as f:
+                return fun(f.name)
+
     def resolve_image_exif(self, image_url):
         """
         Get the EXIF metadata about an image URL as :py:class:`ocrd_models.ocrd_exif.OcrdExif`
@@ -467,19 +481,7 @@ class Workspace():
         Returns:
             :py:class:`ocrd_models.ocrd_exif.OcrdExif`
         """
-        if not image_url:
-            # avoid "finding" just any file
-            raise ValueError(f"'image_url' must be a non-empty string, not '{image_url}' ({type(image_url)})")
-        try:
-            f = next(self.mets.find_files(local_filename=str(image_url)))
-            return exif_from_filename(f.local_filename)
-        except StopIteration:
-            try:
-                f = next(self.mets.find_files(url=str(image_url)))
-                return exif_from_filename(self.download_file(f).local_filename)
-            except StopIteration:
-                with download_temporary_file(image_url) as f:
-                    return exif_from_filename(f.name)
+        return self._apply_mets_file(image_url, exif_from_filename)
 
     @deprecated(version='1.0.0', reason="Use workspace.image_from_page and workspace.image_from_segment")
     def resolve_image_as_pil(self, image_url, coords=None):
@@ -498,22 +500,9 @@ class Workspace():
         return self._resolve_image_as_pil(image_url, coords)
 
     def _resolve_image_as_pil(self, image_url, coords=None):
-        if not image_url:
-            # avoid "finding" just any file
-            raise Exception("Cannot resolve empty image path")
         log = getLogger('ocrd.workspace._resolve_image_as_pil')
-        with pushd_popd(self.directory):
-            try:
-                f = next(self.mets.find_files(local_filename=str(image_url)))
-                pil_image = Image.open(f.local_filename)
-            except StopIteration:
-                try:
-                    f = next(self.mets.find_files(url=str(image_url)))
-                    pil_image = Image.open(self.download_file(f).local_filename)
-                except StopIteration:
-                    with download_temporary_file(image_url) as f:
-                        pil_image = Image.open(f.name)
-            pil_image.load() # alloc and give up the FD
+        pil_image = self._apply_mets_file(image_url, Image.open)
+        pil_image.load() # alloc and give up the FD
 
         # Pillow does not properly support higher color depths
         # (e.g. 16-bit or 32-bit or floating point grayscale),
@@ -788,16 +777,14 @@ class Workspace():
             raise Exception('Found no AlternativeImage that satisfies all requirements ' +
                             'filename="%s" in page "%s"' % (
                                 filename, page_id))
-        if not all(feature in page_coords['features']
-                   for feature in feature_selector.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'selector="%s" in page "%s"' % (
-                                feature_selector, page_id))
-        if any(feature in page_coords['features']
-               for feature in feature_filter.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'filter="%s" in page "%s"' % (
-                                feature_filter, page_id))
+        if (not all(feature in page_coords['features']
+                    for feature in feature_selector.split(',') if feature) or
+            any(feature in page_coords['features']
+                for feature in feature_filter.split(',') if feature)):
+            raise Exception('Found no AlternativeImage that satisfies all requirements' +
+                            ' selector="%s"' % feature_selector +
+                            ' filter="%s"' % feature_filter +
+                            ' in page "%s"' % page_id)
         # ensure DPI will be set in image meta-data again
         if 'DPI' in page_coords:
             dpi = page_coords['DPI']
@@ -1049,16 +1036,14 @@ class Workspace():
             raise Exception('Found no AlternativeImage that satisfies all requirements ' +
                             'filename="%s" in segment "%s"' % (
                                 filename, segment.id))
-        if not all(feature in segment_coords['features']
-                   for feature in feature_selector.split(',') if feature):
+        if (not all(feature in segment_coords['features']
+                    for feature in feature_selector.split(',') if feature) or
+            any(feature in segment_coords['features']
+                for feature in feature_filter.split(',') if feature)):
             raise Exception('Found no AlternativeImage that satisfies all requirements' +
-                            'selector="%s" in segment "%s"' % (
-                                feature_selector, segment.id))
-        if any(feature in segment_coords['features']
-               for feature in feature_filter.split(',') if feature):
-            raise Exception('Found no AlternativeImage that satisfies all requirements ' +
-                            'filter="%s" in segment "%s"' % (
-                                feature_filter, segment.id))
+                            ' selector="%s"' % feature_selector +
+                            ' filter="%s"' % feature_filter +
+                            ' in segment "%s"' % segment.id)
         # ensure DPI will be set in image meta-data again
         if 'DPI' in segment_coords:
             dpi = segment_coords['DPI']

{ocrd-3.3.2 → ocrd-3.4.1/src/ocrd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.3.2
+Version: 3.4.1
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd_network/constants.py

@@ -11,12 +11,16 @@ OCRD_ALL_TOOL_JSON = "ocrd-all-tool.json"
 SERVER_ALL_PAGES_PLACEHOLDER = "all_pages"
 
 
-class AgentType(str, Enum):
+class StrEnum(str, Enum):
+    def __str__(self):
+        return self.value
+
+class AgentType(StrEnum):
     PROCESSING_WORKER = "worker"
     PROCESSOR_SERVER = "server"
 
 
-class DeployType(str, Enum):
+class DeployType(StrEnum):
     # Deployed by the Processing Server config file
     DOCKER = "docker"
     NATIVE = "native"
@@ -26,7 +30,7 @@ class DeployType(str, Enum):
 
 
 # TODO: Make the states uppercase
-class JobState(str, Enum):
+class JobState(StrEnum):
     # The processing job is cached inside the Processing Server requests cache
     cached = "CACHED"
     # The processing job was cancelled due to failed dependencies
@@ -43,7 +47,7 @@ class JobState(str, Enum):
     unset = "UNSET"
 
 
-class NetworkLoggingDirs(str, Enum):
+class NetworkLoggingDirs(StrEnum):
     METS_SERVERS = "mets_servers"
     PROCESSING_JOBS = "processing_jobs"
     PROCESSING_SERVERS = "processing_servers"
@@ -51,7 +55,7 @@ class NetworkLoggingDirs(str, Enum):
     PROCESSOR_SERVERS = "processor_servers"
 
 
-class ServerApiTags(str, Enum):
+class ServerApiTags(StrEnum):
     ADMIN = "admin"
     DISCOVERY = "discovery"
     PROCESSING = "processing"

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd_network/logging_utils.py

@@ -9,11 +9,22 @@ def configure_file_handler_with_formatter(logger: Logger, log_file: Path, mode:
     file_handler = FileHandler(filename=log_file, mode=mode)
     file_handler.setFormatter(Formatter(LOG_FORMAT))
     logger.addHandler(file_handler)
+    try:
+        log_file.chmod(0o666)
+    except PermissionError:
+        # if the file exists the permissions are supposed to already fit
+        pass
 
 
 def get_root_logging_dir(module_name: NetworkLoggingDirs) -> Path:
     module_log_dir = Path(config.OCRD_NETWORK_LOGS_ROOT_DIR, module_name.value)
-    module_log_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        module_log_dir.mkdir(parents=True, exist_ok=True)
+        module_log_dir.chmod(0o777)
+    except PermissionError:
+        # if the folder exists the permissions are supposed to already fit
+        pass
+
     return module_log_dir
 
 

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd_utils/config.py

@@ -277,13 +277,23 @@ config.add(name="OCRD_NETWORK_SOCKETS_ROOT_DIR",
            description="The root directory where all mets server related socket files are created",
            parser=lambda val: Path(val),
            default=(True, Path(gettempdir(), "ocrd_network_sockets")))
-config.OCRD_NETWORK_SOCKETS_ROOT_DIR.mkdir(parents=True, exist_ok=True)
+config.OCRD_NETWORK_SOCKETS_ROOT_DIR.mkdir(mode=0o777, parents=True, exist_ok=True)
+try:
+    config.OCRD_NETWORK_SOCKETS_ROOT_DIR.chmod(0o777)
+except PermissionError:
+    # if the folder exists the permissions are supposed to already fit
+    pass
 
 config.add(name="OCRD_NETWORK_LOGS_ROOT_DIR",
            description="The root directory where all ocrd_network related file logs are stored",
            parser=lambda val: Path(val),
            default=(True, Path(gettempdir(), "ocrd_network_logs")))
-config.OCRD_NETWORK_LOGS_ROOT_DIR.mkdir(parents=True, exist_ok=True)
+config.OCRD_NETWORK_LOGS_ROOT_DIR.mkdir(mode=0o777, parents=True, exist_ok=True)
+try:
+    config.OCRD_NETWORK_LOGS_ROOT_DIR.chmod(0o777)
+except PermissionError:
+    # if the folder exists the permissions are supposed to already fit
+    pass
 
 config.add("HOME",
     description="Directory to look for `ocrd_logging.conf`, fallback for unset XDG variables.",

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd_utils/logging.py

@@ -32,6 +32,7 @@ import logging
 import logging.config
 from pathlib import Path
 import sys
+from os import chmod
 
 from .constants import LOG_FORMAT, LOG_TIMEFMT
 from .config import config
@@ -166,6 +167,15 @@ def initLogging(builtin_only=False, force_reinit=False, silent=not config.OCRD_L
         if not silent:
             print(f"[LOGGING] Picked up logging config at {config_file}", file=sys.stderr)
         logging.config.fileConfig(config_file)
+        # Set permission of processing-server logfile to 666 to prevent possible permission erros while logging
+        try:
+            network_logger = logging.getLogger("ocrd_network")
+            for handler in network_logger.handlers:
+                if isinstance(handler, logging.FileHandler):
+                    chmod(handler.baseFilename, 0o666)
+        except PermissionError:
+            # if the file exists the permissions are supposed to already fit
+            pass
     else:
         if not silent:
             print("[LOGGING] Initializing logging with built-in defaults", file=sys.stderr)

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd_utils/os.py

@@ -254,7 +254,7 @@ def guess_media_type(input_file : str, fallback : str = None, application_xml :
     if mimetype is None:
         mimetype = EXT_TO_MIME.get(''.join(Path(input_file).suffixes), fallback)
     if mimetype is None:
-        raise ValueError("Could not determine MIME type of input_file must")
+        raise ValueError(f"Could not determine MIME type of {input_file}")
     if mimetype == 'application/xml':
         mimetype = application_xml
     return mimetype

ocrd-3.3.2/VERSION

@@ -1 +0,0 @@
-3.3.2

{ocrd-3.3.2 → ocrd-3.4.1}/src/ocrd/cli/workspace.py

@@ -474,11 +474,11 @@ def workspace_find(ctx, file_grp, mimetype, page_id, file_id, output_field, incl
                 if wait:
                     time.sleep(wait)
             if undo_download and f.url and f.local_filename:
-                f.local_filename = None
                 modified_mets = True
                 if not keep_files:
                     ctx.log.debug("rm %s [cwd=%s]", f.local_filename, workspace.directory)
                     unlink(f.local_filename)
+                f.local_filename = None
             ret_entry = [f.ID if field == 'pageId' else str(getattr(f, field)) or '' for field in output_field]
             ret.append(ret_entry)
     if modified_mets: