ocrd 3.0.0a2__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

ocrd/processor/builtin/dummy_processor.py

@@ -1,6 +1,6 @@
 # pylint: disable=missing-module-docstring,invalid-name
-from os.path import join, basename
-from typing import Optional, Union
+from os.path import join
+from typing import Optional
 
 import click
 
@@ -10,7 +10,6 @@ from ocrd.processor.ocrd_page_result import OcrdPageResult
 from ocrd_models.ocrd_file import OcrdFileType
 from ocrd_models.ocrd_page import OcrdPage, to_xml
 from ocrd_utils import (
-    getLogger,
     make_file_id,
     MIME_TO_EXT,
     MIMETYPE_PAGE,
@@ -20,8 +19,6 @@ from ocrd_utils import (
 )
 from ocrd_modelfactory import page_from_file
 
-OCRD_TOOL = parse_json_string_with_comments(resource_string(__package__ + '.dummy', 'ocrd-tool.json'))
-
 class DummyProcessor(Processor):
     """
     Bare-bones processor creates PAGE-XML and optionally copies file from input group to output group
@@ -76,17 +73,13 @@ class DummyProcessor(Processor):
             super().process_page_file(input_file)
 
     @property
-    def metadata(self):
-        return OCRD_TOOL
+    def metadata_filename(self):
+        return 'processor/builtin/dummy/ocrd-tool.json'
 
     @property
     def executable(self):
         return 'ocrd-dummy'
 
-    @property
-    def version(self):
-        return '0.0.3'
-
 @click.command()
 @ocrd_cli_options
 def cli(*args, **kwargs):

ocrd/processor/helpers.py

@@ -1,21 +1,18 @@
 """
 Helper methods for running and documenting processors
 """
-from os import chdir, getcwd
 from time import perf_counter, process_time
 from functools import lru_cache
 import json
 import inspect
 from subprocess import run
-from typing import List
+from typing import List, Optional
 
-from click import wrap_text
-from ocrd.workspace import Workspace
+from ..workspace import Workspace
 from ocrd_utils import freeze_args, getLogger, config, setOverrideLogLevel, getLevelName, sparkline
 
 
 __all__ = [
-    'generate_processor_help',
     'run_cli',
     'run_processor'
 ]
@@ -39,10 +36,7 @@ def run_processor(
         log_level=None,
         input_file_grp=None,
         output_file_grp=None,
-        show_resource=None,
-        list_resources=False,
         parameter=None,
-        parameter_override=None,
         working_dir=None,
         mets_server_url=None,
         instance_caching=False
@@ -84,7 +78,7 @@ def run_processor(
     log.debug("Running processor %s", processorClass)
 
     processor = get_processor(
-        processor_class=processorClass,
+        processorClass,
         parameter=parameter,
         workspace=None,
         page_id=page_id,
@@ -102,7 +96,7 @@ def run_processor(
     t0_cpu = process_time()
     if any(x in config.OCRD_PROFILE for x in ['RSS', 'PSS']):
         backend = 'psutil_pss' if 'PSS' in config.OCRD_PROFILE else 'psutil'
-        from memory_profiler import memory_usage
+        from memory_profiler import memory_usage # pylint: disable=import-outside-toplevel
         try:
             mem_usage = memory_usage(proc=(processor.process_workspace, [workspace], {}),
                                      # only run process once
@@ -212,156 +206,15 @@ def run_cli(
     if not log_filename:
         result = run(args, check=False)
     else:
-        with open(log_filename, 'a') as file_desc:
+        with open(log_filename, 'a', encoding='utf-8') as file_desc:
             result = run(args, check=False, stdout=file_desc, stderr=file_desc)
     return result.returncode
 
 
-def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None):
-    """Generate a string describing the full CLI of this processor including params.
-
-    Args:
-         ocrd_tool (dict): this processor's ``tools`` section of the module's ``ocrd-tool.json``
-         processor_instance (object, optional): the processor implementation
-             (for adding any module/class/function docstrings)
-        subcommand (string): 'worker' or 'server'
-    """
-    doc_help = ''
-    if processor_instance:
-        module = inspect.getmodule(processor_instance)
-        if module and module.__doc__:
-            doc_help += '\n' + inspect.cleandoc(module.__doc__) + '\n'
-        if processor_instance.__doc__:
-            doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + '\n'
-        if processor_instance.process_workspace.__doc__:
-            doc_help += '\n' + inspect.cleandoc(processor_instance.process_workspace.__doc__) + '\n'
-        if processor_instance.process.__doc__:
-            doc_help += '\n' + inspect.cleandoc(processor_instance.process.__doc__) + '\n'
-        if doc_help:
-            doc_help = '\n\n' + wrap_text(doc_help, width=72,
-                                          initial_indent='  > ',
-                                          subsequent_indent='  > ',
-                                          preserve_paragraphs=True)
-    subcommands = '''\
-    worker      Start a processing worker rather than do local processing
-    server      Start a processor server rather than do local processing
-'''
-
-    processing_worker_options = '''\
-  --queue                         The RabbitMQ server address in format
-                                  "amqp://{user}:{pass}@{host}:{port}/{vhost}"
-                                  [amqp://admin:admin@localhost:5672]
-  --database                      The MongoDB server address in format
-                                  "mongodb://{host}:{port}"
-                                  [mongodb://localhost:27018]
-  --log-filename                  Filename to redirect STDOUT/STDERR to,
-                                  if specified.
-'''
-
-    processing_server_options = '''\
-  --address                       The Processor server address in format
-                                  "{host}:{port}"
-  --database                      The MongoDB server address in format
-                                  "mongodb://{host}:{port}"
-                                  [mongodb://localhost:27018]
-'''
-
-    processing_options = '''\
-  -m, --mets URL-PATH             URL or file path of METS to process [./mets.xml]
-  -w, --working-dir PATH          Working directory of local workspace [dirname(URL-PATH)]
-  -I, --input-file-grp USE        File group(s) used as input
-  -O, --output-file-grp USE       File group(s) used as output
-  -g, --page-id ID                Physical page ID(s) to process instead of full document []
-  --overwrite                     Remove existing output pages/images
-                                  (with "--page-id", remove only those).
-                                  Short-hand for OCRD_EXISTING_OUTPUT=OVERWRITE
-  --debug                         Abort on any errors with full stack trace.
-                                  Short-hand for OCRD_MISSING_OUTPUT=ABORT
-  --profile                       Enable profiling
-  --profile-file PROF-PATH        Write cProfile stats to PROF-PATH. Implies "--profile"
-  -p, --parameter JSON-PATH       Parameters, either verbatim JSON string
-                                  or JSON file path
-  -P, --param-override KEY VAL    Override a single JSON object key-value pair,
-                                  taking precedence over --parameter
-  -U, --mets-server-url URL       URL of a METS Server for parallel incremental access to METS
-                                  If URL starts with http:// start an HTTP server there,
-                                  otherwise URL is a path to an on-demand-created unix socket
-  -l, --log-level [OFF|ERROR|WARN|INFO|DEBUG|TRACE]
-                                  Override log level globally [INFO]
-'''
-
-    information_options = '''\
-  -C, --show-resource RESNAME     Dump the content of processor resource RESNAME
-  -L, --list-resources            List names of processor resources
-  -J, --dump-json                 Dump tool description as JSON
-  -D, --dump-module-dir           Show the 'module' resource location path for this processor
-  -h, --help                      Show this message
-  -V, --version                   Show version
-'''
-
-    parameter_help = ''
-    if 'parameters' not in ocrd_tool or not ocrd_tool['parameters']:
-        parameter_help = '  NONE\n'
-    else:
-        def wrap(s):
-            return wrap_text(s, initial_indent=' '*3,
-                             subsequent_indent=' '*4,
-                             width=72, preserve_paragraphs=True)
-        for param_name, param in ocrd_tool['parameters'].items():
-            parameter_help += wrap('"%s" [%s%s]' % (
-                param_name,
-                param['type'],
-                ' - REQUIRED' if 'required' in param and param['required'] else
-                ' - %s' % json.dumps(param['default']) if 'default' in param else ''))
-            parameter_help += '\n ' + wrap(param['description'])
-            if 'enum' in param:
-                parameter_help += '\n ' + wrap('Possible values: %s' % json.dumps(param['enum']))
-            parameter_help += "\n"
-
-    if not subcommand:
-        return f'''\
-Usage: {ocrd_tool['executable']} [worker|server] [OPTIONS]
-
-  {ocrd_tool['description']}{doc_help}
-
-Subcommands:
-{subcommands}
-Options for processing:
-{processing_options}
-Options for information:
-{information_options}
-Parameters:
-{parameter_help}
-'''
-    elif subcommand == 'worker':
-        return f'''\
-Usage: {ocrd_tool['executable']} worker [OPTIONS]
-
-  Run {ocrd_tool['executable']} as a processing worker.
-
-  {ocrd_tool['description']}{doc_help}
-
-Options:
-{processing_worker_options}
-'''
-    elif subcommand == 'server':
-        return f'''\
-Usage: {ocrd_tool['executable']} server [OPTIONS]
-
-  Run {ocrd_tool['executable']} as a processor sever.
 
-  {ocrd_tool['description']}{doc_help}
-
-Options:
-{processing_server_options}
-'''
-    else:
-        pass
-
-
-# Taken from https://github.com/OCR-D/core/pull/884
-@freeze_args
-@lru_cache(maxsize=config.OCRD_MAX_PROCESSOR_CACHE)
+# not decorated here but at runtime (on first use)
+#@freeze_args
+#@lru_cache(maxsize=config.OCRD_MAX_PROCESSOR_CACHE)
 def get_cached_processor(parameter: dict, processor_class):
     """
     Call this function to get back an instance of a processor.
@@ -374,16 +227,13 @@ def get_cached_processor(parameter: dict, processor_class):
         Otherwise, an instance of the `:py:class:~ocrd.Processor` is returned.
     """
     if processor_class:
-        dict_params = dict(parameter) if parameter else None
-        processor = processor_class(None, parameter=dict_params)
-        processor.setup()
+        processor = processor_class(None, parameter=dict(parameter))
         return processor
     return None
 
-
 def get_processor(
         processor_class,
-        parameter: dict,
+        parameter: Optional[dict] = None,
         workspace: Workspace = None,
         page_id: str = None,
         input_file_grp: List[str] = None,
@@ -391,11 +241,24 @@ def get_processor(
         instance_caching: bool = False,
 ):
     if processor_class:
+        if parameter is None:
+            parameter = {}
         if instance_caching:
+            global get_cached_processor
+            if not hasattr(get_cached_processor, '__wrapped__'):
+                # first call: wrap
+                if processor_class.max_instances < 0:
+                    maxsize = config.OCRD_MAX_PROCESSOR_CACHE
+                else:
+                    maxsize = min(config.OCRD_MAX_PROCESSOR_CACHE, processor_class.max_instances)
+                # wrapping in call cache
+                # wrapping dict into frozendict (from https://github.com/OCR-D/core/pull/884)
+                get_cached_processor = freeze_args(lru_cache(maxsize=maxsize)(get_cached_processor))
             processor = get_cached_processor(parameter, processor_class)
         else:
+            # avoid passing workspace already (deprecated chdir behaviour)
             processor = processor_class(None, parameter=parameter)
-            processor.setup()
+        # set current processing parameters
         processor.workspace = workspace
         processor.page_id = page_id
         processor.input_file_grp = input_file_grp

ocrd/processor/ocrd_page_result.py

@@ -1,15 +1,15 @@
 from dataclasses import dataclass, field
-from typing import List
+from typing import List, Union
 from ocrd_models.ocrd_page import OcrdPage
 from PIL.Image import Image
 
-from ocrd_models.ocrd_page_generateds import AlternativeImageType
+from ocrd_models.ocrd_page_generateds import AlternativeImageType, PageType
 
 @dataclass
 class OcrdPageResultImage():
     pil : Image
     file_id_suffix : str
-    alternative_image : AlternativeImageType
+    alternative_image : Union[AlternativeImageType, PageType]
 
 @dataclass
 class OcrdPageResult():

ocrd/resolver.py

@@ -18,7 +18,6 @@ from ocrd_utils import (
 )
 from ocrd.workspace import Workspace
 from ocrd_models import OcrdMets
-from ocrd_models.constants import NAMESPACES as NS
 from ocrd_models.utils import handle_oai_response
 
 class Resolver():
@@ -310,5 +309,3 @@ class Resolver():
                         raise ValueError("--mets '%s' has a directory part inconsistent with --directory '%s'" % (mets_url, directory))
 
         return str(Path(directory).resolve()), str(mets_url), str(mets_basename), mets_server_url
-
-

ocrd/resource_manager.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 from os.path import join
-from os import environ, listdir, makedirs, getcwd, path, unlink
+from os import environ, listdir, getcwd, unlink
 from shutil import copytree, rmtree, copy
 from fnmatch import filter as apply_glob
 from datetime import datetime
@@ -13,14 +13,18 @@ from gdown.parse_url import parse_url as gparse_url
 from gdown.download import get_url_from_gdrive_confirmation
 from yaml import safe_load, safe_dump
 
+# pylint: disable=wrong-import-position
+
 # https://github.com/OCR-D/core/issues/867
 # https://stackoverflow.com/questions/50900727/skip-converting-entities-while-loading-a-yaml-string-using-pyyaml
 import yaml.constructor
-yaml.constructor.SafeConstructor.yaml_constructors[u'tag:yaml.org,2002:timestamp'] = \
-    yaml.constructor.SafeConstructor.yaml_constructors[u'tag:yaml.org,2002:str']
+yaml.constructor.SafeConstructor.yaml_constructors['tag:yaml.org,2002:timestamp'] = \
+    yaml.constructor.SafeConstructor.yaml_constructors['tag:yaml.org,2002:str']
+
+# pylint: enable=wrong-import-position
 
 from ocrd_validators import OcrdResourceListValidator
-from ocrd_utils import getLogger, directory_size, get_moduledir, EXT_TO_MIME, nth_url_segment, guess_media_type, config
+from ocrd_utils import getLogger, directory_size, get_moduledir, guess_media_type, config
 from ocrd_utils.os import get_processor_resource_types, list_all_resources, pushd_popd, get_ocrd_tool_json
 from .constants import RESOURCE_LIST_FILENAME, RESOURCE_USER_LIST_COMMENT
 
@@ -248,7 +252,7 @@ class OcrdResourceManager:
                         if "Content-Disposition" not in r.headers:
                             url = get_url_from_gdrive_confirmation(r.text)
                 except RuntimeError as e:
-                    log.warning("Cannot unwrap Google Drive URL: ", e)
+                    log.warning("Cannot unwrap Google Drive URL: %s", e)
             with open(filename, 'wb') as f:
                 with requests.get(url, stream=True) as r:
                     r.raise_for_status()

ocrd/workspace.py

@@ -1,7 +1,7 @@
 import io
 from os import makedirs, unlink, listdir, path
 from pathlib import Path
-from shutil import move, copyfileobj
+from shutil import copyfileobj
 from re import sub
 from tempfile import NamedTemporaryFile
 from contextlib import contextmanager
@@ -43,7 +43,6 @@ from ocrd_utils import (
     MIME_TO_PIL,
     MIMETYPE_PAGE,
     REGEX_PREFIX,
-    config
 )
 
 from .workspace_backup import WorkspaceBackupManager
@@ -96,8 +95,8 @@ class Workspace():
             if self.is_remote:
                 mets = ClientSideOcrdMets(mets_server_url, self.directory)
                 if mets.workspace_path != self.directory:
-                    raise ValueError(f"METS server {mets_server_url} workspace directory {mets.workspace_path} differs "
-                            f"from local workspace directory {self.directory}. These are not the same workspaces.")
+                    raise ValueError(f"METS server {mets_server_url} workspace directory '{mets.workspace_path}' differs "
+                            f"from local workspace directory '{self.directory}'. These are not the same workspaces.")
             else:
                 mets = OcrdMets(filename=self.mets_target)
         self.mets = mets
@@ -111,7 +110,7 @@ class Workspace():
 
     def __repr__(self):
         return 'Workspace[remote=%s, directory=%s, baseurl=%s, file_groups=%s, files=%s]' % (
-            not not self.is_remote,
+            self.is_remote,
             self.directory,
             self.baseurl,
             self.mets.file_groups,
@@ -648,7 +647,7 @@ class Workspace():
         log = getLogger('ocrd.workspace.image_from_page')
         page_image_info = self.resolve_image_exif(page.imageFilename)
         page_image = self._resolve_image_as_pil(page.imageFilename)
-        page_coords = dict()
+        page_coords = {}
         # use identity as initial affine coordinate transform:
         page_coords['transform'] = np.eye(3)
         # interim bbox (updated with each change to the transform):
@@ -1091,7 +1090,7 @@ class Workspace():
             The (absolute) path of the created file.
         """
         log = getLogger('ocrd.workspace.save_image_file')
-        saveargs = dict()
+        saveargs = {}
         if 'dpi' in image.info:
             saveargs['dpi'] = image.info['dpi']
         image_bytes = io.BytesIO()
@@ -1168,9 +1167,9 @@ def _reflect(log, name, orientation, segment_image, segment_coords, segment_xywh
     # Transpose in affine coordinate transform:
     # (consistent with image transposition or AlternativeImage below)
     transposition = {
-        90: Image.ROTATE_90,
-        180: Image.ROTATE_180,
-        270: Image.ROTATE_270
+        90: Image.Transpose.ROTATE_90,
+        180: Image.Transpose.ROTATE_180,
+        270: Image.Transpose.ROTATE_270
     }.get(orientation) # no default
     segment_coords['transform'] = transpose_coordinates(
         segment_coords['transform'], transposition,
@@ -1238,5 +1237,5 @@ def _scale(log, name, factor, segment_image, segment_coords, segment_xywh, **kwa
         segment_image = segment_image.resize((int(segment_image.width * factor),
                                               int(segment_image.height * factor)),
                                              # slowest, but highest quality:
-                                             Image.BICUBIC)
+                                             Image.Resampling.BICUBIC)
     return segment_image, segment_coords, segment_xywh

ocrd/workspace_backup.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from os import makedirs
-from os.path import join, basename, getsize, abspath
+from os.path import join, basename, getsize
 from glob import glob
 from shutil import copy
 import hashlib

{ocrd-3.0.0a2.dist-info → ocrd-3.0.0b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0a2
+Version: 3.0.0b2
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -94,17 +94,12 @@ complete stack of OCR-D-related software.
 
 The easiest way to install is via `pip`:
 
-```sh
-pip install ocrd
+    pip install ocrd
 
-# or just the functionality you need, e.g.
-
-pip install ocrd_modelfactory
-```
 
 All Python software released by [OCR-D](https://github.com/OCR-D) requires Python 3.8 or higher.
 
-**NOTE** Some OCR-D-Tools (or even test cases) _might_ reveal an unintended behavior if you have specific environment modifications, like:
+> **NOTE** Some OCR-D tools (or even test cases) _might_ reveal an unintended behavior if you have specific environment modifications, like:
 * using a custom build of [ImageMagick](https://github.com/ImageMagick/ImageMagick), whose format delegates are different from what OCR-D supposes
 * custom Python logging configurations in your personal account
 
@@ -129,7 +124,6 @@ Almost all behaviour of the OCR-D/core software is configured via CLI options an
 
 Some parts of the software are configured via environment variables:
 
-* `OCRD_METS_CACHING`: If set to `true`, access to the METS file is cached, speeding in-memory search and modification.
 * `OCRD_PROFILE`: This variable configures the built-in CPU and memory profiling. If empty, no profiling is done. Otherwise expected to contain any of the following tokens:
   * `CPU`: Enable CPU profiling of processor runs
   * `RSS`: Enable RSS memory profiling
@@ -142,18 +136,46 @@ Some parts of the software are configured via environment variables:
 * `XDG_CONFIG_HOME`: Directory to look for `./ocrd/resources.yml` (i.e. `ocrd resmgr` user database) – defaults to `$HOME/.config`.
 * `XDG_DATA_HOME`: Directory to look for `./ocrd-resources/*` (i.e. `ocrd resmgr` data location) – defaults to `$HOME/.local/share`.
 
-* `OCRD_DOWNLOAD_RETRIES`: Number of times to retry failed attempts for downloads of workspace files.
+* `OCRD_DOWNLOAD_RETRIES`: Number of times to retry failed attempts for downloads of resources or workspace files.
 * `OCRD_DOWNLOAD_TIMEOUT`: Timeout in seconds for connecting or reading (comma-separated) when downloading.
 
+* `OCRD_MISSING_INPUT`: How to deal with missing input files (for some fileGrp/pageId) during processing:
+  * `SKIP`: ignore and proceed with next page's input
+  * `ABORT`: throw `MissingInputFile` exception
+
+* `OCRD_MISSING_OUTPUT`: How to deal with missing output files (for some fileGrp/pageId) during processing:
+  * `SKIP`: ignore and proceed processing next page
+  * `COPY`: fall back to copying input PAGE to output fileGrp for page
+  * `ABORT`: re-throw whatever caused processing to fail
+
+* `OCRD_MAX_MISSING_OUTPUTS`: Maximal rate of skipped/fallback pages among all processed pages before aborting (decimal fraction, ignored if negative).
+
+* `OCRD_EXISTING_OUTPUT`: How to deal with already existing output files (for some fileGrp/pageId) during processing:
+  * `SKIP`: ignore and proceed processing next page
+  * `OVERWRITE`: force writing result to output fileGrp for page
+  * `ABORT`: re-throw `FileExistsError` exception
+
+
 * `OCRD_METS_CACHING`: Whether to enable in-memory storage of OcrdMets data structures for speedup during processing or workspace operations.
 
 * `OCRD_MAX_PROCESSOR_CACHE`: Maximum number of processor instances (for each set of parameters) to be kept in memory (including loaded models) for processing workers or processor servers.
 
+* `OCRD_MAX_PARALLEL_PAGES`: Maximum number of processor threads for page-parallel processing (within each Processor's selected page range, independent of the number of Processing Workers or Processor Servers). If set `>1`, then a METS Server must be used for METS synchronisation.
+
+* `OCRD_PROCESSING_PAGE_TIMEOUT`: Timeout in seconds for processing a single page. If set >0, when exceeded, the same as OCRD_MISSING_OUTPUT applies.
+
 * `OCRD_NETWORK_SERVER_ADDR_PROCESSING`: Default address of Processing Server to connect to (for `ocrd network client processing`).
 * `OCRD_NETWORK_SERVER_ADDR_WORKFLOW`: Default address of Workflow Server to connect to (for `ocrd network client workflow`).
 * `OCRD_NETWORK_SERVER_ADDR_WORKSPACE`: Default address of Workspace Server to connect to (for `ocrd network client workspace`).
 * `OCRD_NETWORK_RABBITMQ_CLIENT_CONNECT_ATTEMPTS`: Number of attempts for a worker to create its queue. Helpful if the rabbitmq-server needs time to be fully started.
 
+* `OCRD_NETWORK_CLIENT_POLLING_SLEEP`: How many seconds to sleep before trying `ocrd network client` again.
+* `OCRD_NETWORK_CLIENT_POLLING_TIMEOUT`: Timeout for a blocking `ocrd network client` (in seconds).
+
+* `OCRD_NETWORK_SOCKETS_ROOT_DIR`: The root directory where all mets server related socket files are created.
+* `OCRD_NETWORK_LOGS_ROOT_DIR`: The root directory where all ocrd_network related file logs are stored.
+
+
 
 ## Packages