ocrd 3.0.0a2__tar.gz → 3.0.0b1__tar.gz

{ocrd-3.0.0a2/src/ocrd.egg-info → ocrd-3.0.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0a2
+Version: 3.0.0b1
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0

ocrd-3.0.0b1/VERSION

@@ -0,0 +1 @@
+3.0.0b1

{ocrd-3.0.0a2 → ocrd-3.0.0b1}/src/ocrd/cli/__init__.py

@@ -10,6 +10,36 @@ import click
 
 from ocrd_utils import config
 
+# pylint: disable=wrong-import-position
+
+def command_with_replaced_help(*replacements):
+
+    class CommandWithReplacedHelp(click.Command):
+        def get_help(self, ctx):
+            newhelp = super().get_help(ctx)
+            for replacement in replacements:
+                newhelp = re.sub(*replacement, newhelp)
+            # print(newhelp)
+            return newhelp
+
+    return CommandWithReplacedHelp
+
+# pylint: enable=wrong-import-position
+
+from ..decorators import ocrd_loglevel
+from .ocrd_tool import ocrd_tool_cli
+from .workspace import workspace_cli
+from .process import process_cli
+from .bashlib import bashlib_cli
+from .validate import validate_cli
+from .resmgr import resmgr_cli
+from .zip import zip_cli
+from .log import log_cli
+from .network import network_cli
+
+
+__all__ = ['cli']
+
 _epilog = f"""
 
 \b
@@ -41,6 +71,10 @@ Variables:
 \b
 {config.describe('OCRD_MAX_PROCESSOR_CACHE')}
 \b
+{config.describe('OCRD_NETWORK_CLIENT_POLLING_SLEEP')}
+\b
+{config.describe('OCRD_NETWORK_CLIENT_POLLING_TIMEOUT')}
+\b
 {config.describe('OCRD_NETWORK_SERVER_ADDR_PROCESSING')}
 \b
 {config.describe('OCRD_NETWORK_SERVER_ADDR_WORKFLOW')}
@@ -60,30 +94,6 @@ Variables:
 {config.describe('OCRD_LOGGING_DEBUG')}
 """
 
-def command_with_replaced_help(*replacements):
-
-    class CommandWithReplacedHelp(click.Command):
-        def get_help(self, ctx):
-            help = super().get_help(ctx)
-            for replacement in replacements:
-                help = re.sub(*replacement, help)
-            # print(help)
-            return help
-
-    return CommandWithReplacedHelp
-
-
-from ..decorators import ocrd_loglevel
-from .ocrd_tool import ocrd_tool_cli
-from .workspace import workspace_cli
-from .process import process_cli
-from .bashlib import bashlib_cli
-from .validate import validate_cli
-from .resmgr import resmgr_cli
-from .zip import zip_cli
-from .log import log_cli
-from .network import network_cli
-
 @click.group(epilog=_epilog)
 @click.version_option(package_name='ocrd')
 @ocrd_loglevel
@@ -101,5 +111,3 @@ cli.add_command(validate_cli)
 cli.add_command(log_cli)
 cli.add_command(resmgr_cli)
 cli.add_command(network_cli)
-
-__all__ = ['cli']

{ocrd-3.0.0a2 → ocrd-3.0.0b1}/src/ocrd/cli/bashlib.py

@@ -8,7 +8,6 @@ OCR-D CLI: bash library
 """
 from __future__ import print_function
 import sys
-from os.path import isfile
 import click
 
 from ocrd.constants import BASHLIB_FILENAME
@@ -23,15 +22,7 @@ from ocrd.decorators import (
     ocrd_loglevel,
     ocrd_cli_wrap_processor
 )
-from ocrd_utils import (
-    is_local_filename,
-    get_local_filename,
-    initLogging,
-    getLogger,
-    make_file_id,
-    config
-)
-from ocrd.resolver import Resolver
+from ocrd_utils import make_file_id
 from ocrd.processor import Processor
 
 # ----------------------------------------------------------------------
@@ -82,6 +73,8 @@ def bashlib_constants(name):
         print(val)
 
 @bashlib_cli.command('input-files')
+@click.option('--ocrd-tool', help="path to ocrd-tool.json of processor to feed", default=None)
+@click.option('--executable', help="name of processor executable in ocrd-tool.json", default=None)
 @click.option('-m', '--mets', help="METS to process", default=DEFAULT_METS_BASENAME)
 @click.option('-w', '--working-dir', help="Working Directory")
 @click.option('-I', '--input-file-grp', help='File group(s) used as input.', default=None)
@@ -96,7 +89,7 @@ def bashlib_constants(name):
 @parameter_option
 @parameter_override_option
 @ocrd_loglevel
-def bashlib_input_files(**kwargs):
+def bashlib_input_files(ocrd_tool, executable, **kwargs):
     """
     List input files for processing
 
@@ -108,12 +101,6 @@ def bashlib_input_files(**kwargs):
     (The printing format is one associative array initializer per line.)
     """
     class BashlibProcessor(Processor):
-        @property
-        def ocrd_tool(self):
-            return {'executable': '', 'steps': ['']}
-        @property
-        def version(self):
-            return '1.0'
         # go half way of the normal run_processor / process_workspace call tree
         # by just delegating to process_workspace, overriding process_page_file
         # to ensure all input files exist locally (without persisting them in the METS)
@@ -129,4 +116,31 @@ def bashlib_input_files(**kwargs):
                 print(f"[{field}]='{value}'", end=' ')
             output_file_id = make_file_id(input_files[0], kwargs['output_file_grp'])
             print(f"[outputFileId]='{output_file_id}'")
-    ocrd_cli_wrap_processor(BashlibProcessor, **kwargs)
+    if ocrd_tool and executable:
+        class FullBashlibProcessor(BashlibProcessor):
+            @property
+            def metadata_location(self):
+                # needed for metadata loading and validation mechanism
+                return ocrd_tool
+            @property
+            def executable(self):
+                # needed for ocrd_tool lookup
+                return executable
+    else:
+        # we have no true metadata file, so fill in just to make it work
+        class FullBashlibProcessor(BashlibProcessor):
+            @property
+            def ocrd_tool(self):
+                # needed to satisfy the validator
+                return {'executable': '',
+                        # required now
+                        'input_file_grp_cardinality': 1,
+                        'output_file_grp_cardinality': 1,
+                        'steps': ['']
+                }
+            @property
+            def version(self):
+                # needed to satisfy the validator and wrapper
+                return '1.0'
+
+    ocrd_cli_wrap_processor(FullBashlibProcessor, **kwargs)

{ocrd-3.0.0a2 → ocrd-3.0.0b1}/src/ocrd/cli/ocrd_tool.py

@@ -17,7 +17,6 @@ from ocrd.decorators import parameter_option, parameter_override_option
 from ocrd.processor import Processor
 from ocrd_utils import (
     set_json_key_value_overrides,
-    VERSION as OCRD_VERSION,
     parse_json_string_or_file,
     parse_json_string_with_comments as loads
 )
@@ -29,23 +28,26 @@ class OcrdToolCtx():
         self.filename = filename
         with codecs.open(filename, encoding='utf-8') as f:
             self.content = f.read()
+            # perhaps the validator should _always_ run (for default expansion)
+            # so validate command only for the report?
             self.json = loads(self.content)
+        self.tool_name = ''
 
         class BashProcessor(Processor):
             @property
-            def metadata(inner_self):
+            def metadata(inner_self): # pylint: disable=no-self-argument,arguments-renamed
                 return self.json
             @property
-            def executable(inner_self):
+            def executable(inner_self): # pylint: disable=no-self-argument,arguments-renamed
                 return self.tool_name
             @property
-            def moduledir(inner_self):
+            def moduledir(inner_self): # pylint: disable=no-self-argument,arguments-renamed
                 return os.path.dirname(self.filename)
             # set docstrings to empty
             __doc__ = None
             # HACK: override the module-level docstring, too
             getmodule(OcrdToolCtx).__doc__ = None
-            def process(inner_self):
+            def process(inner_self): # pylint: disable=no-self-argument,arguments-renamed
                 return super()
 
         self.processor = BashProcessor

{ocrd-3.0.0a2 → ocrd-3.0.0b1}/src/ocrd/cli/workspace.py

@@ -6,7 +6,7 @@ OCR-D CLI: workspace management
     :nested: full
 """
 import os
-from os import getcwd, rmdir, unlink
+from os import rmdir, unlink
 from os.path import dirname, relpath, normpath, exists, join, isabs, isdir
 from pathlib import Path
 from json import loads, dumps
@@ -14,7 +14,6 @@ import sys
 from glob import glob   # XXX pathlib.Path.glob does not support absolute globs
 import re
 import time
-import numpy as np
 
 import click
 
@@ -118,7 +117,7 @@ def workspace_validate(ctx, mets_url, download, skip, page_textequiv_consistency
 @workspace_cli.command('clone', cls=command_with_replaced_help(
     (r' \[WORKSPACE_DIR\]', ''))) # XXX deprecated argument
 @click.option('-f', '--clobber-mets', help="Overwrite existing METS file", default=False, is_flag=True)
-@click.option('-a', '--download', is_flag=True, help="Download all files and change location in METS file after cloning")
+@click.option('-a', '--download', is_flag=True, help="Download all selected files and add local path references in METS file afterwards")
 @click.argument('mets_url')
 @mets_find_options
 # XXX deprecated
@@ -129,8 +128,10 @@ def workspace_clone(ctx, clobber_mets, download, file_grp, file_id, page_id, mim
     Create a workspace from METS_URL and return the directory
 
     METS_URL can be a URL, an absolute path or a path relative to $PWD.
-    If METS_URL is not provided, use --mets accordingly.
     METS_URL can also be an OAI-PMH GetRecord URL wrapping a METS file.
+
+    Additional options pertain to the selection of files / fileGrps / pages
+    to be downloaded, if --download is used.
     """
     LOG = getLogger('ocrd.cli.workspace.clone')
     if workspace_dir:
@@ -143,6 +144,7 @@ def workspace_clone(ctx, clobber_mets, download, file_grp, file_id, page_id, mim
         mets_basename=ctx.mets_basename,
         clobber_mets=clobber_mets,
         download=download,
+        fileGrp=file_grp,
         ID=file_id,
         pageId=page_id,
         mimetype=mimetype,
@@ -408,7 +410,7 @@ def workspace_cli_bulk_add(ctx, regex, mimetype, page_id, file_id, url, local_fi
         if dry_run:
             log.info('workspace.add_file(%s)' % file_dict)
         else:
-            workspace.add_file(fileGrp, ignore=ignore, force=force, **file_dict)
+            workspace.add_file(fileGrp, ignore=ignore, force=force, **file_dict) # pylint: disable=redundant-keyword-arg
 
     # save changes to disk
     workspace.save_mets()
@@ -452,7 +454,7 @@ def workspace_find(ctx, file_grp, mimetype, page_id, file_id, output_field, incl
     snake_to_camel = {"file_id": "ID", "page_id": "pageId", "file_grp": "fileGrp"}
     output_field = [snake_to_camel.get(x, x) for x in output_field]
     modified_mets = False
-    ret = list()
+    ret = []
     workspace = Workspace(
         ctx.resolver,
         directory=ctx.directory,
@@ -748,7 +750,7 @@ def set_id(ctx, id):   # pylint: disable=redefined-builtin
 
 @workspace_cli.command('update-page')
 @click.option('--set', 'attr_value_pairs', help=f"set mets:div ATTR to VALUE. possible keys: {METS_PAGE_DIV_ATTRIBUTE.names()}", metavar="ATTR VALUE", nargs=2, multiple=True)
-@click.option('--order', help="[DEPRECATED - use --set ATTR VALUE", metavar='ORDER')               
+@click.option('--order', help="[DEPRECATED - use --set ATTR VALUE", metavar='ORDER')
 @click.option('--orderlabel', help="DEPRECATED - use --set ATTR VALUE", metavar='ORDERLABEL')
 @click.option('--contentids', help="DEPRECATED - use --set ATTR VALUE", metavar='ORDERLABEL')
 @click.argument('PAGE_ID')
@@ -757,7 +759,7 @@ def update_page(ctx, attr_value_pairs, order, orderlabel, contentids, page_id):
     """
     Update the @ID, @ORDER, @ORDERLABEL, @LABEL or @CONTENTIDS attributes of the mets:div with @ID=PAGE_ID
     """
-    update_kwargs = {k: v for k, v in attr_value_pairs}
+    update_kwargs = dict(attr_value_pairs)
     if order:
         update_kwargs['ORDER'] = order
     if orderlabel:

{ocrd-3.0.0a2 → ocrd-3.0.0b1}/src/ocrd/decorators/__init__.py

@@ -1,4 +1,5 @@
 import sys
+from contextlib import nullcontext
 
 from ocrd_utils import (
     config,
@@ -9,6 +10,7 @@ from ocrd_utils import (
     parse_json_string_with_comments,
     set_json_key_value_overrides,
     parse_json_string_or_file,
+    redirect_stderr_and_stdout_to_file,
 )
 from ocrd_validators import WorkspaceValidator
 from ocrd_network import ProcessingWorker, ProcessorServer, AgentType
@@ -104,10 +106,10 @@ def ocrd_cli_wrap_processor(
         kwargs['parameter'] = parse_json_string_or_file(*kwargs['parameter'],
                                                         resolve_preset_file=resolve)
     else:
-        kwargs['parameter'] = dict()
+        kwargs['parameter'] = {}
     # Merge parameter overrides and parameters
     if 'parameter_override' in kwargs:
-        set_json_key_value_overrides(kwargs['parameter'], *kwargs['parameter_override'])
+        set_json_key_value_overrides(kwargs['parameter'], *kwargs.pop('parameter_override'))
     # Assert -I / -O
     if not kwargs['input_file_grp']:
         raise ValueError('-I/--input-file-grp is required')
@@ -140,17 +142,21 @@ def ocrd_cli_wrap_processor(
         print("Profiling...")
         pr = cProfile.Profile()
         pr.enable()
-        def exit():
+        def goexit():
             pr.disable()
             print("Profiling completed")
             if profile_file:
-                with open(profile_file, 'wb') as f:
-                    pr.dump_stats(profile_file)
+                pr.dump_stats(profile_file)
             s = io.StringIO()
             pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats()
             print(s.getvalue())
-        atexit.register(exit)
-    run_processor(processorClass, mets_url=mets, workspace=workspace, **kwargs)
+        atexit.register(goexit)
+    if log_filename:
+        log_ctx = redirect_stderr_and_stdout_to_file(log_filename)
+    else:
+        log_ctx = nullcontext()
+    with log_ctx:
+        run_processor(processorClass, mets_url=mets, workspace=workspace, **kwargs)
 
 
 def check_and_run_network_agent(ProcessorClass, subcommand: str, address: str, database: str, queue: str):

{ocrd-3.0.0a2 → ocrd-3.0.0b1}/src/ocrd/lib.bash

@@ -299,6 +299,8 @@ ocrd__wrap () {
         eval "ocrd__files[$i]=ocrd__file$i"
         let ++i
     done < <(ocrd bashlib input-files \
+                  --ocrd-tool $OCRD_TOOL_JSON \
+                  --executable $OCRD_TOOL_NAME \
                   -m "${ocrd__argv[mets_file]}" \
                   -I "${ocrd__argv[input_file_grp]}" \
                   -O "${ocrd__argv[output_file_grp]}" \

{ocrd-3.0.0a2 → ocrd-3.0.0b1}/src/ocrd/mets_server.py

@@ -21,7 +21,7 @@ from pydantic import BaseModel, Field, ValidationError
 import uvicorn
 
 from ocrd_models import OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent
-from ocrd_utils import getLogger, deprecated_alias
+from ocrd_utils import getLogger
 
 
 #
@@ -236,7 +236,7 @@ class ClientSideOcrdMets:
             agent_dict["_type"] = agent_dict.pop("type")
         return [ClientSideOcrdAgent(None, **agent_dict) for agent_dict in agent_dicts]
 
-    def add_agent(self, *args, **kwargs):
+    def add_agent(self, **kwargs):
         if not self.multiplexing_mode:
             return self.session.request("POST", f"{self.url}/agent", json=OcrdAgentModel.create(**kwargs).dict())
         else:
@@ -403,7 +403,6 @@ class OcrdMetsServer:
     @staticmethod
     def kill_process(mets_server_pid: int):
         subprocess_run(args=["kill", "-s", "SIGINT", f"{mets_server_pid}"], shell=False, universal_newlines=True)
-        return
 
     def shutdown(self):
         if self.is_uds: