ocrd 3.0.0b6__py3-none-any.whl → 3.0.1__py3-none-any.whl

ocrd/cli/__init__.py

@@ -16,7 +16,7 @@ def command_with_replaced_help(*replacements):
 
     class CommandWithReplacedHelp(click.Command):
         def get_help(self, ctx):
-            newhelp = super().get_help(ctx)
+            newhelp : str = super().get_help(ctx)
             for replacement in replacements:
                 newhelp = re.sub(*replacement, newhelp)
             # print(newhelp)
@@ -83,6 +83,8 @@ Variables:
 \b
 {config.describe('OCRD_NETWORK_RABBITMQ_CLIENT_CONNECT_ATTEMPTS')}
 \b
+{config.describe('OCRD_NETWORK_RABBITMQ_HEARTBEAT')}
+\b
 {config.describe('OCRD_PROFILE_FILE')}
 \b
 {config.describe('OCRD_PROFILE', wrap_text=False)}

ocrd/decorators/__init__.py

@@ -48,6 +48,9 @@ def ocrd_cli_wrap_processor(
     # ocrd_network params end #
     **kwargs
 ):
+    # init logging handlers so no imported libs can preempt ours
+    initLogging()
+
     # FIXME: remove workspace arg entirely
     processor = processorClass(None)
     if not sys.argv[1:]:
@@ -89,8 +92,6 @@ def ocrd_cli_wrap_processor(
         # Used for checking/starting network agents for the WebAPI architecture
         check_and_run_network_agent(processorClass, subcommand, address, database, queue)
 
-    # from here: single-run processing context
-    initLogging()
     if 'parameter' in kwargs:
         # Disambiguate parameter file/literal, and resolve file
         def resolve(name):

ocrd/mets_server.py

@@ -1,8 +1,10 @@
 """
 # METS server functionality
 """
+import os
 import re
 from os import _exit, chmod
+import signal
 from typing import Dict, Optional, Union, List, Tuple
 from time import sleep
 from pathlib import Path
@@ -155,13 +157,13 @@ class ClientSideOcrdMets:
         Request writing the changes to the file system
         """
         if not self.multiplexing_mode:
-            self.session.request("PUT", url=self.url)
+            return self.session.request("PUT", url=self.url).text
         else:
-            self.session.request(
+            return self.session.request(
                 "POST",
                 self.url,
                 json=MpxReq.save(self.ws_dir_path)
-            )
+            ).json()["text"]
 
     def stop(self):
         """
@@ -169,14 +171,13 @@ class ClientSideOcrdMets:
         """
         try:
             if not self.multiplexing_mode:
-                self.session.request("DELETE", self.url)
-                return
+                return self.session.request("DELETE", self.url).text
             else:
-                self.session.request(
+                return self.session.request(
                     "POST",
                     self.url,
                     json=MpxReq.stop(self.ws_dir_path)
-                )
+                ).json()["text"]
         except ConnectionError:
             # Expected because we exit the process without returning
             pass
@@ -323,7 +324,7 @@ class ClientSideOcrdMets:
 
 
 class MpxReq:
-    """This class wrapps the request bodies needed for the tcp forwarding
+    """This class wraps the request bodies needed for the tcp forwarding
 
     For every mets-server-call like find_files or workspace_path a special request_body is
     needed to call `MetsServerProxy.forward_tcp_request`. These are created by this functions.
@@ -346,12 +347,12 @@ class MpxReq:
     @staticmethod
     def save(ws_dir_path: str) -> Dict:
         return MpxReq.__args_wrapper(
-            ws_dir_path, method_type="PUT", response_type="empty", request_url="", request_data={})
+            ws_dir_path, method_type="PUT", response_type="text", request_url="", request_data={})
 
     @staticmethod
     def stop(ws_dir_path: str) -> Dict:
         return MpxReq.__args_wrapper(
-            ws_dir_path, method_type="DELETE", response_type="empty", request_url="", request_data={})
+            ws_dir_path, method_type="DELETE", response_type="text", request_url="", request_data={})
 
     @staticmethod
     def reload(ws_dir_path: str) -> Dict:
@@ -428,18 +429,24 @@ class OcrdMetsServer:
 
     @staticmethod
     def kill_process(mets_server_pid: int):
-        subprocess_run(args=["kill", "-s", "SIGINT", f"{mets_server_pid}"], shell=False, universal_newlines=True)
+        os.kill(mets_server_pid, signal.SIGINT)
+        sleep(3)
+        try:
+            os.kill(mets_server_pid, signal.SIGKILL)
+        except ProcessLookupError as e:
+            pass
 
     def shutdown(self):
+        pid = os.getpid()
+        self.log.info(f"Shutdown method of mets server[{pid}] invoked, sending SIGTERM signal.")
+        os.kill(pid, signal.SIGTERM)
         if self.is_uds:
             if Path(self.url).exists():
-                self.log.debug(f'UDS socket {self.url} still exists, removing it')
+                self.log.warning(f"Due to a server shutdown, removing the existing UDS socket file: {self.url}")
                 Path(self.url).unlink()
-        # os._exit because uvicorn catches SystemExit raised by sys.exit
-        _exit(0)
 
     def startup(self):
-        self.log.info("Starting up METS server")
+        self.log.info(f"Configuring the Mets Server")
 
         workspace = self.workspace
 
@@ -465,32 +472,49 @@ class OcrdMetsServer:
             """
             Write current changes to the file system
             """
-            return workspace.save_mets()
+            workspace.save_mets()
+            response = Response(content="The Mets Server is writing changes to disk.", media_type='text/plain')
+            self.log.info(f"PUT / -> {response.__dict__}")
+            return response
 
         @app.delete(path='/')
-        async def stop():
+        def stop():
             """
             Stop the mets server
             """
-            getLogger('ocrd.models.ocrd_mets').info(f'Shutting down METS Server {self.url}')
             workspace.save_mets()
+            response = Response(content="The Mets Server will shut down soon...", media_type='text/plain')
             self.shutdown()
+            self.log.info(f"DELETE / -> {response.__dict__}")
+            return response
 
         @app.post(path='/reload')
-        async def workspace_reload_mets():
+        def workspace_reload_mets():
             """
             Reload mets file from the file system
             """
             workspace.reload_mets()
-            return Response(content=f'Reloaded from {workspace.directory}', media_type="text/plain")
+            response = Response(content=f"Reloaded from {workspace.directory}", media_type='text/plain')
+            self.log.info(f"POST /reload -> {response.__dict__}")
+            return response
 
         @app.get(path='/unique_identifier', response_model=str)
         async def unique_identifier():
-            return Response(content=workspace.mets.unique_identifier, media_type='text/plain')
+            response = Response(content=workspace.mets.unique_identifier, media_type='text/plain')
+            self.log.info(f"GET /unique_identifier -> {response.__dict__}")
+            return response
 
         @app.get(path='/workspace_path', response_model=str)
         async def workspace_path():
-            return Response(content=workspace.directory, media_type="text/plain")
+            response = Response(content=workspace.directory, media_type="text/plain")
+            self.log.info(f"GET /workspace_path -> {response.__dict__}")
+            return response
+
+        @app.get(path='/physical_pages', response_model=OcrdPageListModel)
+        async def physical_pages():
+            response = {'physical_pages': workspace.mets.physical_pages}
+            self.log.info(f"GET /physical_pages -> {response}")
+            return response
 
         @app.get(path='/physical_pages', response_model=OcrdPageListModel)
         async def physical_pages():
@@ -498,18 +522,24 @@ class OcrdMetsServer:
 
         @app.get(path='/file_groups', response_model=OcrdFileGroupListModel)
         async def file_groups():
-            return {'file_groups': workspace.mets.file_groups}
+            response = {'file_groups': workspace.mets.file_groups}
+            self.log.info(f"GET /file_groups -> {response}")
+            return response
 
         @app.get(path='/agent', response_model=OcrdAgentListModel)
         async def agents():
-            return OcrdAgentListModel.create(workspace.mets.agents)
+            response = OcrdAgentListModel.create(workspace.mets.agents)
+            self.log.info(f"GET /agent -> {response.__dict__}")
+            return response
 
         @app.post(path='/agent', response_model=OcrdAgentModel)
         async def add_agent(agent: OcrdAgentModel):
             kwargs = agent.dict()
             kwargs['_type'] = kwargs.pop('type')
             workspace.mets.add_agent(**kwargs)
-            return agent
+            response = agent
+            self.log.info(f"POST /agent -> {response.__dict__}")
+            return response
 
         @app.get(path="/file", response_model=OcrdFileListModel)
         async def find_files(
@@ -526,7 +556,9 @@ class OcrdMetsServer:
             found = workspace.mets.find_all_files(
                 fileGrp=file_grp, ID=file_id, pageId=page_id, mimetype=mimetype, local_filename=local_filename, url=url
             )
-            return OcrdFileListModel.create(found)
+            response = OcrdFileListModel.create(found)
+            self.log.info(f"GET /file -> {response.__dict__}")
+            return response
 
         @app.post(path='/file', response_model=OcrdFileModel)
         async def add_file(
@@ -549,7 +581,9 @@ class OcrdMetsServer:
             # Add to workspace
             kwargs = file_resource.dict()
             workspace.add_file(**kwargs, force=force)
-            return file_resource
+            response = file_resource
+            self.log.info(f"POST /file -> {response.__dict__}")
+            return response
 
         # ------------- #
 
@@ -557,9 +591,6 @@ class OcrdMetsServer:
             # Create socket and change to world-readable and -writable to avoid permission errors
             self.log.debug(f"chmod 0o677 {self.url}")
             server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-            if Path(self.url).exists() and not is_socket_in_use(self.url):
-                # remove leftover unused socket which blocks startup
-                Path(self.url).unlink()
             server.bind(self.url)  # creates the socket file
             atexit.register(self.shutdown)
             server.close()
@@ -571,16 +602,5 @@ class OcrdMetsServer:
         uvicorn_kwargs['log_config'] = None
         uvicorn_kwargs['access_log'] = False
 
-        self.log.debug("Starting uvicorn")
+        self.log.info("Starting the uvicorn Mets Server")
         uvicorn.run(app, **uvicorn_kwargs)
-
-
-def is_socket_in_use(socket_path):
-    if Path(socket_path).exists():
-        client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        try:
-            client.connect(socket_path)
-        except OSError:
-            return False
-        client.close()
-        return True

ocrd/processor/base.py

@@ -18,10 +18,11 @@ from os import getcwd
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union, get_args
 import sys
+import logging
+import logging.handlers
 import inspect
 import tarfile
 import io
-import weakref
 from collections import defaultdict
 from frozendict import frozendict
 # concurrent.futures is buggy in py38,
@@ -129,7 +130,8 @@ class DummyExecutor:
     def __init__(self, initializer=None, initargs=(), **kwargs):
         initializer(*initargs)
     def shutdown(self, **kwargs):
-        pass
+        # allow gc to catch processor instance (unless cached)
+        _page_worker_set_ctxt(None, None)
     def submit(self, fn, *args, **kwargs) -> DummyFuture:
         return DummyFuture(fn, *args, **kwargs)
 
@@ -158,12 +160,12 @@ class Processor():
 
     max_workers : int = -1
     """
-    maximum number of processor threads for page-parallel processing (ignored if negative),
+    maximum number of processor forks for page-parallel processing (ignored if negative),
     to be applied on top of :py:data:`~ocrd_utils.config.OCRD_MAX_PARALLEL_PAGES` (i.e.
     whatever is smaller).
 
     (Override this if you know how many pages fit into processing units - GPU shaders / CPU cores
-    - at once, or if your class is not thread-safe.)
+    - at once, or if your class already creates threads prior to forking, e.g. during ``setup``.)
     """
 
     max_page_seconds : int = -1
@@ -366,12 +368,14 @@ class Processor():
         self._base_logger = getLogger('ocrd.processor.base')
         if parameter is not None:
             self.parameter = parameter
-        # ensure that shutdown gets called at destruction
-        self._finalizer = weakref.finalize(self, self.shutdown)
         # workaround for deprecated#72 (@deprecated decorator does not work for subclasses):
         setattr(self, 'process',
                 deprecated(version='3.0', reason='process() should be replaced with process_page_pcgts() or process_page_file() or process_workspace()')(getattr(self, 'process')))
 
+    def __del__(self):
+        self._base_logger.debug("shutting down %s in %s", repr(self), mp.current_process().name)
+        self.shutdown()
+
     def show_help(self, subcommand=None):
         """
         Print a usage description including the standard CLI and all of this processor's ocrd-tool
@@ -502,7 +506,7 @@ class Processor():
                 # set up multitasking
                 max_workers = max(0, config.OCRD_MAX_PARALLEL_PAGES)
                 if self.max_workers > 0 and self.max_workers < config.OCRD_MAX_PARALLEL_PAGES:
-                    self._base_logger.info("limiting number of threads from %d to %d", max_workers, self.max_workers)
+                    self._base_logger.info("limiting number of workers from %d to %d", max_workers, self.max_workers)
                     max_workers = self.max_workers
                 if max_workers > 1:
                     assert isinstance(workspace.mets, ClientSideOcrdMets), \
@@ -514,22 +518,31 @@ class Processor():
 
                 if max_workers > 1:
                     executor_cls = ProcessPoolExecutor
+                    log_queue = mp.Queue()
                 else:
                     executor_cls = DummyExecutor
+                    log_queue = None
                 executor = executor_cls(
                     max_workers=max_workers or 1,
                     # only forking method avoids pickling
                     context=mp.get_context('fork'),
                     # share processor instance as global to avoid pickling
                     initializer=_page_worker_set_ctxt,
-                    initargs=(self,),
+                    initargs=(self, log_queue),
                 )
+                if max_workers > 1:
+                    # forward messages from log queue (in subprocesses) to all root handlers
+                    log_listener = logging.handlers.QueueListener(log_queue, *logging.root.handlers, respect_handler_level=True)
+                    log_listener.start()
                 try:
                     self._base_logger.debug("started executor %s with %d workers", str(executor), max_workers or 1)
                     tasks = self.process_workspace_submit_tasks(executor, max_seconds)
                     stats = self.process_workspace_handle_tasks(tasks)
                 finally:
                     executor.shutdown(kill_workers=True, wait=False)
+                    if max_workers > 1:
+                        log_listener.stop()
+                        del log_listener
 
             except NotImplementedError:
                 # fall back to deprecated method
@@ -1109,13 +1122,16 @@ in Processor.process_workspace. Forking allows inheriting global
 objects, and with the METS Server we do not mutate the local
 processor instance anyway.
 """
-def _page_worker_set_ctxt(processor):
+def _page_worker_set_ctxt(processor, log_queue):
     """
     Overwrites `ocrd.processor.base._page_worker_processor` instance
     for sharing with subprocesses in ProcessPoolExecutor initializer.
     """
     global _page_worker_processor
     _page_worker_processor = processor
+    if log_queue:
+        # replace all log handlers with just one queue handler
+        logging.root.handlers = [logging.handlers.QueueHandler(log_queue)]
 
 def _page_worker(timeout, *input_files):
     """

ocrd/processor/builtin/dummy/ocrd-tool.json

@@ -16,6 +16,26 @@
           "description": "Whether to actually copy files (true) or just create PAGE-XML as a side effect (false)"
         }
       }
+    },
+    "ocrd-filter": {
+      "executable": "ocrd-filter",
+      "description": "Bare-bones processor can be dynamically configured to remove segments based on XPath queries",
+      "steps": ["recognition/post-correction"],
+      "categories": ["Quality assurance"],
+      "input_file_grp_cardinality": 1,
+      "output_file_grp_cardinality": 1,
+      "parameters": {
+        "select": {
+          "type": "string",
+          "default": "//*[ends-with(local-name(),'Region')]",
+          "description": "Which segments to select for removal. An XPath 2.0 query expression (path and optional predicates), with 'pc' as namespace prefix for PAGE-XML and our extension functions (see help text). Only selection of segment hierarchy elements is allowed (so e.g. `*` would be equivalent to `pc:NoiseRegion|pc:LineDrawingRegion|pc:AdvertRegion|pc:ImageRegion|pc:ChartRegion|pc:MusicRegion|pc:GraphicRegion|pc:UnknownRegion|pc:CustomRegion|pc:SeparatorRegion|pc:MathsRegion|pc:TextRegion|pc:MapRegion|pc:ChemRegion|pc:TableRegion|pc:TextLine|pc:Word|pc:Glyph`, but `pc:MetadataItem` or `pc:Border` or `pc:Coords` would not match).\nFor example, to remove words or glyphs with low text confidence, select '(pc:Word|pc:Glyph)[pc:TextEquiv/@conf < 0.7]'. Or low layout confidence, '*[pc:Coords/@conf < 0.7]'.\nTo remove high pixel-to-character rate, select '*[pc:pixelarea(.) div string-length(pc:textequiv(.)) > 10000]'."
+        },
+        "plot": {
+          "type": "boolean",
+          "default": false,
+          "description": "Whether to extract an image for each filtered segment and write to the output fileGrp."
+        }
+      }
     }
   }
 }

ocrd/processor/builtin/dummy_processor.py

@@ -13,9 +13,6 @@ from ocrd_utils import (
     make_file_id,
     MIME_TO_EXT,
     MIMETYPE_PAGE,
-    parse_json_string_with_comments,
-    resource_string,
-    config
 )
 from ocrd_modelfactory import page_from_file
 

ocrd/processor/builtin/filter_processor.py

@@ -0,0 +1,108 @@
+# pylint: disable=missing-module-docstring,invalid-name
+from typing import Optional
+
+from lxml import etree
+import click
+
+from ocrd import Processor, OcrdPageResult, OcrdPageResultImage
+from ocrd.decorators import ocrd_cli_options, ocrd_cli_wrap_processor
+from ocrd_models import OcrdPage
+
+_SEGTYPES = [
+    "NoiseRegion",
+    "LineDrawingRegion",
+    "AdvertRegion",
+    "ImageRegion",
+    "ChartRegion",
+    "MusicRegion",
+    "GraphicRegion",
+    "UnknownRegion",
+    "CustomRegion",
+    "SeparatorRegion",
+    "MathsRegion",
+    "TextRegion",
+    "MapRegion",
+    "ChemRegion",
+    "TableRegion",
+    "TextLine",
+    "Word",
+    "Glyph"
+]
+
+class FilterProcessor(Processor):
+    def process_page_pcgts(self, *input_pcgts: Optional[OcrdPage], page_id: Optional[str] = None) -> OcrdPageResult:
+        """
+        Remove PAGE segment hierarchy elements based on flexible selection criteria.
+
+        Open and deserialise PAGE input file, then iterate over the segment hierarchy
+        down to the level required for ``select`` (which could be multiple levels at once).
+
+        Remove any segments matching XPath query ``select`` from that hierarchy (and from
+        the `ReadingOrder` if it is a region type).
+
+        \b
+        Besides full XPath 2.0 syntax, this supports extra predicates:
+        - `pc:pixelarea()` for the number of pixels of the bounding box (or sum area on node sets),
+        - `pc:textequiv()` for the first TextEquiv unicode string (or concatenated string on node sets).
+
+        If ``plot`` is `true`, then extract and write an image file for all removed segments
+        to the output fileGrp (without reference to the PAGE).
+
+        Produce a new PAGE output file by serialising the resulting hierarchy.
+        """
+        pcgts = input_pcgts[0]
+        result = OcrdPageResult(pcgts)
+        nodes = pcgts.xpath(self.parameter['select'])
+        # get PAGE objects from matching etree nodes
+        # but allow only hierarchy segments
+        segments = [segment for segment in map(pcgts.revmap.get, nodes)
+                    if segment.__class__.__name__.replace('Type', '') in _SEGTYPES]
+        if not(len(segments)):
+            self.logger.info("no matches")
+            return result
+        rodict = pcgts.get_Page().get_ReadingOrderGroups()
+        if self.parameter['plot']:
+            page_image, page_coords, _ = self.workspace.image_from_page(pcgts.get_Page(), page_id)
+        for segment in segments:
+            segtype = segment.original_tagname_
+            self.logger.info("matched %s segment %s", segtype, segment.id)
+            parent = segment.parent_object_
+            partype = parent.__class__.__name__.replace('Type', '')
+            if partype == 'Page':
+                getattr(parent, 'get_' + segtype)().remove(segment)
+            elif partype.endswith('Region'):
+                if segtype.endswith('Region'):
+                    getattr(parent, 'get_' + segtype)().remove(segment)
+                else:
+                    parent.TextLine.remove(segment)
+            elif partype == 'TextLine':
+                parent.Word.remove(segment)
+            elif partype == 'Word':
+                parent.Glyph.remove(segment)
+            else:
+                raise Exception(f"unexpected type ({partype}) of parent for matched segment ({segtype})")
+            segment.parent_object_ = None
+            if segtype.endswith('Region') and segment.id in rodict:
+                # remove from ReadingOrder as well
+                roelem = rodict[segment.id]
+                rorefs = getattr(roelem.parent_object_, roelem.__class__.__name__.replace('Type', ''))
+                rorefs.remove(roelem)
+                roelem.parent_object_ = None
+                del rodict[segment.id]
+            if self.parameter['plot']:
+                segment_image, _ = self.workspace.image_from_segment(segment, page_image, page_coords)
+                result.images.append(OcrdPageResultImage(segment_image, segment.id + '.IMG', None))
+        return result
+
+    @property
+    def metadata_filename(self):
+        return 'processor/builtin/dummy/ocrd-tool.json'
+
+    @property
+    def executable(self):
+        return 'ocrd-filter'
+
+@click.command()
+@ocrd_cli_options
+def cli(*args, **kwargs):
+    return ocrd_cli_wrap_processor(FilterProcessor, *args, **kwargs)

ocrd/resource_manager.py

@@ -23,6 +23,10 @@ yaml.constructor.SafeConstructor.yaml_constructors['tag:yaml.org,2002:timestamp'
 
 # pylint: enable=wrong-import-position
 
+# pylint: enable=wrong-import-position
+
+# pylint: enable=wrong-import-position
+
 from ocrd_validators import OcrdResourceListValidator
 from ocrd_utils import getLogger, directory_size, get_moduledir, guess_media_type, config
 from ocrd_utils.os import get_processor_resource_types, list_all_resources, pushd_popd, get_ocrd_tool_json

{ocrd-3.0.0b6.dist-info → ocrd-3.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.0.0b6
+Version: 3.0.1
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -17,6 +17,7 @@ Requires-Dist: click>=7
 Requires-Dist: cryptography<43.0.0
 Requires-Dist: Deprecated==1.2.0
 Requires-Dist: docker
+Requires-Dist: elementpath
 Requires-Dist: fastapi>=0.78.0
 Requires-Dist: filetype
 Requires-Dist: Flask