ocrd 3.7.0__py3-none-any.whl → 3.8.0__py3-none-any.whl

ocrd/resource_manager.py

@@ -1,3 +1,4 @@
+from logging import Logger
 from pathlib import Path
 from os.path import join
 from os import environ, listdir, getcwd, unlink
@@ -5,12 +6,14 @@ from shutil import copytree, rmtree, copy
 from fnmatch import filter as apply_glob
 from datetime import datetime
 from tarfile import open as open_tarfile
+from typing import Dict, Optional
 from urllib.parse import urlparse, unquote
 from zipfile import ZipFile
 
 import requests
 from gdown.parse_url import parse_url as gparse_url
 from gdown.download import get_url_from_gdrive_confirmation
+from git import Repo
 from yaml import safe_load, safe_dump
 
 # pylint: disable=wrong-import-position
@@ -29,8 +32,9 @@ yaml.constructor.SafeConstructor.yaml_constructors['tag:yaml.org,2002:timestamp'
 
 from ocrd_validators import OcrdResourceListValidator
 from ocrd_utils import getLogger, directory_size, get_moduledir, guess_media_type, config
-from ocrd_utils.os import get_processor_resource_types, list_all_resources, pushd_popd, get_ocrd_tool_json
-from .constants import RESOURCE_LIST_FILENAME, RESOURCE_USER_LIST_COMMENT
+from ocrd_utils.constants import RESOURCES_DIR_SYSTEM, RESOURCE_TYPES, MIME_TO_EXT
+from ocrd_utils.os import get_processor_resource_types, is_git_url, list_all_resources, pushd_popd, get_ocrd_tool_json
+from .constants import RESOURCE_USER_LIST_COMMENT
 
 
 class OcrdResourceManager:
@@ -47,14 +51,23 @@ class OcrdResourceManager:
         self._userdir = userdir
         self.user_list = Path(self.xdg_config_home, 'ocrd', 'resources.yml')
 
+        self.log.info(f"OcrdResourceManager data home path: {self.xdg_data_home}")
+        self.log.info(f"OcrdResourceManager config home path: {self.xdg_config_home}")
+        self.log.info(f"OcrdResourceManager user list path: {self.user_list}")
+
         if not skip_init:
-            self.load_resource_list(Path(RESOURCE_LIST_FILENAME))
             if not self.user_list.exists():
                 if not self.user_list.parent.exists():
                     self.user_list.parent.mkdir(parents=True)
                 self.save_user_list()
             self.load_resource_list(self.user_list)
 
+    def __repr__(self):
+        return f"user_list={str(self.user_list)} " + \
+               f"exists={self.user_list.exists()} " + \
+               f"database: {len(self.database)} executables " + \
+               f"{sum(map(len, self.database.values()))} resources"
+
     @property
     def userdir(self):
         if not self._userdir:
@@ -69,19 +82,22 @@ class OcrdResourceManager:
 
     @property
     def xdg_config_home(self):
-        if self._xdg_config_home:
-            return self._xdg_config_home
-        return config.XDG_CONFIG_HOME
+        if not self._xdg_config_home:
+            self._xdg_config_home = config.XDG_CONFIG_HOME
+        return self._xdg_config_home
 
     def save_user_list(self, database=None):
         if not database:
             database = self.database
+        self.log.info(f"Saving resources to path: {self.user_list}")
+        self._dedup_database()
         with open(self.user_list, 'w', encoding='utf-8') as f:
             f.write(RESOURCE_USER_LIST_COMMENT)
             f.write('\n')
             f.write(safe_dump(database))
 
-    def load_resource_list(self, list_filename, database=None):
+    def load_resource_list(self, list_filename: Path, database=None):
+        self.log.info(f"Loading resources from path: {list_filename}")
         if not database:
             database = self.database
         if list_filename.is_file():
@@ -98,30 +114,36 @@ class OcrdResourceManager:
                 database[executable] = list_loaded[executable] + database[executable]
         return database
 
-    def list_available(self, executable=None, dynamic=True, name=None, database=None, url=None):
+    def _search_executables(self, executable: Optional[str]):
+        skip_executables = ["ocrd-cis-data", "ocrd-import", "ocrd-make"]
+        for exec_dir in environ['PATH'].split(':'):
+            self.log.debug(f"Searching for executables inside path: {exec_dir}")
+            for exec_path in Path(exec_dir).glob(f'{executable}'):
+                if not exec_path.name.startswith('ocrd-'):
+                    self.log.warning(f"OCR-D processor executable '{exec_path}' has no 'ocrd-' prefix")
+                if exec_path.name in skip_executables:
+                    self.log.debug(f"Not an OCR-D processor CLI, skipping '{exec_path}'")
+                    continue
+                self.log.debug(f"Inspecting '{exec_path} --dump-json' for resources")
+                ocrd_tool = get_ocrd_tool_json(exec_path)
+                for res_dict in ocrd_tool.get('resources', ()):
+                    if exec_path.name not in self.database:
+                        self.database[exec_path.name] = []
+                    self.database[exec_path.name].insert(0, res_dict)
+
+    def list_available(
+        self, executable: str = None, dynamic: bool = True, name: str = None, database: Dict = None, url: str = None
+    ):
         """
         List models available for download by processor
         """
         if not database:
             database = self.database
         if not executable:
-            return database.items()
+            return list(database.items())
         if dynamic:
-            skip_executables = ["ocrd-cis-data", "ocrd-import", "ocrd-make"]
-            for exec_dir in environ['PATH'].split(':'):
-                for exec_path in Path(exec_dir).glob(f'{executable}'):
-                    if not exec_path.name.startswith('ocrd-'):
-                        self.log.warning(f"OCR-D processor executable '{exec_path}' has no 'ocrd-' prefix")
-                    if exec_path.name in skip_executables:
-                        self.log.debug(f"Not an OCR-D processor CLI, skipping '{exec_path}'")
-                        continue
-                    self.log.debug(f"Inspecting '{exec_path} --dump-json' for resources")
-                    ocrd_tool = get_ocrd_tool_json(exec_path)
-                    for resdict in ocrd_tool.get('resources', ()):
-                        if exec_path.name not in database:
-                            database[exec_path.name] = []
-                        database[exec_path.name].insert(0, resdict)
-            database = self._dedup_database(database)
+            self._search_executables(executable)
+            self.save_user_list()
         found = False
         ret = []
         for k in database:
@@ -139,7 +161,7 @@ class OcrdResourceManager:
             ret = [(executable, [])]
         return ret
 
-    def list_installed(self, executable=None):
+    def list_installed(self, executable: str = None):
         """
         List installed resources, matching with registry by ``name``
         """
@@ -150,28 +172,24 @@ class OcrdResourceManager:
             # resources we know about
             all_executables = list(self.database.keys())
             # resources in the file system
-            parent_dirs = [join(x, 'ocrd-resources') for x in [self.xdg_data_home, '/usr/local/share']]
+            parent_dirs = [f"{join(self.xdg_data_home, 'ocrd-resources')}", RESOURCES_DIR_SYSTEM]
             for parent_dir in parent_dirs:
                 if Path(parent_dir).exists():
                     all_executables += [x for x in listdir(parent_dir) if x.startswith('ocrd-')]
         for this_executable in set(all_executables):
             reslist = []
-            mimetypes = get_processor_resource_types(this_executable)
             moduledir = get_moduledir(this_executable)
-            for res_filename in list_all_resources(this_executable, moduled=moduledir, xdg_data_home=self.xdg_data_home):
-                res_filename = Path(res_filename)
-                if not '*/*' in mimetypes:
-                    if res_filename.is_dir() and not 'text/directory' in mimetypes:
-                        continue
-                    if res_filename.is_file() and ['text/directory'] == mimetypes:
-                        continue
+            resdict_list = self.list_available(executable=this_executable)[0][1]
+            for res_filename in list_all_resources(this_executable,
+                                                   moduled=moduledir,
+                                                   xdg_data_home=self.xdg_data_home):
+                res_filename = Path(res_filename).resolve()
                 res_name = res_filename.name
                 res_type = 'file' if res_filename.is_file() else 'directory'
                 res_size = res_filename.stat().st_size if res_filename.is_file() else directory_size(res_filename)
-                resdict_list = [x for x in self.database.get(this_executable, []) if x['name'] == res_name]
-                if resdict_list:
-                    resdict = resdict_list[0]
-                elif str(res_filename.parent) == moduledir:
+                if resdict := next((res for res in resdict_list if res['name'] == res_name), False):
+                    pass
+                elif str(res_filename.parent).startswith(moduledir):
                     resdict = {
                         'name': res_name, 
                         'url': str(res_filename), 
@@ -181,28 +199,28 @@ class OcrdResourceManager:
                     }
                 else:
                     resdict = self.add_to_user_database(this_executable, res_filename, resource_type=res_type)
-                resdict['path'] = str(res_filename)
+                # resdict['path'] = str(res_filename)
                 reslist.append(resdict)
             ret.append((this_executable, reslist))
+        self.save_user_list()
         return ret
 
     def add_to_user_database(self, executable, res_filename, url=None, resource_type='file'):
         """
         Add a stub entry to the user resource.yml
         """
-        res_name = Path(res_filename).name
-        self.log.info(f"{executable} resource '{res_name}' ({str(res_filename)}) not a known resource, "
-                      f"creating stub in {self.user_list}'")
+        res_name = res_filename.name
         if Path(res_filename).is_dir():
             res_size = directory_size(res_filename)
         else:
             res_size = Path(res_filename).stat().st_size
-        with open(self.user_list, 'r', encoding='utf-8') as f:
-            user_database = safe_load(f) or {}
+        user_database = self.load_resource_list(self.user_list)
         if executable not in user_database:
             user_database[executable] = []
         resources_found = self.list_available(executable=executable, name=res_name, database=user_database)[0][1]
         if not resources_found:
+            self.log.info(f"{executable} resource '{res_name}' ({str(res_filename)}) not a known resource, "
+                          f"creating stub in {self.user_list}'")
             resdict = {
                 'name': res_name,
                 'url': url if url else '???',
@@ -222,20 +240,45 @@ class OcrdResourceManager:
     def default_resource_dir(self):
         return self.location_to_resource_dir('data')
 
-    def location_to_resource_dir(self, location):
-        return '/usr/local/share/ocrd-resources' if location == 'system' else \
-                join(self.xdg_data_home, 'ocrd-resources') if location == 'data' else \
-                getcwd()
+    def location_to_resource_dir(self, location: str) -> str:
+        if location == 'data':
+            return join(self.xdg_data_home, 'ocrd-resources')
+        if location == 'system':
+            return RESOURCES_DIR_SYSTEM
+        return getcwd()
 
-    def resource_dir_to_location(self, resource_path):
+    def resource_dir_to_location(self, resource_path: Path) -> str:
         resource_path = str(resource_path)
-        return 'system' if resource_path.startswith('/usr/local/share/ocrd-resources') else \
-               'data' if resource_path.startswith(join(self.xdg_data_home, 'ocrd-resources')) else \
-               'cwd' if resource_path.startswith(getcwd()) else \
-               resource_path
+        if resource_path.startswith(RESOURCES_DIR_SYSTEM):
+            return 'system'
+        if resource_path.startswith(join(self.xdg_data_home, 'ocrd-resources')):
+            return 'data'
+        if resource_path.startswith(getcwd()):
+            return 'cwd'
+        return resource_path
+
+    def build_resource_dest_dir(self, location: str, executable: str) -> Path:
+        if location == 'module':
+            base_dir = get_moduledir(executable)
+            if not base_dir:
+                base_dir = self.location_to_resource_dir('data')
+        else:
+            base_dir = self.location_to_resource_dir(location)
+        no_subdir = location in ['cwd', 'module']
+        dest_dir = Path(base_dir) if no_subdir else Path(base_dir, executable)
+        return dest_dir
+
+    @staticmethod
+    def remove_resource(log: Logger, resource_path: Path):
+        if resource_path.is_dir():
+            log.info(f"Removing existing target resource directory {resource_path}")
+            rmtree(str(resource_path))
+        else:
+            log.info(f"Removing existing target resource file {resource_path}")
+            unlink(str(resource_path))
 
     @staticmethod
-    def parameter_usage(name, usage='as-is'):
+    def parameter_usage(name: str, usage: str = 'as-is') -> str:
         if usage == 'as-is':
             return name
         elif usage == 'without-extension':
@@ -243,8 +286,7 @@ class OcrdResourceManager:
         raise ValueError(f"No such usage '{usage}'")
 
     @staticmethod
-    def _download_impl(url, filename, progress_cb=None, size=None):
-        log = getLogger('ocrd.resource_manager._download_impl')
+    def _download_impl(log: Logger, url: str, filename):
         log.info(f"Downloading {url} to {filename}")
         try:
             gdrive_file_id, is_gdrive_download_link = gparse_url(url, warning=False)
@@ -256,36 +298,36 @@ class OcrdResourceManager:
                         if "Content-Disposition" not in r.headers:
                             url = get_url_from_gdrive_confirmation(r.text)
                 except RuntimeError as e:
-                    log.warning("Cannot unwrap Google Drive URL: %s", e)
-            with open(filename, 'wb') as f:
-                with requests.get(url, stream=True) as r:
-                    r.raise_for_status()
-                    for data in r.iter_content(chunk_size=4096):
-                        if progress_cb:
-                            progress_cb(len(data))
-                        f.write(data)
+                    log.warning(f"Cannot unwrap Google Drive URL: {e}")
+            if is_git_url(url):
+                log.info("Cloning a git repository")
+                repo = Repo.clone_from(url, filename, depth=1)
+                # keep only the checkout
+                rmtree(join(filename, '.git'))
+            else:
+                with open(filename, 'wb') as f:
+                    with requests.get(url, stream=True) as r:
+                        r.raise_for_status()
+                        for data in r.iter_content(chunk_size=4096):
+                            f.write(data)
         except Exception as e:
             rmtree(filename, ignore_errors=True)
             Path(filename).unlink(missing_ok=True)
             raise e
 
     @staticmethod
-    def _copy_file(src, dst, progress_cb=None):
-        log = getLogger('ocrd.resource_manager._copy_file')
+    def _copy_file(log: Logger, src, dst):
         log.info(f"Copying file {src} to {dst}")
         with open(dst, 'wb') as f_out, open(src, 'rb') as f_in:
             while True:
                 chunk = f_in.read(4096)
                 if chunk:
                     f_out.write(chunk)
-                    if progress_cb:
-                        progress_cb(len(chunk))
                 else:
                     break
 
     @staticmethod
-    def _copy_dir(src, dst, progress_cb=None):
-        log = getLogger('ocrd.resource_manager._copy_dir')
+    def _copy_dir(log: Logger, src, dst):
         log.info(f"Copying dir recursively from {src} to {dst}")
         if not Path(src).is_dir():
             raise ValueError(f"The source is not a directory: {src}")
@@ -293,76 +335,123 @@ class OcrdResourceManager:
         for child in Path(src).rglob('*'):
             child_dst = Path(dst) / child.relative_to(src)
             if Path(child).is_dir():
-                OcrdResourceManager._copy_dir(child, child_dst, progress_cb)
+                OcrdResourceManager._copy_dir(log, child, child_dst)
             else:
-                OcrdResourceManager._copy_file(child, child_dst, progress_cb)
+                OcrdResourceManager._copy_file(log, child, child_dst)
 
     @staticmethod
-    def _copy_impl(src_filename, filename, progress_cb=None):
-        log = getLogger('ocrd.resource_manager._copy_impl')
+    def _copy_impl(log: Logger, src_filename, filename):
         log.info(f"Copying {src_filename} to {filename}")
         if Path(src_filename).is_dir():
-            OcrdResourceManager._copy_dir(src_filename, filename, progress_cb)
+            OcrdResourceManager._copy_dir(log, src_filename, filename)
         else:
-            OcrdResourceManager._copy_file(src_filename, filename, progress_cb)
+            OcrdResourceManager._copy_file(log, src_filename, filename)
+
+    @staticmethod
+    def _extract_archive(log: Logger, tempdir: Path, path_in_archive: str, fpath: Path, archive_fname: str):
+        Path('out').mkdir()
+        with pushd_popd('out'):
+            mimetype = guess_media_type(f'../{archive_fname}', fallback='application/octet-stream')
+            log.info(f"Extracting {mimetype} archive to {tempdir}/out")
+            if mimetype == 'application/zip':
+                with ZipFile(f'../{archive_fname}', 'r') as zipf:
+                    zipf.extractall()
+            elif mimetype in ('application/gzip', 'application/x-xz'):
+                with open_tarfile(f'../{archive_fname}', 'r:*') as tar:
+                    tar.extractall()
+            else:
+                raise RuntimeError(f"Unable to handle extraction of {mimetype} archive")
+            log.info(f"Copying '{path_in_archive}' from archive to {fpath}")
+            if Path(path_in_archive).is_dir():
+                copytree(path_in_archive, str(fpath))
+            else:
+                copy(path_in_archive, str(fpath))
+
+    def copy_resource(
+        self, log: Logger, url: str, fpath: Path, resource_type: str = 'file', path_in_archive: str = '.'
+    ) -> Path:
+        """
+        Copy a local resource to another destination
+        """
+        if resource_type == 'archive':
+            archive_fname = 'download.tar.xx'
+            with pushd_popd(tempdir=True) as tempdir:
+                self._copy_impl(log, url, archive_fname)
+                self._extract_archive(log, tempdir, path_in_archive, fpath, archive_fname)
+        else:
+            self._copy_impl(log, url, fpath)
+        return fpath
+
+    def download_resource(
+        self, log: Logger, url: str, fpath: Path, resource_type: str = 'file', path_in_archive: str = '.'
+    ) -> Path:
+        """
+        Download a resource by URL to a destination directory
+        """
+        if resource_type == 'archive':
+            archive_fname = 'download.tar.xx'
+            with pushd_popd(tempdir=True) as tempdir:
+                self._download_impl(log, url, archive_fname)
+                self._extract_archive(log, tempdir, path_in_archive, fpath, archive_fname)
+        else:
+            self._download_impl(log, url, fpath)
+        return fpath
 
     # TODO Proper caching (make head request for size, If-Modified etc)
-    def download(
-        self, executable, url, basedir, overwrite=False, no_subdir=False, name=None, resource_type='file',
-        path_in_archive='.', progress_cb=None,
-    ):
+    def handle_resource(
+        self, res_dict: Dict, executable: str, dest_dir: Path, any_url: str, overwrite: bool = False,
+        resource_type: str = 'file', path_in_archive: str = '.'
+    ) -> Optional[Path]:
         """
-        Download a resource by URL
+        Download or Copy a resource by URL to a destination directory
         """
-        log = getLogger('ocrd.resource_manager.download')
-        destdir = Path(basedir) if no_subdir else Path(basedir, executable)
-        if not name:
-            url_parsed = urlparse(url)
-            name = Path(unquote(url_parsed.path)).name
-        fpath = Path(destdir, name)
-        is_url = url.startswith('https://') or url.startswith('http://')
+        log = getLogger('ocrd.resource_manager.handle_resource')
+        registered = "registered" if "size" in res_dict else "unregistered"
+        resource_type = res_dict.get('type', resource_type)
+        resource_name = res_dict.get('name', None)
+        path_in_archive = res_dict.get('path_in_archive', path_in_archive)
+
+        if resource_type not in RESOURCE_TYPES:
+            raise ValueError(f"Unknown resource type: {resource_type}, must be one of: {RESOURCE_TYPES}")
+        if any_url:
+            res_dict['url'] = any_url
+        if not resource_name:
+            url_parsed = urlparse(res_dict['url'])
+            resource_name = Path(unquote(url_parsed.path)).name
+            if resource_type == 'archive' and path_in_archive != '.':
+                resource_name = Path(path_in_archive).name
+        if res_dict['url'] == '???':
+            log.warning(f"Skipping user resource {resource_name} since download url is: {res_dict['url']}")
+            return None
+
+        fpath = Path(dest_dir, resource_name)
         if fpath.exists():
             if not overwrite:
                 fpath_type = 'Directory' if fpath.is_dir() else 'File'
                 log.warning(f"{fpath_type} {fpath} already exists but --overwrite is not set, skipping the download")
                 # raise FileExistsError(f"{fpath_type} {fpath} already exists but --overwrite is not set")
                 return fpath
-            if fpath.is_dir():
-                log.info(f"Removing existing target directory {fpath}")
-                rmtree(str(fpath))
-            else:
-                log.info(f"Removing existing target file {fpath}")
-                unlink(str(fpath))
-        destdir.mkdir(parents=True, exist_ok=True)
-        if resource_type in ('file', 'directory'):
-            if is_url:
-                self._download_impl(url, fpath, progress_cb)
-            else:
-                self._copy_impl(url, fpath, progress_cb)
-        elif resource_type == 'archive':
-            archive_fname = 'download.tar.xx'
-            with pushd_popd(tempdir=True) as tempdir:
-                if is_url:
-                    self._download_impl(url, archive_fname, progress_cb)
-                else:
-                    self._copy_impl(url, archive_fname, progress_cb)
-                Path('out').mkdir()
-                with pushd_popd('out'):
-                    mimetype = guess_media_type(f'../{archive_fname}', fallback='application/octet-stream')
-                    log.info(f"Extracting {mimetype} archive to {tempdir}/out")
-                    if mimetype == 'application/zip':
-                        with ZipFile(f'../{archive_fname}', 'r') as zipf:
-                            zipf.extractall()
-                    elif mimetype in ('application/gzip', 'application/x-xz'):
-                        with open_tarfile(f'../{archive_fname}', 'r:*') as tar:
-                            tar.extractall()
-                    else:
-                        raise RuntimeError(f"Unable to handle extraction of {mimetype} archive {url}")
-                    log.info(f"Copying '{path_in_archive}' from archive to {fpath}")
-                    if Path(path_in_archive).is_dir():
-                        copytree(path_in_archive, str(fpath))
-                    else:
-                        copy(path_in_archive, str(fpath))
+            self.remove_resource(log, resource_path=fpath)
+        dest_dir.mkdir(parents=True, exist_ok=True)
+
+        # TODO @mehmedGIT: Consider properly handling cases for invalid URLs.
+        if res_dict['url'].startswith('https://') or res_dict['url'].startswith('http://'):
+            log.info(f"Downloading {registered} resource '{resource_name}' ({res_dict['url']})")
+            if 'size' not in res_dict:
+                with requests.head(res_dict['url']) as r:
+                    res_dict['size'] = int(r.headers.get('content-length', 0))
+            fpath = self.download_resource(log, res_dict['url'], fpath, resource_type, path_in_archive)
+        else:
+            log.info(f"Copying {registered} resource '{resource_name}' ({res_dict['url']})")
+            urlpath = Path(res_dict['url'])
+            res_dict['url'] = str(urlpath.resolve())
+            res_dict['size'] = directory_size(urlpath) if Path(urlpath).is_dir() else urlpath.stat().st_size
+            fpath = self.copy_resource(log, res_dict['url'], fpath, resource_type, path_in_archive)
+
+        if registered == 'unregistered':
+            self.add_to_user_database(executable, fpath, url=res_dict['url'])
+        self.save_user_list()
+        log.info(f"Installed resource {res_dict['url']} under {fpath}")
         return fpath
 
     def _dedup_database(self, database=None, dedup_key='name'):

{ocrd-3.7.0.dist-info → ocrd-3.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ocrd
-Version: 3.7.0
+Version: 3.8.0
 Summary: OCR-D framework
 Author-email: Konstantin Baierer <unixprog@gmail.com>
 License: Apache License 2.0
@@ -16,12 +16,13 @@ Requires-Dist: beanie~=1.7
 Requires-Dist: click>=7
 Requires-Dist: cryptography<43.0.0
 Requires-Dist: Deprecated==1.2.0
-Requires-Dist: docker
+Requires-Dist: docker>=7.1.0
 Requires-Dist: elementpath
 Requires-Dist: fastapi>=0.78.0
 Requires-Dist: filetype
 Requires-Dist: Flask
 Requires-Dist: frozendict>=2.4.0
+Requires-Dist: gitpython
 Requires-Dist: gdown
 Requires-Dist: httpx>=0.22.0
 Requires-Dist: jsonschema>=4
@@ -68,6 +69,9 @@ Requires-Dist: shapely>=2; python_version >= "3.9"
 * [Command line tools](#command-line-tools)
 	* [`ocrd` CLI](#ocrd-cli)
 	* [`ocrd-dummy` CLI](#ocrd-dummy-cli)
+	* [`ocrd-filter` CLI](#ocrd-filter-cli)
+	* [`ocrd-command` CLI](#ocrd-command-cli)
+	* [`ocrd-merge` CLI](#ocrd-merge-cli)
 * [Configuration](#configuration)
 * [Packages](#packages)
 	* [ocrd_utils](#ocrd_utils)
@@ -76,7 +80,6 @@ Requires-Dist: shapely>=2; python_version >= "3.9"
 	* [ocrd_validators](#ocrd_validators)
 	* [ocrd_network](#ocrd_network)
 	* [ocrd](#ocrd)
-* [bash library](#bash-library)
 * [Testing](#testing)
 * [See Also](#see-also)
 
@@ -121,6 +124,22 @@ supported flags, options and arguments.
 
 A minimal [OCR-D processor](https://ocr-d.de/en/user_guide#using-the-ocr-d-processors) that copies from `-I/-input-file-grp` to `-O/-output-file-grp`
 
+### `ocrd-filter` CLI
+
+A simple [OCR-D processor](https://ocr-d.de/en/user_guide#using-the-ocr-d-processors) that removes segments in PAGE-XML files from `-I/-input-file-grp` to `-O/-output-file-grp` with arbitrary selection based on powerful XPath 2.0 expressions.
+
+### `ocrd-command` CLI
+
+A simple [OCR-D processor](https://ocr-d.de/en/user_guide#using-the-ocr-d-processors) that runs arbitrary shell commands to transform PAGE-XML files from `-I/-input-file-grp` to `-O/-output-file-grp` (in effect "wrapping" them for OCR-D).
+
+### `ocrd-merge` CLI
+
+A simple [OCR-D processor](https://ocr-d.de/en/user_guide#using-the-ocr-d-processors) that (for every page) joins PAGE-XML files from multiple `-I/-input-file-grp` into a single `-O/-output-file-grp`, ensuring that
+- `Border` polygons are joined
+- all regions are concatenated, while
+- ensuring segment identifiers do not clash,
+- and the reading order simply gets concatenated.
+
 ## Configuration
 
 Almost all behaviour of the OCR-D/core software is configured via CLI options and flags, which can be listed with the `--help` flag that all CLI support.