ocrd 3.1.1__py3-none-any.whl → 3.2.0__py3-none-any.whl

ocrd_utils/os.py

@@ -8,6 +8,7 @@ __all__ = [
     'get_ocrd_tool_json',
     'get_moduledir',
     'get_processor_resource_types',
+    'get_env_locations',
     'guess_media_type',
     'pushd_popd',
     'unzip_file_to_dir',
@@ -15,28 +16,30 @@ __all__ = [
     'redirect_stderr_and_stdout_to_file',
 ]
 
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 from tempfile import TemporaryDirectory, gettempdir
 from functools import lru_cache
 from contextlib import contextmanager, redirect_stderr, redirect_stdout
 from shutil import which
 from json import loads
 from json.decoder import JSONDecodeError
-from os import getcwd, chdir, stat, chmod, umask, environ
+from os import getcwd, chdir, stat, chmod, umask, environ, PathLike
 from pathlib import Path
 from os.path import abspath as abspath_, join
 from zipfile import ZipFile
 from subprocess import run, PIPE
 from mimetypes import guess_type as mimetypes_guess
 from filetype import guess as filetype_guess
+from fnmatch import filter as apply_glob
 
 from atomicwrites import atomic_write as atomic_write_, AtomicWriter
 
-from .constants import EXT_TO_MIME
+from .constants import EXT_TO_MIME, MIME_TO_EXT, RESOURCE_LOCATIONS, RESOURCES_DIR_SYSTEM
 from .config import config
 from .logging import getLogger
 from .introspect import resource_string
 
-def abspath(url):
+def abspath(url : str) -> str:
     """
     Get a full path to a file or file URL
 
@@ -47,7 +50,7 @@ def abspath(url):
     return abspath_(url)
 
 @contextmanager
-def pushd_popd(newcwd=None, tempdir=False):
+def pushd_popd(newcwd : Union[str, PathLike] = None, tempdir : bool = False) -> Iterator[PathLike]:
     if newcwd and tempdir:
         raise Exception("pushd_popd can accept either newcwd or tempdir, not both")
     try:
@@ -67,7 +70,7 @@ def pushd_popd(newcwd=None, tempdir=False):
     finally:
         chdir(oldcwd)
 
-def unzip_file_to_dir(path_to_zip, output_directory):
+def unzip_file_to_dir(path_to_zip : Union[str, PathLike], output_directory : str) -> None:
     """
     Extract a ZIP archive to a directory
     """
@@ -75,7 +78,7 @@ def unzip_file_to_dir(path_to_zip, output_directory):
         z.extractall(output_directory)
 
 @lru_cache()
-def get_ocrd_tool_json(executable):
+def get_ocrd_tool_json(executable : str) -> Dict[str, Any]:
     """
     Get the ``ocrd-tool`` description of ``executable``.
     """
@@ -90,11 +93,11 @@ def get_ocrd_tool_json(executable):
         except (JSONDecodeError, OSError) as e:
             getLogger('ocrd.utils.get_ocrd_tool_json').error(f'{executable} --dump-json produced invalid JSON: {e}')
     if 'resource_locations' not in ocrd_tool:
-        ocrd_tool['resource_locations'] = ['data', 'cwd', 'system', 'module']
+        ocrd_tool['resource_locations'] = RESOURCE_LOCATIONS
     return ocrd_tool
 
 @lru_cache()
-def get_moduledir(executable):
+def get_moduledir(executable : str) -> str:
     moduledir = None
     try:
         ocrd_all_moduledir = loads(resource_string('ocrd', 'ocrd-all-module-dir.json'))
@@ -106,57 +109,80 @@ def get_moduledir(executable):
             getLogger('ocrd.utils.get_moduledir').error(f'{executable} --dump-module-dir failed: {e}')
     return moduledir
 
-def list_resource_candidates(executable, fname, cwd=getcwd(), moduled=None, xdg_data_home=None):
+def get_env_locations(executable: str) -> List[str]:
+    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
+    if processor_path_var in environ:
+        return environ[processor_path_var].split(':')
+    return []
+
+def list_resource_candidates(executable : str, fname : str, cwd : Optional[str] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
     """
     Generate candidates for processor resources according to
     https://ocr-d.de/en/spec/ocrd_tool#file-parameters
     """
+    if cwd is None:
+        cwd = getcwd()
     candidates = []
     candidates.append(join(cwd, fname))
-    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
-    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
-    if processor_path_var in environ:
-        candidates += [join(x, fname) for x in environ[processor_path_var].split(':')]
+    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
+    for processor_path in get_env_locations(executable):
+        candidates.append(join(processor_path, fname))
     candidates.append(join(xdg_data_home, 'ocrd-resources', executable, fname))
-    candidates.append(join('/usr/local/share/ocrd-resources', executable, fname))
+    candidates.append(join(RESOURCES_DIR_SYSTEM, executable, fname))
     if moduled:
         candidates.append(join(moduled, fname))
     return candidates
 
-def list_all_resources(executable, moduled=None, xdg_data_home=None):
+def list_all_resources(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
     """
     List all processor resources in the filesystem according to
-    https://ocr-d.de/en/spec/ocrd_tool#file-parameters
+    https://ocr-d.de/en/spec/ocrd_tool#resource-parameters
     """
-    candidates = []
+    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
+    if ocrd_tool is None:
+        ocrd_tool = get_ocrd_tool_json(executable)
+    # processor we're looking for might not be installed, hence the fallbacks
     try:
-        resource_locations = get_ocrd_tool_json(executable)['resource_locations']
-    except FileNotFoundError:
-        # processor we're looking for resource_locations of is not installed.
+        mimetypes = get_processor_resource_types(executable, ocrd_tool=ocrd_tool)
+    except KeyError:
+        mimetypes = ['*/*']
+    try:
+        resource_locations = ocrd_tool['resource_locations']
+    except KeyError:
         # Assume the default
-        resource_locations = ['data', 'cwd', 'system', 'module']
-    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
-    # XXX cwd would list too many false positives
+        resource_locations = RESOURCE_LOCATIONS
+    try:
+        # fixme: if resources_list contains directories, their "suffix" will interfere
+        # (e.g. dirname without dot means we falsely match files without suffix)
+        resource_suffixes = [Path(res['name']).suffix
+                             for res in ocrd_tool['resources']]
+    except KeyError:
+        resource_suffixes = []
+    logger = getLogger('ocrd.utils.list_all_resources')
+    candidates = []
+    # cwd would list too many false positives:
     # if 'cwd' in resource_locations:
-    #     cwd_candidate = join(getcwd(), 'ocrd-resources', executable)
-    #     if Path(cwd_candidate).exists():
-    #         candidates.append(cwd_candidate)
-    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
-    if processor_path_var in environ:
-        for processor_path in environ[processor_path_var].split(':'):
-            if Path(processor_path).is_dir():
-                candidates += Path(processor_path).iterdir()
+    #     cwddir = Path.cwd()
+    #     candidates.append(cwddir.itertree())
+    # but we do not use this anyway:
+    # relative paths are tried w.r.t. CWD
+    # prior to list_all_resources resolution.
+    for processor_path in get_env_locations(executable):
+        processor_path = Path(processor_path)
+        if processor_path.is_dir():
+            candidates += processor_path.iterdir()
     if 'data' in resource_locations:
         datadir = Path(xdg_data_home, 'ocrd-resources', executable)
         if datadir.is_dir():
             candidates += datadir.iterdir()
     if 'system' in resource_locations:
-        systemdir = Path('/usr/local/share/ocrd-resources', executable)
+        systemdir = Path(RESOURCES_DIR_SYSTEM, executable)
         if systemdir.is_dir():
             candidates += systemdir.iterdir()
     if 'module' in resource_locations and moduled:
         # recurse fully
-        for resource in itertree(Path(moduled)):
+        moduled = Path(moduled)
+        for resource in moduled.iterdir():
             if resource.is_dir():
                 continue
             if any(resource.match(pattern) for pattern in
@@ -164,17 +190,66 @@ def list_all_resources(executable, moduled=None, xdg_data_home=None):
                    # code and data; `is_resource()` only singles out
                    # files over directories; but we want data files only
                    # todo: more code and cache exclusion patterns!
-                   ['*.py', '*.py[cod]', '*~', 'ocrd-tool.json', 
+                   ['*.py', '*.py[cod]', '*~', '.*.swp', '*.swo',
+                    '__pycache__/*', '*.egg-info/*', '*.egg',
+                    'copyright.txt', 'LICENSE*', 'README.md', 'MANIFEST',
+                    'TAGS', '.DS_Store',
+                    # C extensions
+                    '*.so',
+                    # translations
+                    '*.mo', '*.pot',
+                    '*.log', '*.orig', '*.BAK',
+                    '.git/*',
+                    # our stuff
+                    'ocrd-tool.json',
                     'environment.pickle', 'resource_list.yml', 'lib.bash']):
+                logger.debug("ignoring module candidate '%s'", resource)
                 continue
             candidates.append(resource)
-    # recurse once
-    for parent in candidates:
-        if parent.is_dir() and parent.name != '.git':
-            candidates += parent.iterdir()
-    return sorted([str(x) for x in candidates])
+    if mimetypes != ['*/*']:
+        logger.debug("matching candidates for %s by content-type %s", executable, str(mimetypes))
+    def valid_resource_type(path):
+        if '*/*' in mimetypes:
+            return True
+        if path.is_dir():
+            if not 'text/directory' in mimetypes:
+                logger.debug("ignoring directory candidate '%s'", path)
+                return False
+            if path.name in ['.git']:
+                logger.debug("ignoring directory candidate '%s'", path)
+                return False
+            return True
+        if not path.is_file():
+            logger.warning("ignoring non-file, non-directory candidate '%s'", path)
+            return False
+        res_mimetype = guess_media_type(path, fallback='')
+        if res_mimetype == 'application/json':
+            # always accept, regardless of configured mimetypes:
+            # needed for distributing or sharing parameter preset files
+            return True
+        if ['text/directory'] == mimetypes:
+            logger.debug("ignoring non-directory candidate '%s'", path)
+            return False
+        if 'application/octet-stream' in mimetypes:
+            # catch-all type - do not enforce anything
+            return True
+        if path.suffix in resource_suffixes:
+            return True
+        if any(path.suffix == MIME_TO_EXT.get(mime, None)
+               for mime in mimetypes):
+            return True
+        if not res_mimetype:
+            logger.warning("cannot determine content type of candidate '%s'", path)
+            return True
+        if any(apply_glob([res_mimetype], mime)
+               for mime in mimetypes):
+            return True
+        logger.debug("ignoring %s candidate '%s'", res_mimetype, path)
+        return False
+    candidates = sorted(filter(valid_resource_type, candidates))
+    return map(str, candidates)
 
-def get_processor_resource_types(executable, ocrd_tool=None):
+def get_processor_resource_types(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None) -> List[str]:
     """
     Determine what type of resource parameters a processor needs.
 
@@ -186,13 +261,16 @@ def get_processor_resource_types(executable, ocrd_tool=None):
         if not which(executable):
             return ['*/*']
         ocrd_tool = get_ocrd_tool_json(executable)
-    if not next((True for p in ocrd_tool.get('parameters', {}).values() if 'content-type' in p), False):
-        # None of the parameters for this processor are resources (or not
-        # the resource parameters are not properly declared, so output both
-        # directories and files
+    mime_types = [mime
+                  for param in ocrd_tool.get('parameters', {}).values()
+                  if param['type'] == 'string' and param.get('format', '') == 'uri' and 'content-type' in param
+                  for mime in param['content-type'].split(',')]
+    if not len(mime_types):
+        # None of the parameters for this processor are resources
+        # (or the parameters' resource types are not properly declared,)
+        # so output both directories and files
         return ['*/*']
-    return [p['content-type'] for p in ocrd_tool['parameters'].values()
-            if 'content-type' in p]
+    return mime_types
 
 # ht @pabs3
 # https://github.com/untitaker/python-atomicwrites/issues/42
@@ -211,12 +289,12 @@ class AtomicWriterPerms(AtomicWriter):
         return f
 
 @contextmanager
-def atomic_write(fpath):
+def atomic_write(fpath : str) -> Iterator[str]:
     with atomic_write_(fpath, writer_cls=AtomicWriterPerms, overwrite=True) as f:
         yield f
 
 
-def is_file_in_directory(directory, file):
+def is_file_in_directory(directory : Union[str, PathLike], file : Union[str, PathLike]) -> bool:
     """
     Return True if ``file`` is in ``directory`` (by checking that all components of ``directory`` are in ``file.parts``)
     """
@@ -224,7 +302,7 @@ def is_file_in_directory(directory, file):
     file = Path(file)
     return list(file.parts)[:len(directory.parts)] == list(directory.parts)
 
-def itertree(path):
+def itertree(path : Union[str, PathLike]) -> PathLike:
     """
     Generate a list of paths by recursively enumerating ``path``
     """
@@ -235,14 +313,14 @@ def itertree(path):
             yield from itertree(subpath)
     yield path
 
-def directory_size(path):
+def directory_size(path : Union[str, PathLike]) -> int:
     """
     Calculates size of all files in directory ``path``
     """
     path = Path(path)
     return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file())
 
-def guess_media_type(input_file : str, fallback : str = None, application_xml : str = 'application/xml'):
+def guess_media_type(input_file : str, fallback : Optional[str] = None, application_xml : str = 'application/xml') -> str:
     """
     Guess the media type of a file path
     """
@@ -254,7 +332,7 @@ def guess_media_type(input_file : str, fallback : str = None, application_xml :
     if mimetype is None:
         mimetype = EXT_TO_MIME.get(''.join(Path(input_file).suffixes), fallback)
     if mimetype is None:
-        raise ValueError("Could not determine MIME type of input_file must")
+        raise ValueError("Could not determine MIME type of input_file '%s'", str(input_file))
     if mimetype == 'application/xml':
         mimetype = application_xml
     return mimetype

ocrd_validators/ocrd_tool.schema.yml

@@ -142,18 +142,21 @@ properties:
                     description: List the allowed values if a fixed list.
                   content-type:
                     type: string
-                    default: 'application/octet-stream'
                     description: >
+                      If parameter is reference to file (type=string format=uri):
                       The media type of resources this processor expects for
                       this parameter. Most processors use files for resources
                       (e.g.  `*.traineddata` for `ocrd-tesserocr-recognize`)
                       while others use directories of files (e.g. `default` for
-                      `ocrd-eynollah-segment`).  If a parameter requires
-                      directories, it must set `content-type` to
+                      `ocrd-eynollah-segment`).
+                      If a parameter requires directories, it must set this to
                       `text/directory`.
                   cacheable:
                     type: boolean
-                    description: "If parameter is reference to file: Whether the file should be cached, e.g. because it is large and won't change."
+                    description: >
+                      If parameter is reference to file (type=string format=uri):
+                      Whether the file should be cached, e.g. because it is large
+                      and won't change.
                     default: false
           description:
             description: Concise description of what the tool does

ocrd_validators/workspace_validator.py

@@ -7,6 +7,7 @@ from pathlib import Path
 
 from ocrd_utils import getLogger, MIMETYPE_PAGE, pushd_popd, is_local_filename, DEFAULT_METS_BASENAME
 from ocrd_models import ValidationReport
+from ocrd_models.constants import PAGE_ALTIMG_FEATURES
 from ocrd_modelfactory import page_from_file
 
 from .constants import FILE_GROUP_CATEGORIES, FILE_GROUP_PREFIX
@@ -98,6 +99,9 @@ class WorkspaceValidator():
         self.page_coordinate_consistency = page_coordinate_consistency
         # there will be more options to come
         self.page_checks = [check for check in ['mets_fileid_page_pcgtsid',
+                                                'imagefilename',
+                                                'alternativeimage_filename',
+                                                'alternativeimage_comments',
                                                 'dimension',
                                                 'page',
                                                 'page_xsd']
@@ -118,7 +122,7 @@ class WorkspaceValidator():
             mets_url (string): URL of the METS file
             src_dir (string, None): Directory containing mets file
             skip (list): Validation checks to omit. One or more of 
-                'mets_unique_identifier', 'mets_file_group_names', 
+                'mets_unique_identifier',
                 'mets_files', 'pixel_density', 'dimension', 'url',
                 'multipage', 'page', 'page_xsd', 'mets_xsd', 
                 'mets_fileid_page_pcgtsid'
@@ -145,8 +149,6 @@ class WorkspaceValidator():
             try:
                 if 'mets_unique_identifier' not in self.skip:
                     self._validate_mets_unique_identifier()
-                if 'mets_file_group_names' not in self.skip:
-                    self._validate_mets_file_group_names()
                 if 'mets_files' not in self.skip:
                     self._validate_mets_files()
                 if 'pixel_density' not in self.skip:
@@ -192,7 +194,11 @@ class WorkspaceValidator():
             self.workspace.download_file(f)
             page = page_from_file(f).get_Page()
             imageFilename = page.imageFilename
-            if not self.mets.find_files(url=imageFilename, **self.find_kwargs):
+            if is_local_filename(imageFilename):
+                kwargs = dict(local_filename=imageFilename, **self.find_kwargs)
+            else:
+                kwargs = dict(url=imageFilename, **self.find_kwargs)
+            if not self.mets.find_files(**kwargs):
                 self.report.add_error(f"PAGE '{f.ID}': imageFilename '{imageFilename}' not found in METS")
             if is_local_filename(imageFilename) and not Path(imageFilename).exists():
                 self.report.add_warning(f"PAGE '{f.ID}': imageFilename '{imageFilename}' points to non-existent local file")
@@ -295,6 +301,9 @@ class WorkspaceValidator():
             if f.url and 'url' not in self.skip:
                 if re.match(r'^file:/[^/]', f.url):
                     self.report.add_error(f"File '{f.ID}' has an invalid (Java-specific) file URL '{f.url}'")
+                elif ':' not in f.url:
+                    self.report.add_error(f"File '{f.ID}' has an invalid (non-URI) file URL '{f.url}'")
+                    continue
                 scheme = f.url[0:f.url.index(':')]
                 if scheme not in ('http', 'https', 'file'):
                     self.report.add_warning(f"File '{f.ID}' has non-HTTP, non-file URL '{f.url}'")
@@ -321,17 +330,43 @@ class WorkspaceValidator():
             pcgts = page_from_file(f)
             page = pcgts.get_Page()
             if 'dimension' in self.page_checks:
-                _, _, exif = self.workspace.image_from_page(page, f.pageId)
-                if page.imageHeight != exif.height:
-                    self.report.add_error(f"PAGE '{f.ID}': @imageHeight != image's actual height ({page.imageHeight} != {exif.height})")
-                if page.imageWidth != exif.width:
-                    self.report.add_error(f"PAGE '{f.ID}': @imageWidth != image's actual width ({page.imageWidth} != {exif.width})")
+                img = self.workspace._resolve_image_as_pil(page.imageFilename)
+                if page.imageHeight != img.height:
+                    self.report.add_error(f"PAGE '{f.ID}': @imageHeight != image's actual height ({page.imageHeight} != {img.height})")
+                if page.imageWidth != img.width:
+                    self.report.add_error(f"PAGE '{f.ID}': @imageWidth != image's actual width ({page.imageWidth} != {img.width})")
             if 'imagefilename' in self.page_checks:
                 imageFilename = page.imageFilename
-                if not self.mets.find_files(url=imageFilename):
+                if is_local_filename(imageFilename):
+                    kwargs = dict(local_filename=imageFilename, **self.find_kwargs)
+                else:
+                    kwargs = dict(url=imageFilename, **self.find_kwargs)
+                if not self.mets.find_files(**kwargs):
                     self.report.add_error(f"PAGE '{f.ID}': imageFilename '{imageFilename}' not found in METS")
                 if is_local_filename(imageFilename) and not Path(imageFilename).exists():
                     self.report.add_warning(f"PAGE '{f.ID}': imageFilename '{imageFilename}' points to non-existent local file")
+            if 'alternativeimage_filename' in self.page_checks:
+                for altimg in page.get_AllAlternativeImages():
+                    if is_local_filename(altimg.filename):
+                        kwargs = dict(local_filename=altimg.filename, **self.find_kwargs)
+                    else:
+                        kwargs = dict(url=altimg.filename, **self.find_kwargs)
+                    if not self.mets.find_files(**kwargs):
+                        self.report.add_error(f"PAGE '{f.ID}': {altimg.parent_object_.id} AlternativeImage "
+                                              f"'{altimg.filename}' not found in METS")
+                    if is_local_filename(altimg.filename) and not Path(altimg.filename).exists():
+                        self.report.add_warning(f"PAGE '{f.ID}': {altimg.parent_object_.id} AlternativeImage "
+                                                f"'{altimg.filename}' points to non-existent local file")
+            if 'alternativeimage_comments' in self.page_checks:
+                for altimg in page.get_AllAlternativeImages():
+                    if altimg.comments is None:
+                        self.report.add_error(f"PAGE '{f.ID}': {altimg.parent_object_.id} AlternativeImage "
+                                              f"'{altimg.filename}' features not specified in PAGE")
+                    else:
+                        for feature in altimg.comments.split(','):
+                            if feature not in PAGE_ALTIMG_FEATURES:
+                                self.report.add_error(f"PAGE '{f.ID}': {altimg.parent_object_.id} AlternativeImage "
+                                                      f"'{altimg.filename}' feature '{feature}' not standardized for PAGE")
             if 'mets_fileid_page_pcgtsid' in self.page_checks and pcgts.pcGtsId != f.ID:
                 self.report.add_warning('pc:PcGts/@pcGtsId differs from mets:file/@ID: "%s" !== "%s"' % (pcgts.pcGtsId or '', f.ID or ''))