ocrd 3.7.0__py3-none-any.whl → 3.8.1__py3-none-any.whl

ocrd_utils/constants.py

@@ -5,6 +5,7 @@ from .introspect import dist_version
 from re import compile as regex_compile
 
 __all__ = [
+    'DEFAULT_METS_BASENAME',
     'EXT_TO_MIME',
     'LOG_FORMAT',
     'LOG_TIMEFMT',
@@ -14,7 +15,9 @@ __all__ = [
     'PIL_TO_MIME',
     'REGEX_PREFIX',
     'REGEX_FILE_ID',
+    'RESOURCES_DIR_SYSTEM',
     'RESOURCE_LOCATIONS',
+    'RESOURCE_TYPES',
     'VERSION',
 ]
 
@@ -108,6 +111,8 @@ LOG_FORMAT = r'%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s'
 LOG_TIMEFMT = r'%H:%M:%S'
 
 RESOURCE_LOCATIONS = ['data', 'cwd', 'system', 'module']
+RESOURCE_TYPES = ['file', 'directory', 'archive']
+RESOURCES_DIR_SYSTEM = '/usr/local/share/ocrd-resources'
 
 DEFAULT_METS_BASENAME = 'mets.xml'
 

ocrd_utils/os.py

@@ -5,9 +5,11 @@ __all__ = [
     'abspath',
     'directory_size',
     'is_file_in_directory',
+    'is_git_url',
     'get_ocrd_tool_json',
     'get_moduledir',
     'get_processor_resource_types',
+    'get_env_locations',
     'guess_media_type',
     'pushd_popd',
     'unzip_file_to_dir',
@@ -15,29 +17,30 @@ __all__ = [
     'redirect_stderr_and_stdout_to_file',
 ]
 
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 from tempfile import TemporaryDirectory, gettempdir
 from functools import lru_cache
 from contextlib import contextmanager, redirect_stderr, redirect_stdout
 from shutil import which
 from json import loads
 from json.decoder import JSONDecodeError
-from os import getcwd, chdir, stat, chmod, umask, environ
+from os import getcwd, chdir, stat, chmod, umask, environ, PathLike
 from pathlib import Path
 from os.path import abspath as abspath_, join
 from zipfile import ZipFile
-from subprocess import run, PIPE
+from subprocess import run, PIPE, CalledProcessError
 from mimetypes import guess_type as mimetypes_guess
 from filetype import guess as filetype_guess
+from fnmatch import filter as apply_glob
 
 from atomicwrites import atomic_write as atomic_write_, AtomicWriter
 
-from .constants import EXT_TO_MIME
+from .constants import EXT_TO_MIME, MIME_TO_EXT, RESOURCE_LOCATIONS, RESOURCES_DIR_SYSTEM
 from .config import config
 from .logging import getLogger
 from .introspect import resource_string
 
-
-def abspath(url):
+def abspath(url : str) -> str:
     """
     Get a full path to a file or file URL
 
@@ -49,7 +52,7 @@ def abspath(url):
 
 
 @contextmanager
-def pushd_popd(newcwd=None, tempdir=False):
+def pushd_popd(newcwd : Union[str, PathLike] = None, tempdir : bool = False) -> Iterator[PathLike]:
     if newcwd and tempdir:
         raise Exception("pushd_popd can accept either newcwd or tempdir, not both")
     try:
@@ -69,8 +72,7 @@ def pushd_popd(newcwd=None, tempdir=False):
     finally:
         chdir(oldcwd)
 
-
-def unzip_file_to_dir(path_to_zip, output_directory):
+def unzip_file_to_dir(path_to_zip : Union[str, PathLike], output_directory : str) -> None:
     """
     Extract a ZIP archive to a directory
     """
@@ -79,7 +81,16 @@ def unzip_file_to_dir(path_to_zip, output_directory):
 
 
 @lru_cache()
-def get_ocrd_tool_json(executable):
+def is_git_url(url: str) -> bool:
+    try:
+        run(['git', 'ls-remote', '--exit-code', '-q', '-h', url], check=True)
+    except CalledProcessError:
+        return False
+    return True
+
+
+@lru_cache()
+def get_ocrd_tool_json(executable : str) -> Dict[str, Any]:
     """
     Get the ``ocrd-tool`` description of ``executable``.
     """
@@ -93,12 +104,12 @@ def get_ocrd_tool_json(executable):
         except (JSONDecodeError, OSError) as e:
             getLogger('ocrd.utils.get_ocrd_tool_json').error(f'{executable} --dump-json produced invalid JSON: {e}')
     if 'resource_locations' not in ocrd_tool:
-        ocrd_tool['resource_locations'] = ['data', 'cwd', 'system', 'module']
+        ocrd_tool['resource_locations'] = RESOURCE_LOCATIONS
     return ocrd_tool
 
 
 @lru_cache()
-def get_moduledir(executable):
+def get_moduledir(executable : str) -> str:
     moduledir = None
     try:
         ocrd_all_moduledir = loads(resource_string('ocrd', 'ocrd-all-module-dir.json'))
@@ -110,59 +121,80 @@ def get_moduledir(executable):
             getLogger('ocrd.utils.get_moduledir').error(f'{executable} --dump-module-dir failed: {e}')
     return moduledir
 
+def get_env_locations(executable: str) -> List[str]:
+    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
+    if processor_path_var in environ:
+        return environ[processor_path_var].split(':')
+    return []
 
-def list_resource_candidates(executable, fname, cwd=getcwd(), moduled=None, xdg_data_home=None):
+def list_resource_candidates(executable : str, fname : str, cwd : Optional[str] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
     """
     Generate candidates for processor resources according to
     https://ocr-d.de/en/spec/ocrd_tool#file-parameters
     """
+    if cwd is None:
+        cwd = getcwd()
     candidates = []
     candidates.append(join(cwd, fname))
-    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
-    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
-    if processor_path_var in environ:
-        candidates += [join(x, fname) for x in environ[processor_path_var].split(':')]
+    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
+    for processor_path in get_env_locations(executable):
+        candidates.append(join(processor_path, fname))
     candidates.append(join(xdg_data_home, 'ocrd-resources', executable, fname))
-    candidates.append(join('/usr/local/share/ocrd-resources', executable, fname))
+    candidates.append(join(RESOURCES_DIR_SYSTEM, executable, fname))
     if moduled:
         candidates.append(join(moduled, fname))
     return candidates
 
-
-def list_all_resources(executable, moduled=None, xdg_data_home=None):
+def list_all_resources(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
     """
     List all processor resources in the filesystem according to
-    https://ocr-d.de/en/spec/ocrd_tool#file-parameters
+    https://ocr-d.de/en/spec/ocrd_tool#resource-parameters
     """
-    candidates = []
+    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
+    if ocrd_tool is None:
+        ocrd_tool = get_ocrd_tool_json(executable)
+    # processor we're looking for might not be installed, hence the fallbacks
     try:
-        resource_locations = get_ocrd_tool_json(executable)['resource_locations']
-    except FileNotFoundError:
-        # processor we're looking for resource_locations of is not installed.
+        mimetypes = get_processor_resource_types(executable, ocrd_tool=ocrd_tool)
+    except KeyError:
+        mimetypes = ['*/*']
+    try:
+        resource_locations = ocrd_tool['resource_locations']
+    except KeyError:
         # Assume the default
-        resource_locations = ['data', 'cwd', 'system', 'module']
-    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
-    # XXX cwd would list too many false positives
+        resource_locations = RESOURCE_LOCATIONS
+    try:
+        # fixme: if resources_list contains directories, their "suffix" will interfere
+        # (e.g. dirname without dot means we falsely match files without suffix)
+        resource_suffixes = [Path(res['name']).suffix
+                             for res in ocrd_tool['resources']]
+    except KeyError:
+        resource_suffixes = []
+    logger = getLogger('ocrd.utils.list_all_resources')
+    candidates = []
+    # cwd would list too many false positives:
     # if 'cwd' in resource_locations:
-    #     cwd_candidate = join(getcwd(), 'ocrd-resources', executable)
-    #     if Path(cwd_candidate).exists():
-    #         candidates.append(cwd_candidate)
-    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
-    if processor_path_var in environ:
-        for processor_path in environ[processor_path_var].split(':'):
-            if Path(processor_path).is_dir():
-                candidates += Path(processor_path).iterdir()
+    #     cwddir = Path.cwd()
+    #     candidates.append(cwddir.itertree())
+    # but we do not use this anyway:
+    # relative paths are tried w.r.t. CWD
+    # prior to list_all_resources resolution.
+    for processor_path in get_env_locations(executable):
+        processor_path = Path(processor_path)
+        if processor_path.is_dir():
+            candidates += processor_path.iterdir()
     if 'data' in resource_locations:
         datadir = Path(xdg_data_home, 'ocrd-resources', executable)
         if datadir.is_dir():
             candidates += datadir.iterdir()
     if 'system' in resource_locations:
-        systemdir = Path('/usr/local/share/ocrd-resources', executable)
+        systemdir = Path(RESOURCES_DIR_SYSTEM, executable)
         if systemdir.is_dir():
             candidates += systemdir.iterdir()
     if 'module' in resource_locations and moduled:
         # recurse fully
-        for resource in itertree(Path(moduled)):
+        moduled = Path(moduled)
+        for resource in moduled.iterdir():
             if resource.is_dir():
                 continue
             if any(resource.match(pattern) for pattern in
@@ -170,18 +202,66 @@ def list_all_resources(executable, moduled=None, xdg_data_home=None):
                    # code and data; `is_resource()` only singles out
                    # files over directories; but we want data files only
                    # todo: more code and cache exclusion patterns!
-                   ['*.py', '*.py[cod]', '*~', 'ocrd-tool.json',
+                   ['*.py', '*.py[cod]', '*~', '.*.swp', '*.swo',
+                    '__pycache__/*', '*.egg-info/*', '*.egg',
+                    'copyright.txt', 'LICENSE*', 'README.md', 'MANIFEST',
+                    'TAGS', '.DS_Store',
+                    # C extensions
+                    '*.so',
+                    # translations
+                    '*.mo', '*.pot',
+                    '*.log', '*.orig', '*.BAK',
+                    '.git/*',
+                    # our stuff
+                    'ocrd-tool.json',
                     'environment.pickle', 'resource_list.yml']):
+                logger.debug("ignoring module candidate '%s'", resource)
                 continue
             candidates.append(resource)
-    # recurse once
-    for parent in candidates:
-        if parent.is_dir() and parent.name != '.git':
-            candidates += parent.iterdir()
-    return sorted([str(x) for x in candidates])
-
-
-def get_processor_resource_types(executable, ocrd_tool=None):
+    if mimetypes != ['*/*']:
+        logger.debug("matching candidates for %s by content-type %s", executable, str(mimetypes))
+    def valid_resource_type(path):
+        if '*/*' in mimetypes:
+            return True
+        if path.is_dir():
+            if not 'text/directory' in mimetypes:
+                logger.debug("ignoring directory candidate '%s'", path)
+                return False
+            if path.name in ['.git']:
+                logger.debug("ignoring directory candidate '%s'", path)
+                return False
+            return True
+        if not path.is_file():
+            logger.warning("ignoring non-file, non-directory candidate '%s'", path)
+            return False
+        res_mimetype = guess_media_type(path, fallback='')
+        if res_mimetype == 'application/json':
+            # always accept, regardless of configured mimetypes:
+            # needed for distributing or sharing parameter preset files
+            return True
+        if ['text/directory'] == mimetypes:
+            logger.debug("ignoring non-directory candidate '%s'", path)
+            return False
+        if 'application/octet-stream' in mimetypes:
+            # catch-all type - do not enforce anything
+            return True
+        if path.suffix in resource_suffixes:
+            return True
+        if any(path.suffix == MIME_TO_EXT.get(mime, None)
+               for mime in mimetypes):
+            return True
+        if not res_mimetype:
+            logger.warning("cannot determine content type of candidate '%s'", path)
+            return True
+        if any(apply_glob([res_mimetype], mime)
+               for mime in mimetypes):
+            return True
+        logger.debug("ignoring %s candidate '%s'", res_mimetype, path)
+        return False
+    candidates = sorted(filter(valid_resource_type, candidates))
+    return map(str, candidates)
+
+def get_processor_resource_types(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None) -> List[str]:
     """
     Determine what type of resource parameters a processor needs.
 
@@ -193,13 +273,16 @@ def get_processor_resource_types(executable, ocrd_tool=None):
         if not which(executable):
             return ['*/*']
         ocrd_tool = get_ocrd_tool_json(executable)
-    if not next((True for p in ocrd_tool.get('parameters', {}).values() if 'content-type' in p), False):
-        # None of the parameters for this processor are resources (or not
-        # the resource parameters are not properly declared, so output both
-        # directories and files
+    mime_types = [mime
+                  for param in ocrd_tool.get('parameters', {}).values()
+                  if param['type'] == 'string' and param.get('format', '') == 'uri' and 'content-type' in param
+                  for mime in param['content-type'].split(',')]
+    if not len(mime_types):
+        # None of the parameters for this processor are resources
+        # (or the parameters' resource types are not properly declared,)
+        # so output both directories and files
         return ['*/*']
-    return [p['content-type'] for p in ocrd_tool['parameters'].values()
-            if 'content-type' in p]
+    return mime_types
 
 
 # ht @pabs3
@@ -220,12 +303,12 @@ class AtomicWriterPerms(AtomicWriter):
 
 
 @contextmanager
-def atomic_write(fpath):
+def atomic_write(fpath : str) -> Iterator[str]:
     with atomic_write_(fpath, writer_cls=AtomicWriterPerms, overwrite=True) as f:
         yield f
 
 
-def is_file_in_directory(directory, file):
+def is_file_in_directory(directory : Union[str, PathLike], file : Union[str, PathLike]) -> bool:
     """
     Return True if ``file`` is in ``directory`` (by checking that all components of ``directory`` are in ``file.parts``)
     """
@@ -233,8 +316,7 @@ def is_file_in_directory(directory, file):
     file = Path(file)
     return list(file.parts)[:len(directory.parts)] == list(directory.parts)
 
-
-def itertree(path):
+def itertree(path : Union[str, PathLike]) -> PathLike:
     """
     Generate a list of paths by recursively enumerating ``path``
     """
@@ -245,16 +327,14 @@ def itertree(path):
             yield from itertree(subpath)
     yield path
 
-
-def directory_size(path):
+def directory_size(path : Union[str, PathLike]) -> int:
     """
     Calculates size of all files in directory ``path``
     """
     path = Path(path)
     return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file())
 
-
-def guess_media_type(input_file: str, fallback: str = None, application_xml: str = 'application/xml'):
+def guess_media_type(input_file : str, fallback : Optional[str] = None, application_xml : str = 'application/xml') -> str:
     """
     Guess the media type of a file path
     """
@@ -266,7 +346,7 @@ def guess_media_type(input_file: str, fallback: str = None, application_xml: str
     if mimetype is None:
         mimetype = EXT_TO_MIME.get(''.join(Path(input_file).suffixes), fallback)
     if mimetype is None:
-        raise ValueError(f"Could not determine MIME type of {input_file}")
+        raise ValueError("Could not determine MIME type of input_file '%s'", str(input_file))
     if mimetype == 'application/xml':
         mimetype = application_xml
     return mimetype

ocrd_validators/ocrd_tool.schema.yml

@@ -142,18 +142,21 @@ properties:
                     description: List the allowed values if a fixed list.
                   content-type:
                     type: string
-                    default: 'application/octet-stream'
                     description: >
+                      If parameter is reference to file (type=string format=uri):
                       The media type of resources this processor expects for
                       this parameter. Most processors use files for resources
                       (e.g.  `*.traineddata` for `ocrd-tesserocr-recognize`)
                       while others use directories of files (e.g. `default` for
-                      `ocrd-eynollah-segment`).  If a parameter requires
-                      directories, it must set `content-type` to
+                      `ocrd-eynollah-segment`).
+                      If a parameter requires directories, it must set this to
                       `text/directory`.
                   cacheable:
                     type: boolean
-                    description: "If parameter is reference to file: Whether the file should be cached, e.g. because it is large and won't change."
+                    description: >
+                      If parameter is reference to file (type=string format=uri):
+                      Whether the file should be cached, e.g. because it is large
+                      and won't change.
                     default: false
           description:
             description: Concise description of what the tool does

ocrd/resource_list.yml

@@ -1,61 +0,0 @@
-# List available resources by processor for "ocrd resmgr"
-ocrd-calamari-recognize:
-  # XXX disabled since older ocrd_calamari versions don't support resource resolving
-  #- url: https://qurator-data.de/calamari-models/GT4HistOCR/2019-07-22T15_49+0200/model.tar.xz
-  #  type: archive
-  #  name: qurator-gt4histocr-0.3
-  #  description: Calamari model trained with GT4HistOCR
-  #  size: 116439072
-  #  path_in_archive: '.'
-  #  version_range: '< 1.0.0'
-  - url: https://qurator-data.de/calamari-models/GT4HistOCR/2019-12-11T11_10+0100/model.tar.xz
-    type: archive
-    name: qurator-gt4histocr-1.0
-    description: Calamari model trained with GT4HistOCR
-    size: 90275264
-    path_in_archive: '.'
-    version_range: '>= 1.0.0'
-  - url: https://github.com/Calamari-OCR/calamari_models_experimental/releases/download/v0.0.1-pre1/c1_fraktur19-1.tar.gz
-    type: archive
-    name: zpd-fraktur19
-    description: Model trained on 19th century german fraktur
-    path_in_archive: 'c1_fraktur19-1'
-    size: 86009886
-    version_range: '>= 1.0.0'
-  - url: https://github.com/Calamari-OCR/calamari_models_experimental/releases/download/v0.0.1-pre1/c1_latin-script-hist-3.tar.gz
-    type: archive
-    name: zpd-latin-script-hist-3
-    path_in_archive: 'c1_latin-script-hist-3'
-    description: Model trained on historical latin-script texts
-    size: 88416863
-    version_range: '>= 1.0.0'
-ocrd-cis-ocropy-recognize:
-  - url: https://github.com/zuphilip/ocropy-models/raw/master/en-default.pyrnn.gz
-    name: en-default.pyrnn.gz
-    description: Default ocropy model
-    size: 83826134
-  - url: https://github.com/zuphilip/ocropy-models/raw/master/fraktur.pyrnn.gz
-    name: fraktur.pyrnn.gz
-    description: Default ocropy fraktur model
-    size: 43882365
-  - url: https://github.com/jze/ocropus-model_fraktur/raw/master/fraktur.pyrnn.gz
-    name: fraktur-jze.pyrnn.gz
-    description: ocropy fraktur model by github.com/jze
-    size: 2961298
-  - url: https://github.com/chreul/OCR_Testdata_EarlyPrintedBooks/raw/master/LatinHist-98000.pyrnn.gz
-    name: LatinHist.pyrnn.gz
-    description: ocropy historical latin model by github.com/chreul
-    size: 16989864
-ocrd-sbb-binarize:
-  - url: https://qurator-data.de/sbb_binarization/2021-03-09/models.tar.gz
-    description: updated default models provided by github.com/qurator-spk
-    name: default-2021-03-09
-    type: archive
-    path_in_archive: models
-    size: 133363179
-  - url: https://qurator-data.de/sbb_binarization/models.tar.gz
-    description: default models provided by github.com/qurator-spk
-    name: default
-    type: archive
-    path_in_archive: models
-    size: 1654623597