ocrd 3.6.0__py3-none-any.whl → 3.8.0__py3-none-any.whl

ocrd_utils/os.py

@@ -5,9 +5,11 @@ __all__ = [
     'abspath',
     'directory_size',
     'is_file_in_directory',
+    'is_git_url',
     'get_ocrd_tool_json',
     'get_moduledir',
     'get_processor_resource_types',
+    'get_env_locations',
     'guess_media_type',
     'pushd_popd',
     'unzip_file_to_dir',
@@ -15,29 +17,30 @@ __all__ = [
     'redirect_stderr_and_stdout_to_file',
 ]
 
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 from tempfile import TemporaryDirectory, gettempdir
 from functools import lru_cache
 from contextlib import contextmanager, redirect_stderr, redirect_stdout
 from shutil import which
 from json import loads
 from json.decoder import JSONDecodeError
-from os import getcwd, chdir, stat, chmod, umask, environ
+from os import getcwd, chdir, stat, chmod, umask, environ, PathLike
 from pathlib import Path
 from os.path import abspath as abspath_, join
 from zipfile import ZipFile
-from subprocess import run, PIPE
+from subprocess import run, PIPE, CalledProcessError
 from mimetypes import guess_type as mimetypes_guess
 from filetype import guess as filetype_guess
+from fnmatch import filter as apply_glob
 
 from atomicwrites import atomic_write as atomic_write_, AtomicWriter
 
-from .constants import EXT_TO_MIME
+from .constants import EXT_TO_MIME, MIME_TO_EXT, RESOURCE_LOCATIONS, RESOURCES_DIR_SYSTEM
 from .config import config
 from .logging import getLogger
 from .introspect import resource_string
 
-
-def abspath(url):
+def abspath(url : str) -> str:
     """
     Get a full path to a file or file URL
 
@@ -49,7 +52,7 @@ def abspath(url):
 
 
 @contextmanager
-def pushd_popd(newcwd=None, tempdir=False):
+def pushd_popd(newcwd : Union[str, PathLike] = None, tempdir : bool = False) -> Iterator[PathLike]:
     if newcwd and tempdir:
         raise Exception("pushd_popd can accept either newcwd or tempdir, not both")
     try:
@@ -69,8 +72,7 @@ def pushd_popd(newcwd=None, tempdir=False):
     finally:
         chdir(oldcwd)
 
-
-def unzip_file_to_dir(path_to_zip, output_directory):
+def unzip_file_to_dir(path_to_zip : Union[str, PathLike], output_directory : str) -> None:
     """
     Extract a ZIP archive to a directory
     """
@@ -79,7 +81,16 @@ def unzip_file_to_dir(path_to_zip, output_directory):
 
 
 @lru_cache()
-def get_ocrd_tool_json(executable):
+def is_git_url(url: str) -> bool:
+    try:
+        run(['git', 'ls-remote', '--exit-code', '-q', '-h', url], check=True)
+    except CalledProcessError:
+        return False
+    return True
+
+
+@lru_cache()
+def get_ocrd_tool_json(executable : str) -> Dict[str, Any]:
     """
     Get the ``ocrd-tool`` description of ``executable``.
     """
@@ -93,12 +104,12 @@ def get_ocrd_tool_json(executable):
         except (JSONDecodeError, OSError) as e:
             getLogger('ocrd.utils.get_ocrd_tool_json').error(f'{executable} --dump-json produced invalid JSON: {e}')
     if 'resource_locations' not in ocrd_tool:
-        ocrd_tool['resource_locations'] = ['data', 'cwd', 'system', 'module']
+        ocrd_tool['resource_locations'] = RESOURCE_LOCATIONS
     return ocrd_tool
 
 
 @lru_cache()
-def get_moduledir(executable):
+def get_moduledir(executable : str) -> str:
     moduledir = None
     try:
         ocrd_all_moduledir = loads(resource_string('ocrd', 'ocrd-all-module-dir.json'))
@@ -110,59 +121,80 @@ def get_moduledir(executable):
             getLogger('ocrd.utils.get_moduledir').error(f'{executable} --dump-module-dir failed: {e}')
     return moduledir
 
+def get_env_locations(executable: str) -> List[str]:
+    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
+    if processor_path_var in environ:
+        return environ[processor_path_var].split(':')
+    return []
 
-def list_resource_candidates(executable, fname, cwd=getcwd(), moduled=None, xdg_data_home=None):
+def list_resource_candidates(executable : str, fname : str, cwd : Optional[str] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
     """
     Generate candidates for processor resources according to
     https://ocr-d.de/en/spec/ocrd_tool#file-parameters
     """
+    if cwd is None:
+        cwd = getcwd()
     candidates = []
     candidates.append(join(cwd, fname))
-    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
-    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
-    if processor_path_var in environ:
-        candidates += [join(x, fname) for x in environ[processor_path_var].split(':')]
+    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
+    for processor_path in get_env_locations(executable):
+        candidates.append(join(processor_path, fname))
     candidates.append(join(xdg_data_home, 'ocrd-resources', executable, fname))
-    candidates.append(join('/usr/local/share/ocrd-resources', executable, fname))
+    candidates.append(join(RESOURCES_DIR_SYSTEM, executable, fname))
     if moduled:
         candidates.append(join(moduled, fname))
     return candidates
 
-
-def list_all_resources(executable, moduled=None, xdg_data_home=None):
+def list_all_resources(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
     """
     List all processor resources in the filesystem according to
-    https://ocr-d.de/en/spec/ocrd_tool#file-parameters
+    https://ocr-d.de/en/spec/ocrd_tool#resource-parameters
     """
-    candidates = []
+    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
+    if ocrd_tool is None:
+        ocrd_tool = get_ocrd_tool_json(executable)
+    # processor we're looking for might not be installed, hence the fallbacks
     try:
-        resource_locations = get_ocrd_tool_json(executable)['resource_locations']
-    except FileNotFoundError:
-        # processor we're looking for resource_locations of is not installed.
+        mimetypes = get_processor_resource_types(executable, ocrd_tool=ocrd_tool)
+    except KeyError:
+        mimetypes = ['*/*']
+    try:
+        resource_locations = ocrd_tool['resource_locations']
+    except KeyError:
         # Assume the default
-        resource_locations = ['data', 'cwd', 'system', 'module']
-    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
-    # XXX cwd would list too many false positives
+        resource_locations = RESOURCE_LOCATIONS
+    try:
+        # fixme: if resources_list contains directories, their "suffix" will interfere
+        # (e.g. dirname without dot means we falsely match files without suffix)
+        resource_suffixes = [Path(res['name']).suffix
+                             for res in ocrd_tool['resources']]
+    except KeyError:
+        resource_suffixes = []
+    logger = getLogger('ocrd.utils.list_all_resources')
+    candidates = []
+    # cwd would list too many false positives:
     # if 'cwd' in resource_locations:
-    #     cwd_candidate = join(getcwd(), 'ocrd-resources', executable)
-    #     if Path(cwd_candidate).exists():
-    #         candidates.append(cwd_candidate)
-    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
-    if processor_path_var in environ:
-        for processor_path in environ[processor_path_var].split(':'):
-            if Path(processor_path).is_dir():
-                candidates += Path(processor_path).iterdir()
+    #     cwddir = Path.cwd()
+    #     candidates.append(cwddir.itertree())
+    # but we do not use this anyway:
+    # relative paths are tried w.r.t. CWD
+    # prior to list_all_resources resolution.
+    for processor_path in get_env_locations(executable):
+        processor_path = Path(processor_path)
+        if processor_path.is_dir():
+            candidates += processor_path.iterdir()
     if 'data' in resource_locations:
         datadir = Path(xdg_data_home, 'ocrd-resources', executable)
         if datadir.is_dir():
             candidates += datadir.iterdir()
     if 'system' in resource_locations:
-        systemdir = Path('/usr/local/share/ocrd-resources', executable)
+        systemdir = Path(RESOURCES_DIR_SYSTEM, executable)
         if systemdir.is_dir():
             candidates += systemdir.iterdir()
     if 'module' in resource_locations and moduled:
         # recurse fully
-        for resource in itertree(Path(moduled)):
+        moduled = Path(moduled)
+        for resource in moduled.iterdir():
             if resource.is_dir():
                 continue
             if any(resource.match(pattern) for pattern in
@@ -170,18 +202,66 @@ def list_all_resources(executable, moduled=None, xdg_data_home=None):
                    # code and data; `is_resource()` only singles out
                    # files over directories; but we want data files only
                    # todo: more code and cache exclusion patterns!
-                   ['*.py', '*.py[cod]', '*~', 'ocrd-tool.json',
-                    'environment.pickle', 'resource_list.yml', 'lib.bash']):
+                   ['*.py', '*.py[cod]', '*~', '.*.swp', '*.swo',
+                    '__pycache__/*', '*.egg-info/*', '*.egg',
+                    'copyright.txt', 'LICENSE*', 'README.md', 'MANIFEST',
+                    'TAGS', '.DS_Store',
+                    # C extensions
+                    '*.so',
+                    # translations
+                    '*.mo', '*.pot',
+                    '*.log', '*.orig', '*.BAK',
+                    '.git/*',
+                    # our stuff
+                    'ocrd-tool.json',
+                    'environment.pickle', 'resource_list.yml']):
+                logger.debug("ignoring module candidate '%s'", resource)
                 continue
             candidates.append(resource)
-    # recurse once
-    for parent in candidates:
-        if parent.is_dir() and parent.name != '.git':
-            candidates += parent.iterdir()
-    return sorted([str(x) for x in candidates])
-
-
-def get_processor_resource_types(executable, ocrd_tool=None):
+    if mimetypes != ['*/*']:
+        logger.debug("matching candidates for %s by content-type %s", executable, str(mimetypes))
+    def valid_resource_type(path):
+        if '*/*' in mimetypes:
+            return True
+        if path.is_dir():
+            if not 'text/directory' in mimetypes:
+                logger.debug("ignoring directory candidate '%s'", path)
+                return False
+            if path.name in ['.git']:
+                logger.debug("ignoring directory candidate '%s'", path)
+                return False
+            return True
+        if not path.is_file():
+            logger.warning("ignoring non-file, non-directory candidate '%s'", path)
+            return False
+        res_mimetype = guess_media_type(path, fallback='')
+        if res_mimetype == 'application/json':
+            # always accept, regardless of configured mimetypes:
+            # needed for distributing or sharing parameter preset files
+            return True
+        if ['text/directory'] == mimetypes:
+            logger.debug("ignoring non-directory candidate '%s'", path)
+            return False
+        if 'application/octet-stream' in mimetypes:
+            # catch-all type - do not enforce anything
+            return True
+        if path.suffix in resource_suffixes:
+            return True
+        if any(path.suffix == MIME_TO_EXT.get(mime, None)
+               for mime in mimetypes):
+            return True
+        if not res_mimetype:
+            logger.warning("cannot determine content type of candidate '%s'", path)
+            return True
+        if any(apply_glob([res_mimetype], mime)
+               for mime in mimetypes):
+            return True
+        logger.debug("ignoring %s candidate '%s'", res_mimetype, path)
+        return False
+    candidates = sorted(filter(valid_resource_type, candidates))
+    return map(str, candidates)
+
+def get_processor_resource_types(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None) -> List[str]:
     """
     Determine what type of resource parameters a processor needs.
 
@@ -193,13 +273,16 @@ def get_processor_resource_types(executable, ocrd_tool=None):
         if not which(executable):
             return ['*/*']
         ocrd_tool = get_ocrd_tool_json(executable)
-    if not next((True for p in ocrd_tool.get('parameters', {}).values() if 'content-type' in p), False):
-        # None of the parameters for this processor are resources (or not
-        # the resource parameters are not properly declared, so output both
-        # directories and files
+    mime_types = [mime
+                  for param in ocrd_tool.get('parameters', {}).values()
+                  if param['type'] == 'string' and param.get('format', '') == 'uri' and 'content-type' in param
+                  for mime in param['content-type'].split(',')]
+    if not len(mime_types):
+        # None of the parameters for this processor are resources
+        # (or the parameters' resource types are not properly declared,)
+        # so output both directories and files
         return ['*/*']
-    return [p['content-type'] for p in ocrd_tool['parameters'].values()
-            if 'content-type' in p]
+    return mime_types
 
 
 # ht @pabs3
@@ -220,12 +303,12 @@ class AtomicWriterPerms(AtomicWriter):
 
 
 @contextmanager
-def atomic_write(fpath):
+def atomic_write(fpath : str) -> Iterator[str]:
     with atomic_write_(fpath, writer_cls=AtomicWriterPerms, overwrite=True) as f:
         yield f
 
 
-def is_file_in_directory(directory, file):
+def is_file_in_directory(directory : Union[str, PathLike], file : Union[str, PathLike]) -> bool:
     """
     Return True if ``file`` is in ``directory`` (by checking that all components of ``directory`` are in ``file.parts``)
     """
@@ -233,8 +316,7 @@ def is_file_in_directory(directory, file):
     file = Path(file)
     return list(file.parts)[:len(directory.parts)] == list(directory.parts)
 
-
-def itertree(path):
+def itertree(path : Union[str, PathLike]) -> PathLike:
     """
     Generate a list of paths by recursively enumerating ``path``
     """
@@ -245,16 +327,14 @@ def itertree(path):
             yield from itertree(subpath)
     yield path
 
-
-def directory_size(path):
+def directory_size(path : Union[str, PathLike]) -> int:
     """
     Calculates size of all files in directory ``path``
     """
     path = Path(path)
     return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file())
 
-
-def guess_media_type(input_file: str, fallback: str = None, application_xml: str = 'application/xml'):
+def guess_media_type(input_file : str, fallback : Optional[str] = None, application_xml : str = 'application/xml') -> str:
     """
     Guess the media type of a file path
     """
@@ -266,7 +346,7 @@ def guess_media_type(input_file: str, fallback: str = None, application_xml: str
     if mimetype is None:
         mimetype = EXT_TO_MIME.get(''.join(Path(input_file).suffixes), fallback)
     if mimetype is None:
-        raise ValueError(f"Could not determine MIME type of {input_file}")
+        raise ValueError("Could not determine MIME type of input_file '%s'", str(input_file))
     if mimetype == 'application/xml':
         mimetype = application_xml
     return mimetype

ocrd_validators/ocrd_tool.schema.yml

@@ -142,18 +142,21 @@ properties:
                     description: List the allowed values if a fixed list.
                   content-type:
                     type: string
-                    default: 'application/octet-stream'
                     description: >
+                      If parameter is reference to file (type=string format=uri):
                       The media type of resources this processor expects for
                       this parameter. Most processors use files for resources
                       (e.g.  `*.traineddata` for `ocrd-tesserocr-recognize`)
                       while others use directories of files (e.g. `default` for
-                      `ocrd-eynollah-segment`).  If a parameter requires
-                      directories, it must set `content-type` to
+                      `ocrd-eynollah-segment`).
+                      If a parameter requires directories, it must set this to
                       `text/directory`.
                   cacheable:
                     type: boolean
-                    description: "If parameter is reference to file: Whether the file should be cached, e.g. because it is large and won't change."
+                    description: >
+                      If parameter is reference to file (type=string format=uri):
+                      Whether the file should be cached, e.g. because it is large
+                      and won't change.
                     default: false
           description:
             description: Concise description of what the tool does

ocrd/cli/log.py

@@ -1,56 +0,0 @@
-"""
-OCR-D CLI: Logging
-
-.. click:: ocrd.cli.log:log_cli
-    :prog: ocrd log
-    :nested: full
-"""
-import click
-from ocrd_utils import initLogging, getLogger, getLevelName
-
-
-class LogCtx():
-
-    def __init__(self, name):
-        self.name = name
-
-    def log(self, lvl, *args, **kwargs):
-        logger = getLogger(self.name)
-        logger.log(getLevelName(lvl), *args, **kwargs)
-
-
-pass_log = click.make_pass_decorator(LogCtx)
-
-
-@click.group("log")
-@click.option('-n', '--name', envvar='OCRD_TOOL_NAME', default='log_cli', metavar='LOGGER_NAME',
-              help='Name of the logger', show_default=True)
-@click.pass_context
-def log_cli(ctx, name, *args, **kwargs):
-    """
-    Logging
-
-    Logger name will be ocrd.OCRD_TOOL_NAME where OCRD_TOOL_NAME is normally
-    (when using bashlib) the name of the processor.
-    """
-    initLogging()
-    ctx.obj = LogCtx('ocrd.' + name)
-
-
-def _bind_log_command(lvl):
-    @click.argument('msgs', nargs=-1)
-    @pass_log
-    def _log_wrapper(ctx, msgs):
-        if not msgs:
-            ctx.log(lvl.upper(), '')
-        elif len(msgs) == 1 and msgs[0] == '-':
-            for stdin_line in click.get_text_stream('stdin'):
-                ctx.log(lvl.upper(), stdin_line.rstrip('\n'))
-        else:
-            msg = list(msgs) if '%s' in msgs[0] else ' '.join([x.replace('%', '%%') for x in msgs])
-            ctx.log(lvl.upper(), msg)
-    return _log_wrapper
-
-
-for _lvl in ['trace', 'debug', 'info', 'warning', 'error', 'critical']:
-    log_cli.command(_lvl, help="Log a %s message" % _lvl.upper())(_bind_log_command(_lvl))