ocrd 3.2.0__py3-none-any.whl → 3.3.0__py3-none-any.whl

ocrd_utils/os.py

@@ -8,7 +8,6 @@ __all__ = [
     'get_ocrd_tool_json',
     'get_moduledir',
     'get_processor_resource_types',
-    'get_env_locations',
     'guess_media_type',
     'pushd_popd',
     'unzip_file_to_dir',
@@ -16,30 +15,28 @@ __all__ = [
     'redirect_stderr_and_stdout_to_file',
 ]
 
-from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 from tempfile import TemporaryDirectory, gettempdir
 from functools import lru_cache
 from contextlib import contextmanager, redirect_stderr, redirect_stdout
 from shutil import which
 from json import loads
 from json.decoder import JSONDecodeError
-from os import getcwd, chdir, stat, chmod, umask, environ, PathLike
+from os import getcwd, chdir, stat, chmod, umask, environ
 from pathlib import Path
 from os.path import abspath as abspath_, join
 from zipfile import ZipFile
 from subprocess import run, PIPE
 from mimetypes import guess_type as mimetypes_guess
 from filetype import guess as filetype_guess
-from fnmatch import filter as apply_glob
 
 from atomicwrites import atomic_write as atomic_write_, AtomicWriter
 
-from .constants import EXT_TO_MIME, MIME_TO_EXT, RESOURCE_LOCATIONS, RESOURCES_DIR_SYSTEM
+from .constants import EXT_TO_MIME
 from .config import config
 from .logging import getLogger
 from .introspect import resource_string
 
-def abspath(url : str) -> str:
+def abspath(url):
     """
     Get a full path to a file or file URL
 
@@ -50,7 +47,7 @@ def abspath(url : str) -> str:
     return abspath_(url)
 
 @contextmanager
-def pushd_popd(newcwd : Union[str, PathLike] = None, tempdir : bool = False) -> Iterator[PathLike]:
+def pushd_popd(newcwd=None, tempdir=False):
     if newcwd and tempdir:
         raise Exception("pushd_popd can accept either newcwd or tempdir, not both")
     try:
@@ -70,7 +67,7 @@ def pushd_popd(newcwd : Union[str, PathLike] = None, tempdir : bool = False) ->
     finally:
         chdir(oldcwd)
 
-def unzip_file_to_dir(path_to_zip : Union[str, PathLike], output_directory : str) -> None:
+def unzip_file_to_dir(path_to_zip, output_directory):
     """
     Extract a ZIP archive to a directory
     """
@@ -78,7 +75,7 @@ def unzip_file_to_dir(path_to_zip : Union[str, PathLike], output_directory : str
         z.extractall(output_directory)
 
 @lru_cache()
-def get_ocrd_tool_json(executable : str) -> Dict[str, Any]:
+def get_ocrd_tool_json(executable):
     """
     Get the ``ocrd-tool`` description of ``executable``.
     """
@@ -93,11 +90,11 @@ def get_ocrd_tool_json(executable : str) -> Dict[str, Any]:
         except (JSONDecodeError, OSError) as e:
             getLogger('ocrd.utils.get_ocrd_tool_json').error(f'{executable} --dump-json produced invalid JSON: {e}')
     if 'resource_locations' not in ocrd_tool:
-        ocrd_tool['resource_locations'] = RESOURCE_LOCATIONS
+        ocrd_tool['resource_locations'] = ['data', 'cwd', 'system', 'module']
     return ocrd_tool
 
 @lru_cache()
-def get_moduledir(executable : str) -> str:
+def get_moduledir(executable):
     moduledir = None
     try:
         ocrd_all_moduledir = loads(resource_string('ocrd', 'ocrd-all-module-dir.json'))
@@ -109,80 +106,57 @@ def get_moduledir(executable : str) -> str:
             getLogger('ocrd.utils.get_moduledir').error(f'{executable} --dump-module-dir failed: {e}')
     return moduledir
 
-def get_env_locations(executable: str) -> List[str]:
-    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
-    if processor_path_var in environ:
-        return environ[processor_path_var].split(':')
-    return []
-
-def list_resource_candidates(executable : str, fname : str, cwd : Optional[str] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
+def list_resource_candidates(executable, fname, cwd=getcwd(), moduled=None, xdg_data_home=None):
     """
     Generate candidates for processor resources according to
     https://ocr-d.de/en/spec/ocrd_tool#file-parameters
     """
-    if cwd is None:
-        cwd = getcwd()
     candidates = []
     candidates.append(join(cwd, fname))
-    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
-    for processor_path in get_env_locations(executable):
-        candidates.append(join(processor_path, fname))
+    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
+    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
+    if processor_path_var in environ:
+        candidates += [join(x, fname) for x in environ[processor_path_var].split(':')]
     candidates.append(join(xdg_data_home, 'ocrd-resources', executable, fname))
-    candidates.append(join(RESOURCES_DIR_SYSTEM, executable, fname))
+    candidates.append(join('/usr/local/share/ocrd-resources', executable, fname))
     if moduled:
         candidates.append(join(moduled, fname))
     return candidates
 
-def list_all_resources(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None, moduled : Optional[str] = None, xdg_data_home : Optional[str] = None) -> List[str]:
+def list_all_resources(executable, moduled=None, xdg_data_home=None):
     """
     List all processor resources in the filesystem according to
-    https://ocr-d.de/en/spec/ocrd_tool#resource-parameters
+    https://ocr-d.de/en/spec/ocrd_tool#file-parameters
     """
-    xdg_data_home = xdg_data_home or config.XDG_DATA_HOME
-    if ocrd_tool is None:
-        ocrd_tool = get_ocrd_tool_json(executable)
-    # processor we're looking for might not be installed, hence the fallbacks
-    try:
-        mimetypes = get_processor_resource_types(executable, ocrd_tool=ocrd_tool)
-    except KeyError:
-        mimetypes = ['*/*']
+    candidates = []
     try:
-        resource_locations = ocrd_tool['resource_locations']
-    except KeyError:
+        resource_locations = get_ocrd_tool_json(executable)['resource_locations']
+    except FileNotFoundError:
+        # processor we're looking for resource_locations of is not installed.
         # Assume the default
-        resource_locations = RESOURCE_LOCATIONS
-    try:
-        # fixme: if resources_list contains directories, their "suffix" will interfere
-        # (e.g. dirname without dot means we falsely match files without suffix)
-        resource_suffixes = [Path(res['name']).suffix
-                             for res in ocrd_tool['resources']]
-    except KeyError:
-        resource_suffixes = []
-    logger = getLogger('ocrd.utils.list_all_resources')
-    candidates = []
-    # cwd would list too many false positives:
+        resource_locations = ['data', 'cwd', 'system', 'module']
+    xdg_data_home = config.XDG_DATA_HOME if not xdg_data_home else xdg_data_home
+    # XXX cwd would list too many false positives
     # if 'cwd' in resource_locations:
-    #     cwddir = Path.cwd()
-    #     candidates.append(cwddir.itertree())
-    # but we do not use this anyway:
-    # relative paths are tried w.r.t. CWD
-    # prior to list_all_resources resolution.
-    for processor_path in get_env_locations(executable):
-        processor_path = Path(processor_path)
-        if processor_path.is_dir():
-            candidates += processor_path.iterdir()
+    #     cwd_candidate = join(getcwd(), 'ocrd-resources', executable)
+    #     if Path(cwd_candidate).exists():
+    #         candidates.append(cwd_candidate)
+    processor_path_var = '%s_PATH' % executable.replace('-', '_').upper()
+    if processor_path_var in environ:
+        for processor_path in environ[processor_path_var].split(':'):
+            if Path(processor_path).is_dir():
+                candidates += Path(processor_path).iterdir()
     if 'data' in resource_locations:
         datadir = Path(xdg_data_home, 'ocrd-resources', executable)
         if datadir.is_dir():
             candidates += datadir.iterdir()
     if 'system' in resource_locations:
-        systemdir = Path(RESOURCES_DIR_SYSTEM, executable)
+        systemdir = Path('/usr/local/share/ocrd-resources', executable)
         if systemdir.is_dir():
             candidates += systemdir.iterdir()
     if 'module' in resource_locations and moduled:
         # recurse fully
-        moduled = Path(moduled)
-        for resource in moduled.iterdir():
+        for resource in itertree(Path(moduled)):
             if resource.is_dir():
                 continue
             if any(resource.match(pattern) for pattern in
@@ -190,66 +164,17 @@ def list_all_resources(executable : str, ocrd_tool : Optional[Dict[str, Any]] =
                    # code and data; `is_resource()` only singles out
                    # files over directories; but we want data files only
                    # todo: more code and cache exclusion patterns!
-                   ['*.py', '*.py[cod]', '*~', '.*.swp', '*.swo',
-                    '__pycache__/*', '*.egg-info/*', '*.egg',
-                    'copyright.txt', 'LICENSE*', 'README.md', 'MANIFEST',
-                    'TAGS', '.DS_Store',
-                    # C extensions
-                    '*.so',
-                    # translations
-                    '*.mo', '*.pot',
-                    '*.log', '*.orig', '*.BAK',
-                    '.git/*',
-                    # our stuff
-                    'ocrd-tool.json',
+                   ['*.py', '*.py[cod]', '*~', 'ocrd-tool.json', 
                     'environment.pickle', 'resource_list.yml', 'lib.bash']):
-                logger.debug("ignoring module candidate '%s'", resource)
                 continue
             candidates.append(resource)
-    if mimetypes != ['*/*']:
-        logger.debug("matching candidates for %s by content-type %s", executable, str(mimetypes))
-    def valid_resource_type(path):
-        if '*/*' in mimetypes:
-            return True
-        if path.is_dir():
-            if not 'text/directory' in mimetypes:
-                logger.debug("ignoring directory candidate '%s'", path)
-                return False
-            if path.name in ['.git']:
-                logger.debug("ignoring directory candidate '%s'", path)
-                return False
-            return True
-        if not path.is_file():
-            logger.warning("ignoring non-file, non-directory candidate '%s'", path)
-            return False
-        res_mimetype = guess_media_type(path, fallback='')
-        if res_mimetype == 'application/json':
-            # always accept, regardless of configured mimetypes:
-            # needed for distributing or sharing parameter preset files
-            return True
-        if ['text/directory'] == mimetypes:
-            logger.debug("ignoring non-directory candidate '%s'", path)
-            return False
-        if 'application/octet-stream' in mimetypes:
-            # catch-all type - do not enforce anything
-            return True
-        if path.suffix in resource_suffixes:
-            return True
-        if any(path.suffix == MIME_TO_EXT.get(mime, None)
-               for mime in mimetypes):
-            return True
-        if not res_mimetype:
-            logger.warning("cannot determine content type of candidate '%s'", path)
-            return True
-        if any(apply_glob([res_mimetype], mime)
-               for mime in mimetypes):
-            return True
-        logger.debug("ignoring %s candidate '%s'", res_mimetype, path)
-        return False
-    candidates = sorted(filter(valid_resource_type, candidates))
-    return map(str, candidates)
+    # recurse once
+    for parent in candidates:
+        if parent.is_dir() and parent.name != '.git':
+            candidates += parent.iterdir()
+    return sorted([str(x) for x in candidates])
 
-def get_processor_resource_types(executable : str, ocrd_tool : Optional[Dict[str, Any]] = None) -> List[str]:
+def get_processor_resource_types(executable, ocrd_tool=None):
     """
     Determine what type of resource parameters a processor needs.
 
@@ -261,16 +186,13 @@ def get_processor_resource_types(executable : str, ocrd_tool : Optional[Dict[str
         if not which(executable):
             return ['*/*']
         ocrd_tool = get_ocrd_tool_json(executable)
-    mime_types = [mime
-                  for param in ocrd_tool.get('parameters', {}).values()
-                  if param['type'] == 'string' and param.get('format', '') == 'uri' and 'content-type' in param
-                  for mime in param['content-type'].split(',')]
-    if not len(mime_types):
-        # None of the parameters for this processor are resources
-        # (or the parameters' resource types are not properly declared,)
-        # so output both directories and files
+    if not next((True for p in ocrd_tool.get('parameters', {}).values() if 'content-type' in p), False):
+        # None of the parameters for this processor are resources (or not
+        # the resource parameters are not properly declared, so output both
+        # directories and files
         return ['*/*']
-    return mime_types
+    return [p['content-type'] for p in ocrd_tool['parameters'].values()
+            if 'content-type' in p]
 
 # ht @pabs3
 # https://github.com/untitaker/python-atomicwrites/issues/42
@@ -289,12 +211,12 @@ class AtomicWriterPerms(AtomicWriter):
         return f
 
 @contextmanager
-def atomic_write(fpath : str) -> Iterator[str]:
+def atomic_write(fpath):
     with atomic_write_(fpath, writer_cls=AtomicWriterPerms, overwrite=True) as f:
         yield f
 
 
-def is_file_in_directory(directory : Union[str, PathLike], file : Union[str, PathLike]) -> bool:
+def is_file_in_directory(directory, file):
     """
     Return True if ``file`` is in ``directory`` (by checking that all components of ``directory`` are in ``file.parts``)
     """
@@ -302,7 +224,7 @@ def is_file_in_directory(directory : Union[str, PathLike], file : Union[str, Pat
     file = Path(file)
     return list(file.parts)[:len(directory.parts)] == list(directory.parts)
 
-def itertree(path : Union[str, PathLike]) -> PathLike:
+def itertree(path):
     """
     Generate a list of paths by recursively enumerating ``path``
     """
@@ -313,14 +235,14 @@ def itertree(path : Union[str, PathLike]) -> PathLike:
             yield from itertree(subpath)
     yield path
 
-def directory_size(path : Union[str, PathLike]) -> int:
+def directory_size(path):
     """
     Calculates size of all files in directory ``path``
     """
     path = Path(path)
     return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file())
 
-def guess_media_type(input_file : str, fallback : Optional[str] = None, application_xml : str = 'application/xml') -> str:
+def guess_media_type(input_file : str, fallback : str = None, application_xml : str = 'application/xml'):
     """
     Guess the media type of a file path
     """
@@ -332,7 +254,7 @@ def guess_media_type(input_file : str, fallback : Optional[str] = None, applicat
     if mimetype is None:
         mimetype = EXT_TO_MIME.get(''.join(Path(input_file).suffixes), fallback)
     if mimetype is None:
-        raise ValueError("Could not determine MIME type of input_file '%s'", str(input_file))
+        raise ValueError("Could not determine MIME type of input_file must")
     if mimetype == 'application/xml':
         mimetype = application_xml
     return mimetype

ocrd_validators/ocrd_tool.schema.yml

@@ -142,21 +142,18 @@ properties:
                     description: List the allowed values if a fixed list.
                   content-type:
                     type: string
+                    default: 'application/octet-stream'
                     description: >
-                      If parameter is reference to file (type=string format=uri):
                       The media type of resources this processor expects for
                       this parameter. Most processors use files for resources
                       (e.g.  `*.traineddata` for `ocrd-tesserocr-recognize`)
                       while others use directories of files (e.g. `default` for
-                      `ocrd-eynollah-segment`).
-                      If a parameter requires directories, it must set this to
+                      `ocrd-eynollah-segment`).  If a parameter requires
+                      directories, it must set `content-type` to
                       `text/directory`.
                   cacheable:
                     type: boolean
-                    description: >
-                      If parameter is reference to file (type=string format=uri):
-                      Whether the file should be cached, e.g. because it is large
-                      and won't change.
+                    description: "If parameter is reference to file: Whether the file should be cached, e.g. because it is large and won't change."
                     default: false
           description:
             description: Concise description of what the tool does

ocrd_network/cli/resmgr_server.py

@@ -1,23 +0,0 @@
-import click
-from ocrd_network import ResourceManagerServer, ServerAddressParamType
-
-
-@click.command('resmgr-server')
-@click.option('-a', '--address',
-              help='The URL of the OCR-D resource manager server, format: host:port',
-              type=ServerAddressParamType(),
-              required=True)
-def resource_manager_server_cli(address: str):
-    """
-    Start standalone REST API OCR-D Resource Manager Server
-    """
-    try:
-        # Note, the address is already validated with the type field
-        host, port = address.split(':')
-        resource_manager_server = ResourceManagerServer(
-            host = host,
-            port = int(port)
-        )
-        resource_manager_server.start()
-    except Exception as e:
-        raise Exception("OCR-D Resource Manager Server has failed with error") from e

ocrd_network/resource_manager_server.py

@@ -1,178 +0,0 @@
-from datetime import datetime
-from os import getpid
-from pathlib import Path
-import requests
-from shutil import which
-from typing import Any
-from uvicorn import run as uvicorn_run
-from fastapi import APIRouter, FastAPI, HTTPException, status
-
-from ocrd import OcrdResourceManager
-from ocrd_utils import directory_size, getLogger, get_moduledir, get_ocrd_tool_json, initLogging
-from .logging_utils import configure_file_handler_with_formatter, get_resource_manager_server_logging_file_path
-
-
-class ResourceManagerServer(FastAPI):
-    def __init__(self, host: str, port: int) -> None:
-        self.title = f"OCR-D Resource Manager Server"
-        super().__init__(
-            title=self.title,
-            on_startup=[self.on_startup],
-            on_shutdown=[self.on_shutdown],
-            description=self.title
-        )
-        initLogging()
-        self.log = getLogger("ocrd_network.resource_manager_server")
-        log_file = get_resource_manager_server_logging_file_path(pid=getpid())
-        configure_file_handler_with_formatter(self.log, log_file=log_file, mode="a")
-
-        self.resmgr_instance = OcrdResourceManager()
-
-        self.hostname = host
-        self.port = port
-
-        self.add_api_routes()
-
-    def start(self):
-        uvicorn_run(self, host=self.hostname, port=int(self.port))
-
-    async def on_startup(self):
-        self.log.info(f"Starting {self.title}")
-        pass
-
-    async def on_shutdown(self) -> None:
-        pass
-
-    def add_api_routes(self):
-        base_router = APIRouter()
-        base_router.add_api_route(
-            path="/",
-            endpoint=self.home_page,
-            methods=["GET"],
-            status_code=status.HTTP_200_OK,
-            summary="Get information about the OCR-D Resource Manager Server"
-        )
-        base_router.add_api_route(
-            path="/list_available",
-            endpoint=self.list_available_resources,
-            methods=["GET"],
-            status_code=status.HTTP_200_OK,
-            summary=""
-        )
-        base_router.add_api_route(
-            path="/list_installed",
-            endpoint=self.list_installed_resources,
-            methods=["GET"],
-            status_code=status.HTTP_200_OK,
-            summary=""
-        )
-        base_router.add_api_route(
-            path="/download",
-            endpoint=self.download_resource,
-            methods=["GET"],
-            status_code=status.HTTP_200_OK,
-            summary=""
-        )
-        self.include_router(base_router)
-
-    async def home_page(self):
-        message = f"The home page of the {self.title}"
-        json_message = {
-            "message": message,
-            "time": datetime.now().strftime("%Y-%m-%d %H:%M")
-        }
-        return json_message
-
-    async def list_available_resources(
-        self,
-        executable: Any = None,
-        dynamic: bool = True,
-        name: Any = None,
-        database: Any = None,
-        url: Any = None
-    ):
-        result = self.resmgr_instance.list_available(executable, dynamic, name, database, url)
-        json_message = {
-            "result": result
-        }
-        return json_message
-
-    async def list_installed_resources(self, executable: Any = None):
-        result = self.resmgr_instance.list_available(executable)
-        json_message = {
-            "result": result
-        }
-        return json_message
-
-    async def download_resource(
-        self,
-        executable: str,
-        name: Any = None,
-        location: Any = None,
-        any_url: str = '',
-        no_dynamic: bool = False,
-        resource_type: str = 'file',
-        path_in_archive: str = '.',
-        allow_uninstalled: bool = True,
-        overwrite: bool = True
-    ):
-        resmgr = OcrdResourceManager()
-        response = []
-        if executable != '*' and not name:
-            message = f"Unless EXECUTABLE ('{executable}') is the '*' wildcard, NAME is required"
-            self.log.error(message)
-            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=message)
-        elif executable == '*':
-            executable = None
-        if name == '*':
-            name = None
-        if executable and not which(executable):
-            if not allow_uninstalled:
-                message = (f"Executable '{executable}' is not installed. To download resources anyway, "
-                           f"use the -a/--allow-uninstalled flag")
-                self.log.error(message)
-                raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=message)
-            else:
-                message = f"Executable '{executable}' is not installed, but downloading resources anyway."
-                self.log.info(message)
-                response.append(message)
-        reslist = resmgr.list_available(executable=executable, dynamic=not no_dynamic, name=name)
-        if not any(r[1] for r in reslist):
-            message = f"No resources {name} found in registry for executable {executable}"
-            self.log.info(message)
-            response.append(message)
-            if executable and name:
-                reslist = [(executable, [{
-                    'url': any_url or '???',
-                    'name': name,
-                    'type': resource_type,
-                    'path_in_archive': path_in_archive}]
-                )]
-        for this_executable, this_reslist in reslist:
-            resource_locations = get_ocrd_tool_json(this_executable)['resource_locations']
-            if not location:
-                location = resource_locations[0]
-            elif location not in resource_locations:
-                response.append(
-                    f"The selected --location {location} is not in the {this_executable}'s resource search path, "
-                    f"refusing to install to invalid location. Instead installing to: {resource_locations[0]}")
-            res_dest_dir = resmgr.build_resource_dest_dir(location=location, executable=this_executable)
-            for res_dict in this_reslist:
-                try:
-                    fpath = resmgr.handle_resource(
-                        res_dict=res_dict,
-                        executable=this_executable,
-                        dest_dir=res_dest_dir,
-                        any_url=any_url,
-                        overwrite=overwrite,
-                        resource_type=resource_type,
-                        path_in_archive=path_in_archive
-                    )
-                    if not fpath:
-                        continue
-                except FileExistsError as exc:
-                    response.append(str(exc))
-                usage = res_dict.get('parameter_usage', 'as-is')
-                response.append(f"Use in parameters as '{resmgr.parameter_usage(res_dict['name'], usage)}'")
-        json_message = { "result": response }
-        return json_message