ocrd 3.2.0__py3-none-any.whl → 3.3.1__py3-none-any.whl

ocrd/cli/network.py

@@ -13,7 +13,6 @@ from ocrd_network.cli import (
     processing_server_cli,
     processing_worker_cli,
     processor_server_cli,
-    resource_manager_server_cli
 )
 
 
@@ -29,4 +28,3 @@ network_cli.add_command(client_cli)
 network_cli.add_command(processing_server_cli)
 network_cli.add_command(processing_worker_cli)
 network_cli.add_command(processor_server_cli)
-network_cli.add_command(resource_manager_server_cli)

ocrd/cli/resmgr.py

@@ -18,9 +18,9 @@ from ocrd_utils import (
     getLogger,
     get_moduledir,
     get_ocrd_tool_json,
+    resource_filename,
     initLogging,
     RESOURCE_LOCATIONS,
-    RESOURCE_TYPES
 )
 from ocrd.constants import RESOURCE_USER_LIST_COMMENT
 
@@ -72,13 +72,13 @@ def list_installed(executable=None):
 @click.option('-n', '--any-url', default='', help='URL of unregistered resource to download/copy from')
 @click.option('-D', '--no-dynamic', default=False, is_flag=True,
               help="Whether to skip looking into each processor's --dump-{json,module-dir} for module-level resources")
-@click.option('-t', '--resource-type', type=click.Choice(RESOURCE_TYPES), default='file',
+@click.option('-t', '--resource-type', type=click.Choice(['file', 'directory', 'archive']), default='file',
               help='Type of resource',)
 @click.option('-P', '--path-in-archive', default='.', help='Path to extract in case of archive type')
 @click.option('-a', '--allow-uninstalled', is_flag=True,
               help="Allow installing resources for uninstalled processors",)
 @click.option('-o', '--overwrite', help='Overwrite existing resources', is_flag=True)
-@click.option('-l', '--location', type=click.Choice(RESOURCE_LOCATIONS), default='data',
+@click.option('-l', '--location', type=click.Choice(RESOURCE_LOCATIONS),
               help="Where to store resources - defaults to first location in processor's 'resource_locations' "
                    "list or finally 'data'")
 @click.argument('executable', required=True)
@@ -107,6 +107,8 @@ def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstal
         executable = None
     if name == '*':
         name = None
+    is_url = (any_url.startswith('https://') or any_url.startswith('http://')) if any_url else False
+    is_filename = Path(any_url).exists() if any_url else False
     if executable and not which(executable):
         if not allow_uninstalled:
             log.error(f"Executable '{executable}' is not installed. "
@@ -125,30 +127,65 @@ def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstal
                 'path_in_archive': path_in_archive}]
             )]
     for this_executable, this_reslist in reslist:
-        resource_locations = get_ocrd_tool_json(this_executable)['resource_locations']
-        if not location:
-            location = resource_locations[0]
-        elif location not in resource_locations:
-            log.warning(f"The selected --location {location} is not in the {this_executable}'s resource search path, "
-                        f"refusing to install to invalid location. Instead installing to: {resource_locations[0]}")
-        res_dest_dir = resmgr.build_resource_dest_dir(location=location, executable=this_executable)
-        for res_dict in this_reslist:
+        for resdict in this_reslist:
+            if 'size' in resdict:
+                registered = "registered"
+            else:
+                registered = "unregistered"
+            if any_url:
+                resdict['url'] = any_url
+            if resdict['url'] == '???':
+                log.warning(f"Cannot download user resource {resdict['name']}")
+                continue
+            if resdict['url'].startswith('https://') or resdict['url'].startswith('http://'):
+                log.info(f"Downloading {registered} resource '{resdict['name']}' ({resdict['url']})")
+                if 'size' not in resdict:
+                    with requests.head(resdict['url']) as r:
+                        resdict['size'] = int(r.headers.get('content-length', 0))
+            else:
+                log.info(f"Copying {registered} resource '{resdict['name']}' ({resdict['url']})")
+                urlpath = Path(resdict['url'])
+                resdict['url'] = str(urlpath.resolve())
+                if Path(urlpath).is_dir():
+                    resdict['size'] = directory_size(urlpath)
+                else:
+                    resdict['size'] = urlpath.stat().st_size
+            if not location:
+                location = get_ocrd_tool_json(this_executable)['resource_locations'][0]
+            elif location not in get_ocrd_tool_json(this_executable)['resource_locations']:
+                log.error(f"The selected --location {location} is not in the {this_executable}'s resource search path, "
+                          f"refusing to install to invalid location")
+                sys.exit(1)
+            if location != 'module':
+                basedir = resmgr.location_to_resource_dir(location)
+            else:
+                basedir = get_moduledir(this_executable)
+                if not basedir:
+                    basedir = resmgr.location_to_resource_dir('data')
+
             try:
-                fpath = resmgr.handle_resource(
-                    res_dict=res_dict,
-                    executable=this_executable,
-                    dest_dir=res_dest_dir,
-                    any_url=any_url,
-                    overwrite=overwrite,
-                    resource_type=resource_type,
-                    path_in_archive=path_in_archive
-                )
-                if not fpath:
-                    continue
+                with click.progressbar(length=resdict['size']) as bar:
+                    fpath = resmgr.download(
+                        this_executable,
+                        resdict['url'],
+                        basedir,
+                        name=resdict['name'],
+                        resource_type=resdict.get('type', resource_type),
+                        path_in_archive=resdict.get('path_in_archive', path_in_archive),
+                        overwrite=overwrite,
+                        no_subdir=location in ['cwd', 'module'],
+                        progress_cb=lambda delta: bar.update(delta)
+                    )
+                if registered == 'unregistered':
+                    log.info(f"{this_executable} resource '{name}' ({any_url}) not a known resource, creating stub "
+                             f"in {resmgr.user_list}'")
+                    resmgr.add_to_user_database(this_executable, fpath, url=any_url)
+                resmgr.save_user_list()
+                log.info(f"Installed resource {resdict['url']} under {fpath}")
             except FileExistsError as exc:
                 log.info(str(exc))
-            usage = res_dict.get('parameter_usage', 'as-is')
-            log.info(f"Use in parameters as '{resmgr.parameter_usage(res_dict['name'], usage)}'")
+            log.info(f"Use in parameters as "
+                     f"'{resmgr.parameter_usage(resdict['name'], usage=resdict.get('parameter_usage', 'as-is'))}'")
 
 
 @resmgr_cli.command('migrate')

ocrd/decorators/__init__.py

@@ -107,10 +107,10 @@ def ocrd_cli_wrap_processor(
     if 'parameter_override' in kwargs:
         set_json_key_value_overrides(kwargs['parameter'], *kwargs.pop('parameter_override'))
     # Assert -I / -O
-    if not kwargs['input_file_grp']:
+    if not kwargs.get('input_file_grp', None):
         raise ValueError('-I/--input-file-grp is required')
-    if not kwargs['output_file_grp']:
-        raise ValueError('-O/--output-file-grp is required')
+    if 'output_file_grp' not in kwargs:
+        raise ValueError('-O/--output-file-grp is required') # actually, it may be None
     resolver = Resolver()
     working_dir, mets, _, mets_server_url = \
             resolver.resolve_mets_arguments(working_dir, mets, None, mets_server_url)

ocrd/processor/base.py

@@ -43,14 +43,15 @@ from .ocrd_page_result import OcrdPageResult
 from ocrd_utils import (
     VERSION as OCRD_VERSION,
     MIMETYPE_PAGE,
+    MIME_TO_EXT,
     config,
     getLogger,
     list_resource_candidates,
+    pushd_popd,
     list_all_resources,
     get_processor_resource_types,
     resource_filename,
     parse_json_file_with_comments,
-    pushd_popd,
     make_file_id,
     deprecation_warning
 )
@@ -904,8 +905,9 @@ class Processor():
             cwd = self.old_pwd
         else:
             cwd = getcwd()
-        ret = list(filter(exists, list_resource_candidates(executable, val,
-                                                           cwd=cwd, moduled=self.moduledir)))
+        ret = [cand for cand in list_resource_candidates(executable, val,
+                                                         cwd=cwd, moduled=self.moduledir)
+               if exists(cand)]
         if ret:
             self._base_logger.debug("Resolved %s to absolute path %s" % (val, ret[0]))
             return ret[0]
@@ -936,9 +938,17 @@ class Processor():
         """
         List all resources found in the filesystem and matching content-type by filename suffix
         """
-        for res in list_all_resources(self.executable, ocrd_tool=self.ocrd_tool, moduled=self.moduledir):
+        mimetypes = get_processor_resource_types(None, self.ocrd_tool)
+        for res in list_all_resources(self.ocrd_tool['executable'], moduled=self.moduledir):
             res = Path(res)
-            yield res.name
+            if not '*/*' in mimetypes:
+                if res.is_dir() and not 'text/directory' in mimetypes:
+                    continue
+                # if we do not know all MIME types, then keep the file, otherwise require suffix match
+                if res.is_file() and not any(res.suffix == MIME_TO_EXT.get(mime, res.suffix)
+                                             for mime in mimetypes):
+                    continue
+            yield res
 
     @property
     def module(self):
@@ -1102,7 +1112,7 @@ class Processor():
                 if not ifiles[i]:
                     # could be from non-unique with on_error=skip or from true gap
                     self._base_logger.error(f'Found no file for page {page} in file group {ifg}')
-                    if config.OCRD_MISSING_INPUT == 'abort':
+                    if config.OCRD_MISSING_INPUT == 'ABORT':
                         raise MissingInputFile(ifg, page, mimetype)
             if not any(ifiles):
                 # must be from non-unique with on_error=skip

ocrd/processor/builtin/dummy/ocrd-tool.json

@@ -1,6 +1,7 @@
 {
   "version": "1.0.0",
   "git_url": "https://github.com/OCR-D/core",
+  "dockerhub": "ocrd/core",
   "tools": {
     "ocrd-dummy": {
       "executable": "ocrd-dummy",

ocrd/processor/helpers.py

@@ -276,7 +276,7 @@ def get_processor(
         # set current processing parameters
         processor.workspace = workspace
         processor.page_id = page_id
-        processor.input_file_grp = input_file_grp
-        processor.output_file_grp = output_file_grp
+        processor.input_file_grp = input_file_grp or ''
+        processor.output_file_grp = output_file_grp or ''
         return processor
     raise ValueError("Processor class is not known")