ocrd 3.1.2__py3-none-any.whl → 3.2.0__py3-none-any.whl

ocrd/cli/network.py

@@ -13,6 +13,7 @@ from ocrd_network.cli import (
     processing_server_cli,
     processing_worker_cli,
     processor_server_cli,
+    resource_manager_server_cli
 )
 
 
@@ -28,3 +29,4 @@ network_cli.add_command(client_cli)
 network_cli.add_command(processing_server_cli)
 network_cli.add_command(processing_worker_cli)
 network_cli.add_command(processor_server_cli)
+network_cli.add_command(resource_manager_server_cli)

ocrd/cli/resmgr.py

@@ -18,9 +18,9 @@ from ocrd_utils import (
     getLogger,
     get_moduledir,
     get_ocrd_tool_json,
-    resource_filename,
     initLogging,
     RESOURCE_LOCATIONS,
+    RESOURCE_TYPES
 )
 from ocrd.constants import RESOURCE_USER_LIST_COMMENT
 
@@ -72,13 +72,13 @@ def list_installed(executable=None):
 @click.option('-n', '--any-url', default='', help='URL of unregistered resource to download/copy from')
 @click.option('-D', '--no-dynamic', default=False, is_flag=True,
               help="Whether to skip looking into each processor's --dump-{json,module-dir} for module-level resources")
-@click.option('-t', '--resource-type', type=click.Choice(['file', 'directory', 'archive']), default='file',
+@click.option('-t', '--resource-type', type=click.Choice(RESOURCE_TYPES), default='file',
               help='Type of resource',)
 @click.option('-P', '--path-in-archive', default='.', help='Path to extract in case of archive type')
 @click.option('-a', '--allow-uninstalled', is_flag=True,
               help="Allow installing resources for uninstalled processors",)
 @click.option('-o', '--overwrite', help='Overwrite existing resources', is_flag=True)
-@click.option('-l', '--location', type=click.Choice(RESOURCE_LOCATIONS),
+@click.option('-l', '--location', type=click.Choice(RESOURCE_LOCATIONS), default='data',
               help="Where to store resources - defaults to first location in processor's 'resource_locations' "
                    "list or finally 'data'")
 @click.argument('executable', required=True)
@@ -107,8 +107,6 @@ def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstal
         executable = None
     if name == '*':
         name = None
-    is_url = (any_url.startswith('https://') or any_url.startswith('http://')) if any_url else False
-    is_filename = Path(any_url).exists() if any_url else False
     if executable and not which(executable):
         if not allow_uninstalled:
             log.error(f"Executable '{executable}' is not installed. "
@@ -127,65 +125,30 @@ def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstal
                 'path_in_archive': path_in_archive}]
             )]
     for this_executable, this_reslist in reslist:
-        for resdict in this_reslist:
-            if 'size' in resdict:
-                registered = "registered"
-            else:
-                registered = "unregistered"
-            if any_url:
-                resdict['url'] = any_url
-            if resdict['url'] == '???':
-                log.warning(f"Cannot download user resource {resdict['name']}")
-                continue
-            if resdict['url'].startswith('https://') or resdict['url'].startswith('http://'):
-                log.info(f"Downloading {registered} resource '{resdict['name']}' ({resdict['url']})")
-                if 'size' not in resdict:
-                    with requests.head(resdict['url']) as r:
-                        resdict['size'] = int(r.headers.get('content-length', 0))
-            else:
-                log.info(f"Copying {registered} resource '{resdict['name']}' ({resdict['url']})")
-                urlpath = Path(resdict['url'])
-                resdict['url'] = str(urlpath.resolve())
-                if Path(urlpath).is_dir():
-                    resdict['size'] = directory_size(urlpath)
-                else:
-                    resdict['size'] = urlpath.stat().st_size
-            if not location:
-                location = get_ocrd_tool_json(this_executable)['resource_locations'][0]
-            elif location not in get_ocrd_tool_json(this_executable)['resource_locations']:
-                log.error(f"The selected --location {location} is not in the {this_executable}'s resource search path, "
-                          f"refusing to install to invalid location")
-                sys.exit(1)
-            if location != 'module':
-                basedir = resmgr.location_to_resource_dir(location)
-            else:
-                basedir = get_moduledir(this_executable)
-                if not basedir:
-                    basedir = resmgr.location_to_resource_dir('data')
-
+        resource_locations = get_ocrd_tool_json(this_executable)['resource_locations']
+        if not location:
+            location = resource_locations[0]
+        elif location not in resource_locations:
+            log.warning(f"The selected --location {location} is not in the {this_executable}'s resource search path, "
+                        f"refusing to install to invalid location. Instead installing to: {resource_locations[0]}")
+        res_dest_dir = resmgr.build_resource_dest_dir(location=location, executable=this_executable)
+        for res_dict in this_reslist:
             try:
-                with click.progressbar(length=resdict['size']) as bar:
-                    fpath = resmgr.download(
-                        this_executable,
-                        resdict['url'],
-                        basedir,
-                        name=resdict['name'],
-                        resource_type=resdict.get('type', resource_type),
-                        path_in_archive=resdict.get('path_in_archive', path_in_archive),
-                        overwrite=overwrite,
-                        no_subdir=location in ['cwd', 'module'],
-                        progress_cb=lambda delta: bar.update(delta)
-                    )
-                if registered == 'unregistered':
-                    log.info(f"{this_executable} resource '{name}' ({any_url}) not a known resource, creating stub "
-                             f"in {resmgr.user_list}'")
-                    resmgr.add_to_user_database(this_executable, fpath, url=any_url)
-                resmgr.save_user_list()
-                log.info(f"Installed resource {resdict['url']} under {fpath}")
+                fpath = resmgr.handle_resource(
+                    res_dict=res_dict,
+                    executable=this_executable,
+                    dest_dir=res_dest_dir,
+                    any_url=any_url,
+                    overwrite=overwrite,
+                    resource_type=resource_type,
+                    path_in_archive=path_in_archive
+                )
+                if not fpath:
+                    continue
             except FileExistsError as exc:
                 log.info(str(exc))
-            log.info(f"Use in parameters as "
-                     f"'{resmgr.parameter_usage(resdict['name'], usage=resdict.get('parameter_usage', 'as-is'))}'")
+            usage = res_dict.get('parameter_usage', 'as-is')
+            log.info(f"Use in parameters as '{resmgr.parameter_usage(res_dict['name'], usage)}'")
 
 
 @resmgr_cli.command('migrate')

ocrd/cli/workspace.py

@@ -88,8 +88,8 @@ def workspace_cli(ctx, directory, mets, mets_basename, mets_server_url, backup):
 @pass_workspace
 @click.option('-a', '--download', is_flag=True, help="Download all files")
 @click.option('-s', '--skip', help="Tests to skip", default=[], multiple=True, type=click.Choice(
-    ['imagefilename', 'dimension', 'pixel_density', 'page', 'url', 'page_xsd', 'mets_fileid_page_pcgtsid',
-     'mets_unique_identifier', 'mets_file_group_names', 'mets_files', 'mets_xsd']))
+    ['imagefilename', 'alternativeimage_filename', 'alternativeimage_comments', 'dimension', 'pixel_density', 'page', 'page_xsd',
+     'url', 'mets_fileid_page_pcgtsid', 'mets_unique_identifier', 'mets_files', 'mets_xsd']))
 @click.option('--page-textequiv-consistency', '--page-strictness', help="How strict to check PAGE multi-level textequiv consistency", type=click.Choice(['strict', 'lax', 'fix', 'off']), default='strict')
 @click.option('--page-coordinate-consistency', help="How fierce to check PAGE multi-level coordinate consistency", type=click.Choice(['poly', 'baseline', 'both', 'off']), default='poly')
 @click.argument('mets_url', default=None, required=False)

ocrd/processor/base.py

@@ -43,15 +43,14 @@ from .ocrd_page_result import OcrdPageResult
 from ocrd_utils import (
     VERSION as OCRD_VERSION,
     MIMETYPE_PAGE,
-    MIME_TO_EXT,
     config,
     getLogger,
     list_resource_candidates,
-    pushd_popd,
     list_all_resources,
     get_processor_resource_types,
     resource_filename,
     parse_json_file_with_comments,
+    pushd_popd,
     make_file_id,
     deprecation_warning
 )
@@ -779,10 +778,14 @@ class Processor():
         to handle cases like multiple output fileGrps, non-PAGE input etc.)
         """
         input_pcgts : List[Optional[OcrdPage]] = [None] * len(input_files)
-        assert isinstance(input_files[0], get_args(OcrdFileType))
-        page_id = input_files[0].pageId
+        input_pos = next(i for i, input_file in enumerate(input_files) if input_file is not None)
+        page_id = input_files[input_pos].pageId
         self._base_logger.info("processing page %s", page_id)
         for i, input_file in enumerate(input_files):
+            if input_file is None:
+                grp = self.input_file_grp.split(',')[i]
+                self._base_logger.debug(f"ignoring missing file for input fileGrp {grp} for page {page_id}")
+                continue
             assert isinstance(input_file, get_args(OcrdFileType))
             self._base_logger.debug(f"parsing file {input_file.ID} for page {page_id}")
             try:
@@ -792,7 +795,10 @@ class Processor():
             except ValueError as err:
                 # not PAGE and not an image to generate PAGE for
                 self._base_logger.error(f"non-PAGE input for page {page_id}: {err}")
-        output_file_id = make_file_id(input_files[0], self.output_file_grp)
+        output_file_id = make_file_id(input_files[input_pos], self.output_file_grp)
+        if input_files[input_pos].fileGrp == self.output_file_grp:
+            # input=output fileGrp: re-use ID exactly
+            output_file_id = input_files[input_pos].ID
         output_file = next(self.workspace.mets.find_files(ID=output_file_id), None)
         if output_file and config.OCRD_EXISTING_OUTPUT != 'OVERWRITE':
             # short-cut avoiding useless computation:
@@ -898,9 +904,8 @@ class Processor():
             cwd = self.old_pwd
         else:
             cwd = getcwd()
-        ret = [cand for cand in list_resource_candidates(executable, val,
-                                                         cwd=cwd, moduled=self.moduledir)
-               if exists(cand)]
+        ret = list(filter(exists, list_resource_candidates(executable, val,
+                                                           cwd=cwd, moduled=self.moduledir)))
         if ret:
             self._base_logger.debug("Resolved %s to absolute path %s" % (val, ret[0]))
             return ret[0]
@@ -931,17 +936,9 @@ class Processor():
         """
         List all resources found in the filesystem and matching content-type by filename suffix
         """
-        mimetypes = get_processor_resource_types(None, self.ocrd_tool)
-        for res in list_all_resources(self.ocrd_tool['executable'], moduled=self.moduledir):
+        for res in list_all_resources(self.executable, ocrd_tool=self.ocrd_tool, moduled=self.moduledir):
             res = Path(res)
-            if not '*/*' in mimetypes:
-                if res.is_dir() and not 'text/directory' in mimetypes:
-                    continue
-                # if we do not know all MIME types, then keep the file, otherwise require suffix match
-                if res.is_file() and not any(res.suffix == MIME_TO_EXT.get(mime, res.suffix)
-                                             for mime in mimetypes):
-                    continue
-            yield res
+            yield res.name
 
     @property
     def module(self):