ocrd 3.7.0__py3-none-any.whl → 3.8.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. ocrd/cli/network.py +2 -0
  2. ocrd/cli/resmgr.py +29 -65
  3. ocrd/constants.py +0 -2
  4. ocrd/ocrd-all-tool.json +25 -0
  5. ocrd/processor/base.py +6 -16
  6. ocrd/processor/builtin/dummy/ocrd-tool.json +25 -0
  7. ocrd/processor/builtin/merge_processor.py +131 -0
  8. ocrd/processor/builtin/param_command_header2unordered.json +7 -0
  9. ocrd/processor/builtin/param_command_heading2unordered.json +7 -0
  10. ocrd/processor/builtin/param_command_lines2orientation.json +6 -0
  11. ocrd/processor/builtin/param_command_page-update-version.json +5 -0
  12. ocrd/processor/builtin/param_command_transkribus-to-prima.json +8 -0
  13. ocrd/processor/builtin/shell_processor.py +128 -0
  14. ocrd/resource_manager.py +213 -124
  15. {ocrd-3.7.0.dist-info → ocrd-3.8.1.dist-info}/METADATA +22 -3
  16. {ocrd-3.7.0.dist-info → ocrd-3.8.1.dist-info}/RECORD +34 -26
  17. {ocrd-3.7.0.dist-info → ocrd-3.8.1.dist-info}/entry_points.txt +2 -0
  18. ocrd_models/ocrd_agent.py +3 -3
  19. ocrd_network/__init__.py +1 -0
  20. ocrd_network/cli/__init__.py +2 -0
  21. ocrd_network/cli/resmgr_server.py +23 -0
  22. ocrd_network/constants.py +3 -0
  23. ocrd_network/logging_utils.py +5 -0
  24. ocrd_network/resource_manager_server.py +182 -0
  25. ocrd_network/runtime_data/connection_clients.py +1 -1
  26. ocrd_network/runtime_data/hosts.py +43 -16
  27. ocrd_network/runtime_data/network_agents.py +15 -1
  28. ocrd_utils/__init__.py +5 -1
  29. ocrd_utils/constants.py +5 -0
  30. ocrd_utils/os.py +141 -61
  31. ocrd_validators/ocrd_tool.schema.yml +7 -4
  32. ocrd/resource_list.yml +0 -61
  33. {ocrd-3.7.0.dist-info → ocrd-3.8.1.dist-info}/LICENSE +0 -0
  34. {ocrd-3.7.0.dist-info → ocrd-3.8.1.dist-info}/WHEEL +0 -0
  35. {ocrd-3.7.0.dist-info → ocrd-3.8.1.dist-info}/top_level.txt +0 -0
ocrd/cli/network.py CHANGED
@@ -12,6 +12,7 @@ from ocrd_network.cli import (
12
12
  client_cli,
13
13
  processing_server_cli,
14
14
  processing_worker_cli,
15
+ resource_manager_server_cli
15
16
  )
16
17
 
17
18
 
@@ -26,3 +27,4 @@ def network_cli():
26
27
  network_cli.add_command(client_cli)
27
28
  network_cli.add_command(processing_server_cli)
28
29
  network_cli.add_command(processing_worker_cli)
30
+ network_cli.add_command(resource_manager_server_cli)
ocrd/cli/resmgr.py CHANGED
@@ -20,6 +20,7 @@ from ocrd_utils import (
20
20
  get_ocrd_tool_json,
21
21
  initLogging,
22
22
  RESOURCE_LOCATIONS,
23
+ RESOURCE_TYPES
23
24
  )
24
25
  from ocrd.constants import RESOURCE_USER_LIST_COMMENT
25
26
 
@@ -70,16 +71,16 @@ def list_installed(executable=None):
70
71
  @resmgr_cli.command('download')
71
72
  @click.option('-n', '--any-url', default='', help='URL of unregistered resource to download/copy from')
72
73
  @click.option('-D', '--no-dynamic', default=False, is_flag=True,
73
- help="Whether to skip looking into each processor's --dump-{json,module-dir} for module-level resources")
74
- @click.option('-t', '--resource-type', type=click.Choice(['file', 'directory', 'archive']), default='file',
75
- help='Type of resource',)
76
- @click.option('-P', '--path-in-archive', default='.', help='Path to extract in case of archive type')
74
+ help="Skip looking into each processor's --dump-{json,module-dir} module-registered resources")
75
+ @click.option('-t', '--resource-type', type=click.Choice(RESOURCE_TYPES), default='file',
76
+ help='Type of resource (when unregistered or incomplete)',)
77
+ @click.option('-P', '--path-in-archive', default='.', help='Path to extract in case of archive type (when unregistered or incomplete)')
77
78
  @click.option('-a', '--allow-uninstalled', is_flag=True,
78
- help="Allow installing resources for uninstalled processors",)
79
+ help="Allow installing resources for not installed processors",)
79
80
  @click.option('-o', '--overwrite', help='Overwrite existing resources', is_flag=True)
80
- @click.option('-l', '--location', type=click.Choice(RESOURCE_LOCATIONS),
81
+ @click.option('-l', '--location', type=click.Choice(RESOURCE_LOCATIONS),
81
82
  help="Where to store resources - defaults to first location in processor's 'resource_locations' "
82
- "list or finally 'data'")
83
+ "list, i.e. usually 'data'")
83
84
  @click.argument('executable', required=True)
84
85
  @click.argument('name', required=False)
85
86
  def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstalled, overwrite, location, executable,
@@ -106,8 +107,6 @@ def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstal
106
107
  executable = None
107
108
  if name == '*':
108
109
  name = None
109
- is_url = (any_url.startswith('https://') or any_url.startswith('http://')) if any_url else False
110
- is_filename = Path(any_url).exists() if any_url else False
111
110
  if executable and not which(executable):
112
111
  if not allow_uninstalled:
113
112
  log.error(f"Executable '{executable}' is not installed. "
@@ -126,65 +125,30 @@ def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstal
126
125
  'path_in_archive': path_in_archive}]
127
126
  )]
128
127
  for this_executable, this_reslist in reslist:
129
- for resdict in this_reslist:
130
- if 'size' in resdict:
131
- registered = "registered"
132
- else:
133
- registered = "unregistered"
134
- if any_url:
135
- resdict['url'] = any_url
136
- if resdict['url'] == '???':
137
- log.warning(f"Cannot download user resource {resdict['name']}")
138
- continue
139
- if resdict['url'].startswith('https://') or resdict['url'].startswith('http://'):
140
- log.info(f"Downloading {registered} resource '{resdict['name']}' ({resdict['url']})")
141
- if 'size' not in resdict:
142
- with requests.head(resdict['url']) as r:
143
- resdict['size'] = int(r.headers.get('content-length', 0))
144
- else:
145
- log.info(f"Copying {registered} resource '{resdict['name']}' ({resdict['url']})")
146
- urlpath = Path(resdict['url'])
147
- resdict['url'] = str(urlpath.resolve())
148
- if Path(urlpath).is_dir():
149
- resdict['size'] = directory_size(urlpath)
150
- else:
151
- resdict['size'] = urlpath.stat().st_size
152
- if not location:
153
- location = get_ocrd_tool_json(this_executable)['resource_locations'][0]
154
- elif location not in get_ocrd_tool_json(this_executable)['resource_locations']:
155
- log.error(f"The selected --location {location} is not in the {this_executable}'s resource search path, "
156
- f"refusing to install to invalid location")
157
- sys.exit(1)
158
- if location != 'module':
159
- basedir = resmgr.location_to_resource_dir(location)
160
- else:
161
- basedir = get_moduledir(this_executable)
162
- if not basedir:
163
- basedir = resmgr.location_to_resource_dir('data')
164
-
128
+ resource_locations = get_ocrd_tool_json(this_executable)['resource_locations']
129
+ if not location:
130
+ location = resource_locations[0]
131
+ elif location not in resource_locations:
132
+ log.warning(f"The selected --location {location} is not in the {this_executable}'s resource search path, "
133
+ f"refusing to install to invalid location. Instead installing to: {resource_locations[0]}")
134
+ res_dest_dir = resmgr.build_resource_dest_dir(location=location, executable=this_executable)
135
+ for res_dict in this_reslist:
165
136
  try:
166
- with click.progressbar(length=resdict['size']) as bar:
167
- fpath = resmgr.download(
168
- this_executable,
169
- resdict['url'],
170
- basedir,
171
- name=resdict['name'],
172
- resource_type=resdict.get('type', resource_type),
173
- path_in_archive=resdict.get('path_in_archive', path_in_archive),
174
- overwrite=overwrite,
175
- no_subdir=location in ['cwd', 'module'],
176
- progress_cb=lambda delta: bar.update(delta)
177
- )
178
- if registered == 'unregistered':
179
- log.info(f"{this_executable} resource '{name}' ({any_url}) not a known resource, creating stub "
180
- f"in {resmgr.user_list}'")
181
- resmgr.add_to_user_database(this_executable, fpath, url=any_url)
182
- resmgr.save_user_list()
183
- log.info(f"Installed resource {resdict['url']} under {fpath}")
137
+ fpath = resmgr.handle_resource(
138
+ res_dict=res_dict,
139
+ executable=this_executable,
140
+ dest_dir=res_dest_dir,
141
+ any_url=any_url,
142
+ overwrite=overwrite,
143
+ resource_type=resource_type,
144
+ path_in_archive=path_in_archive
145
+ )
146
+ if not fpath:
147
+ continue
184
148
  except FileExistsError as exc:
185
149
  log.info(str(exc))
186
- log.info(f"Use in parameters as "
187
- f"'{resmgr.parameter_usage(resdict['name'], usage=resdict.get('parameter_usage', 'as-is'))}'")
150
+ usage = res_dict.get('parameter_usage', 'as-is')
151
+ log.info(f"Use in parameters as '{resmgr.parameter_usage(res_dict['name'], usage)}'")
188
152
 
189
153
 
190
154
  @resmgr_cli.command('migrate')
ocrd/constants.py CHANGED
@@ -9,7 +9,6 @@ __all__ = [
9
9
  'DOWNLOAD_DIR',
10
10
  'DEFAULT_REPOSITORY_URL',
11
11
  'BASHLIB_FILENAME',
12
- 'RESOURCE_LIST_FILENAME',
13
12
  'BACKUP_DIR',
14
13
  'RESOURCE_USER_LIST_COMMENT',
15
14
  ]
@@ -19,6 +18,5 @@ DEFAULT_UPLOAD_FOLDER = '/tmp/uploads-ocrd-core'
19
18
  DOWNLOAD_DIR = '/tmp/ocrd-core-downloads'
20
19
  DEFAULT_REPOSITORY_URL = 'http://localhost:5000/'
21
20
  BASHLIB_FILENAME = resource_filename(__package__, 'lib.bash')
22
- RESOURCE_LIST_FILENAME = resource_filename(__package__, 'resource_list.yml')
23
21
  RESOURCE_USER_LIST_COMMENT = "# OCR-D private resource list (consider sending a PR with your own resources to OCR-D/core)"
24
22
  BACKUP_DIR = '.backup'
ocrd/ocrd-all-tool.json CHANGED
@@ -41,5 +41,30 @@
41
41
  "description": "Whether to extract an image for each filtered segment and write to the output fileGrp."
42
42
  }
43
43
  }
44
+ },
45
+ "ocrd-command": {
46
+ "executable": "ocrd-command",
47
+ "description": "Bare-bones processor runs shell commands to process PAGE files",
48
+ "steps": ["recognition/text-recognition", "recognition/font-identification", "recognition/post-correction", "layout/segmentation", "layout/analysis"],
49
+ "categories": [],
50
+ "input_file_grp_cardinality": [1, -1],
51
+ "output_file_grp_cardinality": 1,
52
+ "parameters": {
53
+ "command": {
54
+ "type": "string",
55
+ "default": "cat @INFILE > @OUTFILE",
56
+ "description": "Shell command to operate on PAGE files, with @INFILE as place-holder for the input file path(s), and @OUTFILE as place-holder for the output file path. If running on multiple input fileGrps, then @INFILE must be repeated as many times."
57
+ }
58
+ }
59
+ },
60
+ "ocrd-merge": {
61
+ "executable": "ocrd-merge",
62
+ "description": "Bare-bones processor merges annotations from multiple fileGrps",
63
+ "steps": ["layout/segmentation"],
64
+ "categories": [],
65
+ "input_file_grp_cardinality": [1, -1],
66
+ "output_file_grp_cardinality": 1,
67
+ "parameters": {
68
+ }
44
69
  }
45
70
  }
ocrd/processor/base.py CHANGED
@@ -42,15 +42,14 @@ from .ocrd_page_result import OcrdPageResult
42
42
  from ocrd_utils import (
43
43
  VERSION as OCRD_VERSION,
44
44
  MIMETYPE_PAGE,
45
- MIME_TO_EXT,
46
45
  config,
47
46
  getLogger,
48
47
  list_resource_candidates,
49
- pushd_popd,
50
48
  list_all_resources,
51
49
  get_processor_resource_types,
52
50
  resource_filename,
53
51
  parse_json_file_with_comments,
52
+ pushd_popd,
54
53
  make_file_id,
55
54
  deprecation_warning
56
55
  )
@@ -608,7 +607,7 @@ class Processor():
608
607
  """
609
608
  Ensure all input files for a single page are
610
609
  downloaded to the workspace, then schedule
611
- :py:meth:`.process_process_file` to be run on
610
+ :py:meth:`.process_page_file` to be run on
612
611
  them via `executor` (enforcing a per-page time
613
612
  limit of `max_seconds`).
614
613
 
@@ -935,9 +934,8 @@ class Processor():
935
934
  cwd = self.old_pwd
936
935
  else:
937
936
  cwd = getcwd()
938
- ret = [cand for cand in list_resource_candidates(executable, val,
939
- cwd=cwd, moduled=self.moduledir)
940
- if exists(cand)]
937
+ ret = list(filter(exists, list_resource_candidates(executable, val,
938
+ cwd=cwd, moduled=self.moduledir)))
941
939
  if ret:
942
940
  self._base_logger.debug("Resolved %s to absolute path %s" % (val, ret[0]))
943
941
  return ret[0]
@@ -968,17 +966,9 @@ class Processor():
968
966
  """
969
967
  List all resources found in the filesystem and matching content-type by filename suffix
970
968
  """
971
- mimetypes = get_processor_resource_types(None, self.ocrd_tool)
972
- for res in list_all_resources(self.ocrd_tool['executable'], moduled=self.moduledir):
969
+ for res in list_all_resources(self.executable, ocrd_tool=self.ocrd_tool, moduled=self.moduledir):
973
970
  res = Path(res)
974
- if '*/*' not in mimetypes:
975
- if res.is_dir() and 'text/directory' not in mimetypes:
976
- continue
977
- # if we do not know all MIME types, then keep the file, otherwise require suffix match
978
- if res.is_file() and not any(res.suffix == MIME_TO_EXT.get(mime, res.suffix)
979
- for mime in mimetypes):
980
- continue
981
- yield res
971
+ yield res.name
982
972
 
983
973
  @property
984
974
  def module(self):
@@ -37,6 +37,31 @@
37
37
  "description": "Whether to extract an image for each filtered segment and write to the output fileGrp."
38
38
  }
39
39
  }
40
+ },
41
+ "ocrd-command": {
42
+ "executable": "ocrd-command",
43
+ "description": "Bare-bones processor runs shell commands to process PAGE files",
44
+ "steps": ["recognition/text-recognition", "recognition/font-identification", "recognition/post-correction", "layout/segmentation", "layout/analysis"],
45
+ "categories": [],
46
+ "input_file_grp_cardinality": [1, -1],
47
+ "output_file_grp_cardinality": 1,
48
+ "parameters": {
49
+ "command": {
50
+ "type": "string",
51
+ "default": "cat @INFILE > @OUTFILE",
52
+ "description": "Shell command to operate on PAGE files, with @INFILE as place-holder for the input file path(s), and @OUTFILE as place-holder for the output file path. If running on multiple input fileGrps, then @INFILE must be repeated as many times."
53
+ }
54
+ }
55
+ },
56
+ "ocrd-merge": {
57
+ "executable": "ocrd-merge",
58
+ "description": "Bare-bones processor merges annotations from multiple fileGrps",
59
+ "steps": ["layout/segmentation"],
60
+ "categories": [],
61
+ "input_file_grp_cardinality": [1, -1],
62
+ "output_file_grp_cardinality": 1,
63
+ "parameters": {
64
+ }
40
65
  }
41
66
  }
42
67
  }
@@ -0,0 +1,131 @@
1
+ # pylint: disable=missing-module-docstring,invalid-name
2
+ from typing import Optional
3
+ from itertools import count
4
+ from collections import OrderedDict as odict
5
+
6
+ import click
7
+
8
+ from ocrd import Processor, OcrdPageResult
9
+ from ocrd.decorators import ocrd_cli_options, ocrd_cli_wrap_processor
10
+ from ocrd_modelfactory import page_from_file
11
+ from ocrd_models import OcrdPage
12
+ from ocrd_models.ocrd_page import (
13
+ BorderType,
14
+ CoordsType,
15
+ ReadingOrderType,
16
+ UnorderedGroupType,
17
+ )
18
+ from ocrd_utils import bbox_from_points
19
+
20
+ _SEGTYPES = [
21
+ "NoiseRegion",
22
+ "LineDrawingRegion",
23
+ "AdvertRegion",
24
+ "ImageRegion",
25
+ "ChartRegion",
26
+ "MusicRegion",
27
+ "GraphicRegion",
28
+ "UnknownRegion",
29
+ "CustomRegion",
30
+ "SeparatorRegion",
31
+ "MathsRegion",
32
+ "TextRegion",
33
+ "MapRegion",
34
+ "ChemRegion",
35
+ "TableRegion",
36
+ "TextLine",
37
+ "Word",
38
+ "Glyph"
39
+ ]
40
+
41
+
42
+ def get_border_bbox(pcgts):
43
+ if pcgts.Page.Border is None:
44
+ return [0, 0, pcgts.Page.imageWidth, pcgts.Page.imageHeight]
45
+ return bbox_from_points(pcgts.Page.Border.Coords.points)
46
+
47
+ def rename_segments(pcgts, start=1):
48
+ renamed = {}
49
+ rodict = pcgts.Page.get_ReadingOrderGroups()
50
+ # get everything that has an identifier
51
+ nodes = pcgts.xpath("//*[@id]")
52
+ # filter segments
53
+ segments = [segment for segment in map(pcgts.revmap.get, nodes)
54
+ # get PAGE objects from matching etree nodes
55
+ # but allow only hierarchy segments
56
+ if segment.__class__.__name__.replace('Type', '') in _SEGTYPES]
57
+ # count segments and rename them
58
+ # fixme: or perhaps better to have each segment type named and counted differently?
59
+ num = 0
60
+ regions = []
61
+ for num, segment in zip(count(start=start), segments):
62
+ segtype = segment.original_tagname_
63
+ #parent = segment.parent_object_
64
+ newname = "seg%011d" % num
65
+ assert not segment.id in renamed
66
+ if segtype.endswith('Region') and segment.id in rodict:
67
+ # update reading order
68
+ roelem = rodict[segment.id]
69
+ roelem.regionRef = newname
70
+ renamed[segment.id] = newname
71
+ segment.id = newname
72
+ return num
73
+
74
+ class MergeProcessor(Processor):
75
+ def process_page_pcgts(self, *input_pcgts: Optional[OcrdPage], page_id: Optional[str] = None) -> OcrdPageResult:
76
+ """
77
+ Merge PAGE segment hierarchy elements from all input file groups.
78
+
79
+ For each page, open and deserialise PAGE input files. Rename all elements
80
+ of the segment hierarchy to new (clash-free) identifers. Redefine the
81
+ `Border` coordinates as the convex hull of all input borders. Then add all
82
+ regions from all input files, concatenating them into a single `ReadingOrder`
83
+ in the order of input file groups.
84
+
85
+ Produce a new PAGE output file by serialising the resulting hierarchy.
86
+ """
87
+ actual_pcgts = list(filter(None, input_pcgts))
88
+ assert len(set(pcgts.Page.imageFilename for pcgts in actual_pcgts)) == 1, \
89
+ "input files must all reference the same @imageFilename"
90
+ # create new PAGE for image
91
+ result = OcrdPageResult(page_from_file(actual_pcgts[0].Page.imageFilename))
92
+ # unify Border
93
+ borders = [get_border_bbox(pcgts) for pcgts in actual_pcgts]
94
+ minx, miny, maxx, maxy = zip(*borders)
95
+ minx = min(minx)
96
+ miny = min(miny)
97
+ maxx = max(maxx)
98
+ maxy = max(maxy)
99
+ result.pcgts.Page.set_Border(
100
+ BorderType(CoordsType(
101
+ points=f"{minx},{miny} {maxx},{miny} {maxx},{maxy} {minx},{maxy}")))
102
+ # rename all segments
103
+ num = 1
104
+ for pcgts in actual_pcgts:
105
+ num = rename_segments(pcgts, num)
106
+ # concatenate all regions
107
+ ug = UnorderedGroupType(id="merged")
108
+ result.pcgts.Page.set_ReadingOrder(ReadingOrderType(UnorderedGroup=ug))
109
+ for pcgts in actual_pcgts:
110
+ for region in pcgts.Page.get_AllRegions():
111
+ adder = getattr(result.pcgts.Page, 'add_' + region.original_tagname_)
112
+ adder(region)
113
+ if pcgts.Page.ReadingOrder:
114
+ group = pcgts.Page.ReadingOrder.OrderedGroup or pcgts.Page.ReadingOrder.UnorderedGroup
115
+ adder = getattr(ug, 'add_' + group.original_tagname_)
116
+ adder(group)
117
+ return result
118
+
119
+ @property
120
+ def metadata_filename(self):
121
+ return 'processor/builtin/dummy/ocrd-tool.json'
122
+
123
+ @property
124
+ def executable(self):
125
+ return 'ocrd-merge'
126
+
127
+
128
+ @click.command()
129
+ @ocrd_cli_options
130
+ def cli(*args, **kwargs):
131
+ return ocrd_cli_wrap_processor(MergeProcessor, *args, **kwargs)
@@ -0,0 +1,7 @@
1
+ {
2
+ # requires https://github.com/bertsky/workflow-configuration installed
3
+ # partitions PAGE-XML ReadingOrder from single OrderedGroup to top
4
+ # UnorderedGroup divided into OrderedGroups starting at every @type=header
5
+ # text regions.
6
+ "command": "page-header2unordered @INFILE > @OUTFILE"
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ # requires https://github.com/bertsky/workflow-configuration installed
3
+ # partitions PAGE-XML ReadingOrder from single OrderedGroup to top
4
+ # UnorderedGroup divided into OrderedGroups starting at every @type=heading
5
+ # text regions.
6
+ "command": "page-heading2unordered @INFILE > @OUTFILE"
7
+ }
@@ -0,0 +1,6 @@
1
+ {
2
+ # requires https://github.com/bertsky/workflow-configuration installed
3
+ # retrieves lines from all paragraphs and geometrically determines
4
+ # their average skew, annotating the result under /Page/@orientation
5
+ "command": "page-lines2orientation @INFILE > @OUTFILE"
6
+ }
@@ -0,0 +1,5 @@
1
+ {
2
+ # requires https://github.com/PRImA-Research-Lab/prima-page-converter installed
3
+ # with the main JAR file copied to /usr/local/share/
4
+ "command": "java -jar /usr/local/share/PageConverter.jar -source-xml @INFILE -convert-to LATEST -target-xml @OUTFILE"
5
+ }
@@ -0,0 +1,8 @@
1
+ {
2
+ # requires https://github.com/kba/transkribus-to-prima installed
3
+ # converts PAGE from Transkribus dialect to PRImA standard
4
+ # also runs various fixes often necessary to make this work:
5
+ # - ensuring coordinates range within the image size
6
+ # - ensuring segment identifiers do not start with numbers
7
+ "command": "page-fix-coordinates @INFILE - | sed -e 's/ id=\"/ id=\"id/' -e 's/regionRef=\"/regionRef=\"id/' | transkribus-to-prima -V - @OUTFILE"
8
+ }
@@ -0,0 +1,128 @@
1
+ # pylint: disable=missing-module-docstring,invalid-name
2
+ from typing import Optional, get_args
3
+ import os
4
+ import subprocess
5
+ from tempfile import TemporaryDirectory
6
+
7
+ import click
8
+
9
+ from ocrd.decorators import ocrd_cli_options, ocrd_cli_wrap_processor
10
+ from ocrd import Processor
11
+ from ocrd_models import OcrdPage, OcrdFileType
12
+ from ocrd_models.ocrd_page import to_xml
13
+ from ocrd_modelfactory import page_from_file
14
+ from ocrd_utils import config, make_file_id, MIMETYPE_PAGE
15
+
16
+
17
+ class ShellProcessor(Processor):
18
+
19
+ def setup(self):
20
+ command = self.parameter['command']
21
+ if '@INFILE' not in command:
22
+ raise Exception("command parameter requires @INFILE pattern")
23
+ if '@OUTFILE' not in command:
24
+ raise Exception("command parameter requires @OUTFILE pattern")
25
+
26
+ def process_page_file(self, *input_files: Optional[OcrdFileType]) -> None:
27
+ """
28
+ Process PAGE files via arbitrary command line on the shell.
29
+
30
+ \b
31
+ For each selected physical page of the workspace, pass ``command``
32
+ to the shell, replacing:
33
+ - the string ``@INFILE`` with the PAGE input file path,
34
+ - the string ``@OUTFILE`` with the PAGE output file path.
35
+
36
+ Modify the resulting PAGE output file with our new `@pcGtsId` and
37
+ metadata.
38
+ """
39
+ input_paths: List[str] = [""] * len(input_files)
40
+ input_pos = next(i for i, input_file in enumerate(input_files)
41
+ if input_file is not None)
42
+ page_id = input_files[input_pos].pageId
43
+ self._base_logger.info("processing page %s", page_id)
44
+ for i, input_file in enumerate(input_files):
45
+ grp = self.input_file_grp.split(',')[i]
46
+ if input_file is None:
47
+ self._base_logger.debug(f"ignoring missing file for input fileGrp {grp} for page {page_id}")
48
+ continue
49
+ assert isinstance(input_file, get_args(OcrdFileType))
50
+ if not input_file.local_filename:
51
+ self._base_logger.error(f'No local file exists for page {page_id} in file group {grp}')
52
+ if config.OCRD_MISSING_INPUT == 'ABORT':
53
+ raise MissingInputFile(grp, page_id, input_file.mimetype)
54
+ continue
55
+ self._base_logger.debug(f"parsing file {input_file.ID} for page {page_id}")
56
+ if os.path.exists(input_file.local_filename):
57
+ input_paths[i] = input_file.local_filename
58
+ else:
59
+ self._base_logger.error(f"non-existing local file for input fileGrp {grp} for page {page_id}")
60
+ if not any(input_paths):
61
+ self._base_logger.warning(f'skipping page {page_id}')
62
+ return
63
+ output_file_id = make_file_id(input_files[input_pos], self.output_file_grp)
64
+ if input_files[input_pos].fileGrp == self.output_file_grp:
65
+ # input=output fileGrp: re-use ID exactly
66
+ output_file_id = input_files[input_pos].ID
67
+ output_file = next(self.workspace.mets.find_files(ID=output_file_id), None)
68
+ if output_file and config.OCRD_EXISTING_OUTPUT != 'OVERWRITE':
69
+ # short-cut avoiding useless computation:
70
+ raise FileExistsError(
71
+ f"A file with ID=={output_file_id} already exists {output_file} and neither force nor ignore are set"
72
+ )
73
+ command = self.parameter['command']
74
+ with TemporaryDirectory(suffix=page_id) as tmpdir:
75
+ out_path = os.path.join(tmpdir, output_file_id + ".xml")
76
+ # remove quotation around filename patterns, if any
77
+ command = command.replace('"@INFILE"', '@INFILE').replace('"@OUTFILE"', '@OUTFILE')
78
+ command = command.replace("'@INFILE'", '@INFILE').replace("'@OUTFILE'", '@OUTFILE')
79
+ # replace filename patterns with actual paths, quoted
80
+ for in_path in input_paths:
81
+ command = command.replace('@INFILE', '"' + in_path + '"', 1)
82
+ command = command.replace('@OUTFILE', '"' + out_path + '"')
83
+ # execute command pattern
84
+ self.logger.debug("Running command: '%s'", command)
85
+ # pylint: disable=subprocess-run-check
86
+ result = subprocess.run(command, shell=True,
87
+ universal_newlines=True,
88
+ stdout=subprocess.PIPE,
89
+ stderr=subprocess.PIPE)
90
+ self.logger.debug("Command for %s returned: %d", page_id, result.returncode)
91
+ if result.stdout:
92
+ self.logger.info("Command for %s stdout: %s", page_id, result.stdout)
93
+ if result.stderr:
94
+ self.logger.warning("Command for %s stderr: %s", page_id, result.stderr)
95
+ if result.returncode != 0:
96
+ self.logger.error("Command for %s failed", page_id)
97
+ return
98
+ try:
99
+ result = page_from_file(out_path)
100
+ assert isinstance(result, OcrdPage)
101
+ except ValueError as err:
102
+ # not PAGE and not an image to generate PAGE for
103
+ self._base_logger.error(f"non-PAGE output for page {page_id}: {err}")
104
+ return
105
+ result.set_pcGtsId(output_file_id)
106
+ self.add_metadata(result)
107
+ self.workspace.add_file(
108
+ file_id=output_file_id,
109
+ file_grp=self.output_file_grp,
110
+ page_id=page_id,
111
+ local_filename=os.path.join(self.output_file_grp, output_file_id + '.xml'),
112
+ mimetype=MIMETYPE_PAGE,
113
+ content=to_xml(result),
114
+ )
115
+
116
+ @property
117
+ def metadata_filename(self):
118
+ return 'processor/builtin/dummy/ocrd-tool.json'
119
+
120
+ @property
121
+ def executable(self):
122
+ return 'ocrd-command'
123
+
124
+
125
+ @click.command()
126
+ @ocrd_cli_options
127
+ def cli(*args, **kwargs):
128
+ return ocrd_cli_wrap_processor(ShellProcessor, *args, **kwargs)