digiflow 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
digiflow/__init__.py ADDED
@@ -0,0 +1,22 @@
1
+ from .digflow_identifier import *
2
+ from .digiflow_io import *
3
+ from .digiflow_metadata import *
4
+ from .digiflow_generate import *
5
+ from .digiflow_export import *
6
+ from .digiflow_validate import (
7
+ ddb_validation,
8
+ ddb_xslt_validation,
9
+ DDB_IGNORE_RULES_BASIC,
10
+ DDB_IGNORE_RULES_MVW,
11
+ DDB_IGNORE_RULES_NEWSPAPERS,
12
+ DDB_MEDIA_SCH,
13
+ DDB_NEWSP_SCH,
14
+ CLI_DIR,
15
+ PATH_CLI,
16
+ SCH_DIR,
17
+ PATH_MEDIA_SCH,
18
+ PATH_NEWSP_SCH,
19
+ DIGIS_MULTIVOLUME,
20
+ DIGIS_NEWSPAPER,
21
+ TMP_SCH_FILE_NAME,
22
+ )
@@ -0,0 +1,374 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Module for Handling of persistent Identifiers"""
3
+
4
+ import enum
5
+ import functools
6
+ import os
7
+ import re
8
+
9
+ from collections import (
10
+ namedtuple
11
+ )
12
+ from pathlib import (
13
+ Path
14
+ )
15
+ from typing import (
16
+ List,
17
+ )
18
+
19
+ from .digiflow_metadata import (
20
+ MetsReader,
21
+ XMLNS,
22
+ )
23
+
24
+
25
+ ##################
26
+ # script constants
27
+ #
28
+ # URN PATTERN
29
+ URN_PATTERN_ULB = 'urn:nbn:de:gbv:3:{}-{}-{}-{}'
30
+ DEFAULT_PATTERN_ULB = r"^urn\:nbn\:de\:gbv\:3\:\d\-\d{3,}.*\d+$"
31
+ DEFAULT_GRANULAR_PAGE_FRAGMENT = '/fragment/page'
32
+ # kitodo2 system id
33
+ KITODO2_ID = '1192015415'
34
+
35
+ ###############
36
+ # Pruefziffer mapping
37
+ # Map Chars to Ints
38
+ # cf. http://www.persistent-identifier.de/?link=316
39
+ PZ_MAP = {}
40
+ PZ_MAP["0"] = 1
41
+ PZ_MAP["1"] = 2
42
+ PZ_MAP["2"] = 3
43
+ PZ_MAP["3"] = 4
44
+ PZ_MAP["4"] = 5
45
+ PZ_MAP["5"] = 6
46
+ PZ_MAP["6"] = 7
47
+ PZ_MAP["7"] = 8
48
+ PZ_MAP["8"] = 9
49
+ PZ_MAP["u"] = 11
50
+ PZ_MAP["r"] = 12
51
+ PZ_MAP["n"] = 13
52
+ PZ_MAP["b"] = 14
53
+ PZ_MAP["d"] = 15
54
+ PZ_MAP["e"] = 16
55
+ PZ_MAP[":"] = 17
56
+ PZ_MAP["a"] = 18
57
+ PZ_MAP["c"] = 19
58
+ PZ_MAP["f"] = 21
59
+ PZ_MAP["g"] = 22
60
+ PZ_MAP["h"] = 23
61
+ PZ_MAP["i"] = 24
62
+ PZ_MAP["j"] = 25
63
+ PZ_MAP["l"] = 26
64
+ PZ_MAP["L"] = 26
65
+ PZ_MAP["m"] = 27
66
+ PZ_MAP["o"] = 28
67
+ PZ_MAP["p"] = 29
68
+ PZ_MAP["q"] = 31
69
+ PZ_MAP["s"] = 32
70
+ PZ_MAP["t"] = 33
71
+ PZ_MAP["v"] = 34
72
+ PZ_MAP["w"] = 35
73
+ PZ_MAP["x"] = 36
74
+ PZ_MAP["X"] = 36
75
+ PZ_MAP["y"] = 37
76
+ PZ_MAP["Y"] = 37
77
+ PZ_MAP["z"] = 38
78
+ PZ_MAP["Z"] = 38
79
+ PZ_MAP["-"] = 39
80
+ PZ_MAP["9"] = 41
81
+ PZ_MAP["k"] = 42
82
+ PZ_MAP["_"] = 43
83
+ PZ_MAP["/"] = 45
84
+ PZ_MAP["."] = 47
85
+ PZ_MAP["+"] = 45
86
+
87
+
88
+ class DigiFlowURNExistsException(Exception):
89
+ """Mark URN already exists"""
90
+
91
+
92
+ class GranularURNExistsException(Exception):
93
+ """Mark Granular URN exists"""
94
+
95
+
96
+ class GranularURNException(Exception):
97
+ """Mark generic granular URN Exception"""
98
+
99
+
100
+ class URNType(enum.IntEnum):
101
+ '''
102
+ Represent different level of identifiers
103
+ Represent URN on work-level, i.e. for a
104
+ * work-level: identifies digitalization objects like
105
+ monographiesy, c- and f-stages or newspaper years/issues
106
+ * page-level: physical page with complete URN
107
+ * page-fragment-level: physical page
108
+ with fragemented URN derived from work-level URN
109
+ * page-fragment-focus: box within physical page
110
+ with fragemented URN derived from work-level URN
111
+
112
+ Please note, that there was also a granular URN
113
+ with corresponded to only printed pages, but this
114
+ version was not mentioned recently
115
+ '''
116
+
117
+ WORK = 1
118
+ GRANULAR_PAGE = 2
119
+ GRANULAR_FRAGMENT_PAGE = 3
120
+ GRANULAR_FRAGMENT_PAGE_FOCUS = 4
121
+ MISSING = 5
122
+ UNKNOWN = 6
123
+
124
+
125
+ def generate_urn_pz(urn):
126
+ """
127
+ Generate Checkdigit for persistent identifiers
128
+ see: http://www.persistent-identifier.de/?link=316
129
+ """
130
+
131
+ # map + reduce
132
+ mapped_digits_str = functools.reduce(lambda c, p: c + p, [str(PZ_MAP[c]) for c in urn])
133
+
134
+ # sum
135
+ pzs = sum((i * int(c) for i, c in enumerate(mapped_digits_str, 1)))
136
+
137
+ # divide by last element of digits mapped from urn
138
+ divisor = mapped_digits_str[-1]
139
+ quotient = int(pzs / int(divisor))
140
+
141
+ # return last digit
142
+ return str(quotient)[-1:][0]
143
+
144
+
145
+ # Container Struct
146
+ GranularContainer = namedtuple('GranularContainer', ['id', 'type', 'urn'])
147
+
148
+ class ProcessReport:
149
+ """Handle outcomes concerning URN handling"""
150
+
151
+ def __init__(self, added:List[GranularContainer], existed:List[GranularContainer]) -> None:
152
+ self.added: List[GranularContainer] = added
153
+ self.existed: List[GranularContainer] = existed
154
+ self._replaced = None
155
+
156
+ @property
157
+ def replaced(self):
158
+ """Get entries for which a mapping exist
159
+ for existed and added entries, which
160
+ means they were replaced"""
161
+ if self._replaced is None:
162
+ self._replaced = {}
163
+ for _add in self.added:
164
+ for _exs in self.existed:
165
+ if _add.id == _exs.id:
166
+ self._replaced[_add.id] = f"{_exs.urn} => {_add.urn}"
167
+ return self._replaced
168
+
169
+
170
+ def _determine_types(elem_cnts:List) -> List[GranularContainer]:
171
+ _types = []
172
+ for _el in elem_cnts:
173
+ cnt_id = _el.attrib['ID']
174
+ gurn = _el.get('CONTENTIDS')
175
+ if not gurn:
176
+ _type = URNType.MISSING
177
+ else:
178
+ if re.match(r".*\-p\d{4}\-\d$", gurn):
179
+ _type = URNType.GRANULAR_PAGE
180
+ elif re.match(r".*\/fragment\/page=\d{4,}$", gurn):
181
+ _type = URNType.GRANULAR_FRAGMENT_PAGE
182
+ else:
183
+ _type = URNType.UNKNOWN
184
+ _types.append(GranularContainer(cnt_id, _type, gurn))
185
+ return _types
186
+
187
+
188
+ def _determine_main_type(cnts:List[GranularContainer]) -> URNType:
189
+ _map = {}
190
+ for cnt in cnts:
191
+ if cnt.type in _map:
192
+ _map[cnt.type] = _map[cnt.type] + 1
193
+ else:
194
+ _map[cnt.type] = 1
195
+ return max(_map.items(), key=lambda t: t[1])[0]
196
+
197
+
198
+ def enrich_urn_granular(xml_tree,
199
+ urn_main,
200
+ page_num=None,
201
+ padd_left=4, do_sanitize=False, do_replace=False):
202
+ """
203
+ Enrich URN granular at physical containers
204
+ default: Granular URN 2.0 - GRANULAR_FRAGMENT_PAGE
205
+ infix '/fragment/page={fptr:href[last]}' in div@CONTENTIDS
206
+ """
207
+
208
+ add = []
209
+ exs = []
210
+ cnts_total = _get_phys_containers(xml_tree)
211
+ cnt_types = _determine_types(cnts_total)
212
+ exs = cnt_types
213
+ cnt_urn_fits = [e for e in cnts_total if does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
214
+ urn_type = _determine_main_type(cnt_types)
215
+ if do_replace:
216
+ cnt_urn_fits = []
217
+ cnt_diffs = len(cnts_total) - len(cnt_urn_fits)
218
+ # if no granular urns exist at all, then all must get new urn ...
219
+ if not cnt_urn_fits:
220
+ # identify insertion point
221
+ phys_divs = xml_tree.findall('.//mets:structMap[@TYPE="PHYSICAL"]/mets:div/mets:div', XMLNS)
222
+ # alert invalid data
223
+ if len(phys_divs) == 0:
224
+ raise GranularURNException(f"No phys pages in {xml_tree.base}!")
225
+ # insert URNTYPE.GRANULAR_FRAGMENT_PAGE for each physical container
226
+ for phys_div in phys_divs:
227
+ add.append(_insert_page_fragment(phys_div, urn_main, padd_left))
228
+ urn_type = URNType.GRANULAR_FRAGMENT_PAGE
229
+ # if differences between phys containers exist, then only some must be
230
+ # inserted / generated depending on existing urn type
231
+ # might happend with additional images (colorchecker, post scans, ...)
232
+ elif cnt_diffs:
233
+ # here we pass physical containers, since we *might* require it
234
+ # for sanitizing CONTENTIDS for latest added structure
235
+ try:
236
+ add += insert_granular_urn(urn_type, cnts_total, urn_main, page_num, padd_left)
237
+ except GranularURNExistsException as exc:
238
+ if do_sanitize:
239
+ (_, _, report) = _guess_urn_granular_from_existing(xml_tree, urn_main)
240
+ add += report.added
241
+ else:
242
+ raise GranularURNExistsException(exc) from exc
243
+ _report = ProcessReport(added=add,existed=exs)
244
+ return (xml_tree, urn_type, _report)
245
+
246
+
247
+ def _guess_urn_granular_from_existing(xml_tree, urn_main):
248
+ """Here we run in a seldom particular failure
249
+ driven by a missing granularen URN - let's try to fix 'em just one time!
250
+ """
251
+ cnts_total = _get_phys_containers(xml_tree)
252
+ phys_divs = [e for e in cnts_total if does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
253
+ last_phys_div = cnts_total[-1]
254
+ len_phys_div = len(cnts_total)
255
+ if str(len_phys_div) == last_phys_div.attrib['ORDER']:
256
+ contentids = sorted([i.attrib['CONTENTIDS'] for i in phys_divs]).pop()
257
+ page_num = int(contentids.split('-')[-2][1:]) + 1
258
+ xml_tree, urn_type, rep = enrich_urn_granular(xml_tree, urn_main, page_num=page_num)
259
+ _part_rep = ProcessReport(added=rep.added, existed=[])
260
+ return (xml_tree, urn_type, _part_rep)
261
+ raise GranularURNException(f"Inconsistent Granular URN '{xml_tree.base}'")
262
+
263
+
264
+ def _insert_page_fragment(phys_div, main_urn: str, padd:int):
265
+ order = int(phys_div.attrib['ORDER'])
266
+ pf_o = f"{order:0{padd}}"
267
+ the_gran_urn = f"{main_urn}{DEFAULT_GRANULAR_PAGE_FRAGMENT}={ pf_o}"
268
+ phys_div.attrib['CONTENTIDS'] = the_gran_urn
269
+ return GranularContainer(id=phys_div.attrib['ID'],
270
+ type=URNType.GRANULAR_FRAGMENT_PAGE,
271
+ urn=the_gran_urn)
272
+
273
+
274
+ def does_fit(e, pattern=DEFAULT_PATTERN_ULB):
275
+ """
276
+ Proper NBN URN consists of
277
+ * prefix like urn:nbn:de:gbv:3
278
+ * some subseqent numerical collection number
279
+ * additional identifiers (sementic IDs, PPNs, ...)
280
+ * suffix alike 'fragment' or '-p' for granularity
281
+ * probably numerial exemplar count
282
+ * final singular checkdigit
283
+ """
284
+ if 'CONTENTIDS' not in e.attrib:
285
+ return False
286
+ _id = e.attrib['CONTENTIDS']
287
+ return re.match(pattern, _id) is not None
288
+
289
+
290
+ def insert_granular_urn(urn_type, phys_conts, main_urn, page_num=None, padd_left=4):
291
+ """
292
+ Take care of inconsistent physical containers!
293
+
294
+ Physical containers missing CONTENTIDS
295
+ http://digital.bibliothek.uni-halle.de/hd/oai/?verb=GetRecord&metadataPrefix=mets&mode=xml&identifier=10595
296
+
297
+ Granular URN missing in between, therefore ORDER and granular URN do not match in the end
298
+ https://digitale.bibliothek.uni-halle.de/vd16/oai?verb=GetRecord&metadataPrefix=mets&identifier=993531
299
+ """
300
+
301
+ inserts = []
302
+ els_missing = [e for e in phys_conts if not does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
303
+ if urn_type == URNType.GRANULAR_PAGE:
304
+ for el_misses in els_missing:
305
+ # crashes if @ORDER cant be parsed as integer
306
+ # calculate pXXXX from attr 'ORDER'
307
+ # or, -if provided-, from parameter page_num
308
+ p_number = page_num if page_num else int(el_misses.attrib['ORDER'])
309
+ raw_gurn = f"{main_urn}-p{p_number:0{padd_left}}-"
310
+ new_gurn = f"{raw_gurn}{generate_urn_pz(raw_gurn)}"
311
+ el_id = el_misses.attrib['ID']
312
+ # check if new granular URN is *really* unique
313
+ # raise Exception otherwise ...
314
+ cnts_with_gurn = [
315
+ e for e in phys_conts
316
+ if 'CONTENTIDS' in e.attrib and re.match(r".*-p\d+-\d$", e.attrib['CONTENTIDS'])]
317
+ existing_ones = [e for e in cnts_with_gurn
318
+ if e.attrib['CONTENTIDS'] == new_gurn]
319
+ if existing_ones:
320
+ pat = "calculated granular URN '{}' already used by: '{}'!"
321
+ exsting_ids = [e.attrib['ID'] for e in existing_ones]
322
+ msg_fail = pat.format(new_gurn, exsting_ids)
323
+ raise GranularURNExistsException(msg_fail)
324
+ el_misses.attrib['CONTENTIDS'] = new_gurn
325
+ inserts.append(GranularContainer(id=el_id, type=URNType.GRANULAR_FRAGMENT_PAGE, urn=new_gurn))
326
+ return inserts
327
+
328
+
329
+ def _get_phys_containers(xml_tree):
330
+ """Get all physical containers that contain @ORDER attribute"""
331
+ return xml_tree.findall(
332
+ './/mets:structMap[@TYPE="PHYSICAL"]//mets:div[@ORDER]', XMLNS)
333
+
334
+
335
+ def enrich_urn_kitodo2(process_path:'str|Path', collection='1', system_id=KITODO2_ID, exemplar='1',
336
+ force_overwrite=False):
337
+ """Enrich URN for Kitodo2 if not existing"""
338
+
339
+ _urns = []
340
+ read = MetsReader(process_path)
341
+ ulb_urns = read.findall('.//goobi:metadata[@name="ulb_urn"]', read.primary_dmd)
342
+ if ulb_urns and not force_overwrite:
343
+ raise GranularURNExistsException(f"Detected existing URN {ulb_urns}!")
344
+
345
+ identifiers = read.xpath('.//goobi:metadata[@name="CatalogIDSource" and(not(@anchorId))]/text()')
346
+ if len(identifiers) != 1:
347
+ raise GranularURNException(f"Ambigious/invalid base URN {identifiers}!")
348
+
349
+ # format persistent identifier (aka URN) with provided info
350
+ urn = URN_PATTERN_ULB.format(collection, system_id, identifiers[0], exemplar)
351
+
352
+ # generate pruefziffer
353
+ urn += generate_urn_pz(urn)
354
+
355
+ # insert
356
+ read.insert_into_prime_mods('{http://meta.goobi.org/v1.5.1/}metadata',
357
+ {'name': 'ulb_urn'}, urn)
358
+ read.write()
359
+ _urns.append(urn)
360
+
361
+ # multi volume?
362
+ process_path = os.path.dirname(process_path)
363
+ path_meta_anchor = os.path.join(process_path, 'meta_anchor.xml')
364
+ if os.path.exists(path_meta_anchor):
365
+ ppn_c_stage = read.xpath('.//*[@anchorId="true"]/text()')
366
+ urn_host = f'urn:nbn:de:gbv:3:{collection}-{system_id}-{ppn_c_stage[0]}-{exemplar}'
367
+ pz_c_stage = generate_urn_pz(urn_host)
368
+ urn_host += pz_c_stage
369
+ _proc_host = MetsReader(path_meta_anchor)
370
+ _proc_host.insert_into_prime_mods('{http://meta.goobi.org/v1.5.1/}metadata',
371
+ {'name': 'ulb_urn'}, urn_host)
372
+ _proc_host.write()
373
+ _urns.append(urn_host)
374
+ return _urns
@@ -0,0 +1,302 @@
1
+ """
2
+ For the export of processes to share_it,
3
+ the generated data should be enriched and packed into a SAF.
4
+ The SAF file is a simple ZIP file,
5
+ but it is structured according to a specified structure.
6
+
7
+ Implements task #5735
8
+ """
9
+
10
+ import os
11
+ import pathlib
12
+ import shutil
13
+ import subprocess
14
+ import sys
15
+ import tempfile
16
+
17
+ from lxml import etree as ET
18
+
19
+ from digiflow import (
20
+ write_xml_file,
21
+ )
22
+
23
+
24
+ # default export mappings
25
+ SRC_FULLTEXT = 'FULLTEXT'
26
+ KITODO2_MAX = 'max'
27
+ SAF_METS_XML = 'mets.xml'
28
+ SRC_MAX = 'MAX'
29
+ SRC_IMAGE_FOOTER = 'IMAGE_FOOTER'
30
+ BUNDLE_PREVIEW = 'BUNDLE_BRANDED_PREVIEW__'
31
+ SRC_PREVIEW = 'BUNDLE_BRANDED_PREVIEW__'
32
+ BUNDLE_THUMBNAIL = 'BUNDLE_THUMBNAIL__'
33
+ SRC_THUMBS = 'BUNDLE_THUMBNAIL__'
34
+ DIR_IMAGES = [KITODO2_MAX, SRC_MAX, SRC_IMAGE_FOOTER]
35
+ DIR_DERIVATES = [SRC_PREVIEW, SRC_THUMBS]
36
+ DEFAULT_EXPORT_MAPPINGS = {'.xml': SAF_METS_XML,
37
+ '.pdf': None,
38
+ SRC_IMAGE_FOOTER: None,
39
+ SRC_THUMBS: None,
40
+ SRC_PREVIEW: None,
41
+ SRC_FULLTEXT: None
42
+ }
43
+ CONTENTS_FILE_MAPPINGS = {
44
+ SRC_FULLTEXT : 'FULLTEXT_OCR',
45
+ SRC_MAX : ''
46
+ }
47
+
48
+ class DigiFlowExportError(Exception):
49
+ """Mark Export Exception"""
50
+
51
+
52
+ class ExportMapping:
53
+
54
+ def __init__(self, path_source, path_target):
55
+ if not os.path.exists(path_source):
56
+ raise DigiFlowExportError("Invalid source path '{}'!'".format(path_source))
57
+ if not os.path.isabs(path_source):
58
+ raise DigiFlowExportError("Source path '{}' is not absolute!".format(path_source))
59
+ path_target_dir = os.path.dirname(path_target)
60
+ if not os.path.exists(path_target_dir):
61
+ os.makedirs(path_target_dir)
62
+ self.path_source = path_source
63
+ self.path_target = path_target
64
+ _path_src = pathlib.Path(path_source)
65
+ self.src_bundle = _path_src.parent.stem
66
+
67
+ def get(self):
68
+ """Access mapping"""
69
+ return (self.path_source, self.path_target)
70
+
71
+ def copy(self):
72
+ """copy from source to target"""
73
+ shutil.copy(self.path_source, self.path_target)
74
+
75
+ def __repr__(self) -> str:
76
+ return f"{self.path_source} => {self.path_target}"
77
+
78
+ def __eq__(self, __o: object) -> bool:
79
+ return self.path_source == __o.path_source and self.path_target == __o.path_target
80
+
81
+ def __lt__(self, __o: object) -> bool:
82
+ self_src_base = os.path.basename(self.path_source)
83
+ other_src_base = os.path.basename(__o.path_source)
84
+ return self_src_base <= other_src_base
85
+
86
+
87
+ def process(export_mappings, work_dir, archive_name,
88
+ target_collection, target_data_dir):
89
+ """process digitalizates with images"""
90
+
91
+ for mapping in export_mappings:
92
+ mapping.copy()
93
+ _handle_dublin_core_dummy(work_dir)
94
+ _handle_dublin_core_derivates(work_dir)
95
+ _handle_collections_file(work_dir, target_collection)
96
+ #_handle_contents_file(work_dir)
97
+ _handle_contents_file(work_dir, export_mappings)
98
+ (the_tmp_path, the_filesize) = _compress(os.path.dirname(work_dir), archive_name)
99
+ path_export_processing = _move_to_tmp_file(the_tmp_path, target_data_dir)
100
+ return (path_export_processing, the_filesize)
101
+
102
+
103
+ def map_contents(src_dir, dst_dir, export_map=None):
104
+ """
105
+ Create Mappings for source dir file entries to
106
+ future destination dir entries.
107
+
108
+ Serve as intermediate "SAF Building construction site".
109
+
110
+ All matching entries equal or below source dir get copied
111
+ in flatten manner into default SAF tmp dir 'item_000'.
112
+ If no explicite export-mapping provided, files are
113
+ copied as-they-are, for example
114
+
115
+ Otherwise rename file accordingly. I.e., search for a
116
+ file with extension '.xml' and turn it into 'mets.xml'
117
+ """
118
+
119
+ mappings = []
120
+ if not export_map:
121
+ export_map = {'.xml': SAF_METS_XML}
122
+
123
+ source_paths = [os.path.join(curr, a_file)
124
+ for curr, _, files
125
+ in os.walk(src_dir)
126
+ for a_file in files]
127
+
128
+ for src, dst in export_map.items():
129
+ for source_path in source_paths:
130
+ if src in source_path:
131
+ # for each individual file review
132
+ if not dst:
133
+ _default_dst = os.path.basename(source_path)
134
+ dst_path = os.path.join(dst_dir, _default_dst)
135
+ else:
136
+ dst_path = os.path.join(dst_dir, dst)
137
+ _mapping = ExportMapping(source_path, dst_path)
138
+ # prevent re-addition
139
+ if _mapping not in mappings:
140
+ mappings.append(_mapping)
141
+
142
+ return mappings
143
+
144
+
145
+ # def _handle_contents_file(working_item_dir, only_mets=False):
146
+ def _handle_contents_file(working_item_dir, export_mappings):
147
+ contents_file_path = os.path.join(working_item_dir, "contents")
148
+ with open(contents_file_path, 'a', encoding='UTF-8') as contents_file:
149
+ for mapping in sorted(export_mappings):
150
+ _target_base = pathlib.Path(mapping.path_target)
151
+ _name = _target_base.name
152
+ _src_bundle = mapping.src_bundle
153
+ if _name == SAF_METS_XML:
154
+ contents_file.write(f"{SAF_METS_XML}\tbundle:METS_BACKUP\n")
155
+ elif _name.endswith('.pdf'):
156
+ contents_file.write(f"{_name}\n")
157
+ elif _name.endswith('.pdf.txt'):
158
+ contents_file.write(f"{_name}\tbundle:TEXT\n")
159
+ elif _src_bundle == SRC_FULLTEXT and _is_alto(_target_base):
160
+ contents_file.write(f"{_name}\tbundle:FULLTEXT_OCR\n")
161
+ elif _src_bundle in DIR_IMAGES:
162
+ image_row = _render_row(_name, working_item_dir)
163
+ contents_file.write(image_row)
164
+ # we know them, but we do *not* handle 'em
165
+ # since they're already dealt at main image
166
+ elif _src_bundle in DIR_DERIVATES:
167
+ continue
168
+ else:
169
+ raise DigiFlowExportError(f"can't handle {mapping}!")
170
+
171
+
172
+ def _render_row(img_label, working_item_dir):
173
+ """
174
+ Render image entry in contents file
175
+ extend with optional virtual derivates
176
+ for thumbnails and default images
177
+ identified by their path segments
178
+
179
+ Format
180
+ 1.jpg bundle:MAX_IMAGE virtual:BUNDLE_BRANDED_PREVIEW__1.jpg/preview;BUNDLE_THUMBNAIL__1.jpg/thumbnail
181
+ """
182
+ row = f"{img_label}\tbundle:MAX_IMAGE"
183
+ all_jpgs = [_i for _i in os.listdir(working_item_dir)
184
+ if _i.endswith('.jpg')]
185
+ previews = [_i for _i in all_jpgs if BUNDLE_PREVIEW in _i]
186
+ thumbs = [_i for _i in all_jpgs if BUNDLE_THUMBNAIL in _i]
187
+ if len(previews) > 0 and len(thumbs) > 0:
188
+ row = f"{row}\tvirtual:{BUNDLE_PREVIEW}{img_label}/preview;{BUNDLE_THUMBNAIL}{img_label}/thumbnail"
189
+ return row + "\n"
190
+
191
+
192
+ def _is_alto(path_file) -> bool:
193
+ if not isinstance(path_file, str):
194
+ path_file = str(path_file)
195
+ if not path_file.endswith('.xml'):
196
+ return False
197
+ try:
198
+ xml_tree = ET.parse(path_file).getroot()
199
+ namespace = xml_tree.xpath('namespace-uri(.)')
200
+ return "alto" in namespace.lower()
201
+ except ET.ParseError as _err:
202
+ raise DigiFlowExportError(f"Unknown OCR-Format: {_err.args}") from _err
203
+
204
+
205
+ def _compress(work_dir, archive_name):
206
+ """
207
+ Switched implementation since unable to de-compress zip64Format
208
+ created with shutil.make_archive by Share_it
209
+ """
210
+ zip_size = -1
211
+ zip_file_path = os.path.join(os.path.dirname(work_dir), archive_name) + '.zip'
212
+ previous_dir = os.getcwd()
213
+ os.chdir(work_dir)
214
+ cmd = 'zip -q -r {} item_000'.format(zip_file_path)
215
+ subprocess.run(cmd, shell=True, check=True)
216
+ os.chmod(zip_file_path, 0o666)
217
+ zip_size = int(os.path.getsize(zip_file_path) / 1024 / 1024)
218
+ os.chdir(previous_dir)
219
+ return (zip_file_path, "{}MB".format(zip_size))
220
+
221
+
222
+ def _move_to_tmp_file(the_file_path, destination):
223
+ """
224
+ Move propably very large export data with masked name to external drive
225
+ """
226
+ abs_dstination = os.path.abspath(destination)
227
+ zip_export_path = os.path.join(abs_dstination, os.path.basename(the_file_path))
228
+ export_processing = zip_export_path + '.processing'
229
+ if not os.path.isdir(abs_dstination):
230
+ os.makedirs(abs_dstination, exist_ok=True)
231
+ shutil.move(the_file_path, export_processing)
232
+ return export_processing
233
+
234
+
235
+ def _handle_dublin_core_dummy(work_dir):
236
+ dc_dummy_path = os.path.join(work_dir, "dublin_core.xml")
237
+ dublin_core = ET.Element('dublin_core')
238
+ el_title = ET.Element('dcvalue')
239
+ el_title.set('element', 'title')
240
+ el_title.set('qualifier', 'none')
241
+ el_title.text = 'DUMMY'
242
+ el_date = ET.Element('dcvalue')
243
+ el_date.set('element', 'date')
244
+ el_date.set('qualifier', 'issued')
245
+ el_date.text = '1982'
246
+ dublin_core.append(el_title)
247
+ dublin_core.append(el_date)
248
+ write_xml_file(dublin_core, dc_dummy_path)
249
+
250
+
251
+ def _handle_dublin_core_derivates(work_dir):
252
+ """
253
+ <?xml version="1.0" encoding="UTF-8"?>
254
+ <dublin_core schema="local">
255
+ <dcvalue element="picturegroup" qualifier="thumbnail">EXTERNAL</dcvalue>
256
+ <dcvalue element="picturegroup" qualifier="preview">EXTERNAL</dcvalue>
257
+ </dublin_core>
258
+ """
259
+ dc_path = os.path.join(work_dir, "metadata_local.xml")
260
+ dublin_core = ET.Element('dublin_core', {'schema': 'local'})
261
+ el_pict = ET.Element('dcvalue')
262
+ el_pict.set('element', 'picturegroup')
263
+ el_pict.set('qualifier', 'thumbnail')
264
+ el_pict.text = 'EXTERNAL'
265
+ el_prev = ET.Element('dcvalue')
266
+ el_prev.set('element', 'picturegroup')
267
+ el_prev.set('qualifier', 'preview')
268
+ el_prev.text = 'EXTERNAL'
269
+ dublin_core.append(el_pict)
270
+ dublin_core.append(el_prev)
271
+ write_xml_file(dublin_core, dc_path)
272
+
273
+
274
+ def _handle_collections_file(work_dir, collections):
275
+ if collections:
276
+ collections_path = os.path.join(work_dir, "collections")
277
+ with open(collections_path, 'a') as collections_file:
278
+ collections_file.write(collections)
279
+ else:
280
+ raise DigiFlowExportError("No collections data provided - invalid share_it export!")
281
+
282
+
283
+ def export_data_from(process_metafile_path,
284
+ collection,
285
+ saf_final_name,
286
+ export_dst,
287
+ export_map=DEFAULT_EXPORT_MAPPINGS,
288
+ tmp_saf_dir=None):
289
+ """
290
+ Main entry point to prepare, create and export specified data
291
+ related to provided digitalization item process metadatafile_path
292
+ """
293
+
294
+ source_path_dir = os.path.dirname(process_metafile_path)
295
+ tmp_dir = tempfile.gettempdir()
296
+ prefix = 'opendata-working-'
297
+ if tmp_saf_dir:
298
+ tmp_dir = tmp_saf_dir
299
+ with tempfile.TemporaryDirectory(prefix=prefix, dir=tmp_dir) as tmp_dir:
300
+ work_dir = os.path.join(tmp_dir, saf_final_name, 'item_000')
301
+ _mappings = map_contents(source_path_dir, work_dir, export_map)
302
+ return process(_mappings, work_dir, saf_final_name, collection, export_dst)