digiflow 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- digiflow/__init__.py +22 -0
- digiflow/digflow_identifier.py +374 -0
- digiflow/digiflow_export.py +302 -0
- digiflow/digiflow_generate.py +366 -0
- digiflow/digiflow_io.py +1063 -0
- digiflow/digiflow_metadata.py +808 -0
- digiflow/digiflow_validate.py +333 -0
- digiflow/resources/digilife.ini +90 -0
- digiflow/resources/sch/ddb_validierung_mets-mods-ap-digitalisierte-medien.sch +2127 -0
- digiflow/resources/sch/ddb_validierung_mets-mods-ap-digitalisierte-zeitungen.sch +1562 -0
- digiflow/resources/xsd/alto_4-2.xsd +1105 -0
- digiflow/resources/xsd/mets_1-12.xsd +1780 -0
- digiflow/resources/xsd/mix_2-0.xsd +2570 -0
- digiflow/resources/xsd/mods_3-7.xsd +1486 -0
- digiflow/resources/xsd/page_2019-07-15.xsd +2644 -0
- digiflow/resources/xsd/xlink.xsd +75 -0
- digiflow/resources/xsd/xml.xsd +146 -0
- digiflow/schematron/schxslt-cli.jar +0 -0
- digiflow-3.0.0.dist-info/LICENCE +21 -0
- digiflow-3.0.0.dist-info/METADATA +15 -0
- digiflow-3.0.0.dist-info/RECORD +23 -0
- digiflow-3.0.0.dist-info/WHEEL +5 -0
- digiflow-3.0.0.dist-info/top_level.txt +1 -0
digiflow/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from .digflow_identifier import *
|
|
2
|
+
from .digiflow_io import *
|
|
3
|
+
from .digiflow_metadata import *
|
|
4
|
+
from .digiflow_generate import *
|
|
5
|
+
from .digiflow_export import *
|
|
6
|
+
from .digiflow_validate import (
|
|
7
|
+
ddb_validation,
|
|
8
|
+
ddb_xslt_validation,
|
|
9
|
+
DDB_IGNORE_RULES_BASIC,
|
|
10
|
+
DDB_IGNORE_RULES_MVW,
|
|
11
|
+
DDB_IGNORE_RULES_NEWSPAPERS,
|
|
12
|
+
DDB_MEDIA_SCH,
|
|
13
|
+
DDB_NEWSP_SCH,
|
|
14
|
+
CLI_DIR,
|
|
15
|
+
PATH_CLI,
|
|
16
|
+
SCH_DIR,
|
|
17
|
+
PATH_MEDIA_SCH,
|
|
18
|
+
PATH_NEWSP_SCH,
|
|
19
|
+
DIGIS_MULTIVOLUME,
|
|
20
|
+
DIGIS_NEWSPAPER,
|
|
21
|
+
TMP_SCH_FILE_NAME,
|
|
22
|
+
)
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Module for Handling of persistent Identifiers"""
|
|
3
|
+
|
|
4
|
+
import enum
|
|
5
|
+
import functools
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from collections import (
|
|
10
|
+
namedtuple
|
|
11
|
+
)
|
|
12
|
+
from pathlib import (
|
|
13
|
+
Path
|
|
14
|
+
)
|
|
15
|
+
from typing import (
|
|
16
|
+
List,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
from .digiflow_metadata import (
|
|
20
|
+
MetsReader,
|
|
21
|
+
XMLNS,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
##################
|
|
26
|
+
# script constants
|
|
27
|
+
#
|
|
28
|
+
# URN PATTERN
|
|
29
|
+
URN_PATTERN_ULB = 'urn:nbn:de:gbv:3:{}-{}-{}-{}'
|
|
30
|
+
DEFAULT_PATTERN_ULB = r"^urn\:nbn\:de\:gbv\:3\:\d\-\d{3,}.*\d+$"
|
|
31
|
+
DEFAULT_GRANULAR_PAGE_FRAGMENT = '/fragment/page'
|
|
32
|
+
# kitodo2 system id
|
|
33
|
+
KITODO2_ID = '1192015415'
|
|
34
|
+
|
|
35
|
+
###############
|
|
36
|
+
# Pruefziffer mapping
|
|
37
|
+
# Map Chars to Ints
|
|
38
|
+
# cf. http://www.persistent-identifier.de/?link=316
|
|
39
|
+
PZ_MAP = {}
|
|
40
|
+
PZ_MAP["0"] = 1
|
|
41
|
+
PZ_MAP["1"] = 2
|
|
42
|
+
PZ_MAP["2"] = 3
|
|
43
|
+
PZ_MAP["3"] = 4
|
|
44
|
+
PZ_MAP["4"] = 5
|
|
45
|
+
PZ_MAP["5"] = 6
|
|
46
|
+
PZ_MAP["6"] = 7
|
|
47
|
+
PZ_MAP["7"] = 8
|
|
48
|
+
PZ_MAP["8"] = 9
|
|
49
|
+
PZ_MAP["u"] = 11
|
|
50
|
+
PZ_MAP["r"] = 12
|
|
51
|
+
PZ_MAP["n"] = 13
|
|
52
|
+
PZ_MAP["b"] = 14
|
|
53
|
+
PZ_MAP["d"] = 15
|
|
54
|
+
PZ_MAP["e"] = 16
|
|
55
|
+
PZ_MAP[":"] = 17
|
|
56
|
+
PZ_MAP["a"] = 18
|
|
57
|
+
PZ_MAP["c"] = 19
|
|
58
|
+
PZ_MAP["f"] = 21
|
|
59
|
+
PZ_MAP["g"] = 22
|
|
60
|
+
PZ_MAP["h"] = 23
|
|
61
|
+
PZ_MAP["i"] = 24
|
|
62
|
+
PZ_MAP["j"] = 25
|
|
63
|
+
PZ_MAP["l"] = 26
|
|
64
|
+
PZ_MAP["L"] = 26
|
|
65
|
+
PZ_MAP["m"] = 27
|
|
66
|
+
PZ_MAP["o"] = 28
|
|
67
|
+
PZ_MAP["p"] = 29
|
|
68
|
+
PZ_MAP["q"] = 31
|
|
69
|
+
PZ_MAP["s"] = 32
|
|
70
|
+
PZ_MAP["t"] = 33
|
|
71
|
+
PZ_MAP["v"] = 34
|
|
72
|
+
PZ_MAP["w"] = 35
|
|
73
|
+
PZ_MAP["x"] = 36
|
|
74
|
+
PZ_MAP["X"] = 36
|
|
75
|
+
PZ_MAP["y"] = 37
|
|
76
|
+
PZ_MAP["Y"] = 37
|
|
77
|
+
PZ_MAP["z"] = 38
|
|
78
|
+
PZ_MAP["Z"] = 38
|
|
79
|
+
PZ_MAP["-"] = 39
|
|
80
|
+
PZ_MAP["9"] = 41
|
|
81
|
+
PZ_MAP["k"] = 42
|
|
82
|
+
PZ_MAP["_"] = 43
|
|
83
|
+
PZ_MAP["/"] = 45
|
|
84
|
+
PZ_MAP["."] = 47
|
|
85
|
+
PZ_MAP["+"] = 45
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class DigiFlowURNExistsException(Exception):
|
|
89
|
+
"""Mark URN already exists"""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class GranularURNExistsException(Exception):
|
|
93
|
+
"""Mark Granular URN exists"""
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class GranularURNException(Exception):
|
|
97
|
+
"""Mark generic granular URN Exception"""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class URNType(enum.IntEnum):
|
|
101
|
+
'''
|
|
102
|
+
Represent different level of identifiers
|
|
103
|
+
Represent URN on work-level, i.e. for a
|
|
104
|
+
* work-level: identifies digitalization objects like
|
|
105
|
+
monographiesy, c- and f-stages or newspaper years/issues
|
|
106
|
+
* page-level: physical page with complete URN
|
|
107
|
+
* page-fragment-level: physical page
|
|
108
|
+
with fragemented URN derived from work-level URN
|
|
109
|
+
* page-fragment-focus: box within physical page
|
|
110
|
+
with fragemented URN derived from work-level URN
|
|
111
|
+
|
|
112
|
+
Please note, that there was also a granular URN
|
|
113
|
+
with corresponded to only printed pages, but this
|
|
114
|
+
version was not mentioned recently
|
|
115
|
+
'''
|
|
116
|
+
|
|
117
|
+
WORK = 1
|
|
118
|
+
GRANULAR_PAGE = 2
|
|
119
|
+
GRANULAR_FRAGMENT_PAGE = 3
|
|
120
|
+
GRANULAR_FRAGMENT_PAGE_FOCUS = 4
|
|
121
|
+
MISSING = 5
|
|
122
|
+
UNKNOWN = 6
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def generate_urn_pz(urn):
|
|
126
|
+
"""
|
|
127
|
+
Generate Checkdigit for persistent identifiers
|
|
128
|
+
see: http://www.persistent-identifier.de/?link=316
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
# map + reduce
|
|
132
|
+
mapped_digits_str = functools.reduce(lambda c, p: c + p, [str(PZ_MAP[c]) for c in urn])
|
|
133
|
+
|
|
134
|
+
# sum
|
|
135
|
+
pzs = sum((i * int(c) for i, c in enumerate(mapped_digits_str, 1)))
|
|
136
|
+
|
|
137
|
+
# divide by last element of digits mapped from urn
|
|
138
|
+
divisor = mapped_digits_str[-1]
|
|
139
|
+
quotient = int(pzs / int(divisor))
|
|
140
|
+
|
|
141
|
+
# return last digit
|
|
142
|
+
return str(quotient)[-1:][0]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# Container Struct
|
|
146
|
+
GranularContainer = namedtuple('GranularContainer', ['id', 'type', 'urn'])
|
|
147
|
+
|
|
148
|
+
class ProcessReport:
|
|
149
|
+
"""Handle outcomes concerning URN handling"""
|
|
150
|
+
|
|
151
|
+
def __init__(self, added:List[GranularContainer], existed:List[GranularContainer]) -> None:
|
|
152
|
+
self.added: List[GranularContainer] = added
|
|
153
|
+
self.existed: List[GranularContainer] = existed
|
|
154
|
+
self._replaced = None
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def replaced(self):
|
|
158
|
+
"""Get entries for which a mapping exist
|
|
159
|
+
for existed and added entries, which
|
|
160
|
+
means they were replaced"""
|
|
161
|
+
if self._replaced is None:
|
|
162
|
+
self._replaced = {}
|
|
163
|
+
for _add in self.added:
|
|
164
|
+
for _exs in self.existed:
|
|
165
|
+
if _add.id == _exs.id:
|
|
166
|
+
self._replaced[_add.id] = f"{_exs.urn} => {_add.urn}"
|
|
167
|
+
return self._replaced
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _determine_types(elem_cnts:List) -> List[GranularContainer]:
|
|
171
|
+
_types = []
|
|
172
|
+
for _el in elem_cnts:
|
|
173
|
+
cnt_id = _el.attrib['ID']
|
|
174
|
+
gurn = _el.get('CONTENTIDS')
|
|
175
|
+
if not gurn:
|
|
176
|
+
_type = URNType.MISSING
|
|
177
|
+
else:
|
|
178
|
+
if re.match(r".*\-p\d{4}\-\d$", gurn):
|
|
179
|
+
_type = URNType.GRANULAR_PAGE
|
|
180
|
+
elif re.match(r".*\/fragment\/page=\d{4,}$", gurn):
|
|
181
|
+
_type = URNType.GRANULAR_FRAGMENT_PAGE
|
|
182
|
+
else:
|
|
183
|
+
_type = URNType.UNKNOWN
|
|
184
|
+
_types.append(GranularContainer(cnt_id, _type, gurn))
|
|
185
|
+
return _types
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _determine_main_type(cnts:List[GranularContainer]) -> URNType:
|
|
189
|
+
_map = {}
|
|
190
|
+
for cnt in cnts:
|
|
191
|
+
if cnt.type in _map:
|
|
192
|
+
_map[cnt.type] = _map[cnt.type] + 1
|
|
193
|
+
else:
|
|
194
|
+
_map[cnt.type] = 1
|
|
195
|
+
return max(_map.items(), key=lambda t: t[1])[0]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def enrich_urn_granular(xml_tree,
|
|
199
|
+
urn_main,
|
|
200
|
+
page_num=None,
|
|
201
|
+
padd_left=4, do_sanitize=False, do_replace=False):
|
|
202
|
+
"""
|
|
203
|
+
Enrich URN granular at physical containers
|
|
204
|
+
default: Granular URN 2.0 - GRANULAR_FRAGMENT_PAGE
|
|
205
|
+
infix '/fragment/page={fptr:href[last]}' in div@CONTENTIDS
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
add = []
|
|
209
|
+
exs = []
|
|
210
|
+
cnts_total = _get_phys_containers(xml_tree)
|
|
211
|
+
cnt_types = _determine_types(cnts_total)
|
|
212
|
+
exs = cnt_types
|
|
213
|
+
cnt_urn_fits = [e for e in cnts_total if does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
|
|
214
|
+
urn_type = _determine_main_type(cnt_types)
|
|
215
|
+
if do_replace:
|
|
216
|
+
cnt_urn_fits = []
|
|
217
|
+
cnt_diffs = len(cnts_total) - len(cnt_urn_fits)
|
|
218
|
+
# if no granular urns exist at all, then all must get new urn ...
|
|
219
|
+
if not cnt_urn_fits:
|
|
220
|
+
# identify insertion point
|
|
221
|
+
phys_divs = xml_tree.findall('.//mets:structMap[@TYPE="PHYSICAL"]/mets:div/mets:div', XMLNS)
|
|
222
|
+
# alert invalid data
|
|
223
|
+
if len(phys_divs) == 0:
|
|
224
|
+
raise GranularURNException(f"No phys pages in {xml_tree.base}!")
|
|
225
|
+
# insert URNTYPE.GRANULAR_FRAGMENT_PAGE for each physical container
|
|
226
|
+
for phys_div in phys_divs:
|
|
227
|
+
add.append(_insert_page_fragment(phys_div, urn_main, padd_left))
|
|
228
|
+
urn_type = URNType.GRANULAR_FRAGMENT_PAGE
|
|
229
|
+
# if differences between phys containers exist, then only some must be
|
|
230
|
+
# inserted / generated depending on existing urn type
|
|
231
|
+
# might happend with additional images (colorchecker, post scans, ...)
|
|
232
|
+
elif cnt_diffs:
|
|
233
|
+
# here we pass physical containers, since we *might* require it
|
|
234
|
+
# for sanitizing CONTENTIDS for latest added structure
|
|
235
|
+
try:
|
|
236
|
+
add += insert_granular_urn(urn_type, cnts_total, urn_main, page_num, padd_left)
|
|
237
|
+
except GranularURNExistsException as exc:
|
|
238
|
+
if do_sanitize:
|
|
239
|
+
(_, _, report) = _guess_urn_granular_from_existing(xml_tree, urn_main)
|
|
240
|
+
add += report.added
|
|
241
|
+
else:
|
|
242
|
+
raise GranularURNExistsException(exc) from exc
|
|
243
|
+
_report = ProcessReport(added=add,existed=exs)
|
|
244
|
+
return (xml_tree, urn_type, _report)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _guess_urn_granular_from_existing(xml_tree, urn_main):
|
|
248
|
+
"""Here we run in a seldom particular failure
|
|
249
|
+
driven by a missing granularen URN - let's try to fix 'em just one time!
|
|
250
|
+
"""
|
|
251
|
+
cnts_total = _get_phys_containers(xml_tree)
|
|
252
|
+
phys_divs = [e for e in cnts_total if does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
|
|
253
|
+
last_phys_div = cnts_total[-1]
|
|
254
|
+
len_phys_div = len(cnts_total)
|
|
255
|
+
if str(len_phys_div) == last_phys_div.attrib['ORDER']:
|
|
256
|
+
contentids = sorted([i.attrib['CONTENTIDS'] for i in phys_divs]).pop()
|
|
257
|
+
page_num = int(contentids.split('-')[-2][1:]) + 1
|
|
258
|
+
xml_tree, urn_type, rep = enrich_urn_granular(xml_tree, urn_main, page_num=page_num)
|
|
259
|
+
_part_rep = ProcessReport(added=rep.added, existed=[])
|
|
260
|
+
return (xml_tree, urn_type, _part_rep)
|
|
261
|
+
raise GranularURNException(f"Inconsistent Granular URN '{xml_tree.base}'")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _insert_page_fragment(phys_div, main_urn: str, padd:int):
|
|
265
|
+
order = int(phys_div.attrib['ORDER'])
|
|
266
|
+
pf_o = f"{order:0{padd}}"
|
|
267
|
+
the_gran_urn = f"{main_urn}{DEFAULT_GRANULAR_PAGE_FRAGMENT}={ pf_o}"
|
|
268
|
+
phys_div.attrib['CONTENTIDS'] = the_gran_urn
|
|
269
|
+
return GranularContainer(id=phys_div.attrib['ID'],
|
|
270
|
+
type=URNType.GRANULAR_FRAGMENT_PAGE,
|
|
271
|
+
urn=the_gran_urn)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def does_fit(e, pattern=DEFAULT_PATTERN_ULB):
|
|
275
|
+
"""
|
|
276
|
+
Proper NBN URN consists of
|
|
277
|
+
* prefix like urn:nbn:de:gbv:3
|
|
278
|
+
* some subseqent numerical collection number
|
|
279
|
+
* additional identifiers (sementic IDs, PPNs, ...)
|
|
280
|
+
* suffix alike 'fragment' or '-p' for granularity
|
|
281
|
+
* probably numerial exemplar count
|
|
282
|
+
* final singular checkdigit
|
|
283
|
+
"""
|
|
284
|
+
if 'CONTENTIDS' not in e.attrib:
|
|
285
|
+
return False
|
|
286
|
+
_id = e.attrib['CONTENTIDS']
|
|
287
|
+
return re.match(pattern, _id) is not None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def insert_granular_urn(urn_type, phys_conts, main_urn, page_num=None, padd_left=4):
|
|
291
|
+
"""
|
|
292
|
+
Take care of inconsistent physical containers!
|
|
293
|
+
|
|
294
|
+
Physical containers missing CONTENTIDS
|
|
295
|
+
http://digital.bibliothek.uni-halle.de/hd/oai/?verb=GetRecord&metadataPrefix=mets&mode=xml&identifier=10595
|
|
296
|
+
|
|
297
|
+
Granular URN missing in between, therefore ORDER and granular URN do not match in the end
|
|
298
|
+
https://digitale.bibliothek.uni-halle.de/vd16/oai?verb=GetRecord&metadataPrefix=mets&identifier=993531
|
|
299
|
+
"""
|
|
300
|
+
|
|
301
|
+
inserts = []
|
|
302
|
+
els_missing = [e for e in phys_conts if not does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
|
|
303
|
+
if urn_type == URNType.GRANULAR_PAGE:
|
|
304
|
+
for el_misses in els_missing:
|
|
305
|
+
# crashes if @ORDER cant be parsed as integer
|
|
306
|
+
# calculate pXXXX from attr 'ORDER'
|
|
307
|
+
# or, -if provided-, from parameter page_num
|
|
308
|
+
p_number = page_num if page_num else int(el_misses.attrib['ORDER'])
|
|
309
|
+
raw_gurn = f"{main_urn}-p{p_number:0{padd_left}}-"
|
|
310
|
+
new_gurn = f"{raw_gurn}{generate_urn_pz(raw_gurn)}"
|
|
311
|
+
el_id = el_misses.attrib['ID']
|
|
312
|
+
# check if new granular URN is *really* unique
|
|
313
|
+
# raise Exception otherwise ...
|
|
314
|
+
cnts_with_gurn = [
|
|
315
|
+
e for e in phys_conts
|
|
316
|
+
if 'CONTENTIDS' in e.attrib and re.match(r".*-p\d+-\d$", e.attrib['CONTENTIDS'])]
|
|
317
|
+
existing_ones = [e for e in cnts_with_gurn
|
|
318
|
+
if e.attrib['CONTENTIDS'] == new_gurn]
|
|
319
|
+
if existing_ones:
|
|
320
|
+
pat = "calculated granular URN '{}' already used by: '{}'!"
|
|
321
|
+
exsting_ids = [e.attrib['ID'] for e in existing_ones]
|
|
322
|
+
msg_fail = pat.format(new_gurn, exsting_ids)
|
|
323
|
+
raise GranularURNExistsException(msg_fail)
|
|
324
|
+
el_misses.attrib['CONTENTIDS'] = new_gurn
|
|
325
|
+
inserts.append(GranularContainer(id=el_id, type=URNType.GRANULAR_FRAGMENT_PAGE, urn=new_gurn))
|
|
326
|
+
return inserts
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _get_phys_containers(xml_tree):
|
|
330
|
+
"""Get all physical containers that contain @ORDER attribute"""
|
|
331
|
+
return xml_tree.findall(
|
|
332
|
+
'.//mets:structMap[@TYPE="PHYSICAL"]//mets:div[@ORDER]', XMLNS)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def enrich_urn_kitodo2(process_path:'str|Path', collection='1', system_id=KITODO2_ID, exemplar='1',
|
|
336
|
+
force_overwrite=False):
|
|
337
|
+
"""Enrich URN for Kitodo2 if not existing"""
|
|
338
|
+
|
|
339
|
+
_urns = []
|
|
340
|
+
read = MetsReader(process_path)
|
|
341
|
+
ulb_urns = read.findall('.//goobi:metadata[@name="ulb_urn"]', read.primary_dmd)
|
|
342
|
+
if ulb_urns and not force_overwrite:
|
|
343
|
+
raise GranularURNExistsException(f"Detected existing URN {ulb_urns}!")
|
|
344
|
+
|
|
345
|
+
identifiers = read.xpath('.//goobi:metadata[@name="CatalogIDSource" and(not(@anchorId))]/text()')
|
|
346
|
+
if len(identifiers) != 1:
|
|
347
|
+
raise GranularURNException(f"Ambigious/invalid base URN {identifiers}!")
|
|
348
|
+
|
|
349
|
+
# format persistent identifier (aka URN) with provided info
|
|
350
|
+
urn = URN_PATTERN_ULB.format(collection, system_id, identifiers[0], exemplar)
|
|
351
|
+
|
|
352
|
+
# generate pruefziffer
|
|
353
|
+
urn += generate_urn_pz(urn)
|
|
354
|
+
|
|
355
|
+
# insert
|
|
356
|
+
read.insert_into_prime_mods('{http://meta.goobi.org/v1.5.1/}metadata',
|
|
357
|
+
{'name': 'ulb_urn'}, urn)
|
|
358
|
+
read.write()
|
|
359
|
+
_urns.append(urn)
|
|
360
|
+
|
|
361
|
+
# multi volume?
|
|
362
|
+
process_path = os.path.dirname(process_path)
|
|
363
|
+
path_meta_anchor = os.path.join(process_path, 'meta_anchor.xml')
|
|
364
|
+
if os.path.exists(path_meta_anchor):
|
|
365
|
+
ppn_c_stage = read.xpath('.//*[@anchorId="true"]/text()')
|
|
366
|
+
urn_host = f'urn:nbn:de:gbv:3:{collection}-{system_id}-{ppn_c_stage[0]}-{exemplar}'
|
|
367
|
+
pz_c_stage = generate_urn_pz(urn_host)
|
|
368
|
+
urn_host += pz_c_stage
|
|
369
|
+
_proc_host = MetsReader(path_meta_anchor)
|
|
370
|
+
_proc_host.insert_into_prime_mods('{http://meta.goobi.org/v1.5.1/}metadata',
|
|
371
|
+
{'name': 'ulb_urn'}, urn_host)
|
|
372
|
+
_proc_host.write()
|
|
373
|
+
_urns.append(urn_host)
|
|
374
|
+
return _urns
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
"""
|
|
2
|
+
For the export of processes to share_it,
|
|
3
|
+
the generated data should be enriched and packed into a SAF.
|
|
4
|
+
The SAF file is a simple ZIP file,
|
|
5
|
+
but it is structured according to a specified structure.
|
|
6
|
+
|
|
7
|
+
Implements task #5735
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import pathlib
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import tempfile
|
|
16
|
+
|
|
17
|
+
from lxml import etree as ET
|
|
18
|
+
|
|
19
|
+
from digiflow import (
|
|
20
|
+
write_xml_file,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# default export mappings
|
|
25
|
+
SRC_FULLTEXT = 'FULLTEXT'
|
|
26
|
+
KITODO2_MAX = 'max'
|
|
27
|
+
SAF_METS_XML = 'mets.xml'
|
|
28
|
+
SRC_MAX = 'MAX'
|
|
29
|
+
SRC_IMAGE_FOOTER = 'IMAGE_FOOTER'
|
|
30
|
+
BUNDLE_PREVIEW = 'BUNDLE_BRANDED_PREVIEW__'
|
|
31
|
+
SRC_PREVIEW = 'BUNDLE_BRANDED_PREVIEW__'
|
|
32
|
+
BUNDLE_THUMBNAIL = 'BUNDLE_THUMBNAIL__'
|
|
33
|
+
SRC_THUMBS = 'BUNDLE_THUMBNAIL__'
|
|
34
|
+
DIR_IMAGES = [KITODO2_MAX, SRC_MAX, SRC_IMAGE_FOOTER]
|
|
35
|
+
DIR_DERIVATES = [SRC_PREVIEW, SRC_THUMBS]
|
|
36
|
+
DEFAULT_EXPORT_MAPPINGS = {'.xml': SAF_METS_XML,
|
|
37
|
+
'.pdf': None,
|
|
38
|
+
SRC_IMAGE_FOOTER: None,
|
|
39
|
+
SRC_THUMBS: None,
|
|
40
|
+
SRC_PREVIEW: None,
|
|
41
|
+
SRC_FULLTEXT: None
|
|
42
|
+
}
|
|
43
|
+
CONTENTS_FILE_MAPPINGS = {
|
|
44
|
+
SRC_FULLTEXT : 'FULLTEXT_OCR',
|
|
45
|
+
SRC_MAX : ''
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
class DigiFlowExportError(Exception):
|
|
49
|
+
"""Mark Export Exception"""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ExportMapping:
|
|
53
|
+
|
|
54
|
+
def __init__(self, path_source, path_target):
|
|
55
|
+
if not os.path.exists(path_source):
|
|
56
|
+
raise DigiFlowExportError("Invalid source path '{}'!'".format(path_source))
|
|
57
|
+
if not os.path.isabs(path_source):
|
|
58
|
+
raise DigiFlowExportError("Source path '{}' is not absolute!".format(path_source))
|
|
59
|
+
path_target_dir = os.path.dirname(path_target)
|
|
60
|
+
if not os.path.exists(path_target_dir):
|
|
61
|
+
os.makedirs(path_target_dir)
|
|
62
|
+
self.path_source = path_source
|
|
63
|
+
self.path_target = path_target
|
|
64
|
+
_path_src = pathlib.Path(path_source)
|
|
65
|
+
self.src_bundle = _path_src.parent.stem
|
|
66
|
+
|
|
67
|
+
def get(self):
|
|
68
|
+
"""Access mapping"""
|
|
69
|
+
return (self.path_source, self.path_target)
|
|
70
|
+
|
|
71
|
+
def copy(self):
|
|
72
|
+
"""copy from source to target"""
|
|
73
|
+
shutil.copy(self.path_source, self.path_target)
|
|
74
|
+
|
|
75
|
+
def __repr__(self) -> str:
|
|
76
|
+
return f"{self.path_source} => {self.path_target}"
|
|
77
|
+
|
|
78
|
+
def __eq__(self, __o: object) -> bool:
|
|
79
|
+
return self.path_source == __o.path_source and self.path_target == __o.path_target
|
|
80
|
+
|
|
81
|
+
def __lt__(self, __o: object) -> bool:
|
|
82
|
+
self_src_base = os.path.basename(self.path_source)
|
|
83
|
+
other_src_base = os.path.basename(__o.path_source)
|
|
84
|
+
return self_src_base <= other_src_base
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def process(export_mappings, work_dir, archive_name,
|
|
88
|
+
target_collection, target_data_dir):
|
|
89
|
+
"""process digitalizates with images"""
|
|
90
|
+
|
|
91
|
+
for mapping in export_mappings:
|
|
92
|
+
mapping.copy()
|
|
93
|
+
_handle_dublin_core_dummy(work_dir)
|
|
94
|
+
_handle_dublin_core_derivates(work_dir)
|
|
95
|
+
_handle_collections_file(work_dir, target_collection)
|
|
96
|
+
#_handle_contents_file(work_dir)
|
|
97
|
+
_handle_contents_file(work_dir, export_mappings)
|
|
98
|
+
(the_tmp_path, the_filesize) = _compress(os.path.dirname(work_dir), archive_name)
|
|
99
|
+
path_export_processing = _move_to_tmp_file(the_tmp_path, target_data_dir)
|
|
100
|
+
return (path_export_processing, the_filesize)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def map_contents(src_dir, dst_dir, export_map=None):
|
|
104
|
+
"""
|
|
105
|
+
Create Mappings for source dir file entries to
|
|
106
|
+
future destination dir entries.
|
|
107
|
+
|
|
108
|
+
Serve as intermediate "SAF Building construction site".
|
|
109
|
+
|
|
110
|
+
All matching entries equal or below source dir get copied
|
|
111
|
+
in flatten manner into default SAF tmp dir 'item_000'.
|
|
112
|
+
If no explicite export-mapping provided, files are
|
|
113
|
+
copied as-they-are, for example
|
|
114
|
+
|
|
115
|
+
Otherwise rename file accordingly. I.e., search for a
|
|
116
|
+
file with extension '.xml' and turn it into 'mets.xml'
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
mappings = []
|
|
120
|
+
if not export_map:
|
|
121
|
+
export_map = {'.xml': SAF_METS_XML}
|
|
122
|
+
|
|
123
|
+
source_paths = [os.path.join(curr, a_file)
|
|
124
|
+
for curr, _, files
|
|
125
|
+
in os.walk(src_dir)
|
|
126
|
+
for a_file in files]
|
|
127
|
+
|
|
128
|
+
for src, dst in export_map.items():
|
|
129
|
+
for source_path in source_paths:
|
|
130
|
+
if src in source_path:
|
|
131
|
+
# for each individual file review
|
|
132
|
+
if not dst:
|
|
133
|
+
_default_dst = os.path.basename(source_path)
|
|
134
|
+
dst_path = os.path.join(dst_dir, _default_dst)
|
|
135
|
+
else:
|
|
136
|
+
dst_path = os.path.join(dst_dir, dst)
|
|
137
|
+
_mapping = ExportMapping(source_path, dst_path)
|
|
138
|
+
# prevent re-addition
|
|
139
|
+
if _mapping not in mappings:
|
|
140
|
+
mappings.append(_mapping)
|
|
141
|
+
|
|
142
|
+
return mappings
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# def _handle_contents_file(working_item_dir, only_mets=False):
|
|
146
|
+
def _handle_contents_file(working_item_dir, export_mappings):
|
|
147
|
+
contents_file_path = os.path.join(working_item_dir, "contents")
|
|
148
|
+
with open(contents_file_path, 'a', encoding='UTF-8') as contents_file:
|
|
149
|
+
for mapping in sorted(export_mappings):
|
|
150
|
+
_target_base = pathlib.Path(mapping.path_target)
|
|
151
|
+
_name = _target_base.name
|
|
152
|
+
_src_bundle = mapping.src_bundle
|
|
153
|
+
if _name == SAF_METS_XML:
|
|
154
|
+
contents_file.write(f"{SAF_METS_XML}\tbundle:METS_BACKUP\n")
|
|
155
|
+
elif _name.endswith('.pdf'):
|
|
156
|
+
contents_file.write(f"{_name}\n")
|
|
157
|
+
elif _name.endswith('.pdf.txt'):
|
|
158
|
+
contents_file.write(f"{_name}\tbundle:TEXT\n")
|
|
159
|
+
elif _src_bundle == SRC_FULLTEXT and _is_alto(_target_base):
|
|
160
|
+
contents_file.write(f"{_name}\tbundle:FULLTEXT_OCR\n")
|
|
161
|
+
elif _src_bundle in DIR_IMAGES:
|
|
162
|
+
image_row = _render_row(_name, working_item_dir)
|
|
163
|
+
contents_file.write(image_row)
|
|
164
|
+
# we know them, but we do *not* handle 'em
|
|
165
|
+
# since they're already dealt at main image
|
|
166
|
+
elif _src_bundle in DIR_DERIVATES:
|
|
167
|
+
continue
|
|
168
|
+
else:
|
|
169
|
+
raise DigiFlowExportError(f"can't handle {mapping}!")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _render_row(img_label, working_item_dir):
|
|
173
|
+
"""
|
|
174
|
+
Render image entry in contents file
|
|
175
|
+
extend with optional virtual derivates
|
|
176
|
+
for thumbnails and default images
|
|
177
|
+
identified by their path segments
|
|
178
|
+
|
|
179
|
+
Format
|
|
180
|
+
1.jpg bundle:MAX_IMAGE virtual:BUNDLE_BRANDED_PREVIEW__1.jpg/preview;BUNDLE_THUMBNAIL__1.jpg/thumbnail
|
|
181
|
+
"""
|
|
182
|
+
row = f"{img_label}\tbundle:MAX_IMAGE"
|
|
183
|
+
all_jpgs = [_i for _i in os.listdir(working_item_dir)
|
|
184
|
+
if _i.endswith('.jpg')]
|
|
185
|
+
previews = [_i for _i in all_jpgs if BUNDLE_PREVIEW in _i]
|
|
186
|
+
thumbs = [_i for _i in all_jpgs if BUNDLE_THUMBNAIL in _i]
|
|
187
|
+
if len(previews) > 0 and len(thumbs) > 0:
|
|
188
|
+
row = f"{row}\tvirtual:{BUNDLE_PREVIEW}{img_label}/preview;{BUNDLE_THUMBNAIL}{img_label}/thumbnail"
|
|
189
|
+
return row + "\n"
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _is_alto(path_file) -> bool:
|
|
193
|
+
if not isinstance(path_file, str):
|
|
194
|
+
path_file = str(path_file)
|
|
195
|
+
if not path_file.endswith('.xml'):
|
|
196
|
+
return False
|
|
197
|
+
try:
|
|
198
|
+
xml_tree = ET.parse(path_file).getroot()
|
|
199
|
+
namespace = xml_tree.xpath('namespace-uri(.)')
|
|
200
|
+
return "alto" in namespace.lower()
|
|
201
|
+
except ET.ParseError as _err:
|
|
202
|
+
raise DigiFlowExportError(f"Unknown OCR-Format: {_err.args}") from _err
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _compress(work_dir, archive_name):
|
|
206
|
+
"""
|
|
207
|
+
Switched implementation since unable to de-compress zip64Format
|
|
208
|
+
created with shutil.make_archive by Share_it
|
|
209
|
+
"""
|
|
210
|
+
zip_size = -1
|
|
211
|
+
zip_file_path = os.path.join(os.path.dirname(work_dir), archive_name) + '.zip'
|
|
212
|
+
previous_dir = os.getcwd()
|
|
213
|
+
os.chdir(work_dir)
|
|
214
|
+
cmd = 'zip -q -r {} item_000'.format(zip_file_path)
|
|
215
|
+
subprocess.run(cmd, shell=True, check=True)
|
|
216
|
+
os.chmod(zip_file_path, 0o666)
|
|
217
|
+
zip_size = int(os.path.getsize(zip_file_path) / 1024 / 1024)
|
|
218
|
+
os.chdir(previous_dir)
|
|
219
|
+
return (zip_file_path, "{}MB".format(zip_size))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _move_to_tmp_file(the_file_path, destination):
|
|
223
|
+
"""
|
|
224
|
+
Move propably very large export data with masked name to external drive
|
|
225
|
+
"""
|
|
226
|
+
abs_dstination = os.path.abspath(destination)
|
|
227
|
+
zip_export_path = os.path.join(abs_dstination, os.path.basename(the_file_path))
|
|
228
|
+
export_processing = zip_export_path + '.processing'
|
|
229
|
+
if not os.path.isdir(abs_dstination):
|
|
230
|
+
os.makedirs(abs_dstination, exist_ok=True)
|
|
231
|
+
shutil.move(the_file_path, export_processing)
|
|
232
|
+
return export_processing
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _handle_dublin_core_dummy(work_dir):
|
|
236
|
+
dc_dummy_path = os.path.join(work_dir, "dublin_core.xml")
|
|
237
|
+
dublin_core = ET.Element('dublin_core')
|
|
238
|
+
el_title = ET.Element('dcvalue')
|
|
239
|
+
el_title.set('element', 'title')
|
|
240
|
+
el_title.set('qualifier', 'none')
|
|
241
|
+
el_title.text = 'DUMMY'
|
|
242
|
+
el_date = ET.Element('dcvalue')
|
|
243
|
+
el_date.set('element', 'date')
|
|
244
|
+
el_date.set('qualifier', 'issued')
|
|
245
|
+
el_date.text = '1982'
|
|
246
|
+
dublin_core.append(el_title)
|
|
247
|
+
dublin_core.append(el_date)
|
|
248
|
+
write_xml_file(dublin_core, dc_dummy_path)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _handle_dublin_core_derivates(work_dir):
|
|
252
|
+
"""
|
|
253
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
254
|
+
<dublin_core schema="local">
|
|
255
|
+
<dcvalue element="picturegroup" qualifier="thumbnail">EXTERNAL</dcvalue>
|
|
256
|
+
<dcvalue element="picturegroup" qualifier="preview">EXTERNAL</dcvalue>
|
|
257
|
+
</dublin_core>
|
|
258
|
+
"""
|
|
259
|
+
dc_path = os.path.join(work_dir, "metadata_local.xml")
|
|
260
|
+
dublin_core = ET.Element('dublin_core', {'schema': 'local'})
|
|
261
|
+
el_pict = ET.Element('dcvalue')
|
|
262
|
+
el_pict.set('element', 'picturegroup')
|
|
263
|
+
el_pict.set('qualifier', 'thumbnail')
|
|
264
|
+
el_pict.text = 'EXTERNAL'
|
|
265
|
+
el_prev = ET.Element('dcvalue')
|
|
266
|
+
el_prev.set('element', 'picturegroup')
|
|
267
|
+
el_prev.set('qualifier', 'preview')
|
|
268
|
+
el_prev.text = 'EXTERNAL'
|
|
269
|
+
dublin_core.append(el_pict)
|
|
270
|
+
dublin_core.append(el_prev)
|
|
271
|
+
write_xml_file(dublin_core, dc_path)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _handle_collections_file(work_dir, collections):
|
|
275
|
+
if collections:
|
|
276
|
+
collections_path = os.path.join(work_dir, "collections")
|
|
277
|
+
with open(collections_path, 'a') as collections_file:
|
|
278
|
+
collections_file.write(collections)
|
|
279
|
+
else:
|
|
280
|
+
raise DigiFlowExportError("No collections data provided - invalid share_it export!")
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def export_data_from(process_metafile_path,
|
|
284
|
+
collection,
|
|
285
|
+
saf_final_name,
|
|
286
|
+
export_dst,
|
|
287
|
+
export_map=DEFAULT_EXPORT_MAPPINGS,
|
|
288
|
+
tmp_saf_dir=None):
|
|
289
|
+
"""
|
|
290
|
+
Main entry point to prepare, create and export specified data
|
|
291
|
+
related to provided digitalization item process metadatafile_path
|
|
292
|
+
"""
|
|
293
|
+
|
|
294
|
+
source_path_dir = os.path.dirname(process_metafile_path)
|
|
295
|
+
tmp_dir = tempfile.gettempdir()
|
|
296
|
+
prefix = 'opendata-working-'
|
|
297
|
+
if tmp_saf_dir:
|
|
298
|
+
tmp_dir = tmp_saf_dir
|
|
299
|
+
with tempfile.TemporaryDirectory(prefix=prefix, dir=tmp_dir) as tmp_dir:
|
|
300
|
+
work_dir = os.path.join(tmp_dir, saf_final_name, 'item_000')
|
|
301
|
+
_mappings = map_contents(source_path_dir, work_dir, export_map)
|
|
302
|
+
return process(_mappings, work_dir, saf_final_name, collection, export_dst)
|