digiflow 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- digiflow-3.0.0/LICENCE +21 -0
- digiflow-3.0.0/MANIFEST.in +5 -0
- digiflow-3.0.0/PKG-INFO +10 -0
- digiflow-3.0.0/README.md +37 -0
- digiflow-3.0.0/pyproject.toml +31 -0
- digiflow-3.0.0/setup.cfg +4 -0
- digiflow-3.0.0/src/digiflow/__init__.py +22 -0
- digiflow-3.0.0/src/digiflow/digflow_identifier.py +374 -0
- digiflow-3.0.0/src/digiflow/digiflow_export.py +302 -0
- digiflow-3.0.0/src/digiflow/digiflow_generate.py +366 -0
- digiflow-3.0.0/src/digiflow/digiflow_io.py +1063 -0
- digiflow-3.0.0/src/digiflow/digiflow_metadata.py +808 -0
- digiflow-3.0.0/src/digiflow/digiflow_validate.py +333 -0
- digiflow-3.0.0/src/digiflow/resources/digilife.ini +90 -0
- digiflow-3.0.0/src/digiflow/resources/sch/ddb_validierung_mets-mods-ap-digitalisierte-medien.sch +2127 -0
- digiflow-3.0.0/src/digiflow/resources/sch/ddb_validierung_mets-mods-ap-digitalisierte-zeitungen.sch +1562 -0
- digiflow-3.0.0/src/digiflow/resources/xsd/alto_4-2.xsd +1105 -0
- digiflow-3.0.0/src/digiflow/resources/xsd/mets_1-12.xsd +1780 -0
- digiflow-3.0.0/src/digiflow/resources/xsd/mix_2-0.xsd +2570 -0
- digiflow-3.0.0/src/digiflow/resources/xsd/mods_3-7.xsd +1486 -0
- digiflow-3.0.0/src/digiflow/resources/xsd/page_2019-07-15.xsd +2644 -0
- digiflow-3.0.0/src/digiflow/resources/xsd/xlink.xsd +75 -0
- digiflow-3.0.0/src/digiflow/resources/xsd/xml.xsd +146 -0
- digiflow-3.0.0/src/digiflow/schematron/schxslt-cli.jar +0 -0
- digiflow-3.0.0/src/digiflow.egg-info/PKG-INFO +10 -0
- digiflow-3.0.0/src/digiflow.egg-info/SOURCES.txt +33 -0
- digiflow-3.0.0/src/digiflow.egg-info/dependency_links.txt +1 -0
- digiflow-3.0.0/src/digiflow.egg-info/requires.txt +4 -0
- digiflow-3.0.0/src/digiflow.egg-info/top_level.txt +1 -0
- digiflow-3.0.0/tests/test_digiflow_generate.py +393 -0
- digiflow-3.0.0/tests/test_digiflow_identifier.py +605 -0
- digiflow-3.0.0/tests/test_digiflow_io.py +1577 -0
- digiflow-3.0.0/tests/test_digiflow_metadata.py +766 -0
- digiflow-3.0.0/tests/test_digiflow_validate.py +517 -0
- digiflow-3.0.0/tests/test_export_saf.py +443 -0
digiflow-3.0.0/LICENCE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Universitäts- und Landesbibliothek Sachsen-Anhalt
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
digiflow-3.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: digiflow
|
|
3
|
+
Version: 3.0.0
|
|
4
|
+
Summary: Father's Little Digitization Workflow Helper
|
|
5
|
+
Author-email: Universitäts- und Landesbibliothek Sachsen-Anhalt <development@bibliothek.uni-halle.de>
|
|
6
|
+
Project-URL: Homepage, https://github.com/ulb-sachsen-anhalt/digital-flow
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Requires-Python: >=3.8
|
|
10
|
+
License-File: LICENCE
|
digiflow-3.0.0/README.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# digital flow
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+
[](https://pypi.org/project/digital-flow)   
|
|
5
|
+
|
|
6
|
+
Father's little helper for internal library digitalization workflows running on Linux-Systems.
|
|
7
|
+
|
|
8
|
+
## Testing
|
|
9
|
+
|
|
10
|
+
Testing with tox uses different Python versions.
|
|
11
|
+
|
|
12
|
+
### Local tests using Tox on Ubuntu 20.04 LTS
|
|
13
|
+
|
|
14
|
+
For running tox locally for different Python Versions, it is required to have them installed
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
apt update
|
|
18
|
+
apt-get -y install software-properties-common
|
|
19
|
+
add-apt-repository -y ppa:deadsnakes/ppa
|
|
20
|
+
apt update
|
|
21
|
+
apt-get -y install python python3.6 python3-pip python-setuptools
|
|
22
|
+
|
|
23
|
+
# activate project's environment
|
|
24
|
+
. ./venv/bin/activate
|
|
25
|
+
pip install tox
|
|
26
|
+
tox
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### Run Tests with toxic containers
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
setup_tox.sh your-python-tox-image
|
|
33
|
+
docker build --tag <your-test-image> --build-arg BASE_IMAGE=<your-python-tox-image> -f Dockerfile.tox .
|
|
34
|
+
docker run --rm <your-test-image> tox
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
This executes `tox` in a closed container without distributioning.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "digiflow"
|
|
7
|
+
version = "3.0.0"
|
|
8
|
+
description = "Father's Little Digitization Workflow Helper"
|
|
9
|
+
requires-python = ">=3.8"
|
|
10
|
+
authors = [
|
|
11
|
+
{name = "Universitäts- und Landesbibliothek Sachsen-Anhalt", email = "development@bibliothek.uni-halle.de"}
|
|
12
|
+
]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"License :: OSI Approved :: MIT License"
|
|
16
|
+
]
|
|
17
|
+
dependencies = [
|
|
18
|
+
"lxml",
|
|
19
|
+
"Pillow",
|
|
20
|
+
"numpy",
|
|
21
|
+
"requests"
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/ulb-sachsen-anhalt/digital-flow"
|
|
26
|
+
|
|
27
|
+
[tool.setuptools]
|
|
28
|
+
include-package-data = true
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
where = ["src"]
|
digiflow-3.0.0/setup.cfg
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from .digflow_identifier import *
|
|
2
|
+
from .digiflow_io import *
|
|
3
|
+
from .digiflow_metadata import *
|
|
4
|
+
from .digiflow_generate import *
|
|
5
|
+
from .digiflow_export import *
|
|
6
|
+
from .digiflow_validate import (
|
|
7
|
+
ddb_validation,
|
|
8
|
+
ddb_xslt_validation,
|
|
9
|
+
DDB_IGNORE_RULES_BASIC,
|
|
10
|
+
DDB_IGNORE_RULES_MVW,
|
|
11
|
+
DDB_IGNORE_RULES_NEWSPAPERS,
|
|
12
|
+
DDB_MEDIA_SCH,
|
|
13
|
+
DDB_NEWSP_SCH,
|
|
14
|
+
CLI_DIR,
|
|
15
|
+
PATH_CLI,
|
|
16
|
+
SCH_DIR,
|
|
17
|
+
PATH_MEDIA_SCH,
|
|
18
|
+
PATH_NEWSP_SCH,
|
|
19
|
+
DIGIS_MULTIVOLUME,
|
|
20
|
+
DIGIS_NEWSPAPER,
|
|
21
|
+
TMP_SCH_FILE_NAME,
|
|
22
|
+
)
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Module for Handling of persistent Identifiers"""
|
|
3
|
+
|
|
4
|
+
import enum
|
|
5
|
+
import functools
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from collections import (
|
|
10
|
+
namedtuple
|
|
11
|
+
)
|
|
12
|
+
from pathlib import (
|
|
13
|
+
Path
|
|
14
|
+
)
|
|
15
|
+
from typing import (
|
|
16
|
+
List,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
from .digiflow_metadata import (
|
|
20
|
+
MetsReader,
|
|
21
|
+
XMLNS,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
##################
|
|
26
|
+
# script constants
|
|
27
|
+
#
|
|
28
|
+
# URN PATTERN
|
|
29
|
+
URN_PATTERN_ULB = 'urn:nbn:de:gbv:3:{}-{}-{}-{}'
|
|
30
|
+
DEFAULT_PATTERN_ULB = r"^urn\:nbn\:de\:gbv\:3\:\d\-\d{3,}.*\d+$"
|
|
31
|
+
DEFAULT_GRANULAR_PAGE_FRAGMENT = '/fragment/page'
|
|
32
|
+
# kitodo2 system id
|
|
33
|
+
KITODO2_ID = '1192015415'
|
|
34
|
+
|
|
35
|
+
###############
|
|
36
|
+
# Pruefziffer mapping
|
|
37
|
+
# Map Chars to Ints
|
|
38
|
+
# cf. http://www.persistent-identifier.de/?link=316
|
|
39
|
+
PZ_MAP = {}
|
|
40
|
+
PZ_MAP["0"] = 1
|
|
41
|
+
PZ_MAP["1"] = 2
|
|
42
|
+
PZ_MAP["2"] = 3
|
|
43
|
+
PZ_MAP["3"] = 4
|
|
44
|
+
PZ_MAP["4"] = 5
|
|
45
|
+
PZ_MAP["5"] = 6
|
|
46
|
+
PZ_MAP["6"] = 7
|
|
47
|
+
PZ_MAP["7"] = 8
|
|
48
|
+
PZ_MAP["8"] = 9
|
|
49
|
+
PZ_MAP["u"] = 11
|
|
50
|
+
PZ_MAP["r"] = 12
|
|
51
|
+
PZ_MAP["n"] = 13
|
|
52
|
+
PZ_MAP["b"] = 14
|
|
53
|
+
PZ_MAP["d"] = 15
|
|
54
|
+
PZ_MAP["e"] = 16
|
|
55
|
+
PZ_MAP[":"] = 17
|
|
56
|
+
PZ_MAP["a"] = 18
|
|
57
|
+
PZ_MAP["c"] = 19
|
|
58
|
+
PZ_MAP["f"] = 21
|
|
59
|
+
PZ_MAP["g"] = 22
|
|
60
|
+
PZ_MAP["h"] = 23
|
|
61
|
+
PZ_MAP["i"] = 24
|
|
62
|
+
PZ_MAP["j"] = 25
|
|
63
|
+
PZ_MAP["l"] = 26
|
|
64
|
+
PZ_MAP["L"] = 26
|
|
65
|
+
PZ_MAP["m"] = 27
|
|
66
|
+
PZ_MAP["o"] = 28
|
|
67
|
+
PZ_MAP["p"] = 29
|
|
68
|
+
PZ_MAP["q"] = 31
|
|
69
|
+
PZ_MAP["s"] = 32
|
|
70
|
+
PZ_MAP["t"] = 33
|
|
71
|
+
PZ_MAP["v"] = 34
|
|
72
|
+
PZ_MAP["w"] = 35
|
|
73
|
+
PZ_MAP["x"] = 36
|
|
74
|
+
PZ_MAP["X"] = 36
|
|
75
|
+
PZ_MAP["y"] = 37
|
|
76
|
+
PZ_MAP["Y"] = 37
|
|
77
|
+
PZ_MAP["z"] = 38
|
|
78
|
+
PZ_MAP["Z"] = 38
|
|
79
|
+
PZ_MAP["-"] = 39
|
|
80
|
+
PZ_MAP["9"] = 41
|
|
81
|
+
PZ_MAP["k"] = 42
|
|
82
|
+
PZ_MAP["_"] = 43
|
|
83
|
+
PZ_MAP["/"] = 45
|
|
84
|
+
PZ_MAP["."] = 47
|
|
85
|
+
PZ_MAP["+"] = 45
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class DigiFlowURNExistsException(Exception):
|
|
89
|
+
"""Mark URN already exists"""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class GranularURNExistsException(Exception):
|
|
93
|
+
"""Mark Granular URN exists"""
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class GranularURNException(Exception):
|
|
97
|
+
"""Mark generic granular URN Exception"""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class URNType(enum.IntEnum):
|
|
101
|
+
'''
|
|
102
|
+
Represent different level of identifiers
|
|
103
|
+
Represent URN on work-level, i.e. for a
|
|
104
|
+
* work-level: identifies digitalization objects like
|
|
105
|
+
monographiesy, c- and f-stages or newspaper years/issues
|
|
106
|
+
* page-level: physical page with complete URN
|
|
107
|
+
* page-fragment-level: physical page
|
|
108
|
+
with fragemented URN derived from work-level URN
|
|
109
|
+
* page-fragment-focus: box within physical page
|
|
110
|
+
with fragemented URN derived from work-level URN
|
|
111
|
+
|
|
112
|
+
Please note, that there was also a granular URN
|
|
113
|
+
with corresponded to only printed pages, but this
|
|
114
|
+
version was not mentioned recently
|
|
115
|
+
'''
|
|
116
|
+
|
|
117
|
+
WORK = 1
|
|
118
|
+
GRANULAR_PAGE = 2
|
|
119
|
+
GRANULAR_FRAGMENT_PAGE = 3
|
|
120
|
+
GRANULAR_FRAGMENT_PAGE_FOCUS = 4
|
|
121
|
+
MISSING = 5
|
|
122
|
+
UNKNOWN = 6
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def generate_urn_pz(urn):
|
|
126
|
+
"""
|
|
127
|
+
Generate Checkdigit for persistent identifiers
|
|
128
|
+
see: http://www.persistent-identifier.de/?link=316
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
# map + reduce
|
|
132
|
+
mapped_digits_str = functools.reduce(lambda c, p: c + p, [str(PZ_MAP[c]) for c in urn])
|
|
133
|
+
|
|
134
|
+
# sum
|
|
135
|
+
pzs = sum((i * int(c) for i, c in enumerate(mapped_digits_str, 1)))
|
|
136
|
+
|
|
137
|
+
# divide by last element of digits mapped from urn
|
|
138
|
+
divisor = mapped_digits_str[-1]
|
|
139
|
+
quotient = int(pzs / int(divisor))
|
|
140
|
+
|
|
141
|
+
# return last digit
|
|
142
|
+
return str(quotient)[-1:][0]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# Container Struct
|
|
146
|
+
GranularContainer = namedtuple('GranularContainer', ['id', 'type', 'urn'])
|
|
147
|
+
|
|
148
|
+
class ProcessReport:
|
|
149
|
+
"""Handle outcomes concerning URN handling"""
|
|
150
|
+
|
|
151
|
+
def __init__(self, added:List[GranularContainer], existed:List[GranularContainer]) -> None:
|
|
152
|
+
self.added: List[GranularContainer] = added
|
|
153
|
+
self.existed: List[GranularContainer] = existed
|
|
154
|
+
self._replaced = None
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def replaced(self):
|
|
158
|
+
"""Get entries for which a mapping exist
|
|
159
|
+
for existed and added entries, which
|
|
160
|
+
means they were replaced"""
|
|
161
|
+
if self._replaced is None:
|
|
162
|
+
self._replaced = {}
|
|
163
|
+
for _add in self.added:
|
|
164
|
+
for _exs in self.existed:
|
|
165
|
+
if _add.id == _exs.id:
|
|
166
|
+
self._replaced[_add.id] = f"{_exs.urn} => {_add.urn}"
|
|
167
|
+
return self._replaced
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _determine_types(elem_cnts:List) -> List[GranularContainer]:
|
|
171
|
+
_types = []
|
|
172
|
+
for _el in elem_cnts:
|
|
173
|
+
cnt_id = _el.attrib['ID']
|
|
174
|
+
gurn = _el.get('CONTENTIDS')
|
|
175
|
+
if not gurn:
|
|
176
|
+
_type = URNType.MISSING
|
|
177
|
+
else:
|
|
178
|
+
if re.match(r".*\-p\d{4}\-\d$", gurn):
|
|
179
|
+
_type = URNType.GRANULAR_PAGE
|
|
180
|
+
elif re.match(r".*\/fragment\/page=\d{4,}$", gurn):
|
|
181
|
+
_type = URNType.GRANULAR_FRAGMENT_PAGE
|
|
182
|
+
else:
|
|
183
|
+
_type = URNType.UNKNOWN
|
|
184
|
+
_types.append(GranularContainer(cnt_id, _type, gurn))
|
|
185
|
+
return _types
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _determine_main_type(cnts:List[GranularContainer]) -> URNType:
|
|
189
|
+
_map = {}
|
|
190
|
+
for cnt in cnts:
|
|
191
|
+
if cnt.type in _map:
|
|
192
|
+
_map[cnt.type] = _map[cnt.type] + 1
|
|
193
|
+
else:
|
|
194
|
+
_map[cnt.type] = 1
|
|
195
|
+
return max(_map.items(), key=lambda t: t[1])[0]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def enrich_urn_granular(xml_tree,
|
|
199
|
+
urn_main,
|
|
200
|
+
page_num=None,
|
|
201
|
+
padd_left=4, do_sanitize=False, do_replace=False):
|
|
202
|
+
"""
|
|
203
|
+
Enrich URN granular at physical containers
|
|
204
|
+
default: Granular URN 2.0 - GRANULAR_FRAGMENT_PAGE
|
|
205
|
+
infix '/fragment/page={fptr:href[last]}' in div@CONTENTIDS
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
add = []
|
|
209
|
+
exs = []
|
|
210
|
+
cnts_total = _get_phys_containers(xml_tree)
|
|
211
|
+
cnt_types = _determine_types(cnts_total)
|
|
212
|
+
exs = cnt_types
|
|
213
|
+
cnt_urn_fits = [e for e in cnts_total if does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
|
|
214
|
+
urn_type = _determine_main_type(cnt_types)
|
|
215
|
+
if do_replace:
|
|
216
|
+
cnt_urn_fits = []
|
|
217
|
+
cnt_diffs = len(cnts_total) - len(cnt_urn_fits)
|
|
218
|
+
# if no granular urns exist at all, then all must get new urn ...
|
|
219
|
+
if not cnt_urn_fits:
|
|
220
|
+
# identify insertion point
|
|
221
|
+
phys_divs = xml_tree.findall('.//mets:structMap[@TYPE="PHYSICAL"]/mets:div/mets:div', XMLNS)
|
|
222
|
+
# alert invalid data
|
|
223
|
+
if len(phys_divs) == 0:
|
|
224
|
+
raise GranularURNException(f"No phys pages in {xml_tree.base}!")
|
|
225
|
+
# insert URNTYPE.GRANULAR_FRAGMENT_PAGE for each physical container
|
|
226
|
+
for phys_div in phys_divs:
|
|
227
|
+
add.append(_insert_page_fragment(phys_div, urn_main, padd_left))
|
|
228
|
+
urn_type = URNType.GRANULAR_FRAGMENT_PAGE
|
|
229
|
+
# if differences between phys containers exist, then only some must be
|
|
230
|
+
# inserted / generated depending on existing urn type
|
|
231
|
+
# might happend with additional images (colorchecker, post scans, ...)
|
|
232
|
+
elif cnt_diffs:
|
|
233
|
+
# here we pass physical containers, since we *might* require it
|
|
234
|
+
# for sanitizing CONTENTIDS for latest added structure
|
|
235
|
+
try:
|
|
236
|
+
add += insert_granular_urn(urn_type, cnts_total, urn_main, page_num, padd_left)
|
|
237
|
+
except GranularURNExistsException as exc:
|
|
238
|
+
if do_sanitize:
|
|
239
|
+
(_, _, report) = _guess_urn_granular_from_existing(xml_tree, urn_main)
|
|
240
|
+
add += report.added
|
|
241
|
+
else:
|
|
242
|
+
raise GranularURNExistsException(exc) from exc
|
|
243
|
+
_report = ProcessReport(added=add,existed=exs)
|
|
244
|
+
return (xml_tree, urn_type, _report)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _guess_urn_granular_from_existing(xml_tree, urn_main):
|
|
248
|
+
"""Here we run in a seldom particular failure
|
|
249
|
+
driven by a missing granularen URN - let's try to fix 'em just one time!
|
|
250
|
+
"""
|
|
251
|
+
cnts_total = _get_phys_containers(xml_tree)
|
|
252
|
+
phys_divs = [e for e in cnts_total if does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
|
|
253
|
+
last_phys_div = cnts_total[-1]
|
|
254
|
+
len_phys_div = len(cnts_total)
|
|
255
|
+
if str(len_phys_div) == last_phys_div.attrib['ORDER']:
|
|
256
|
+
contentids = sorted([i.attrib['CONTENTIDS'] for i in phys_divs]).pop()
|
|
257
|
+
page_num = int(contentids.split('-')[-2][1:]) + 1
|
|
258
|
+
xml_tree, urn_type, rep = enrich_urn_granular(xml_tree, urn_main, page_num=page_num)
|
|
259
|
+
_part_rep = ProcessReport(added=rep.added, existed=[])
|
|
260
|
+
return (xml_tree, urn_type, _part_rep)
|
|
261
|
+
raise GranularURNException(f"Inconsistent Granular URN '{xml_tree.base}'")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _insert_page_fragment(phys_div, main_urn: str, padd:int):
|
|
265
|
+
order = int(phys_div.attrib['ORDER'])
|
|
266
|
+
pf_o = f"{order:0{padd}}"
|
|
267
|
+
the_gran_urn = f"{main_urn}{DEFAULT_GRANULAR_PAGE_FRAGMENT}={ pf_o}"
|
|
268
|
+
phys_div.attrib['CONTENTIDS'] = the_gran_urn
|
|
269
|
+
return GranularContainer(id=phys_div.attrib['ID'],
|
|
270
|
+
type=URNType.GRANULAR_FRAGMENT_PAGE,
|
|
271
|
+
urn=the_gran_urn)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def does_fit(e, pattern=DEFAULT_PATTERN_ULB):
|
|
275
|
+
"""
|
|
276
|
+
Proper NBN URN consists of
|
|
277
|
+
* prefix like urn:nbn:de:gbv:3
|
|
278
|
+
* some subseqent numerical collection number
|
|
279
|
+
* additional identifiers (sementic IDs, PPNs, ...)
|
|
280
|
+
* suffix alike 'fragment' or '-p' for granularity
|
|
281
|
+
* probably numerial exemplar count
|
|
282
|
+
* final singular checkdigit
|
|
283
|
+
"""
|
|
284
|
+
if 'CONTENTIDS' not in e.attrib:
|
|
285
|
+
return False
|
|
286
|
+
_id = e.attrib['CONTENTIDS']
|
|
287
|
+
return re.match(pattern, _id) is not None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def insert_granular_urn(urn_type, phys_conts, main_urn, page_num=None, padd_left=4):
|
|
291
|
+
"""
|
|
292
|
+
Take care of inconsistent physical containers!
|
|
293
|
+
|
|
294
|
+
Physical containers missing CONTENTIDS
|
|
295
|
+
http://digital.bibliothek.uni-halle.de/hd/oai/?verb=GetRecord&metadataPrefix=mets&mode=xml&identifier=10595
|
|
296
|
+
|
|
297
|
+
Granular URN missing in between, therefore ORDER and granular URN do not match in the end
|
|
298
|
+
https://digitale.bibliothek.uni-halle.de/vd16/oai?verb=GetRecord&metadataPrefix=mets&identifier=993531
|
|
299
|
+
"""
|
|
300
|
+
|
|
301
|
+
inserts = []
|
|
302
|
+
els_missing = [e for e in phys_conts if not does_fit(e, pattern=DEFAULT_PATTERN_ULB)]
|
|
303
|
+
if urn_type == URNType.GRANULAR_PAGE:
|
|
304
|
+
for el_misses in els_missing:
|
|
305
|
+
# crashes if @ORDER cant be parsed as integer
|
|
306
|
+
# calculate pXXXX from attr 'ORDER'
|
|
307
|
+
# or, -if provided-, from parameter page_num
|
|
308
|
+
p_number = page_num if page_num else int(el_misses.attrib['ORDER'])
|
|
309
|
+
raw_gurn = f"{main_urn}-p{p_number:0{padd_left}}-"
|
|
310
|
+
new_gurn = f"{raw_gurn}{generate_urn_pz(raw_gurn)}"
|
|
311
|
+
el_id = el_misses.attrib['ID']
|
|
312
|
+
# check if new granular URN is *really* unique
|
|
313
|
+
# raise Exception otherwise ...
|
|
314
|
+
cnts_with_gurn = [
|
|
315
|
+
e for e in phys_conts
|
|
316
|
+
if 'CONTENTIDS' in e.attrib and re.match(r".*-p\d+-\d$", e.attrib['CONTENTIDS'])]
|
|
317
|
+
existing_ones = [e for e in cnts_with_gurn
|
|
318
|
+
if e.attrib['CONTENTIDS'] == new_gurn]
|
|
319
|
+
if existing_ones:
|
|
320
|
+
pat = "calculated granular URN '{}' already used by: '{}'!"
|
|
321
|
+
exsting_ids = [e.attrib['ID'] for e in existing_ones]
|
|
322
|
+
msg_fail = pat.format(new_gurn, exsting_ids)
|
|
323
|
+
raise GranularURNExistsException(msg_fail)
|
|
324
|
+
el_misses.attrib['CONTENTIDS'] = new_gurn
|
|
325
|
+
inserts.append(GranularContainer(id=el_id, type=URNType.GRANULAR_FRAGMENT_PAGE, urn=new_gurn))
|
|
326
|
+
return inserts
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _get_phys_containers(xml_tree):
|
|
330
|
+
"""Get all physical containers that contain @ORDER attribute"""
|
|
331
|
+
return xml_tree.findall(
|
|
332
|
+
'.//mets:structMap[@TYPE="PHYSICAL"]//mets:div[@ORDER]', XMLNS)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def enrich_urn_kitodo2(process_path:'str|Path', collection='1', system_id=KITODO2_ID, exemplar='1',
|
|
336
|
+
force_overwrite=False):
|
|
337
|
+
"""Enrich URN for Kitodo2 if not existing"""
|
|
338
|
+
|
|
339
|
+
_urns = []
|
|
340
|
+
read = MetsReader(process_path)
|
|
341
|
+
ulb_urns = read.findall('.//goobi:metadata[@name="ulb_urn"]', read.primary_dmd)
|
|
342
|
+
if ulb_urns and not force_overwrite:
|
|
343
|
+
raise GranularURNExistsException(f"Detected existing URN {ulb_urns}!")
|
|
344
|
+
|
|
345
|
+
identifiers = read.xpath('.//goobi:metadata[@name="CatalogIDSource" and(not(@anchorId))]/text()')
|
|
346
|
+
if len(identifiers) != 1:
|
|
347
|
+
raise GranularURNException(f"Ambigious/invalid base URN {identifiers}!")
|
|
348
|
+
|
|
349
|
+
# format persistent identifier (aka URN) with provided info
|
|
350
|
+
urn = URN_PATTERN_ULB.format(collection, system_id, identifiers[0], exemplar)
|
|
351
|
+
|
|
352
|
+
# generate pruefziffer
|
|
353
|
+
urn += generate_urn_pz(urn)
|
|
354
|
+
|
|
355
|
+
# insert
|
|
356
|
+
read.insert_into_prime_mods('{http://meta.goobi.org/v1.5.1/}metadata',
|
|
357
|
+
{'name': 'ulb_urn'}, urn)
|
|
358
|
+
read.write()
|
|
359
|
+
_urns.append(urn)
|
|
360
|
+
|
|
361
|
+
# multi volume?
|
|
362
|
+
process_path = os.path.dirname(process_path)
|
|
363
|
+
path_meta_anchor = os.path.join(process_path, 'meta_anchor.xml')
|
|
364
|
+
if os.path.exists(path_meta_anchor):
|
|
365
|
+
ppn_c_stage = read.xpath('.//*[@anchorId="true"]/text()')
|
|
366
|
+
urn_host = f'urn:nbn:de:gbv:3:{collection}-{system_id}-{ppn_c_stage[0]}-{exemplar}'
|
|
367
|
+
pz_c_stage = generate_urn_pz(urn_host)
|
|
368
|
+
urn_host += pz_c_stage
|
|
369
|
+
_proc_host = MetsReader(path_meta_anchor)
|
|
370
|
+
_proc_host.insert_into_prime_mods('{http://meta.goobi.org/v1.5.1/}metadata',
|
|
371
|
+
{'name': 'ulb_urn'}, urn_host)
|
|
372
|
+
_proc_host.write()
|
|
373
|
+
_urns.append(urn_host)
|
|
374
|
+
return _urns
|