sdb-reader 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdb_reader/__init__.py +157 -0
- sdb_reader/ask.py +437 -0
- sdb_reader/ask_record.schema.json +388 -0
- sdb_reader/cli.py +1079 -0
- sdb_reader/clp_statements_de.json +997 -0
- sdb_reader/compliance.py +546 -0
- sdb_reader/current.py +134 -0
- sdb_reader/diff.py +520 -0
- sdb_reader/embedded.py +250 -0
- sdb_reader/extraction.py +506 -0
- sdb_reader/layout.py +894 -0
- sdb_reader/line_breaks.py +40 -0
- sdb_reader/messages.py +394 -0
- sdb_reader/messages_data/de.toml +146 -0
- sdb_reader/messages_data/en.toml +141 -0
- sdb_reader/model.py +321 -0
- sdb_reader/parser.py +4601 -0
- sdb_reader/pictograms.py +374 -0
- sdb_reader/presets.py +110 -0
- sdb_reader/presets.toml +23 -0
- sdb_reader/py.typed +0 -0
- sdb_reader/read_record.schema.json +792 -0
- sdb_reader/reading.py +94 -0
- sdb_reader/scan.py +265 -0
- sdb_reader/schema.py +96 -0
- sdb_reader/serialise.py +463 -0
- sdb_reader/show.py +370 -0
- sdb_reader/statements.py +1011 -0
- sdb_reader/verification.py +322 -0
- sdb_reader/versions.py +31 -0
- sdb_reader-0.9.0.dist-info/METADATA +117 -0
- sdb_reader-0.9.0.dist-info/RECORD +37 -0
- sdb_reader-0.9.0.dist-info/WHEEL +5 -0
- sdb_reader-0.9.0.dist-info/entry_points.txt +2 -0
- sdb_reader-0.9.0.dist-info/licenses/LICENSE +661 -0
- sdb_reader-0.9.0.dist-info/licenses/NOTICE +40 -0
- sdb_reader-0.9.0.dist-info/top_level.txt +1 -0
sdb_reader/layout.py
ADDED
|
@@ -0,0 +1,894 @@
|
|
|
1
|
+
"""Read a PDF page's text layer in print order, from the positions of its words.
|
|
2
|
+
|
|
3
|
+
PyMuPDF's plain text follows the order in which the file draws its text, and
|
|
4
|
+
for a two-column label:value form that is not the order of the page: Akzo
|
|
5
|
+
Nobel's sheets draw every label, then every colon, then every value, so
|
|
6
|
+
"Gefahrenhinweise" and "H412 - Schädlich für Wasserorganismen …" reach the text
|
|
7
|
+
several lines apart, and P101 under "Allgemein:" lands behind the 2.3 heading.
|
|
8
|
+
|
|
9
|
+
This reader puts the words where the page prints them instead (SDB-42 to
|
|
10
|
+
SDB-45):
|
|
11
|
+
|
|
12
|
+
1. words that share a baseline form a row, read left to right;
|
|
13
|
+
2. a wide gap splits a row into cells, the columns of the form or table;
|
|
14
|
+
3. a cell that is a colon, or begins with one, separates a label from its
|
|
15
|
+
value, and so does a cell ending in one before a value; each label:value
|
|
16
|
+
pair becomes one line, "label: value";
|
|
17
|
+
4. the rows below a pair continue it while their cells stay in its columns: a
|
|
18
|
+
label printed over several lines ("Ergänzende / Kennzeichnungselemente")
|
|
19
|
+
stays one label, and each further line of the value stays a line of its own;
|
|
20
|
+
5. a row of one cell -- a heading, a line of a paragraph -- is a line.
|
|
21
|
+
|
|
22
|
+
A form in two columns that prints no colons is read as pairs only where a
|
|
23
|
+
row says so itself (SDB-114): a numbered subsection heading of section 1 with
|
|
24
|
+
its content beside it, and, on a page that has shown one, a contact label
|
|
25
|
+
beside its value.
|
|
26
|
+
|
|
27
|
+
Any other rows of several cells with no label:value pair among them are a
|
|
28
|
+
table, and there the file's own order is kept. A table's cells can hold several lines
|
|
29
|
+
each, a heading over its value ("Überarbeitet am:" over "17.02.2025"), and
|
|
30
|
+
which cells belong together cannot be told from the positions alone; the
|
|
31
|
+
program that wrote the file knew, and wrote them in that order.
|
|
32
|
+
|
|
33
|
+
The output is text, as before, so the parser, the record and the model input
|
|
34
|
+
keep their shape.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import re
|
|
40
|
+
import statistics
|
|
41
|
+
from collections.abc import Sequence
|
|
42
|
+
from dataclasses import dataclass, field
|
|
43
|
+
|
|
44
|
+
import pymupdf
|
|
45
|
+
|
|
46
|
+
from .line_breaks import join_broken_words
|
|
47
|
+
|
|
48
|
+
# Text extraction flags: the same the plain reader uses (page.get_text()), so a
|
|
49
|
+
# word reads the same in both, ligatures and whitespace alike.
|
|
50
|
+
FLAGS = pymupdf.TEXTFLAGS_TEXT
|
|
51
|
+
|
|
52
|
+
# Two words share a row when their vertical centres are this close, as a share
|
|
53
|
+
# of the smaller height. A label and its value sit on one baseline but not
|
|
54
|
+
# always to the point: 274.1 against 274.5 on one supplier's sheets.
|
|
55
|
+
_ROW_TOLERANCE = 0.35
|
|
56
|
+
|
|
57
|
+
# A word more than this many times taller than the row's text is not part of
|
|
58
|
+
# it: a pictogram set in a symbol font ("d", "~" at 33 points) is centred on
|
|
59
|
+
# its caption's line.
|
|
60
|
+
_HEIGHT_RATIO = 2.0
|
|
61
|
+
|
|
62
|
+
# A gap wider than this, as a share of the word height, separates two cells.
|
|
63
|
+
# Measured over 300 sheets: a space between words is 0.2 to 0.3 of the height,
|
|
64
|
+
# a colon set apart from its value 0.6, a column gap 1.0 and more.
|
|
65
|
+
_CELL_GAP = 0.8
|
|
66
|
+
|
|
67
|
+
# Two rows of one label:value pair are closer than this, as a share of the
|
|
68
|
+
# height of their words; the next pair or a heading stands further off. Akzo
|
|
69
|
+
# Nobel sets the lines of a value 11.4 apart and the pairs 15, with words 11.1
|
|
70
|
+
# high. The height, not the font size: it follows the line spacing the font is
|
|
71
|
+
# set with, where the size does not -- one supplier's labels are 8 point type on lines
|
|
72
|
+
# 11.1 apart.
|
|
73
|
+
_CONTINUATION_PITCH = 1.2
|
|
74
|
+
|
|
75
|
+
# Space between two rows that reads as an empty line, as a share of the row
|
|
76
|
+
# height. It becomes one in the text, as an empty paragraph did in the plain
|
|
77
|
+
# reading: a word broken at the foot of a page ("Kenn-") otherwise meets the
|
|
78
|
+
# page number below it, and the line join makes "Kenn1 / 12" of it.
|
|
79
|
+
_BLANK_GAP = 0.8
|
|
80
|
+
|
|
81
|
+
# How far the first line of a right-aligned label may end from its last line,
|
|
82
|
+
# as a share of the row height. Right-aligned lines end within a point of each
|
|
83
|
+
# other; a looser bound took a heading or the end of a table line above a pair
|
|
84
|
+
# for the start of its label ("Handschutz Material: Nitrilkautschuk").
|
|
85
|
+
_HEAD_SLACK = 0.15
|
|
86
|
+
|
|
87
|
+
# How far a continuation may stand left of its column and still belong to it.
|
|
88
|
+
_COLUMN_SLACK = 2.0
|
|
89
|
+
|
|
90
|
+
# Text whose direction is further than this from horizontal is rotated.
|
|
91
|
+
_SKEW = 0.01
|
|
92
|
+
|
|
93
|
+
_COLON = ":"
|
|
94
|
+
|
|
95
|
+
# A page number in a cell of its own: "1/20", "2 / 21", "Seite 3 von 9",
|
|
96
|
+
# "Seite: 1/13", "Seite 3(8)". As a label's only value it has to say "Seite":
|
|
97
|
+
# "Stand: 05/22" is a date.
|
|
98
|
+
_PAGE_NUMBER = re.compile(
|
|
99
|
+
r"(?:Seite\s*:?\s*)?\d{1,3}\s*(?:/|von|\()\s*\d{1,3}\)?", re.IGNORECASE
|
|
100
|
+
)
|
|
101
|
+
_NAMED_PAGE_NUMBER = re.compile(r"Seite\s*:?\s*\d.*", re.IGNORECASE)
|
|
102
|
+
|
|
103
|
+
# A form in two columns without colons (SDB-114): what stands at the left of
|
|
104
|
+
# a row and makes the row a pair with what stands right of it. A numbered
|
|
105
|
+
# subsection heading of section 1, "1.3 Einzelheiten zum Lieferanten, …" (some
|
|
106
|
+
# sheets write "1,3"), or a contact label that is the whole cell.
|
|
107
|
+
_SECTION_ONE_HEADING = re.compile(r"1[.,][1-4]\.?\s+[A-ZÄÖÜ]\w")
|
|
108
|
+
_CONTACT_LABEL = re.compile(
|
|
109
|
+
r"(?:Telefon|Tel\.?|Telefax|Fax|E-?Mail|Internet|Homepage|Web(?:seite|site)?)"
|
|
110
|
+
r"(?:[- ]?(?:Nr\.?|Nummer|Adresse)|nummer|adresse)?",
|
|
111
|
+
re.IGNORECASE,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
# The lines of a heading in the label column of such a form stand further
|
|
115
|
+
# apart than those of a label:value pair, 10.5 with words 8 high, and still
|
|
116
|
+
# closer than an empty line's worth.
|
|
117
|
+
_FORM_PITCH = 1.5
|
|
118
|
+
|
|
119
|
+
# A line the line join (SDB-54) would join to the next: a hyphen after a word
|
|
120
|
+
# character, a bracket or a slash, as join_broken_words reads it.
|
|
121
|
+
_BROKEN = re.compile(r"[\w)\]/]-$")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(slots=True)
|
|
125
|
+
class _Word:
|
|
126
|
+
x0: float
|
|
127
|
+
y0: float
|
|
128
|
+
x1: float
|
|
129
|
+
y1: float
|
|
130
|
+
text: str
|
|
131
|
+
# Where the file draws it: block, line and word, as PyMuPDF numbers them.
|
|
132
|
+
order: tuple[int, int, int]
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def height(self) -> float:
|
|
136
|
+
return self.y1 - self.y0
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def centre(self) -> float:
|
|
140
|
+
return (self.y0 + self.y1) / 2
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass(slots=True)
|
|
144
|
+
class _Cell:
|
|
145
|
+
words: list[_Word]
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def x0(self) -> float:
|
|
149
|
+
return self.words[0].x0
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def x1(self) -> float:
|
|
153
|
+
return self.words[-1].x1
|
|
154
|
+
|
|
155
|
+
@property
|
|
156
|
+
def text(self) -> str:
|
|
157
|
+
return " ".join(word.text for word in self.words)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass(slots=True)
|
|
161
|
+
class _Row:
|
|
162
|
+
words: list[_Word]
|
|
163
|
+
cells: list[_Cell] = field(default_factory=list)
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def y0(self) -> float:
|
|
167
|
+
return min(word.y0 for word in self.words)
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def top(self) -> float:
|
|
171
|
+
"""Where the row's line begins: not moved by a cell set a little higher."""
|
|
172
|
+
return statistics.median(word.y0 for word in self.words)
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def y1(self) -> float:
|
|
176
|
+
return max(word.y1 for word in self.words)
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def height(self) -> float:
|
|
180
|
+
return statistics.median(word.height for word in self.words)
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def text(self) -> str:
|
|
184
|
+
return " ".join(cell.text for cell in self.cells)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(slots=True, frozen=True)
|
|
188
|
+
class _Line:
|
|
189
|
+
"""A line of the text, and where on the page it begins."""
|
|
190
|
+
|
|
191
|
+
text: str
|
|
192
|
+
# None for a line placed by the file's order alone: rotated text, an
|
|
193
|
+
# empty line, a page number written after its pair.
|
|
194
|
+
top: float | None = None
|
|
195
|
+
# Where the row the line stands in begins, and where the line ends.
|
|
196
|
+
x0: float = 0.0
|
|
197
|
+
x1: float = 0.0
|
|
198
|
+
height: float = 0.0
|
|
199
|
+
|
|
200
|
+
def runs_on_into(self, following: _Line) -> bool:
|
|
201
|
+
"""Whether *following* is the next line of this one's text block.
|
|
202
|
+
|
|
203
|
+
Right below it, without an empty line's worth of space between
|
|
204
|
+
(SDB-45), and beginning under its row: in its column, further in where
|
|
205
|
+
a statement hangs its text beside its code ("EUH208- Enthält …
|
|
206
|
+
2-Methyl-" over "2H-isothiazol-3-on"), or back at the code's column
|
|
207
|
+
where it does not ("P101 Ist ärztlicher Rat … Kenn-" over
|
|
208
|
+
"zeichnungsetikett"). Only there does a word broken at the end of this
|
|
209
|
+
line go on.
|
|
210
|
+
"""
|
|
211
|
+
if self.top is None or following.top is None:
|
|
212
|
+
return False
|
|
213
|
+
height = max(self.height, following.height)
|
|
214
|
+
return (
|
|
215
|
+
0 < following.top - self.top <= (1 + _BLANK_GAP) * height
|
|
216
|
+
and self.x0 - _COLUMN_SLACK <= following.x0 < self.x1
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@dataclass(slots=True)
|
|
221
|
+
class _Token:
|
|
222
|
+
"""A piece of a row's text, or the colon between a label and its value."""
|
|
223
|
+
|
|
224
|
+
text: str
|
|
225
|
+
x0: float
|
|
226
|
+
x1: float
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@dataclass(slots=True)
|
|
230
|
+
class _Pair:
|
|
231
|
+
"""A label and its value, as the rows of the form print them."""
|
|
232
|
+
|
|
233
|
+
label: list[str]
|
|
234
|
+
value: list[_Line]
|
|
235
|
+
label_x: float
|
|
236
|
+
top: float
|
|
237
|
+
height: float
|
|
238
|
+
# Where the label column ends (the colon) and the value column begins and
|
|
239
|
+
# ends: the next pair of the row starts its own label column.
|
|
240
|
+
separator_x: float
|
|
241
|
+
value_x: float
|
|
242
|
+
value_end: float
|
|
243
|
+
# Page numbers set beside the pair, written after it.
|
|
244
|
+
trailing: list[str] = field(default_factory=list)
|
|
245
|
+
# A subsection heading with its content beside it (SDB-114): the heading
|
|
246
|
+
# stays a line of its own, as where the content stands below it.
|
|
247
|
+
heading: bool = False
|
|
248
|
+
|
|
249
|
+
def lines(self) -> list[_Line]:
|
|
250
|
+
label = _join_broken(self.label)
|
|
251
|
+
if self.heading:
|
|
252
|
+
return [
|
|
253
|
+
_Line(label, self.top, self.label_x, self.separator_x, self.height),
|
|
254
|
+
*self.value,
|
|
255
|
+
*(_Line(text) for text in self.trailing),
|
|
256
|
+
]
|
|
257
|
+
head = f"{label}:" if label else _COLON
|
|
258
|
+
trailing = [_Line(text) for text in self.trailing]
|
|
259
|
+
if not self.value:
|
|
260
|
+
return [
|
|
261
|
+
_Line(head, self.top, self.label_x, self.label_x, self.height),
|
|
262
|
+
*trailing,
|
|
263
|
+
]
|
|
264
|
+
first = self.value[0]
|
|
265
|
+
return [
|
|
266
|
+
_Line(
|
|
267
|
+
f"{head} {first.text}", first.top, self.label_x, first.x1, first.height
|
|
268
|
+
),
|
|
269
|
+
*self.value[1:],
|
|
270
|
+
*trailing,
|
|
271
|
+
]
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
@dataclass(slots=True)
|
|
275
|
+
class _Pairs:
|
|
276
|
+
"""The pairs of one row, open for the rows below that continue them."""
|
|
277
|
+
|
|
278
|
+
leading: _Line | None
|
|
279
|
+
pairs: list[_Pair]
|
|
280
|
+
last_top: float
|
|
281
|
+
height: float
|
|
282
|
+
# A footer row with a page number continues on the next page, if at all.
|
|
283
|
+
open: bool
|
|
284
|
+
# A pair of a form without colons (SDB-114). The two columns of such a
|
|
285
|
+
# form are set apart from each other: the last line of a heading and the
|
|
286
|
+
# value beside it can stand two points apart, and the row's middle then
|
|
287
|
+
# lies lower than the heading's next line would.
|
|
288
|
+
form: bool = False
|
|
289
|
+
# Where the last line of a form's heading begins: a cell in its column
|
|
290
|
+
# further down is the next label, not a further line of the heading.
|
|
291
|
+
label_top: float = 0.0
|
|
292
|
+
|
|
293
|
+
def lines(self) -> list[_Line]:
|
|
294
|
+
lines = [self.leading] if self.leading else []
|
|
295
|
+
for pair in self.pairs:
|
|
296
|
+
lines.extend(pair.lines())
|
|
297
|
+
return lines
|
|
298
|
+
|
|
299
|
+
def continues(self, row: _Row) -> bool:
|
|
300
|
+
"""Take *row* as further lines of the pairs' labels and values, if it is."""
|
|
301
|
+
if not self.open:
|
|
302
|
+
return False
|
|
303
|
+
# Below the pairs, in type of their size: a pictogram glyph split off
|
|
304
|
+
# their row (_without_outsized) stands beside them, not under them.
|
|
305
|
+
if row.top <= self.last_top or row.height > _HEIGHT_RATIO * self.height:
|
|
306
|
+
return False
|
|
307
|
+
top = row.y0 if self.form else row.top
|
|
308
|
+
pitch = _FORM_PITCH if self.form else _CONTINUATION_PITCH
|
|
309
|
+
if top - self.last_top > pitch * max(self.height, row.height):
|
|
310
|
+
return False
|
|
311
|
+
placed: list[tuple[_Pair, bool, _Cell]] = []
|
|
312
|
+
for cell in row.cells:
|
|
313
|
+
for pair in self.pairs:
|
|
314
|
+
if pair.value_x - _COLUMN_SLACK <= cell.x0 < pair.value_end:
|
|
315
|
+
placed.append((pair, False, cell))
|
|
316
|
+
break
|
|
317
|
+
if (
|
|
318
|
+
cell.x0 >= pair.label_x - _COLUMN_SLACK
|
|
319
|
+
and cell.x1 <= pair.separator_x
|
|
320
|
+
):
|
|
321
|
+
placed.append((pair, True, cell))
|
|
322
|
+
break
|
|
323
|
+
else:
|
|
324
|
+
return False
|
|
325
|
+
if self.form and any(is_label for _, is_label, _ in placed):
|
|
326
|
+
if top - self.label_top > pitch * max(self.height, row.height):
|
|
327
|
+
return False
|
|
328
|
+
self.label_top = top
|
|
329
|
+
for pair in self.pairs:
|
|
330
|
+
labels = [c.text for p, is_label, c in placed if p is pair and is_label]
|
|
331
|
+
values = [c for p, is_label, c in placed if p is pair and not is_label]
|
|
332
|
+
if labels:
|
|
333
|
+
pair.label.append(" ".join(labels))
|
|
334
|
+
if values:
|
|
335
|
+
text = " ".join(c.text for c in values)
|
|
336
|
+
pair.value.append(
|
|
337
|
+
_Line(text, row.top, row.cells[0].x0, values[-1].x1, row.height)
|
|
338
|
+
)
|
|
339
|
+
self.last_top = row.top
|
|
340
|
+
return True
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@dataclass(slots=True)
|
|
344
|
+
class _Table:
|
|
345
|
+
"""Rows of cells without a label:value pair: read in the file's order."""
|
|
346
|
+
|
|
347
|
+
rows: list[_Row]
|
|
348
|
+
|
|
349
|
+
def adjoins(self, row: _Row) -> bool:
|
|
350
|
+
"""Whether *row* stands right below the table, without an empty line."""
|
|
351
|
+
return row.y0 - self.rows[-1].y1 <= _BLANK_GAP * row.height
|
|
352
|
+
|
|
353
|
+
def fits(self, row: _Row) -> bool:
|
|
354
|
+
"""Whether a one-cell *row* is a further line of one of the table's columns.
|
|
355
|
+
|
|
356
|
+
Only right below it: after an empty line's worth of space, a cell in
|
|
357
|
+
one of its columns is the page number of the footer as often as it is
|
|
358
|
+
a further line of the table.
|
|
359
|
+
"""
|
|
360
|
+
if not self.adjoins(row):
|
|
361
|
+
return False
|
|
362
|
+
cell = row.cells[0]
|
|
363
|
+
for column in self._columns():
|
|
364
|
+
if (
|
|
365
|
+
cell.x0 >= column[0] - _COLUMN_SLACK
|
|
366
|
+
and cell.x1 <= column[1] + _COLUMN_SLACK
|
|
367
|
+
):
|
|
368
|
+
return True
|
|
369
|
+
return False
|
|
370
|
+
|
|
371
|
+
def _columns(self) -> list[tuple[float, float]]:
|
|
372
|
+
spans = sorted((c.x0, c.x1) for r in self.rows for c in r.cells)
|
|
373
|
+
columns: list[tuple[float, float]] = []
|
|
374
|
+
for x0, x1 in spans:
|
|
375
|
+
if columns and x0 <= columns[-1][1]:
|
|
376
|
+
columns[-1] = (columns[-1][0], max(columns[-1][1], x1))
|
|
377
|
+
else:
|
|
378
|
+
columns.append((x0, x1))
|
|
379
|
+
return columns
|
|
380
|
+
|
|
381
|
+
def lines(self) -> list[_Line]:
|
|
382
|
+
"""The table's lines in the file's order, each where the page prints it."""
|
|
383
|
+
by_line: dict[tuple[int, int], list[_Word]] = {}
|
|
384
|
+
row_start: dict[tuple[int, int], float] = {}
|
|
385
|
+
for row in self.rows:
|
|
386
|
+
for word in row.words:
|
|
387
|
+
by_line.setdefault(word.order[:2], []).append(word)
|
|
388
|
+
row_start.setdefault(word.order[:2], row.cells[0].x0)
|
|
389
|
+
lines: list[_Line] = []
|
|
390
|
+
for key, words in sorted(by_line.items()):
|
|
391
|
+
words.sort(key=lambda w: w.order)
|
|
392
|
+
lines.append(
|
|
393
|
+
_Line(
|
|
394
|
+
" ".join(w.text for w in words),
|
|
395
|
+
statistics.median(w.y0 for w in words),
|
|
396
|
+
row_start[key],
|
|
397
|
+
words[-1].x1,
|
|
398
|
+
statistics.median(w.height for w in words),
|
|
399
|
+
)
|
|
400
|
+
)
|
|
401
|
+
return lines
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def page_text(page: pymupdf.Page, textpage: pymupdf.TextPage | None = None) -> str:
|
|
405
|
+
"""The text of *page* in the order it is printed.
|
|
406
|
+
|
|
407
|
+
*textpage* is the page's text, extracted with FLAGS; a caller that also
|
|
408
|
+
wants the plain text passes it to read the page once.
|
|
409
|
+
"""
|
|
410
|
+
textpage = textpage or page.get_textpage(flags=FLAGS)
|
|
411
|
+
directions = _Directions(page.get_text("dict", textpage=textpage)["blocks"])
|
|
412
|
+
horizontal: list[_Word] = []
|
|
413
|
+
rotated: dict[tuple[int, int], list[str]] = {}
|
|
414
|
+
for x0, y0, x1, y1, text, block, line, number in page.get_text(
|
|
415
|
+
"words", textpage=textpage
|
|
416
|
+
):
|
|
417
|
+
dx, dy = directions.of(block, line, (x0 + x1) / 2, (y0 + y1) / 2)
|
|
418
|
+
if dx < 1.0 - _SKEW or abs(dy) > _SKEW:
|
|
419
|
+
# Rotated text, a margin note set vertically: kept, line by line in
|
|
420
|
+
# the file's order, after the page.
|
|
421
|
+
rotated.setdefault((block, line), []).append(text)
|
|
422
|
+
continue
|
|
423
|
+
horizontal.append(_Word(x0, y0, x1, y1, text, (block, line, number)))
|
|
424
|
+
|
|
425
|
+
lines = _read_rows(_rows(horizontal), page.rect.width)
|
|
426
|
+
lines.extend(_Line(" ".join(words)) for _, words in sorted(rotated.items()))
|
|
427
|
+
return "\n".join(_with_breaks_kept(lines))
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _with_breaks_kept(lines: list[_Line]) -> list[str]:
|
|
431
|
+
"""The lines' text, an empty line after a hyphen that does not run on.
|
|
432
|
+
|
|
433
|
+
The line join (SDB-54) joins a line ending in a hyphen to whatever line
|
|
434
|
+
follows. That is the next line of the word only where it stands right
|
|
435
|
+
below in the same column; elsewhere -- the next cell of a table in the
|
|
436
|
+
file's order, a pair below a table, the page's end -- an empty line keeps
|
|
437
|
+
them apart: "gewässergefähr-" / "H413: Kann für …" made "gewässergefährH413".
|
|
438
|
+
"""
|
|
439
|
+
text: list[str] = []
|
|
440
|
+
for index, line in enumerate(lines):
|
|
441
|
+
text.append(line.text)
|
|
442
|
+
following = lines[index + 1] if index + 1 < len(lines) else None
|
|
443
|
+
if _BROKEN.search(line.text) and (
|
|
444
|
+
following is None or (following.text and not line.runs_on_into(following))
|
|
445
|
+
):
|
|
446
|
+
text.append("")
|
|
447
|
+
return text
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
class _Directions:
|
|
451
|
+
"""The writing direction of each line, found by where its words stand.
|
|
452
|
+
|
|
453
|
+
PyMuPDF numbers a word's line among every line of its block, but the
|
|
454
|
+
"dict" output leaves out a line whose box is empty -- text set at zero
|
|
455
|
+
height. Counting would then give the next line's direction, or none, so a
|
|
456
|
+
word is matched to the line whose box holds it. A word in no line's box is
|
|
457
|
+
on an empty line and reads as horizontal.
|
|
458
|
+
"""
|
|
459
|
+
|
|
460
|
+
def __init__(self, blocks: Sequence[dict[str, object]]) -> None:
|
|
461
|
+
self._lines: dict[int, list[tuple[tuple[float, ...], tuple[float, float]]]] = {}
|
|
462
|
+
for block in blocks:
|
|
463
|
+
number = block.get("number")
|
|
464
|
+
lines = block.get("lines")
|
|
465
|
+
if not isinstance(number, int) or not isinstance(lines, list):
|
|
466
|
+
continue
|
|
467
|
+
self._lines[number] = [
|
|
468
|
+
(tuple(line["bbox"]), (line["dir"][0], line["dir"][1]))
|
|
469
|
+
for line in lines
|
|
470
|
+
]
|
|
471
|
+
|
|
472
|
+
def of(self, block: int, line: int, x: float, y: float) -> tuple[float, float]:
|
|
473
|
+
lines = self._lines.get(block, [])
|
|
474
|
+
candidates = ([lines[line]] if line < len(lines) else []) + lines
|
|
475
|
+
for (x0, y0, x1, y1), direction in candidates:
|
|
476
|
+
if x0 - 1 <= x <= x1 + 1 and y0 - 1 <= y <= y1 + 1:
|
|
477
|
+
return direction
|
|
478
|
+
return (1.0, 0.0)
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _rows(words: list[_Word]) -> list[_Row]:
|
|
482
|
+
"""Group *words* into rows by their vertical centre, each read left to right."""
|
|
483
|
+
clusters: list[list[_Word]] = []
|
|
484
|
+
current: list[_Word] = []
|
|
485
|
+
centre = 0.0
|
|
486
|
+
for word in sorted(words, key=lambda w: (w.centre, w.x0)):
|
|
487
|
+
if current and abs(word.centre - centre) <= _ROW_TOLERANCE * min(
|
|
488
|
+
word.height, statistics.median(w.height for w in current)
|
|
489
|
+
):
|
|
490
|
+
current.append(word)
|
|
491
|
+
centre = sum(w.centre for w in current) / len(current)
|
|
492
|
+
continue
|
|
493
|
+
if current:
|
|
494
|
+
clusters.append(current)
|
|
495
|
+
current = [word]
|
|
496
|
+
centre = word.centre
|
|
497
|
+
if current:
|
|
498
|
+
clusters.append(current)
|
|
499
|
+
|
|
500
|
+
rows: list[_Row] = []
|
|
501
|
+
for cluster in clusters:
|
|
502
|
+
text, outsized = _without_outsized(cluster)
|
|
503
|
+
for part in (text, outsized):
|
|
504
|
+
if part:
|
|
505
|
+
words_of_row = _joined_touching(
|
|
506
|
+
_without_doubles(sorted(part, key=lambda w: w.x0))
|
|
507
|
+
)
|
|
508
|
+
rows.append(_Row(words_of_row, _cells(words_of_row)))
|
|
509
|
+
return rows
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _without_outsized(words: list[_Word]) -> tuple[list[_Word], list[_Word]]:
|
|
513
|
+
"""Split off the words more than _HEIGHT_RATIO times taller than the row's text.
|
|
514
|
+
|
|
515
|
+
Taller only: a superscript stays in its row. The text's height is the
|
|
516
|
+
median of the lower half, so two pictogram glyphs beside a two-word caption
|
|
517
|
+
do not make the glyphs the norm.
|
|
518
|
+
"""
|
|
519
|
+
heights = sorted(word.height for word in words)
|
|
520
|
+
norm = statistics.median(heights[: (len(heights) + 1) // 2])
|
|
521
|
+
text = [w for w in words if w.height <= _HEIGHT_RATIO * norm]
|
|
522
|
+
return text, [w for w in words if w.height > _HEIGHT_RATIO * norm]
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
# Two pieces of a row end and begin at the same point when they are no
|
|
526
|
+
# further apart than this, as a share of their height. A space between words
|
|
527
|
+
# is 0.2 to 0.3 of the height (see _CELL_GAP); a narrow one after a colon
|
|
528
|
+
# still 0.03.
|
|
529
|
+
_TOUCHING = 0.01
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _joined_touching(words: list[_Word]) -> list[_Word]:
|
|
533
|
+
"""Join the pieces of a word the file draws apart (SDB-99).
|
|
534
|
+
|
|
535
|
+
A file that sets single characters in another font draws a word in pieces:
|
|
536
|
+
"24", ".0", "9", ".2025" for "24.09.2025", each piece a word of its own to
|
|
537
|
+
PyMuPDF and not in the order the page shows. They stand where the page
|
|
538
|
+
prints them, without a space between. The joined word takes the place in
|
|
539
|
+
the file's order of its last piece.
|
|
540
|
+
"""
|
|
541
|
+
joined: list[_Word] = []
|
|
542
|
+
for word in words:
|
|
543
|
+
if joined:
|
|
544
|
+
previous = joined[-1]
|
|
545
|
+
gap = word.x0 - previous.x1
|
|
546
|
+
height = min(word.height, previous.height)
|
|
547
|
+
# Drawn apart: in another line of the file. Words of one line
|
|
548
|
+
# that touch are PyMuPDF's own division, at a space it found.
|
|
549
|
+
if abs(gap) <= _TOUCHING * height and word.order[:2] != previous.order[:2]:
|
|
550
|
+
joined[-1] = _Word(
|
|
551
|
+
previous.x0,
|
|
552
|
+
min(previous.y0, word.y0),
|
|
553
|
+
word.x1,
|
|
554
|
+
max(previous.y1, word.y1),
|
|
555
|
+
previous.text + word.text,
|
|
556
|
+
max(previous.order, word.order),
|
|
557
|
+
)
|
|
558
|
+
continue
|
|
559
|
+
joined.append(word)
|
|
560
|
+
return joined
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def _without_doubles(words: list[_Word]) -> list[_Word]:
|
|
564
|
+
"""Drop a word printed twice on itself, the way some files fake bold type.
|
|
565
|
+
|
|
566
|
+
On itself: the two boxes overlap by most of their width. Two dashes side by
|
|
567
|
+
side ("- - -") are separate words, however narrow.
|
|
568
|
+
"""
|
|
569
|
+
kept: list[_Word] = []
|
|
570
|
+
for word in words:
|
|
571
|
+
if any(
|
|
572
|
+
other.text == word.text
|
|
573
|
+
and abs(other.y0 - word.y0) < word.height / 2
|
|
574
|
+
and min(other.x1, word.x1) - max(other.x0, word.x0)
|
|
575
|
+
> 0.8 * min(other.x1 - other.x0, word.x1 - word.x0)
|
|
576
|
+
for other in kept[-3:]
|
|
577
|
+
):
|
|
578
|
+
continue
|
|
579
|
+
kept.append(word)
|
|
580
|
+
return kept
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _cells(words: list[_Word]) -> list[_Cell]:
|
|
584
|
+
cells: list[_Cell] = []
|
|
585
|
+
for word in words:
|
|
586
|
+
if cells:
|
|
587
|
+
previous = cells[-1].words[-1]
|
|
588
|
+
gap = word.x0 - previous.x1
|
|
589
|
+
if gap <= _CELL_GAP * min(word.height, previous.height):
|
|
590
|
+
cells[-1].words.append(word)
|
|
591
|
+
continue
|
|
592
|
+
cells.append(_Cell([word]))
|
|
593
|
+
return cells
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _tokens(row: _Row) -> list[_Token]:
|
|
597
|
+
"""The row as text and colons, each with the x where it begins and ends.
|
|
598
|
+
|
|
599
|
+
A cell ending in a colon is a label only when the next cell is its value,
|
|
600
|
+
one with no colon of its own: in a heading row over a row of values
|
|
601
|
+
("Überarbeitet am:", "SDB-Nummer:", "Datum der letzten Ausgabe: 15.10.2024")
|
|
602
|
+
the next cell is another heading. Nor is a page number anybody's value:
|
|
603
|
+
"Materialnummer:" in a page header has none, and "Seite 2 von 13" beside it
|
|
604
|
+
is the page's. The label then ends where its last letter does, an average
|
|
605
|
+
letter's width before the colon, so a line above can be matched to it.
|
|
606
|
+
"""
|
|
607
|
+
tokens: list[_Token] = []
|
|
608
|
+
cells = row.cells
|
|
609
|
+
for index, cell in enumerate(cells):
|
|
610
|
+
words = cell.words
|
|
611
|
+
first = words[0]
|
|
612
|
+
if first.text.startswith(_COLON):
|
|
613
|
+
tokens.append(_Token(_COLON, first.x0, first.x0))
|
|
614
|
+
rest = first.text[1:].strip()
|
|
615
|
+
remainder = ([rest] if rest else []) + [w.text for w in words[1:]]
|
|
616
|
+
if remainder:
|
|
617
|
+
start = first.x0 if rest else words[1].x0
|
|
618
|
+
tokens.append(_Token(" ".join(remainder), start, cell.x1))
|
|
619
|
+
continue
|
|
620
|
+
following = cells[index + 1].text if index + 1 < len(cells) else None
|
|
621
|
+
if (
|
|
622
|
+
cell.text.endswith(_COLON)
|
|
623
|
+
and cell.text != _COLON
|
|
624
|
+
and following is not None
|
|
625
|
+
and _COLON not in following
|
|
626
|
+
and not _NAMED_PAGE_NUMBER.fullmatch(following)
|
|
627
|
+
):
|
|
628
|
+
last = words[-1]
|
|
629
|
+
letter = (last.x1 - last.x0) / len(last.text)
|
|
630
|
+
tokens.append(_Token(cell.text[:-1].rstrip(), cell.x0, cell.x1 - letter))
|
|
631
|
+
tokens.append(_Token(_COLON, cell.x1, cell.x1))
|
|
632
|
+
continue
|
|
633
|
+
tokens.append(_Token(cell.text, cell.x0, cell.x1))
|
|
634
|
+
return tokens
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _read_rows(rows: list[_Row], width: float) -> list[_Line]:
|
|
638
|
+
lines: list[_Line] = []
|
|
639
|
+
pairs: _Pairs | None = None
|
|
640
|
+
table: _Table | None = None
|
|
641
|
+
|
|
642
|
+
def close() -> None:
|
|
643
|
+
nonlocal pairs, table
|
|
644
|
+
if pairs is not None:
|
|
645
|
+
lines.extend(pairs.lines())
|
|
646
|
+
pairs = None
|
|
647
|
+
if table is not None:
|
|
648
|
+
lines.extend(table.lines())
|
|
649
|
+
table = None
|
|
650
|
+
|
|
651
|
+
above: _Row | None = None
|
|
652
|
+
|
|
653
|
+
def begin(row: _Row) -> None:
|
|
654
|
+
"""Close what is open, and mark an empty line's worth of space above *row*."""
|
|
655
|
+
close()
|
|
656
|
+
if above is not None and row.y0 - above.y1 > _BLANK_GAP * row.height:
|
|
657
|
+
lines.append(_Line(""))
|
|
658
|
+
|
|
659
|
+
read = [(row, _tokens(row)) for row in rows]
|
|
660
|
+
heads: dict[int, list[str]] = {}
|
|
661
|
+
# Whether the page has shown itself to be a form without colons: a
|
|
662
|
+
# subsection heading of section 1 with its content beside it.
|
|
663
|
+
in_form = False
|
|
664
|
+
for index, (row, tokens) in enumerate(read):
|
|
665
|
+
colons = [i for i, token in enumerate(tokens) if token.text == _COLON]
|
|
666
|
+
# A label whose value is not on its row, right beside a table: set
|
|
667
|
+
# beside the middle of a value of several rows ("Sicherheitshinweise :"
|
|
668
|
+
# between the lines of P101 on SAP's sheets), it belongs to the table.
|
|
669
|
+
bare = colons == [len(tokens) - 1] and table is not None and table.adjoins(row)
|
|
670
|
+
if colons and not bare:
|
|
671
|
+
begin(row)
|
|
672
|
+
pairs = _read_pairs(row, tokens, colons, heads)
|
|
673
|
+
heads = {}
|
|
674
|
+
elif (
|
|
675
|
+
pairs is not None and not pairs.form and not colons and pairs.continues(row)
|
|
676
|
+
):
|
|
677
|
+
# A further line of a label:value pair, whatever it says.
|
|
678
|
+
pass
|
|
679
|
+
elif not colons and (form := _form_pair(row, width, in_form)) is not None:
|
|
680
|
+
# Before a continuation of a form's pair: "Telefon" beside its
|
|
681
|
+
# number right below one is the next pair, not a further line of
|
|
682
|
+
# that one's heading.
|
|
683
|
+
begin(row)
|
|
684
|
+
pairs = form
|
|
685
|
+
in_form = True
|
|
686
|
+
heads = {}
|
|
687
|
+
elif pairs is not None and not colons and pairs.continues(row):
|
|
688
|
+
pass
|
|
689
|
+
elif (
|
|
690
|
+
not colons
|
|
691
|
+
and index + 1 < len(read)
|
|
692
|
+
and (found := _label_heads(row, *read[index + 1])) is not None
|
|
693
|
+
):
|
|
694
|
+
# The first lines of the labels of the row below: set to end on
|
|
695
|
+
# the value's line, a label of two lines starts a row higher.
|
|
696
|
+
heads = found
|
|
697
|
+
elif bare or len(row.cells) > 1 or (table is not None and table.fits(row)):
|
|
698
|
+
if table is None:
|
|
699
|
+
begin(row)
|
|
700
|
+
table = _Table([])
|
|
701
|
+
table.rows.append(row)
|
|
702
|
+
else:
|
|
703
|
+
begin(row)
|
|
704
|
+
lines.append(
|
|
705
|
+
_Line(row.text, row.top, row.cells[0].x0, row.cells[-1].x1, row.height)
|
|
706
|
+
)
|
|
707
|
+
above = row
|
|
708
|
+
close()
|
|
709
|
+
return lines
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _form_pair(row: _Row, width: float, in_form: bool) -> _Pairs | None:
|
|
713
|
+
"""*row* as a pair of a form in two columns without colons (SDB-114).
|
|
714
|
+
|
|
715
|
+
Only where the left cell says so itself: a numbered subsection heading of
|
|
716
|
+
section 1, or, *in_form*, once the page has shown such a heading, a
|
|
717
|
+
contact label. Any other row of several cells without a colon stays a
|
|
718
|
+
table, whose cells cannot be told apart by position.
|
|
719
|
+
|
|
720
|
+
And only where the value begins in the left half of a page *width* wide:
|
|
721
|
+
on a page set in two columns of running text, what stands right of a
|
|
722
|
+
heading is the other column, which begins at the middle.
|
|
723
|
+
"""
|
|
724
|
+
if len(row.cells) < 2 or row.cells[1].x0 >= width / 2:
|
|
725
|
+
return None
|
|
726
|
+
# A colon anywhere in the row: the form prints them, and this row is
|
|
727
|
+
# read by them or not at all ("1.4. Notrufnummer: +49 …" beside a remark).
|
|
728
|
+
if any(_COLON in cell.text for cell in row.cells):
|
|
729
|
+
return None
|
|
730
|
+
left, *right = row.cells
|
|
731
|
+
heading = _SECTION_ONE_HEADING.match(left.text) is not None
|
|
732
|
+
if not heading and not (in_form and _CONTACT_LABEL.fullmatch(left.text)):
|
|
733
|
+
return None
|
|
734
|
+
value_x = right[0].x0
|
|
735
|
+
pair = _Pair(
|
|
736
|
+
label=[left.text],
|
|
737
|
+
value=[
|
|
738
|
+
_Line(
|
|
739
|
+
" ".join(cell.text for cell in right),
|
|
740
|
+
row.top,
|
|
741
|
+
value_x,
|
|
742
|
+
right[-1].x1,
|
|
743
|
+
row.height,
|
|
744
|
+
)
|
|
745
|
+
],
|
|
746
|
+
label_x=left.x0,
|
|
747
|
+
top=row.top,
|
|
748
|
+
height=row.height,
|
|
749
|
+
# A heading runs on below in its column; a contact label is one line,
|
|
750
|
+
# and what stands under it is the next label.
|
|
751
|
+
separator_x=value_x - _COLUMN_SLACK if heading else left.x0,
|
|
752
|
+
value_x=value_x,
|
|
753
|
+
value_end=float("inf"),
|
|
754
|
+
heading=heading,
|
|
755
|
+
)
|
|
756
|
+
return _Pairs(
|
|
757
|
+
leading=None,
|
|
758
|
+
pairs=[pair],
|
|
759
|
+
last_top=row.top,
|
|
760
|
+
height=row.height,
|
|
761
|
+
open=True,
|
|
762
|
+
form=True,
|
|
763
|
+
label_top=row.top,
|
|
764
|
+
)
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def _label_heads(
|
|
768
|
+
above: _Row, row: _Row, tokens: list[_Token]
|
|
769
|
+
) -> dict[int, list[str]] | None:
|
|
770
|
+
"""The cells of *above* that begin the labels of *row*, by the label's colon.
|
|
771
|
+
|
|
772
|
+
Only for a row of pairs right below in type of the same height, every pair
|
|
773
|
+
with a value, and only cells that end where a label ends -- right-aligned
|
|
774
|
+
labels of two lines: "Ausgabedatum/Überarbeitungsd" above "atum: 27.04.2020".
|
|
775
|
+
None if any cell of *above* is not such a head.
|
|
776
|
+
"""
|
|
777
|
+
if row.top - above.top > _CONTINUATION_PITCH * max(row.height, above.height):
|
|
778
|
+
return None
|
|
779
|
+
if abs(row.height - above.height) > _HEAD_SLACK * row.height:
|
|
780
|
+
return None
|
|
781
|
+
colons = [i for i, token in enumerate(tokens) if token.text == _COLON]
|
|
782
|
+
if not colons or colons[0] == 0 or colons[-1] == len(tokens) - 1:
|
|
783
|
+
return None
|
|
784
|
+
heads: dict[int, list[str]] = {}
|
|
785
|
+
for cell in above.cells:
|
|
786
|
+
if cell.text.endswith(_COLON):
|
|
787
|
+
return None
|
|
788
|
+
left = float("-inf")
|
|
789
|
+
for number, colon in enumerate(colons):
|
|
790
|
+
label = tokens[colon - 1]
|
|
791
|
+
# Right-aligned with the label's last letter, or with the colon
|
|
792
|
+
# where the colon is the label's own ("atum:" under a line ending
|
|
793
|
+
# exactly where the colon does).
|
|
794
|
+
ends = [label.x1]
|
|
795
|
+
if tokens[colon].x0 - label.x1 <= _HEAD_SLACK * 4 * row.height:
|
|
796
|
+
ends.append(tokens[colon].x1)
|
|
797
|
+
if (
|
|
798
|
+
label.text != _COLON
|
|
799
|
+
and cell.x0 >= left
|
|
800
|
+
and any(abs(cell.x1 - end) <= _HEAD_SLACK * row.height for end in ends)
|
|
801
|
+
):
|
|
802
|
+
heads.setdefault(colon, []).append(cell.text)
|
|
803
|
+
break
|
|
804
|
+
following = colons[number + 1] - 1 if number + 1 < len(colons) else None
|
|
805
|
+
if following is not None and following > colon + 1:
|
|
806
|
+
left = tokens[following - 1].x1
|
|
807
|
+
else:
|
|
808
|
+
return None
|
|
809
|
+
return heads
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def _read_pairs(
|
|
813
|
+
row: _Row,
|
|
814
|
+
tokens: list[_Token],
|
|
815
|
+
colons: list[int],
|
|
816
|
+
heads: dict[int, list[str]],
|
|
817
|
+
) -> _Pairs:
|
|
818
|
+
"""The row's pairs, open for the rows below.
|
|
819
|
+
|
|
820
|
+
Text before the first label stands on its own; each colon takes the token
|
|
821
|
+
before it as its label and what follows, up to the next label, as its value.
|
|
822
|
+
*heads* are the first lines of labels, from the row above, by their colon.
|
|
823
|
+
"""
|
|
824
|
+
first_label = colons[0] - 1
|
|
825
|
+
leading = (
|
|
826
|
+
_Line(
|
|
827
|
+
" ".join(token.text for token in tokens[:first_label]),
|
|
828
|
+
row.top,
|
|
829
|
+
tokens[0].x0,
|
|
830
|
+
tokens[first_label - 1].x1,
|
|
831
|
+
row.height,
|
|
832
|
+
)
|
|
833
|
+
if first_label > 0
|
|
834
|
+
else None
|
|
835
|
+
)
|
|
836
|
+
pairs: list[_Pair] = []
|
|
837
|
+
has_page = False
|
|
838
|
+
for number, colon in enumerate(colons):
|
|
839
|
+
label_index = colon - 1
|
|
840
|
+
label = tokens[label_index].text if label_index >= 0 else ""
|
|
841
|
+
if label == _COLON:
|
|
842
|
+
label = ""
|
|
843
|
+
# "Telefon:" in one cell and ": 0221" in the next: one colon is enough.
|
|
844
|
+
label = label.rstrip(_COLON).rstrip()
|
|
845
|
+
end = colons[number + 1] - 1 if number + 1 < len(colons) else len(tokens)
|
|
846
|
+
values = [t for t in tokens[colon + 1 : max(end, colon + 1)] if t.text]
|
|
847
|
+
# A page number set in a cell of its own beside a footer pair is not
|
|
848
|
+
# part of the value: on a line of its own, the statement texts read
|
|
849
|
+
# past it as past any page number (SDB-27).
|
|
850
|
+
page = [
|
|
851
|
+
t
|
|
852
|
+
for i, t in enumerate(values)
|
|
853
|
+
if (_PAGE_NUMBER if i else _NAMED_PAGE_NUMBER).fullmatch(t.text)
|
|
854
|
+
]
|
|
855
|
+
values = [t for t in values if t not in page]
|
|
856
|
+
has_page = has_page or bool(page)
|
|
857
|
+
next_label = tokens[end].x0 if end < len(tokens) else float("inf")
|
|
858
|
+
pairs.append(
|
|
859
|
+
_Pair(
|
|
860
|
+
label=[*heads.get(colon, []), *([label] if label else [])],
|
|
861
|
+
value=[
|
|
862
|
+
_Line(
|
|
863
|
+
" ".join(t.text for t in values),
|
|
864
|
+
row.top,
|
|
865
|
+
values[0].x0,
|
|
866
|
+
values[-1].x1,
|
|
867
|
+
row.height,
|
|
868
|
+
)
|
|
869
|
+
]
|
|
870
|
+
if values
|
|
871
|
+
else [],
|
|
872
|
+
top=row.top,
|
|
873
|
+
height=row.height,
|
|
874
|
+
label_x=tokens[label_index].x0
|
|
875
|
+
if label_index >= 0
|
|
876
|
+
else tokens[colon].x0,
|
|
877
|
+
separator_x=tokens[colon].x0,
|
|
878
|
+
value_x=values[0].x0 if values else tokens[colon].x1,
|
|
879
|
+
value_end=next_label - _COLUMN_SLACK,
|
|
880
|
+
trailing=[t.text for t in page],
|
|
881
|
+
)
|
|
882
|
+
)
|
|
883
|
+
return _Pairs(
|
|
884
|
+
leading=leading,
|
|
885
|
+
pairs=pairs,
|
|
886
|
+
last_top=row.top,
|
|
887
|
+
height=row.height,
|
|
888
|
+
open=not has_page,
|
|
889
|
+
)
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def _join_broken(parts: list[str]) -> str:
|
|
893
|
+
"""Join the lines of a label, a word broken at a line end joined back (SDB-54)."""
|
|
894
|
+
return join_broken_words("\n".join(parts)).replace("\n", " ")
|