sdb-reader 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdb_reader/layout.py ADDED
@@ -0,0 +1,894 @@
1
+ """Read a PDF page's text layer in print order, from the positions of its words.
2
+
3
+ PyMuPDF's plain text follows the order in which the file draws its text, and
4
+ for a two-column label:value form that is not the order of the page: Akzo
5
+ Nobel's sheets draw every label, then every colon, then every value, so
6
+ "Gefahrenhinweise" and "H412 - Schädlich für Wasserorganismen …" reach the text
7
+ several lines apart, and P101 under "Allgemein:" lands behind the 2.3 heading.
8
+
9
+ This reader puts the words where the page prints them instead (SDB-42 to
10
+ SDB-45):
11
+
12
+ 1. words that share a baseline form a row, read left to right;
13
+ 2. a wide gap splits a row into cells, the columns of the form or table;
14
+ 3. a cell that is a colon, or begins with one, separates a label from its
15
+ value, and so does a cell ending in one before a value; each label:value
16
+ pair becomes one line, "label: value";
17
+ 4. the rows below a pair continue it while their cells stay in its columns: a
18
+ label printed over several lines ("Ergänzende / Kennzeichnungselemente")
19
+ stays one label, and each further line of the value stays a line of its own;
20
+ 5. a row of one cell -- a heading, a line of a paragraph -- is a line.
21
+
22
+ A form in two columns that prints no colons is read as pairs only where a
23
+ row says so itself (SDB-114): a numbered subsection heading of section 1 with
24
+ its content beside it, and, on a page that has shown one, a contact label
25
+ beside its value.
26
+
27
+ Any other rows of several cells with no label:value pair among them are a
28
+ table, and there the file's own order is kept. A table's cells can hold several lines
29
+ each, a heading over its value ("Überarbeitet am:" over "17.02.2025"), and
30
+ which cells belong together cannot be told from the positions alone; the
31
+ program that wrote the file knew, and wrote them in that order.
32
+
33
+ The output is text, as before, so the parser, the record and the model input
34
+ keep their shape.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import re
40
+ import statistics
41
+ from collections.abc import Sequence
42
+ from dataclasses import dataclass, field
43
+
44
+ import pymupdf
45
+
46
+ from .line_breaks import join_broken_words
47
+
48
+ # Text extraction flags: the same the plain reader uses (page.get_text()), so a
49
+ # word reads the same in both, ligatures and whitespace alike.
50
+ FLAGS = pymupdf.TEXTFLAGS_TEXT
51
+
52
+ # Two words share a row when their vertical centres are this close, as a share
53
+ # of the smaller height. A label and its value sit on one baseline but not
54
+ # always to the point: 274.1 against 274.5 on one supplier's sheets.
55
+ _ROW_TOLERANCE = 0.35
56
+
57
+ # A word more than this many times taller than the row's text is not part of
58
+ # it: a pictogram set in a symbol font ("d", "~" at 33 points) is centred on
59
+ # its caption's line.
60
+ _HEIGHT_RATIO = 2.0
61
+
62
+ # A gap wider than this, as a share of the word height, separates two cells.
63
+ # Measured over 300 sheets: a space between words is 0.2 to 0.3 of the height,
64
+ # a colon set apart from its value 0.6, a column gap 1.0 and more.
65
+ _CELL_GAP = 0.8
66
+
67
+ # Two rows of one label:value pair are closer than this, as a share of the
68
+ # height of their words; the next pair or a heading stands further off. Akzo
69
+ # Nobel sets the lines of a value 11.4 apart and the pairs 15, with words 11.1
70
+ # high. The height, not the font size: it follows the line spacing the font is
71
+ # set with, where the size does not -- one supplier's labels are 8 point type on lines
72
+ # 11.1 apart.
73
+ _CONTINUATION_PITCH = 1.2
74
+
75
+ # Space between two rows that reads as an empty line, as a share of the row
76
+ # height. It becomes one in the text, as an empty paragraph did in the plain
77
+ # reading: a word broken at the foot of a page ("Kenn-") otherwise meets the
78
+ # page number below it, and the line join makes "Kenn1 / 12" of it.
79
+ _BLANK_GAP = 0.8
80
+
81
+ # How far the first line of a right-aligned label may end from its last line,
82
+ # as a share of the row height. Right-aligned lines end within a point of each
83
+ # other; a looser bound took a heading or the end of a table line above a pair
84
+ # for the start of its label ("Handschutz Material: Nitrilkautschuk").
85
+ _HEAD_SLACK = 0.15
86
+
87
+ # How far a continuation may stand left of its column and still belong to it.
88
+ _COLUMN_SLACK = 2.0
89
+
90
+ # Text whose direction is further than this from horizontal is rotated.
91
+ _SKEW = 0.01
92
+
93
+ _COLON = ":"
94
+
95
+ # A page number in a cell of its own: "1/20", "2 / 21", "Seite 3 von 9",
96
+ # "Seite: 1/13", "Seite 3(8)". As a label's only value it has to say "Seite":
97
+ # "Stand: 05/22" is a date.
98
+ _PAGE_NUMBER = re.compile(
99
+ r"(?:Seite\s*:?\s*)?\d{1,3}\s*(?:/|von|\()\s*\d{1,3}\)?", re.IGNORECASE
100
+ )
101
+ _NAMED_PAGE_NUMBER = re.compile(r"Seite\s*:?\s*\d.*", re.IGNORECASE)
102
+
103
+ # A form in two columns without colons (SDB-114): what stands at the left of
104
+ # a row and makes the row a pair with what stands right of it. A numbered
105
+ # subsection heading of section 1, "1.3 Einzelheiten zum Lieferanten, …" (some
106
+ # sheets write "1,3"), or a contact label that is the whole cell.
107
+ _SECTION_ONE_HEADING = re.compile(r"1[.,][1-4]\.?\s+[A-ZÄÖÜ]\w")
108
+ _CONTACT_LABEL = re.compile(
109
+ r"(?:Telefon|Tel\.?|Telefax|Fax|E-?Mail|Internet|Homepage|Web(?:seite|site)?)"
110
+ r"(?:[- ]?(?:Nr\.?|Nummer|Adresse)|nummer|adresse)?",
111
+ re.IGNORECASE,
112
+ )
113
+
114
+ # The lines of a heading in the label column of such a form stand further
115
+ # apart than those of a label:value pair, 10.5 with words 8 high, and still
116
+ # closer than an empty line's worth.
117
+ _FORM_PITCH = 1.5
118
+
119
+ # A line the line join (SDB-54) would join to the next: a hyphen after a word
120
+ # character, a bracket or a slash, as join_broken_words reads it.
121
+ _BROKEN = re.compile(r"[\w)\]/]-$")
122
+
123
+
124
+ @dataclass(slots=True)
125
+ class _Word:
126
+ x0: float
127
+ y0: float
128
+ x1: float
129
+ y1: float
130
+ text: str
131
+ # Where the file draws it: block, line and word, as PyMuPDF numbers them.
132
+ order: tuple[int, int, int]
133
+
134
+ @property
135
+ def height(self) -> float:
136
+ return self.y1 - self.y0
137
+
138
+ @property
139
+ def centre(self) -> float:
140
+ return (self.y0 + self.y1) / 2
141
+
142
+
143
+ @dataclass(slots=True)
144
+ class _Cell:
145
+ words: list[_Word]
146
+
147
+ @property
148
+ def x0(self) -> float:
149
+ return self.words[0].x0
150
+
151
+ @property
152
+ def x1(self) -> float:
153
+ return self.words[-1].x1
154
+
155
+ @property
156
+ def text(self) -> str:
157
+ return " ".join(word.text for word in self.words)
158
+
159
+
160
+ @dataclass(slots=True)
161
+ class _Row:
162
+ words: list[_Word]
163
+ cells: list[_Cell] = field(default_factory=list)
164
+
165
+ @property
166
+ def y0(self) -> float:
167
+ return min(word.y0 for word in self.words)
168
+
169
+ @property
170
+ def top(self) -> float:
171
+ """Where the row's line begins: not moved by a cell set a little higher."""
172
+ return statistics.median(word.y0 for word in self.words)
173
+
174
+ @property
175
+ def y1(self) -> float:
176
+ return max(word.y1 for word in self.words)
177
+
178
+ @property
179
+ def height(self) -> float:
180
+ return statistics.median(word.height for word in self.words)
181
+
182
+ @property
183
+ def text(self) -> str:
184
+ return " ".join(cell.text for cell in self.cells)
185
+
186
+
187
+ @dataclass(slots=True, frozen=True)
188
+ class _Line:
189
+ """A line of the text, and where on the page it begins."""
190
+
191
+ text: str
192
+ # None for a line placed by the file's order alone: rotated text, an
193
+ # empty line, a page number written after its pair.
194
+ top: float | None = None
195
+ # Where the row the line stands in begins, and where the line ends.
196
+ x0: float = 0.0
197
+ x1: float = 0.0
198
+ height: float = 0.0
199
+
200
+ def runs_on_into(self, following: _Line) -> bool:
201
+ """Whether *following* is the next line of this one's text block.
202
+
203
+ Right below it, without an empty line's worth of space between
204
+ (SDB-45), and beginning under its row: in its column, further in where
205
+ a statement hangs its text beside its code ("EUH208- Enthält …
206
+ 2-Methyl-" over "2H-isothiazol-3-on"), or back at the code's column
207
+ where it does not ("P101 Ist ärztlicher Rat … Kenn-" over
208
+ "zeichnungsetikett"). Only there does a word broken at the end of this
209
+ line go on.
210
+ """
211
+ if self.top is None or following.top is None:
212
+ return False
213
+ height = max(self.height, following.height)
214
+ return (
215
+ 0 < following.top - self.top <= (1 + _BLANK_GAP) * height
216
+ and self.x0 - _COLUMN_SLACK <= following.x0 < self.x1
217
+ )
218
+
219
+
220
+ @dataclass(slots=True)
221
+ class _Token:
222
+ """A piece of a row's text, or the colon between a label and its value."""
223
+
224
+ text: str
225
+ x0: float
226
+ x1: float
227
+
228
+
229
+ @dataclass(slots=True)
230
+ class _Pair:
231
+ """A label and its value, as the rows of the form print them."""
232
+
233
+ label: list[str]
234
+ value: list[_Line]
235
+ label_x: float
236
+ top: float
237
+ height: float
238
+ # Where the label column ends (the colon) and the value column begins and
239
+ # ends: the next pair of the row starts its own label column.
240
+ separator_x: float
241
+ value_x: float
242
+ value_end: float
243
+ # Page numbers set beside the pair, written after it.
244
+ trailing: list[str] = field(default_factory=list)
245
+ # A subsection heading with its content beside it (SDB-114): the heading
246
+ # stays a line of its own, as where the content stands below it.
247
+ heading: bool = False
248
+
249
+ def lines(self) -> list[_Line]:
250
+ label = _join_broken(self.label)
251
+ if self.heading:
252
+ return [
253
+ _Line(label, self.top, self.label_x, self.separator_x, self.height),
254
+ *self.value,
255
+ *(_Line(text) for text in self.trailing),
256
+ ]
257
+ head = f"{label}:" if label else _COLON
258
+ trailing = [_Line(text) for text in self.trailing]
259
+ if not self.value:
260
+ return [
261
+ _Line(head, self.top, self.label_x, self.label_x, self.height),
262
+ *trailing,
263
+ ]
264
+ first = self.value[0]
265
+ return [
266
+ _Line(
267
+ f"{head} {first.text}", first.top, self.label_x, first.x1, first.height
268
+ ),
269
+ *self.value[1:],
270
+ *trailing,
271
+ ]
272
+
273
+
274
+ @dataclass(slots=True)
275
+ class _Pairs:
276
+ """The pairs of one row, open for the rows below that continue them."""
277
+
278
+ leading: _Line | None
279
+ pairs: list[_Pair]
280
+ last_top: float
281
+ height: float
282
+ # A footer row with a page number continues on the next page, if at all.
283
+ open: bool
284
+ # A pair of a form without colons (SDB-114). The two columns of such a
285
+ # form are set apart from each other: the last line of a heading and the
286
+ # value beside it can stand two points apart, and the row's middle then
287
+ # lies lower than the heading's next line would.
288
+ form: bool = False
289
+ # Where the last line of a form's heading begins: a cell in its column
290
+ # further down is the next label, not a further line of the heading.
291
+ label_top: float = 0.0
292
+
293
+ def lines(self) -> list[_Line]:
294
+ lines = [self.leading] if self.leading else []
295
+ for pair in self.pairs:
296
+ lines.extend(pair.lines())
297
+ return lines
298
+
299
+ def continues(self, row: _Row) -> bool:
300
+ """Take *row* as further lines of the pairs' labels and values, if it is."""
301
+ if not self.open:
302
+ return False
303
+ # Below the pairs, in type of their size: a pictogram glyph split off
304
+ # their row (_without_outsized) stands beside them, not under them.
305
+ if row.top <= self.last_top or row.height > _HEIGHT_RATIO * self.height:
306
+ return False
307
+ top = row.y0 if self.form else row.top
308
+ pitch = _FORM_PITCH if self.form else _CONTINUATION_PITCH
309
+ if top - self.last_top > pitch * max(self.height, row.height):
310
+ return False
311
+ placed: list[tuple[_Pair, bool, _Cell]] = []
312
+ for cell in row.cells:
313
+ for pair in self.pairs:
314
+ if pair.value_x - _COLUMN_SLACK <= cell.x0 < pair.value_end:
315
+ placed.append((pair, False, cell))
316
+ break
317
+ if (
318
+ cell.x0 >= pair.label_x - _COLUMN_SLACK
319
+ and cell.x1 <= pair.separator_x
320
+ ):
321
+ placed.append((pair, True, cell))
322
+ break
323
+ else:
324
+ return False
325
+ if self.form and any(is_label for _, is_label, _ in placed):
326
+ if top - self.label_top > pitch * max(self.height, row.height):
327
+ return False
328
+ self.label_top = top
329
+ for pair in self.pairs:
330
+ labels = [c.text for p, is_label, c in placed if p is pair and is_label]
331
+ values = [c for p, is_label, c in placed if p is pair and not is_label]
332
+ if labels:
333
+ pair.label.append(" ".join(labels))
334
+ if values:
335
+ text = " ".join(c.text for c in values)
336
+ pair.value.append(
337
+ _Line(text, row.top, row.cells[0].x0, values[-1].x1, row.height)
338
+ )
339
+ self.last_top = row.top
340
+ return True
341
+
342
+
343
+ @dataclass(slots=True)
344
+ class _Table:
345
+ """Rows of cells without a label:value pair: read in the file's order."""
346
+
347
+ rows: list[_Row]
348
+
349
+ def adjoins(self, row: _Row) -> bool:
350
+ """Whether *row* stands right below the table, without an empty line."""
351
+ return row.y0 - self.rows[-1].y1 <= _BLANK_GAP * row.height
352
+
353
+ def fits(self, row: _Row) -> bool:
354
+ """Whether a one-cell *row* is a further line of one of the table's columns.
355
+
356
+ Only right below it: after an empty line's worth of space, a cell in
357
+ one of its columns is the page number of the footer as often as it is
358
+ a further line of the table.
359
+ """
360
+ if not self.adjoins(row):
361
+ return False
362
+ cell = row.cells[0]
363
+ for column in self._columns():
364
+ if (
365
+ cell.x0 >= column[0] - _COLUMN_SLACK
366
+ and cell.x1 <= column[1] + _COLUMN_SLACK
367
+ ):
368
+ return True
369
+ return False
370
+
371
+ def _columns(self) -> list[tuple[float, float]]:
372
+ spans = sorted((c.x0, c.x1) for r in self.rows for c in r.cells)
373
+ columns: list[tuple[float, float]] = []
374
+ for x0, x1 in spans:
375
+ if columns and x0 <= columns[-1][1]:
376
+ columns[-1] = (columns[-1][0], max(columns[-1][1], x1))
377
+ else:
378
+ columns.append((x0, x1))
379
+ return columns
380
+
381
+ def lines(self) -> list[_Line]:
382
+ """The table's lines in the file's order, each where the page prints it."""
383
+ by_line: dict[tuple[int, int], list[_Word]] = {}
384
+ row_start: dict[tuple[int, int], float] = {}
385
+ for row in self.rows:
386
+ for word in row.words:
387
+ by_line.setdefault(word.order[:2], []).append(word)
388
+ row_start.setdefault(word.order[:2], row.cells[0].x0)
389
+ lines: list[_Line] = []
390
+ for key, words in sorted(by_line.items()):
391
+ words.sort(key=lambda w: w.order)
392
+ lines.append(
393
+ _Line(
394
+ " ".join(w.text for w in words),
395
+ statistics.median(w.y0 for w in words),
396
+ row_start[key],
397
+ words[-1].x1,
398
+ statistics.median(w.height for w in words),
399
+ )
400
+ )
401
+ return lines
402
+
403
+
404
+ def page_text(page: pymupdf.Page, textpage: pymupdf.TextPage | None = None) -> str:
405
+ """The text of *page* in the order it is printed.
406
+
407
+ *textpage* is the page's text, extracted with FLAGS; a caller that also
408
+ wants the plain text passes it to read the page once.
409
+ """
410
+ textpage = textpage or page.get_textpage(flags=FLAGS)
411
+ directions = _Directions(page.get_text("dict", textpage=textpage)["blocks"])
412
+ horizontal: list[_Word] = []
413
+ rotated: dict[tuple[int, int], list[str]] = {}
414
+ for x0, y0, x1, y1, text, block, line, number in page.get_text(
415
+ "words", textpage=textpage
416
+ ):
417
+ dx, dy = directions.of(block, line, (x0 + x1) / 2, (y0 + y1) / 2)
418
+ if dx < 1.0 - _SKEW or abs(dy) > _SKEW:
419
+ # Rotated text, a margin note set vertically: kept, line by line in
420
+ # the file's order, after the page.
421
+ rotated.setdefault((block, line), []).append(text)
422
+ continue
423
+ horizontal.append(_Word(x0, y0, x1, y1, text, (block, line, number)))
424
+
425
+ lines = _read_rows(_rows(horizontal), page.rect.width)
426
+ lines.extend(_Line(" ".join(words)) for _, words in sorted(rotated.items()))
427
+ return "\n".join(_with_breaks_kept(lines))
428
+
429
+
430
+ def _with_breaks_kept(lines: list[_Line]) -> list[str]:
431
+ """The lines' text, an empty line after a hyphen that does not run on.
432
+
433
+ The line join (SDB-54) joins a line ending in a hyphen to whatever line
434
+ follows. That is the next line of the word only where it stands right
435
+ below in the same column; elsewhere -- the next cell of a table in the
436
+ file's order, a pair below a table, the page's end -- an empty line keeps
437
+ them apart: "gewässergefähr-" / "H413: Kann für …" made "gewässergefährH413".
438
+ """
439
+ text: list[str] = []
440
+ for index, line in enumerate(lines):
441
+ text.append(line.text)
442
+ following = lines[index + 1] if index + 1 < len(lines) else None
443
+ if _BROKEN.search(line.text) and (
444
+ following is None or (following.text and not line.runs_on_into(following))
445
+ ):
446
+ text.append("")
447
+ return text
448
+
449
+
450
+ class _Directions:
451
+ """The writing direction of each line, found by where its words stand.
452
+
453
+ PyMuPDF numbers a word's line among every line of its block, but the
454
+ "dict" output leaves out a line whose box is empty -- text set at zero
455
+ height. Counting would then give the next line's direction, or none, so a
456
+ word is matched to the line whose box holds it. A word in no line's box is
457
+ on an empty line and reads as horizontal.
458
+ """
459
+
460
+ def __init__(self, blocks: Sequence[dict[str, object]]) -> None:
461
+ self._lines: dict[int, list[tuple[tuple[float, ...], tuple[float, float]]]] = {}
462
+ for block in blocks:
463
+ number = block.get("number")
464
+ lines = block.get("lines")
465
+ if not isinstance(number, int) or not isinstance(lines, list):
466
+ continue
467
+ self._lines[number] = [
468
+ (tuple(line["bbox"]), (line["dir"][0], line["dir"][1]))
469
+ for line in lines
470
+ ]
471
+
472
+ def of(self, block: int, line: int, x: float, y: float) -> tuple[float, float]:
473
+ lines = self._lines.get(block, [])
474
+ candidates = ([lines[line]] if line < len(lines) else []) + lines
475
+ for (x0, y0, x1, y1), direction in candidates:
476
+ if x0 - 1 <= x <= x1 + 1 and y0 - 1 <= y <= y1 + 1:
477
+ return direction
478
+ return (1.0, 0.0)
479
+
480
+
481
+ def _rows(words: list[_Word]) -> list[_Row]:
482
+ """Group *words* into rows by their vertical centre, each read left to right."""
483
+ clusters: list[list[_Word]] = []
484
+ current: list[_Word] = []
485
+ centre = 0.0
486
+ for word in sorted(words, key=lambda w: (w.centre, w.x0)):
487
+ if current and abs(word.centre - centre) <= _ROW_TOLERANCE * min(
488
+ word.height, statistics.median(w.height for w in current)
489
+ ):
490
+ current.append(word)
491
+ centre = sum(w.centre for w in current) / len(current)
492
+ continue
493
+ if current:
494
+ clusters.append(current)
495
+ current = [word]
496
+ centre = word.centre
497
+ if current:
498
+ clusters.append(current)
499
+
500
+ rows: list[_Row] = []
501
+ for cluster in clusters:
502
+ text, outsized = _without_outsized(cluster)
503
+ for part in (text, outsized):
504
+ if part:
505
+ words_of_row = _joined_touching(
506
+ _without_doubles(sorted(part, key=lambda w: w.x0))
507
+ )
508
+ rows.append(_Row(words_of_row, _cells(words_of_row)))
509
+ return rows
510
+
511
+
512
+ def _without_outsized(words: list[_Word]) -> tuple[list[_Word], list[_Word]]:
513
+ """Split off the words more than _HEIGHT_RATIO times taller than the row's text.
514
+
515
+ Taller only: a superscript stays in its row. The text's height is the
516
+ median of the lower half, so two pictogram glyphs beside a two-word caption
517
+ do not make the glyphs the norm.
518
+ """
519
+ heights = sorted(word.height for word in words)
520
+ norm = statistics.median(heights[: (len(heights) + 1) // 2])
521
+ text = [w for w in words if w.height <= _HEIGHT_RATIO * norm]
522
+ return text, [w for w in words if w.height > _HEIGHT_RATIO * norm]
523
+
524
+
525
+ # Two pieces of a row end and begin at the same point when they are no
526
+ # further apart than this, as a share of their height. A space between words
527
+ # is 0.2 to 0.3 of the height (see _CELL_GAP); a narrow one after a colon
528
+ # still 0.03.
529
+ _TOUCHING = 0.01
530
+
531
+
532
+ def _joined_touching(words: list[_Word]) -> list[_Word]:
533
+ """Join the pieces of a word the file draws apart (SDB-99).
534
+
535
+ A file that sets single characters in another font draws a word in pieces:
536
+ "24", ".0", "9", ".2025" for "24.09.2025", each piece a word of its own to
537
+ PyMuPDF and not in the order the page shows. They stand where the page
538
+ prints them, without a space between. The joined word takes the place in
539
+ the file's order of its last piece.
540
+ """
541
+ joined: list[_Word] = []
542
+ for word in words:
543
+ if joined:
544
+ previous = joined[-1]
545
+ gap = word.x0 - previous.x1
546
+ height = min(word.height, previous.height)
547
+ # Drawn apart: in another line of the file. Words of one line
548
+ # that touch are PyMuPDF's own division, at a space it found.
549
+ if abs(gap) <= _TOUCHING * height and word.order[:2] != previous.order[:2]:
550
+ joined[-1] = _Word(
551
+ previous.x0,
552
+ min(previous.y0, word.y0),
553
+ word.x1,
554
+ max(previous.y1, word.y1),
555
+ previous.text + word.text,
556
+ max(previous.order, word.order),
557
+ )
558
+ continue
559
+ joined.append(word)
560
+ return joined
561
+
562
+
563
+ def _without_doubles(words: list[_Word]) -> list[_Word]:
564
+ """Drop a word printed twice on itself, the way some files fake bold type.
565
+
566
+ On itself: the two boxes overlap by most of their width. Two dashes side by
567
+ side ("- - -") are separate words, however narrow.
568
+ """
569
+ kept: list[_Word] = []
570
+ for word in words:
571
+ if any(
572
+ other.text == word.text
573
+ and abs(other.y0 - word.y0) < word.height / 2
574
+ and min(other.x1, word.x1) - max(other.x0, word.x0)
575
+ > 0.8 * min(other.x1 - other.x0, word.x1 - word.x0)
576
+ for other in kept[-3:]
577
+ ):
578
+ continue
579
+ kept.append(word)
580
+ return kept
581
+
582
+
583
+ def _cells(words: list[_Word]) -> list[_Cell]:
584
+ cells: list[_Cell] = []
585
+ for word in words:
586
+ if cells:
587
+ previous = cells[-1].words[-1]
588
+ gap = word.x0 - previous.x1
589
+ if gap <= _CELL_GAP * min(word.height, previous.height):
590
+ cells[-1].words.append(word)
591
+ continue
592
+ cells.append(_Cell([word]))
593
+ return cells
594
+
595
+
596
+ def _tokens(row: _Row) -> list[_Token]:
597
+ """The row as text and colons, each with the x where it begins and ends.
598
+
599
+ A cell ending in a colon is a label only when the next cell is its value,
600
+ one with no colon of its own: in a heading row over a row of values
601
+ ("Überarbeitet am:", "SDB-Nummer:", "Datum der letzten Ausgabe: 15.10.2024")
602
+ the next cell is another heading. Nor is a page number anybody's value:
603
+ "Materialnummer:" in a page header has none, and "Seite 2 von 13" beside it
604
+ is the page's. The label then ends where its last letter does, an average
605
+ letter's width before the colon, so a line above can be matched to it.
606
+ """
607
+ tokens: list[_Token] = []
608
+ cells = row.cells
609
+ for index, cell in enumerate(cells):
610
+ words = cell.words
611
+ first = words[0]
612
+ if first.text.startswith(_COLON):
613
+ tokens.append(_Token(_COLON, first.x0, first.x0))
614
+ rest = first.text[1:].strip()
615
+ remainder = ([rest] if rest else []) + [w.text for w in words[1:]]
616
+ if remainder:
617
+ start = first.x0 if rest else words[1].x0
618
+ tokens.append(_Token(" ".join(remainder), start, cell.x1))
619
+ continue
620
+ following = cells[index + 1].text if index + 1 < len(cells) else None
621
+ if (
622
+ cell.text.endswith(_COLON)
623
+ and cell.text != _COLON
624
+ and following is not None
625
+ and _COLON not in following
626
+ and not _NAMED_PAGE_NUMBER.fullmatch(following)
627
+ ):
628
+ last = words[-1]
629
+ letter = (last.x1 - last.x0) / len(last.text)
630
+ tokens.append(_Token(cell.text[:-1].rstrip(), cell.x0, cell.x1 - letter))
631
+ tokens.append(_Token(_COLON, cell.x1, cell.x1))
632
+ continue
633
+ tokens.append(_Token(cell.text, cell.x0, cell.x1))
634
+ return tokens
635
+
636
+
637
+ def _read_rows(rows: list[_Row], width: float) -> list[_Line]:
638
+ lines: list[_Line] = []
639
+ pairs: _Pairs | None = None
640
+ table: _Table | None = None
641
+
642
+ def close() -> None:
643
+ nonlocal pairs, table
644
+ if pairs is not None:
645
+ lines.extend(pairs.lines())
646
+ pairs = None
647
+ if table is not None:
648
+ lines.extend(table.lines())
649
+ table = None
650
+
651
+ above: _Row | None = None
652
+
653
+ def begin(row: _Row) -> None:
654
+ """Close what is open, and mark an empty line's worth of space above *row*."""
655
+ close()
656
+ if above is not None and row.y0 - above.y1 > _BLANK_GAP * row.height:
657
+ lines.append(_Line(""))
658
+
659
+ read = [(row, _tokens(row)) for row in rows]
660
+ heads: dict[int, list[str]] = {}
661
+ # Whether the page has shown itself to be a form without colons: a
662
+ # subsection heading of section 1 with its content beside it.
663
+ in_form = False
664
+ for index, (row, tokens) in enumerate(read):
665
+ colons = [i for i, token in enumerate(tokens) if token.text == _COLON]
666
+ # A label whose value is not on its row, right beside a table: set
667
+ # beside the middle of a value of several rows ("Sicherheitshinweise :"
668
+ # between the lines of P101 on SAP's sheets), it belongs to the table.
669
+ bare = colons == [len(tokens) - 1] and table is not None and table.adjoins(row)
670
+ if colons and not bare:
671
+ begin(row)
672
+ pairs = _read_pairs(row, tokens, colons, heads)
673
+ heads = {}
674
+ elif (
675
+ pairs is not None and not pairs.form and not colons and pairs.continues(row)
676
+ ):
677
+ # A further line of a label:value pair, whatever it says.
678
+ pass
679
+ elif not colons and (form := _form_pair(row, width, in_form)) is not None:
680
+ # Before a continuation of a form's pair: "Telefon" beside its
681
+ # number right below one is the next pair, not a further line of
682
+ # that one's heading.
683
+ begin(row)
684
+ pairs = form
685
+ in_form = True
686
+ heads = {}
687
+ elif pairs is not None and not colons and pairs.continues(row):
688
+ pass
689
+ elif (
690
+ not colons
691
+ and index + 1 < len(read)
692
+ and (found := _label_heads(row, *read[index + 1])) is not None
693
+ ):
694
+ # The first lines of the labels of the row below: set to end on
695
+ # the value's line, a label of two lines starts a row higher.
696
+ heads = found
697
+ elif bare or len(row.cells) > 1 or (table is not None and table.fits(row)):
698
+ if table is None:
699
+ begin(row)
700
+ table = _Table([])
701
+ table.rows.append(row)
702
+ else:
703
+ begin(row)
704
+ lines.append(
705
+ _Line(row.text, row.top, row.cells[0].x0, row.cells[-1].x1, row.height)
706
+ )
707
+ above = row
708
+ close()
709
+ return lines
710
+
711
+
712
+ def _form_pair(row: _Row, width: float, in_form: bool) -> _Pairs | None:
713
+ """*row* as a pair of a form in two columns without colons (SDB-114).
714
+
715
+ Only where the left cell says so itself: a numbered subsection heading of
716
+ section 1, or, *in_form*, once the page has shown such a heading, a
717
+ contact label. Any other row of several cells without a colon stays a
718
+ table, whose cells cannot be told apart by position.
719
+
720
+ And only where the value begins in the left half of a page *width* wide:
721
+ on a page set in two columns of running text, what stands right of a
722
+ heading is the other column, which begins at the middle.
723
+ """
724
+ if len(row.cells) < 2 or row.cells[1].x0 >= width / 2:
725
+ return None
726
+ # A colon anywhere in the row: the form prints them, and this row is
727
+ # read by them or not at all ("1.4. Notrufnummer: +49 …" beside a remark).
728
+ if any(_COLON in cell.text for cell in row.cells):
729
+ return None
730
+ left, *right = row.cells
731
+ heading = _SECTION_ONE_HEADING.match(left.text) is not None
732
+ if not heading and not (in_form and _CONTACT_LABEL.fullmatch(left.text)):
733
+ return None
734
+ value_x = right[0].x0
735
+ pair = _Pair(
736
+ label=[left.text],
737
+ value=[
738
+ _Line(
739
+ " ".join(cell.text for cell in right),
740
+ row.top,
741
+ value_x,
742
+ right[-1].x1,
743
+ row.height,
744
+ )
745
+ ],
746
+ label_x=left.x0,
747
+ top=row.top,
748
+ height=row.height,
749
+ # A heading runs on below in its column; a contact label is one line,
750
+ # and what stands under it is the next label.
751
+ separator_x=value_x - _COLUMN_SLACK if heading else left.x0,
752
+ value_x=value_x,
753
+ value_end=float("inf"),
754
+ heading=heading,
755
+ )
756
+ return _Pairs(
757
+ leading=None,
758
+ pairs=[pair],
759
+ last_top=row.top,
760
+ height=row.height,
761
+ open=True,
762
+ form=True,
763
+ label_top=row.top,
764
+ )
765
+
766
+
767
+ def _label_heads(
768
+ above: _Row, row: _Row, tokens: list[_Token]
769
+ ) -> dict[int, list[str]] | None:
770
+ """The cells of *above* that begin the labels of *row*, by the label's colon.
771
+
772
+ Only for a row of pairs right below in type of the same height, every pair
773
+ with a value, and only cells that end where a label ends -- right-aligned
774
+ labels of two lines: "Ausgabedatum/Überarbeitungsd" above "atum: 27.04.2020".
775
+ None if any cell of *above* is not such a head.
776
+ """
777
+ if row.top - above.top > _CONTINUATION_PITCH * max(row.height, above.height):
778
+ return None
779
+ if abs(row.height - above.height) > _HEAD_SLACK * row.height:
780
+ return None
781
+ colons = [i for i, token in enumerate(tokens) if token.text == _COLON]
782
+ if not colons or colons[0] == 0 or colons[-1] == len(tokens) - 1:
783
+ return None
784
+ heads: dict[int, list[str]] = {}
785
+ for cell in above.cells:
786
+ if cell.text.endswith(_COLON):
787
+ return None
788
+ left = float("-inf")
789
+ for number, colon in enumerate(colons):
790
+ label = tokens[colon - 1]
791
+ # Right-aligned with the label's last letter, or with the colon
792
+ # where the colon is the label's own ("atum:" under a line ending
793
+ # exactly where the colon does).
794
+ ends = [label.x1]
795
+ if tokens[colon].x0 - label.x1 <= _HEAD_SLACK * 4 * row.height:
796
+ ends.append(tokens[colon].x1)
797
+ if (
798
+ label.text != _COLON
799
+ and cell.x0 >= left
800
+ and any(abs(cell.x1 - end) <= _HEAD_SLACK * row.height for end in ends)
801
+ ):
802
+ heads.setdefault(colon, []).append(cell.text)
803
+ break
804
+ following = colons[number + 1] - 1 if number + 1 < len(colons) else None
805
+ if following is not None and following > colon + 1:
806
+ left = tokens[following - 1].x1
807
+ else:
808
+ return None
809
+ return heads
810
+
811
+
812
+ def _read_pairs(
813
+ row: _Row,
814
+ tokens: list[_Token],
815
+ colons: list[int],
816
+ heads: dict[int, list[str]],
817
+ ) -> _Pairs:
818
+ """The row's pairs, open for the rows below.
819
+
820
+ Text before the first label stands on its own; each colon takes the token
821
+ before it as its label and what follows, up to the next label, as its value.
822
+ *heads* are the first lines of labels, from the row above, by their colon.
823
+ """
824
+ first_label = colons[0] - 1
825
+ leading = (
826
+ _Line(
827
+ " ".join(token.text for token in tokens[:first_label]),
828
+ row.top,
829
+ tokens[0].x0,
830
+ tokens[first_label - 1].x1,
831
+ row.height,
832
+ )
833
+ if first_label > 0
834
+ else None
835
+ )
836
+ pairs: list[_Pair] = []
837
+ has_page = False
838
+ for number, colon in enumerate(colons):
839
+ label_index = colon - 1
840
+ label = tokens[label_index].text if label_index >= 0 else ""
841
+ if label == _COLON:
842
+ label = ""
843
+ # "Telefon:" in one cell and ": 0221" in the next: one colon is enough.
844
+ label = label.rstrip(_COLON).rstrip()
845
+ end = colons[number + 1] - 1 if number + 1 < len(colons) else len(tokens)
846
+ values = [t for t in tokens[colon + 1 : max(end, colon + 1)] if t.text]
847
+ # A page number set in a cell of its own beside a footer pair is not
848
+ # part of the value: on a line of its own, the statement texts read
849
+ # past it as past any page number (SDB-27).
850
+ page = [
851
+ t
852
+ for i, t in enumerate(values)
853
+ if (_PAGE_NUMBER if i else _NAMED_PAGE_NUMBER).fullmatch(t.text)
854
+ ]
855
+ values = [t for t in values if t not in page]
856
+ has_page = has_page or bool(page)
857
+ next_label = tokens[end].x0 if end < len(tokens) else float("inf")
858
+ pairs.append(
859
+ _Pair(
860
+ label=[*heads.get(colon, []), *([label] if label else [])],
861
+ value=[
862
+ _Line(
863
+ " ".join(t.text for t in values),
864
+ row.top,
865
+ values[0].x0,
866
+ values[-1].x1,
867
+ row.height,
868
+ )
869
+ ]
870
+ if values
871
+ else [],
872
+ top=row.top,
873
+ height=row.height,
874
+ label_x=tokens[label_index].x0
875
+ if label_index >= 0
876
+ else tokens[colon].x0,
877
+ separator_x=tokens[colon].x0,
878
+ value_x=values[0].x0 if values else tokens[colon].x1,
879
+ value_end=next_label - _COLUMN_SLACK,
880
+ trailing=[t.text for t in page],
881
+ )
882
+ )
883
+ return _Pairs(
884
+ leading=leading,
885
+ pairs=pairs,
886
+ last_top=row.top,
887
+ height=row.height,
888
+ open=not has_page,
889
+ )
890
+
891
+
892
+ def _join_broken(parts: list[str]) -> str:
893
+ """Join the lines of a label, a word broken at a line end joined back (SDB-54)."""
894
+ return join_broken_words("\n".join(parts)).replace("\n", " ")