PyMuPDF 1.24.11__cp38-abi3-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. PyMuPDF-1.24.11.dist-info/COPYING +1 -0
  2. PyMuPDF-1.24.11.dist-info/METADATA +83 -0
  3. PyMuPDF-1.24.11.dist-info/README.md +60 -0
  4. PyMuPDF-1.24.11.dist-info/RECORD +99 -0
  5. PyMuPDF-1.24.11.dist-info/WHEEL +4 -0
  6. PyMuPDF-1.24.11.dist-info/entry_points.txt +3 -0
  7. fitz/__init__.py +11 -0
  8. fitz/table.py +2 -0
  9. fitz/utils.py +2 -0
  10. pymupdf/__init__.py +22122 -0
  11. pymupdf/__main__.py +1146 -0
  12. pymupdf/_apply_pages.py +253 -0
  13. pymupdf/_build.py +1 -0
  14. pymupdf/_extra.pyd +0 -0
  15. pymupdf/_mupdf.pyd +0 -0
  16. pymupdf/extra.py +214 -0
  17. pymupdf/mupdf-devel/include/mupdf/fitz/archive.h +444 -0
  18. pymupdf/mupdf-devel/include/mupdf/fitz/band-writer.h +117 -0
  19. pymupdf/mupdf-devel/include/mupdf/fitz/bidi.h +90 -0
  20. pymupdf/mupdf-devel/include/mupdf/fitz/bitmap.h +175 -0
  21. pymupdf/mupdf-devel/include/mupdf/fitz/buffer.h +250 -0
  22. pymupdf/mupdf-devel/include/mupdf/fitz/color.h +427 -0
  23. pymupdf/mupdf-devel/include/mupdf/fitz/compress.h +88 -0
  24. pymupdf/mupdf-devel/include/mupdf/fitz/compressed-buffer.h +185 -0
  25. pymupdf/mupdf-devel/include/mupdf/fitz/config.h +223 -0
  26. pymupdf/mupdf-devel/include/mupdf/fitz/context.h +1006 -0
  27. pymupdf/mupdf-devel/include/mupdf/fitz/crypt.h +270 -0
  28. pymupdf/mupdf-devel/include/mupdf/fitz/device.h +654 -0
  29. pymupdf/mupdf-devel/include/mupdf/fitz/display-list.h +142 -0
  30. pymupdf/mupdf-devel/include/mupdf/fitz/document.h +1108 -0
  31. pymupdf/mupdf-devel/include/mupdf/fitz/export.h +52 -0
  32. pymupdf/mupdf-devel/include/mupdf/fitz/filter.h +263 -0
  33. pymupdf/mupdf-devel/include/mupdf/fitz/font.h +815 -0
  34. pymupdf/mupdf-devel/include/mupdf/fitz/geometry.h +818 -0
  35. pymupdf/mupdf-devel/include/mupdf/fitz/getopt.h +128 -0
  36. pymupdf/mupdf-devel/include/mupdf/fitz/glyph-cache.h +96 -0
  37. pymupdf/mupdf-devel/include/mupdf/fitz/glyph.h +81 -0
  38. pymupdf/mupdf-devel/include/mupdf/fitz/hash.h +126 -0
  39. pymupdf/mupdf-devel/include/mupdf/fitz/heap-imp.h +163 -0
  40. pymupdf/mupdf-devel/include/mupdf/fitz/heap.h +140 -0
  41. pymupdf/mupdf-devel/include/mupdf/fitz/image.h +443 -0
  42. pymupdf/mupdf-devel/include/mupdf/fitz/link.h +130 -0
  43. pymupdf/mupdf-devel/include/mupdf/fitz/log.h +61 -0
  44. pymupdf/mupdf-devel/include/mupdf/fitz/outline.h +228 -0
  45. pymupdf/mupdf-devel/include/mupdf/fitz/output-svg.h +64 -0
  46. pymupdf/mupdf-devel/include/mupdf/fitz/output.h +415 -0
  47. pymupdf/mupdf-devel/include/mupdf/fitz/path.h +447 -0
  48. pymupdf/mupdf-devel/include/mupdf/fitz/pixmap.h +501 -0
  49. pymupdf/mupdf-devel/include/mupdf/fitz/pool.h +68 -0
  50. pymupdf/mupdf-devel/include/mupdf/fitz/separation.h +138 -0
  51. pymupdf/mupdf-devel/include/mupdf/fitz/shade.h +233 -0
  52. pymupdf/mupdf-devel/include/mupdf/fitz/store.h +456 -0
  53. pymupdf/mupdf-devel/include/mupdf/fitz/story-writer.h +209 -0
  54. pymupdf/mupdf-devel/include/mupdf/fitz/story.h +232 -0
  55. pymupdf/mupdf-devel/include/mupdf/fitz/stream.h +646 -0
  56. pymupdf/mupdf-devel/include/mupdf/fitz/string-util.h +286 -0
  57. pymupdf/mupdf-devel/include/mupdf/fitz/structured-text.h +365 -0
  58. pymupdf/mupdf-devel/include/mupdf/fitz/system.h +459 -0
  59. pymupdf/mupdf-devel/include/mupdf/fitz/text.h +209 -0
  60. pymupdf/mupdf-devel/include/mupdf/fitz/track-usage.h +57 -0
  61. pymupdf/mupdf-devel/include/mupdf/fitz/transition.h +76 -0
  62. pymupdf/mupdf-devel/include/mupdf/fitz/tree.h +62 -0
  63. pymupdf/mupdf-devel/include/mupdf/fitz/types.h +41 -0
  64. pymupdf/mupdf-devel/include/mupdf/fitz/util.h +151 -0
  65. pymupdf/mupdf-devel/include/mupdf/fitz/version.h +31 -0
  66. pymupdf/mupdf-devel/include/mupdf/fitz/write-pixmap.h +499 -0
  67. pymupdf/mupdf-devel/include/mupdf/fitz/writer.h +266 -0
  68. pymupdf/mupdf-devel/include/mupdf/fitz/xml.h +397 -0
  69. pymupdf/mupdf-devel/include/mupdf/fitz.h +97 -0
  70. pymupdf/mupdf-devel/include/mupdf/helpers/mu-office-lib.h +666 -0
  71. pymupdf/mupdf-devel/include/mupdf/helpers/mu-threads.h +280 -0
  72. pymupdf/mupdf-devel/include/mupdf/helpers/pkcs7-openssl.h +48 -0
  73. pymupdf/mupdf-devel/include/mupdf/html.h +53 -0
  74. pymupdf/mupdf-devel/include/mupdf/memento.h +423 -0
  75. pymupdf/mupdf-devel/include/mupdf/pdf/annot.h +965 -0
  76. pymupdf/mupdf-devel/include/mupdf/pdf/clean.h +48 -0
  77. pymupdf/mupdf-devel/include/mupdf/pdf/cmap.h +145 -0
  78. pymupdf/mupdf-devel/include/mupdf/pdf/crypt.h +107 -0
  79. pymupdf/mupdf-devel/include/mupdf/pdf/document.h +842 -0
  80. pymupdf/mupdf-devel/include/mupdf/pdf/event.h +167 -0
  81. pymupdf/mupdf-devel/include/mupdf/pdf/font.h +168 -0
  82. pymupdf/mupdf-devel/include/mupdf/pdf/form.h +380 -0
  83. pymupdf/mupdf-devel/include/mupdf/pdf/image-rewriter.h +78 -0
  84. pymupdf/mupdf-devel/include/mupdf/pdf/interpret.h +492 -0
  85. pymupdf/mupdf-devel/include/mupdf/pdf/javascript.h +43 -0
  86. pymupdf/mupdf-devel/include/mupdf/pdf/name-table.h +596 -0
  87. pymupdf/mupdf-devel/include/mupdf/pdf/object.h +435 -0
  88. pymupdf/mupdf-devel/include/mupdf/pdf/page.h +316 -0
  89. pymupdf/mupdf-devel/include/mupdf/pdf/parse.h +61 -0
  90. pymupdf/mupdf-devel/include/mupdf/pdf/resource.h +133 -0
  91. pymupdf/mupdf-devel/include/mupdf/pdf/xref.h +300 -0
  92. pymupdf/mupdf-devel/include/mupdf/pdf.h +57 -0
  93. pymupdf/mupdf-devel/include/mupdf/ucdn.h +452 -0
  94. pymupdf/mupdf-devel/lib/mupdfcpp64.lib +0 -0
  95. pymupdf/mupdf.py +58506 -0
  96. pymupdf/mupdfcpp64.dll +0 -0
  97. pymupdf/pymupdf.py +2 -0
  98. pymupdf/table.py +2355 -0
  99. pymupdf/utils.py +5880 -0
pymupdf/table.py ADDED
@@ -0,0 +1,2355 @@
1
+ """
2
+ Copyright (C) 2023 Artifex Software, Inc.
3
+
4
+ This file is part of PyMuPDF.
5
+
6
+ PyMuPDF is free software: you can redistribute it and/or modify it under the
7
+ terms of the GNU Affero General Public License as published by the Free
8
+ Software Foundation, either version 3 of the License, or (at your option)
9
+ any later version.
10
+
11
+ PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY
12
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13
+ FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
14
+ details.
15
+
16
+ You should have received a copy of the GNU Affero General Public License
17
+ along with MuPDF. If not, see <https://www.gnu.org/licenses/agpl-3.0.en.html>
18
+
19
+ Alternative licensing terms are available from the licensor.
20
+ For commercial licensing, see <https://www.artifex.com/> or contact
21
+ Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco,
22
+ CA 94129, USA, for further information.
23
+
24
+ ---------------------------------------------------------------------
25
+ Portions of this code have been ported from pdfplumber, see
26
+ https://pypi.org/project/pdfplumber/.
27
+
28
+ The ported code is under the following MIT license:
29
+
30
+ ---------------------------------------------------------------------
31
+ The MIT License (MIT)
32
+
33
+ Copyright (c) 2015, Jeremy Singer-Vine
34
+
35
+ Permission is hereby granted, free of charge, to any person obtaining a copy
36
+ of this software and associated documentation files (the "Software"), to deal
37
+ in the Software without restriction, including without limitation the rights
38
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
39
+ copies of the Software, and to permit persons to whom the Software is
40
+ furnished to do so, subject to the following conditions:
41
+
42
+ The above copyright notice and this permission notice shall be included in all
43
+ copies or substantial portions of the Software.
44
+
45
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
46
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
47
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
48
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
49
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
50
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
51
+ SOFTWARE.
52
+ ---------------------------------------------------------------------
53
+ Also see here: https://github.com/jsvine/pdfplumber/blob/stable/LICENSE.txt
54
+ ---------------------------------------------------------------------
55
+
56
+ The porting mainly pertains to files "table.py" and relevant parts of
57
+ "utils/text.py" within pdfplumber's repository on Github.
58
+ With respect to "text.py", we have removed functions or features that are not
59
+ used by table processing. Examples are:
60
+
61
+ * the text search function
62
+ * simple text extraction
63
+ * text extraction by lines
64
+
65
+ Original pdfplumber code does neither detect, nor identify table headers.
66
+ This PyMuPDF port adds respective code to the 'Table' class as method '_get_header'.
67
+ This is implemented as new class TableHeader with the properties:
68
+ * bbox: A tuple for the header's bbox
69
+ * cells: A tuple for each bbox of a column header
70
+ * names: A list of strings with column header text
71
+ * external: A bool indicating whether the header is outside the table cells.
72
+
73
+ """
74
+
75
+ import inspect
76
+ import itertools
77
+ import string
78
+ import html
79
+ from collections.abc import Sequence
80
+ from dataclasses import dataclass
81
+ from operator import itemgetter
82
+
83
+ # -------------------------------------------------------------------
84
+ # Start of PyMuPDF interface code
85
+ # -------------------------------------------------------------------
86
+ from . import (
87
+ Rect,
88
+ Matrix,
89
+ TEXTFLAGS_TEXT,
90
+ TOOLS,
91
+ EMPTY_RECT,
92
+ sRGB_to_pdf,
93
+ Point,
94
+ message,
95
+ )
96
+
97
+ EDGES = [] # vector graphics from PyMuPDF
98
+ CHARS = [] # text characters from PyMuPDF
99
+ TEXTPAGE = None
100
+ white_spaces = set(string.whitespace) # for checking white space only cells
101
+ # -------------------------------------------------------------------
102
+ # End of PyMuPDF interface code
103
+ # -------------------------------------------------------------------
104
+
105
+
106
+ class UnsetFloat(float):
107
+ pass
108
+
109
+
110
+ NON_NEGATIVE_SETTINGS = [
111
+ "snap_tolerance",
112
+ "snap_x_tolerance",
113
+ "snap_y_tolerance",
114
+ "join_tolerance",
115
+ "join_x_tolerance",
116
+ "join_y_tolerance",
117
+ "edge_min_length",
118
+ "min_words_vertical",
119
+ "min_words_horizontal",
120
+ "intersection_tolerance",
121
+ "intersection_x_tolerance",
122
+ "intersection_y_tolerance",
123
+ ]
124
+
125
+
126
+ TABLE_STRATEGIES = ["lines", "lines_strict", "text", "explicit"]
127
+ UNSET = UnsetFloat(0)
128
+ DEFAULT_SNAP_TOLERANCE = 3
129
+ DEFAULT_JOIN_TOLERANCE = 3
130
+ DEFAULT_MIN_WORDS_VERTICAL = 3
131
+ DEFAULT_MIN_WORDS_HORIZONTAL = 1
132
+ DEFAULT_X_TOLERANCE = 3
133
+ DEFAULT_Y_TOLERANCE = 3
134
+ DEFAULT_X_DENSITY = 7.25
135
+ DEFAULT_Y_DENSITY = 13
136
+ bbox_getter = itemgetter("x0", "top", "x1", "bottom")
137
+
138
+
139
+ LIGATURES = {
140
+ "ff": "ff",
141
+ "ffi": "ffi",
142
+ "ffl": "ffl",
143
+ "fi": "fi",
144
+ "fl": "fl",
145
+ "st": "st",
146
+ "ſt": "st",
147
+ }
148
+
149
+
150
+ def to_list(collection) -> list:
151
+ if isinstance(collection, list):
152
+ return collection
153
+ elif isinstance(collection, Sequence):
154
+ return list(collection)
155
+ elif hasattr(collection, "to_dict"):
156
+ res = collection.to_dict("records") # pragma: nocover
157
+ return res
158
+ else:
159
+ return list(collection)
160
+
161
+
162
+ class TextMap:
163
+ """
164
+ A TextMap maps each unicode character in the text to an individual `char`
165
+ object (or, in the case of layout-implied whitespace, `None`).
166
+ """
167
+
168
+ def __init__(self, tuples=None) -> None:
169
+ self.tuples = tuples
170
+ self.as_string = "".join(map(itemgetter(0), tuples))
171
+
172
+ def match_to_dict(
173
+ self,
174
+ m,
175
+ main_group: int = 0,
176
+ return_groups: bool = True,
177
+ return_chars: bool = True,
178
+ ) -> dict:
179
+ subset = self.tuples[m.start(main_group) : m.end(main_group)]
180
+ chars = [c for (text, c) in subset if c is not None]
181
+ x0, top, x1, bottom = objects_to_bbox(chars)
182
+
183
+ result = {
184
+ "text": m.group(main_group),
185
+ "x0": x0,
186
+ "top": top,
187
+ "x1": x1,
188
+ "bottom": bottom,
189
+ }
190
+
191
+ if return_groups:
192
+ result["groups"] = m.groups()
193
+
194
+ if return_chars:
195
+ result["chars"] = chars
196
+
197
+ return result
198
+
199
+
200
+ class WordMap:
201
+ """
202
+ A WordMap maps words->chars.
203
+ """
204
+
205
+ def __init__(self, tuples) -> None:
206
+ self.tuples = tuples
207
+
208
+ def to_textmap(
209
+ self,
210
+ layout: bool = False,
211
+ layout_width=0,
212
+ layout_height=0,
213
+ layout_width_chars: int = 0,
214
+ layout_height_chars: int = 0,
215
+ x_density=DEFAULT_X_DENSITY,
216
+ y_density=DEFAULT_Y_DENSITY,
217
+ x_shift=0,
218
+ y_shift=0,
219
+ y_tolerance=DEFAULT_Y_TOLERANCE,
220
+ use_text_flow: bool = False,
221
+ presorted: bool = False,
222
+ expand_ligatures: bool = True,
223
+ ) -> TextMap:
224
+ """
225
+ Given a list of (word, chars) tuples (i.e., a WordMap), return a list of
226
+ (char-text, char) tuples (i.e., a TextMap) that can be used to mimic the
227
+ structural layout of the text on the page(s), using the following approach:
228
+
229
+ - Sort the words by (doctop, x0) if not already sorted.
230
+
231
+ - Calculate the initial doctop for the starting page.
232
+
233
+ - Cluster the words by doctop (taking `y_tolerance` into account), and
234
+ iterate through them.
235
+
236
+ - For each cluster, calculate the distance between that doctop and the
237
+ initial doctop, in points, minus `y_shift`. Divide that distance by
238
+ `y_density` to calculate the minimum number of newlines that should come
239
+ before this cluster. Append that number of newlines *minus* the number of
240
+ newlines already appended, with a minimum of one.
241
+
242
+ - Then for each cluster, iterate through each word in it. Divide each
243
+ word's x0, minus `x_shift`, by `x_density` to calculate the minimum
244
+ number of characters that should come before this cluster. Append that
245
+ number of spaces *minus* the number of characters and spaces already
246
+ appended, with a minimum of one. Then append the word's text.
247
+
248
+ - At the termination of each line, add more spaces if necessary to
249
+ mimic `layout_width`.
250
+
251
+ - Finally, add newlines to the end if necessary to mimic to
252
+ `layout_height`.
253
+
254
+ Note: This approach currently works best for horizontal, left-to-right
255
+ text, but will display all words regardless of orientation. There is room
256
+ for improvement in better supporting right-to-left text, as well as
257
+ vertical text.
258
+ """
259
+ _textmap = []
260
+
261
+ if not len(self.tuples):
262
+ return TextMap(_textmap)
263
+
264
+ expansions = LIGATURES if expand_ligatures else {}
265
+
266
+ if layout:
267
+ if layout_width_chars:
268
+ if layout_width:
269
+ raise ValueError(
270
+ "`layout_width` and `layout_width_chars` cannot both be set."
271
+ )
272
+ else:
273
+ layout_width_chars = int(round(layout_width / x_density))
274
+
275
+ if layout_height_chars:
276
+ if layout_height:
277
+ raise ValueError(
278
+ "`layout_height` and `layout_height_chars` cannot both be set."
279
+ )
280
+ else:
281
+ layout_height_chars = int(round(layout_height / y_density))
282
+
283
+ blank_line = [(" ", None)] * layout_width_chars
284
+ else:
285
+ blank_line = []
286
+
287
+ num_newlines = 0
288
+
289
+ words_sorted_doctop = (
290
+ self.tuples
291
+ if presorted or use_text_flow
292
+ else sorted(self.tuples, key=lambda x: float(x[0]["doctop"]))
293
+ )
294
+
295
+ first_word = words_sorted_doctop[0][0]
296
+ doctop_start = first_word["doctop"] - first_word["top"]
297
+
298
+ for i, ws in enumerate(
299
+ cluster_objects(
300
+ words_sorted_doctop, lambda x: float(x[0]["doctop"]), y_tolerance
301
+ )
302
+ ):
303
+ y_dist = (
304
+ (ws[0][0]["doctop"] - (doctop_start + y_shift)) / y_density
305
+ if layout
306
+ else 0
307
+ )
308
+ num_newlines_prepend = max(
309
+ # At least one newline, unless this iis the first line
310
+ int(i > 0),
311
+ # ... or as many as needed to get the imputed "distance" from the top
312
+ round(y_dist) - num_newlines,
313
+ )
314
+
315
+ for i in range(num_newlines_prepend):
316
+ if not len(_textmap) or _textmap[-1][0] == "\n":
317
+ _textmap += blank_line
318
+ _textmap.append(("\n", None))
319
+
320
+ num_newlines += num_newlines_prepend
321
+
322
+ line_len = 0
323
+
324
+ line_words_sorted_x0 = (
325
+ ws
326
+ if presorted or use_text_flow
327
+ else sorted(ws, key=lambda x: float(x[0]["x0"]))
328
+ )
329
+
330
+ for word, chars in line_words_sorted_x0:
331
+ x_dist = (word["x0"] - x_shift) / x_density if layout else 0
332
+ num_spaces_prepend = max(min(1, line_len), round(x_dist) - line_len)
333
+ _textmap += [(" ", None)] * num_spaces_prepend
334
+ line_len += num_spaces_prepend
335
+
336
+ for c in chars:
337
+ letters = expansions.get(c["text"], c["text"])
338
+ for letter in letters:
339
+ _textmap.append((letter, c))
340
+ line_len += 1
341
+
342
+ # Append spaces at end of line
343
+ if layout:
344
+ _textmap += [(" ", None)] * (layout_width_chars - line_len)
345
+
346
+ # Append blank lines at end of text
347
+ if layout:
348
+ num_newlines_append = layout_height_chars - (num_newlines + 1)
349
+ for i in range(num_newlines_append):
350
+ if i > 0:
351
+ _textmap += blank_line
352
+ _textmap.append(("\n", None))
353
+
354
+ # Remove terminal newline
355
+ if _textmap[-1] == ("\n", None):
356
+ _textmap = _textmap[:-1]
357
+
358
+ return TextMap(_textmap)
359
+
360
+
361
+ class WordExtractor:
362
+ def __init__(
363
+ self,
364
+ x_tolerance=DEFAULT_X_TOLERANCE,
365
+ y_tolerance=DEFAULT_Y_TOLERANCE,
366
+ keep_blank_chars: bool = False,
367
+ use_text_flow=False,
368
+ horizontal_ltr=True, # Should words be read left-to-right?
369
+ vertical_ttb=False, # Should vertical words be read top-to-bottom?
370
+ extra_attrs=None,
371
+ split_at_punctuation=False,
372
+ expand_ligatures=True,
373
+ ):
374
+ self.x_tolerance = x_tolerance
375
+ self.y_tolerance = y_tolerance
376
+ self.keep_blank_chars = keep_blank_chars
377
+ self.use_text_flow = use_text_flow
378
+ self.horizontal_ltr = horizontal_ltr
379
+ self.vertical_ttb = vertical_ttb
380
+ self.extra_attrs = [] if extra_attrs is None else extra_attrs
381
+
382
+ # Note: string.punctuation = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
383
+ self.split_at_punctuation = (
384
+ string.punctuation
385
+ if split_at_punctuation is True
386
+ else (split_at_punctuation or "")
387
+ )
388
+
389
+ self.expansions = LIGATURES if expand_ligatures else {}
390
+
391
+ def merge_chars(self, ordered_chars: list):
392
+ x0, top, x1, bottom = objects_to_bbox(ordered_chars)
393
+ doctop_adj = ordered_chars[0]["doctop"] - ordered_chars[0]["top"]
394
+ upright = ordered_chars[0]["upright"]
395
+ direction = 1 if (self.horizontal_ltr if upright else self.vertical_ttb) else -1
396
+
397
+ matrix = ordered_chars[0]["matrix"]
398
+
399
+ rotation = 0
400
+ if not upright and matrix[1] < 0:
401
+ ordered_chars = reversed(ordered_chars)
402
+ rotation = 270
403
+
404
+ if matrix[0] < 0 and matrix[3] < 0:
405
+ rotation = 180
406
+ elif matrix[1] > 0:
407
+ rotation = 90
408
+
409
+ word = {
410
+ "text": "".join(
411
+ self.expansions.get(c["text"], c["text"]) for c in ordered_chars
412
+ ),
413
+ "x0": x0,
414
+ "x1": x1,
415
+ "top": top,
416
+ "doctop": top + doctop_adj,
417
+ "bottom": bottom,
418
+ "upright": upright,
419
+ "direction": direction,
420
+ "rotation": rotation,
421
+ }
422
+
423
+ for key in self.extra_attrs:
424
+ word[key] = ordered_chars[0][key]
425
+
426
+ return word
427
+
428
+ def char_begins_new_word(
429
+ self,
430
+ prev_char,
431
+ curr_char,
432
+ ) -> bool:
433
+ """This method takes several factors into account to determine if
434
+ `curr_char` represents the beginning of a new word:
435
+
436
+ - Whether the text is "upright" (i.e., non-rotated)
437
+ - Whether the user has specified that horizontal text runs
438
+ left-to-right (default) or right-to-left, as represented by
439
+ self.horizontal_ltr
440
+ - Whether the user has specified that vertical text the text runs
441
+ top-to-bottom (default) or bottom-to-top, as represented by
442
+ self.vertical_ttb
443
+ - The x0, top, x1, and bottom attributes of prev_char and
444
+ curr_char
445
+ - The self.x_tolerance and self.y_tolerance settings. Note: In
446
+ this case, x/y refer to those directions for non-rotated text.
447
+ For vertical text, they are flipped. A more accurate terminology
448
+ might be "*intra*line character distance tolerance" and
449
+ "*inter*line character distance tolerance"
450
+
451
+ An important note: The *intra*line distance is measured from the
452
+ *end* of the previous character to the *beginning* of the current
453
+ character, while the *inter*line distance is measured from the
454
+ *top* of the previous character to the *top* of the next
455
+ character. The reasons for this are partly repository-historical,
456
+ and partly logical, as successive text lines' bounding boxes often
457
+ overlap slightly (and we don't want that overlap to be interpreted
458
+ as the two lines being the same line).
459
+
460
+ The upright-ness of the character determines the attributes to
461
+ compare, while horizontal_ltr/vertical_ttb determine the direction
462
+ of the comparison.
463
+ """
464
+
465
+ # Note: Due to the grouping step earlier in the process,
466
+ # curr_char["upright"] will always equal prev_char["upright"].
467
+ if curr_char["upright"]:
468
+ x = self.x_tolerance
469
+ y = self.y_tolerance
470
+ ay = prev_char["top"]
471
+ cy = curr_char["top"]
472
+ if self.horizontal_ltr:
473
+ ax = prev_char["x0"]
474
+ bx = prev_char["x1"]
475
+ cx = curr_char["x0"]
476
+ else:
477
+ ax = -prev_char["x1"]
478
+ bx = -prev_char["x0"]
479
+ cx = -curr_char["x1"]
480
+
481
+ else:
482
+ x = self.y_tolerance
483
+ y = self.x_tolerance
484
+ ay = prev_char["x0"]
485
+ cy = curr_char["x0"]
486
+ if self.vertical_ttb:
487
+ ax = prev_char["top"]
488
+ bx = prev_char["bottom"]
489
+ cx = curr_char["top"]
490
+ else:
491
+ ax = -prev_char["bottom"]
492
+ bx = -prev_char["top"]
493
+ cx = -curr_char["bottom"]
494
+
495
+ return bool(
496
+ # Intraline test
497
+ (cx < ax)
498
+ or (cx > bx + x)
499
+ # Interline test
500
+ or (cy > ay + y)
501
+ )
502
+
503
+ def iter_chars_to_words(self, ordered_chars):
504
+ current_word: list = []
505
+
506
+ def start_next_word(new_char=None):
507
+ nonlocal current_word
508
+
509
+ if current_word:
510
+ yield current_word
511
+
512
+ current_word = [] if new_char is None else [new_char]
513
+
514
+ for char in ordered_chars:
515
+ text = char["text"]
516
+
517
+ if not self.keep_blank_chars and text.isspace():
518
+ yield from start_next_word(None)
519
+
520
+ elif text in self.split_at_punctuation:
521
+ yield from start_next_word(char)
522
+ yield from start_next_word(None)
523
+
524
+ elif current_word and self.char_begins_new_word(current_word[-1], char):
525
+ yield from start_next_word(char)
526
+
527
+ else:
528
+ current_word.append(char)
529
+
530
+ # Finally, after all chars processed
531
+ if current_word:
532
+ yield current_word
533
+
534
+ def iter_sort_chars(self, chars):
535
+ def upright_key(x) -> int:
536
+ return -int(x["upright"])
537
+
538
+ for upright_cluster in cluster_objects(list(chars), upright_key, 0):
539
+ upright = upright_cluster[0]["upright"]
540
+ cluster_key = "doctop" if upright else "x0"
541
+
542
+ # Cluster by line
543
+ subclusters = cluster_objects(
544
+ upright_cluster, itemgetter(cluster_key), self.y_tolerance
545
+ )
546
+
547
+ for sc in subclusters:
548
+ # Sort within line
549
+ sort_key = "x0" if upright else "doctop"
550
+ to_yield = sorted(sc, key=itemgetter(sort_key))
551
+
552
+ # Reverse order if necessary
553
+ if not (self.horizontal_ltr if upright else self.vertical_ttb):
554
+ yield from reversed(to_yield)
555
+ else:
556
+ yield from to_yield
557
+
558
+ def iter_extract_tuples(self, chars):
559
+ ordered_chars = chars if self.use_text_flow else self.iter_sort_chars(chars)
560
+
561
+ grouping_key = itemgetter("upright", *self.extra_attrs)
562
+ grouped_chars = itertools.groupby(ordered_chars, grouping_key)
563
+
564
+ for keyvals, char_group in grouped_chars:
565
+ for word_chars in self.iter_chars_to_words(char_group):
566
+ yield (self.merge_chars(word_chars), word_chars)
567
+
568
+ def extract_wordmap(self, chars) -> WordMap:
569
+ return WordMap(list(self.iter_extract_tuples(chars)))
570
+
571
+ def extract_words(self, chars: list) -> list:
572
+ words = list(word for word, word_chars in self.iter_extract_tuples(chars))
573
+ return words
574
+
575
+
576
+ def extract_words(chars: list, **kwargs) -> list:
577
+ return WordExtractor(**kwargs).extract_words(chars)
578
+
579
+
580
+ TEXTMAP_KWARGS = inspect.signature(WordMap.to_textmap).parameters.keys()
581
+ WORD_EXTRACTOR_KWARGS = inspect.signature(WordExtractor).parameters.keys()
582
+
583
+
584
+ def chars_to_textmap(chars: list, **kwargs) -> TextMap:
585
+ kwargs.update({"presorted": True})
586
+
587
+ extractor = WordExtractor(
588
+ **{k: kwargs[k] for k in WORD_EXTRACTOR_KWARGS if k in kwargs}
589
+ )
590
+ wordmap = extractor.extract_wordmap(chars)
591
+ textmap = wordmap.to_textmap(
592
+ **{k: kwargs[k] for k in TEXTMAP_KWARGS if k in kwargs}
593
+ )
594
+
595
+ return textmap
596
+
597
+
598
+ def extract_text(chars: list, **kwargs) -> str:
599
+ chars = to_list(chars)
600
+ if len(chars) == 0:
601
+ return ""
602
+
603
+ if kwargs.get("layout"):
604
+ return chars_to_textmap(chars, **kwargs).as_string
605
+ else:
606
+ y_tolerance = kwargs.get("y_tolerance", DEFAULT_Y_TOLERANCE)
607
+ extractor = WordExtractor(
608
+ **{k: kwargs[k] for k in WORD_EXTRACTOR_KWARGS if k in kwargs}
609
+ )
610
+ words = extractor.extract_words(chars)
611
+ if words:
612
+ rotation = words[0]["rotation"] # rotation cannot change within a cell
613
+ else:
614
+ rotation = 0
615
+
616
+ if rotation == 90:
617
+ words.sort(key=lambda w: (w["x1"], -w["top"]))
618
+ lines = " ".join([w["text"] for w in words])
619
+ elif rotation == 270:
620
+ words.sort(key=lambda w: (-w["x1"], w["top"]))
621
+ lines = " ".join([w["text"] for w in words])
622
+ else:
623
+ lines = cluster_objects(words, itemgetter("doctop"), y_tolerance)
624
+ lines = "\n".join(" ".join(word["text"] for word in line) for line in lines)
625
+ if rotation == 180: # needs extra treatment
626
+ lines = "".join([(c if c != "\n" else " ") for c in reversed(lines)])
627
+
628
+ return lines
629
+
630
+
631
+ def collate_line(
632
+ line_chars: list,
633
+ tolerance=DEFAULT_X_TOLERANCE,
634
+ ) -> str:
635
+ coll = ""
636
+ last_x1 = None
637
+ for char in sorted(line_chars, key=itemgetter("x0")):
638
+ if (last_x1 is not None) and (char["x0"] > (last_x1 + tolerance)):
639
+ coll += " "
640
+ last_x1 = char["x1"]
641
+ coll += char["text"]
642
+ return coll
643
+
644
+
645
+ def dedupe_chars(chars: list, tolerance=1) -> list:
646
+ """
647
+ Removes duplicate chars — those sharing the same text, fontname, size,
648
+ and positioning (within `tolerance`) as other characters in the set.
649
+ """
650
+ key = itemgetter("fontname", "size", "upright", "text")
651
+ pos_key = itemgetter("doctop", "x0")
652
+
653
+ def yield_unique_chars(chars: list):
654
+ sorted_chars = sorted(chars, key=key)
655
+ for grp, grp_chars in itertools.groupby(sorted_chars, key=key):
656
+ for y_cluster in cluster_objects(
657
+ list(grp_chars), itemgetter("doctop"), tolerance
658
+ ):
659
+ for x_cluster in cluster_objects(
660
+ y_cluster, itemgetter("x0"), tolerance
661
+ ):
662
+ yield sorted(x_cluster, key=pos_key)[0]
663
+
664
+ deduped = yield_unique_chars(chars)
665
+ return sorted(deduped, key=chars.index)
666
+
667
+
668
+ def line_to_edge(line):
669
+ edge = dict(line)
670
+ edge["orientation"] = "h" if (line["top"] == line["bottom"]) else "v"
671
+ return edge
672
+
673
+
674
+ def rect_to_edges(rect) -> list:
675
+ top, bottom, left, right = [dict(rect) for x in range(4)]
676
+ top.update(
677
+ {
678
+ "object_type": "rect_edge",
679
+ "height": 0,
680
+ "y0": rect["y1"],
681
+ "bottom": rect["top"],
682
+ "orientation": "h",
683
+ }
684
+ )
685
+ bottom.update(
686
+ {
687
+ "object_type": "rect_edge",
688
+ "height": 0,
689
+ "y1": rect["y0"],
690
+ "top": rect["top"] + rect["height"],
691
+ "doctop": rect["doctop"] + rect["height"],
692
+ "orientation": "h",
693
+ }
694
+ )
695
+ left.update(
696
+ {
697
+ "object_type": "rect_edge",
698
+ "width": 0,
699
+ "x1": rect["x0"],
700
+ "orientation": "v",
701
+ }
702
+ )
703
+ right.update(
704
+ {
705
+ "object_type": "rect_edge",
706
+ "width": 0,
707
+ "x0": rect["x1"],
708
+ "orientation": "v",
709
+ }
710
+ )
711
+ return [top, bottom, left, right]
712
+
713
+
714
+ def curve_to_edges(curve) -> list:
715
+ point_pairs = zip(curve["pts"], curve["pts"][1:])
716
+ return [
717
+ {
718
+ "object_type": "curve_edge",
719
+ "x0": min(p0[0], p1[0]),
720
+ "x1": max(p0[0], p1[0]),
721
+ "top": min(p0[1], p1[1]),
722
+ "doctop": min(p0[1], p1[1]) + (curve["doctop"] - curve["top"]),
723
+ "bottom": max(p0[1], p1[1]),
724
+ "width": abs(p0[0] - p1[0]),
725
+ "height": abs(p0[1] - p1[1]),
726
+ "orientation": "v" if p0[0] == p1[0] else ("h" if p0[1] == p1[1] else None),
727
+ }
728
+ for p0, p1 in point_pairs
729
+ ]
730
+
731
+
732
+ def obj_to_edges(obj) -> list:
733
+ t = obj["object_type"]
734
+ if "_edge" in t:
735
+ return [obj]
736
+ elif t == "line":
737
+ return [line_to_edge(obj)]
738
+ else:
739
+ return {"rect": rect_to_edges, "curve": curve_to_edges}[t](obj)
740
+
741
+
742
+ def filter_edges(
743
+ edges,
744
+ orientation=None,
745
+ edge_type=None,
746
+ min_length=1,
747
+ ) -> list:
748
+ if orientation not in ("v", "h", None):
749
+ raise ValueError("Orientation must be 'v' or 'h'")
750
+
751
+ def test(e) -> bool:
752
+ dim = "height" if e["orientation"] == "v" else "width"
753
+ et_correct = e["object_type"] == edge_type if edge_type is not None else True
754
+ orient_correct = orientation is None or e["orientation"] == orientation
755
+ return bool(et_correct and orient_correct and (e[dim] >= min_length))
756
+
757
+ return list(filter(test, edges))
758
+
759
+
760
+ def cluster_list(xs, tolerance=0) -> list:
761
+ if tolerance == 0:
762
+ return [[x] for x in sorted(xs)]
763
+ if len(xs) < 2:
764
+ return [[x] for x in sorted(xs)]
765
+ groups = []
766
+ xs = list(sorted(xs))
767
+ current_group = [xs[0]]
768
+ last = xs[0]
769
+ for x in xs[1:]:
770
+ if x <= (last + tolerance):
771
+ current_group.append(x)
772
+ else:
773
+ groups.append(current_group)
774
+ current_group = [x]
775
+ last = x
776
+ groups.append(current_group)
777
+ return groups
778
+
779
+
780
+ def make_cluster_dict(values, tolerance) -> dict:
781
+ clusters = cluster_list(list(set(values)), tolerance)
782
+
783
+ nested_tuples = [
784
+ [(val, i) for val in value_cluster] for i, value_cluster in enumerate(clusters)
785
+ ]
786
+
787
+ return dict(itertools.chain(*nested_tuples))
788
+
789
+
790
+ def cluster_objects(xs, key_fn, tolerance) -> list:
791
+ if not callable(key_fn):
792
+ key_fn = itemgetter(key_fn)
793
+
794
+ values = map(key_fn, xs)
795
+ cluster_dict = make_cluster_dict(values, tolerance)
796
+
797
+ get_0, get_1 = itemgetter(0), itemgetter(1)
798
+
799
+ cluster_tuples = sorted(((x, cluster_dict.get(key_fn(x))) for x in xs), key=get_1)
800
+
801
+ grouped = itertools.groupby(cluster_tuples, key=get_1)
802
+
803
+ return [list(map(get_0, v)) for k, v in grouped]
804
+
805
+
806
+ def move_object(obj, axis: str, value):
807
+ assert axis in ("h", "v")
808
+ if axis == "h":
809
+ new_items = [
810
+ ("x0", obj["x0"] + value),
811
+ ("x1", obj["x1"] + value),
812
+ ]
813
+ if axis == "v":
814
+ new_items = [
815
+ ("top", obj["top"] + value),
816
+ ("bottom", obj["bottom"] + value),
817
+ ]
818
+ if "doctop" in obj:
819
+ new_items += [("doctop", obj["doctop"] + value)]
820
+ if "y0" in obj:
821
+ new_items += [
822
+ ("y0", obj["y0"] - value),
823
+ ("y1", obj["y1"] - value),
824
+ ]
825
+ return obj.__class__(tuple(obj.items()) + tuple(new_items))
826
+
827
+
828
+ def snap_objects(objs, attr: str, tolerance) -> list:
829
+ axis = {"x0": "h", "x1": "h", "top": "v", "bottom": "v"}[attr]
830
+ list_objs = list(objs)
831
+ clusters = cluster_objects(list_objs, itemgetter(attr), tolerance)
832
+ avgs = [sum(map(itemgetter(attr), cluster)) / len(cluster) for cluster in clusters]
833
+ snapped_clusters = [
834
+ [move_object(obj, axis, avg - obj[attr]) for obj in cluster]
835
+ for cluster, avg in zip(clusters, avgs)
836
+ ]
837
+ return list(itertools.chain(*snapped_clusters))
838
+
839
+
840
+ def snap_edges(
841
+ edges,
842
+ x_tolerance=DEFAULT_SNAP_TOLERANCE,
843
+ y_tolerance=DEFAULT_SNAP_TOLERANCE,
844
+ ):
845
+ """
846
+ Given a list of edges, snap any within `tolerance` pixels of one another
847
+ to their positional average.
848
+ """
849
+ by_orientation = {"v": [], "h": []}
850
+ for e in edges:
851
+ by_orientation[e["orientation"]].append(e)
852
+
853
+ snapped_v = snap_objects(by_orientation["v"], "x0", x_tolerance)
854
+ snapped_h = snap_objects(by_orientation["h"], "top", y_tolerance)
855
+ return snapped_v + snapped_h
856
+
857
+
858
+ def resize_object(obj, key: str, value):
859
+ assert key in ("x0", "x1", "top", "bottom")
860
+ old_value = obj[key]
861
+ diff = value - old_value
862
+ new_items = [
863
+ (key, value),
864
+ ]
865
+ if key == "x0":
866
+ assert value <= obj["x1"]
867
+ new_items.append(("width", obj["x1"] - value))
868
+ elif key == "x1":
869
+ assert value >= obj["x0"]
870
+ new_items.append(("width", value - obj["x0"]))
871
+ elif key == "top":
872
+ assert value <= obj["bottom"]
873
+ new_items.append(("doctop", obj["doctop"] + diff))
874
+ new_items.append(("height", obj["height"] - diff))
875
+ if "y1" in obj:
876
+ new_items.append(("y1", obj["y1"] - diff))
877
+ elif key == "bottom":
878
+ assert value >= obj["top"]
879
+ new_items.append(("height", obj["height"] + diff))
880
+ if "y0" in obj:
881
+ new_items.append(("y0", obj["y0"] - diff))
882
+ return obj.__class__(tuple(obj.items()) + tuple(new_items))
883
+
884
+
885
+ def join_edge_group(edges, orientation: str, tolerance=DEFAULT_JOIN_TOLERANCE):
886
+ """
887
+ Given a list of edges along the same infinite line, join those that
888
+ are within `tolerance` pixels of one another.
889
+ """
890
+ if orientation == "h":
891
+ min_prop, max_prop = "x0", "x1"
892
+ elif orientation == "v":
893
+ min_prop, max_prop = "top", "bottom"
894
+ else:
895
+ raise ValueError("Orientation must be 'v' or 'h'")
896
+
897
+ sorted_edges = list(sorted(edges, key=itemgetter(min_prop)))
898
+ joined = [sorted_edges[0]]
899
+ for e in sorted_edges[1:]:
900
+ last = joined[-1]
901
+ if e[min_prop] <= (last[max_prop] + tolerance):
902
+ if e[max_prop] > last[max_prop]:
903
+ # Extend current edge to new extremity
904
+ joined[-1] = resize_object(last, max_prop, e[max_prop])
905
+ else:
906
+ # Edge is separate from previous edges
907
+ joined.append(e)
908
+
909
+ return joined
910
+
911
+
912
+ def merge_edges(
913
+ edges,
914
+ snap_x_tolerance,
915
+ snap_y_tolerance,
916
+ join_x_tolerance,
917
+ join_y_tolerance,
918
+ ):
919
+ """
920
+ Using the `snap_edges` and `join_edge_group` methods above,
921
+ merge a list of edges into a more "seamless" list.
922
+ """
923
+
924
+ def get_group(edge):
925
+ if edge["orientation"] == "h":
926
+ return ("h", edge["top"])
927
+ else:
928
+ return ("v", edge["x0"])
929
+
930
+ if snap_x_tolerance > 0 or snap_y_tolerance > 0:
931
+ edges = snap_edges(edges, snap_x_tolerance, snap_y_tolerance)
932
+
933
+ _sorted = sorted(edges, key=get_group)
934
+ edge_groups = itertools.groupby(_sorted, key=get_group)
935
+ edge_gen = (
936
+ join_edge_group(
937
+ items, k[0], (join_x_tolerance if k[0] == "h" else join_y_tolerance)
938
+ )
939
+ for k, items in edge_groups
940
+ )
941
+ edges = list(itertools.chain(*edge_gen))
942
+ return edges
943
+
944
+
945
+ def bbox_to_rect(bbox) -> dict:
946
+ """
947
+ Return the rectangle (i.e a dict with keys "x0", "top", "x1",
948
+ "bottom") for an object.
949
+ """
950
+ return {"x0": bbox[0], "top": bbox[1], "x1": bbox[2], "bottom": bbox[3]}
951
+
952
+
953
+ def objects_to_rect(objects) -> dict:
954
+ """
955
+ Given an iterable of objects, return the smallest rectangle (i.e. a
956
+ dict with "x0", "top", "x1", and "bottom" keys) that contains them
957
+ all.
958
+ """
959
+ return bbox_to_rect(objects_to_bbox(objects))
960
+
961
+
962
+ def merge_bboxes(bboxes):
963
+ """
964
+ Given an iterable of bounding boxes, return the smallest bounding box
965
+ that contains them all.
966
+ """
967
+ x0, top, x1, bottom = zip(*bboxes)
968
+ return (min(x0), min(top), max(x1), max(bottom))
969
+
970
+
971
+ def objects_to_bbox(objects):
972
+ """
973
+ Given an iterable of objects, return the smallest bounding box that
974
+ contains them all.
975
+ """
976
+ return merge_bboxes(map(bbox_getter, objects))
977
+
978
+
979
+ def words_to_edges_h(words, word_threshold: int = DEFAULT_MIN_WORDS_HORIZONTAL):
980
+ """
981
+ Find (imaginary) horizontal lines that connect the tops
982
+ of at least `word_threshold` words.
983
+ """
984
+ by_top = cluster_objects(words, itemgetter("top"), 1)
985
+ large_clusters = filter(lambda x: len(x) >= word_threshold, by_top)
986
+ rects = list(map(objects_to_rect, large_clusters))
987
+ if len(rects) == 0:
988
+ return []
989
+ min_x0 = min(map(itemgetter("x0"), rects))
990
+ max_x1 = max(map(itemgetter("x1"), rects))
991
+
992
+ edges = []
993
+ for r in rects:
994
+ edges += [
995
+ # Top of text
996
+ {
997
+ "x0": min_x0,
998
+ "x1": max_x1,
999
+ "top": r["top"],
1000
+ "bottom": r["top"],
1001
+ "width": max_x1 - min_x0,
1002
+ "orientation": "h",
1003
+ },
1004
+ # For each detected row, we also add the 'bottom' line. This will
1005
+ # generate extra edges, (some will be redundant with the next row
1006
+ # 'top' line), but this catches the last row of every table.
1007
+ {
1008
+ "x0": min_x0,
1009
+ "x1": max_x1,
1010
+ "top": r["bottom"],
1011
+ "bottom": r["bottom"],
1012
+ "width": max_x1 - min_x0,
1013
+ "orientation": "h",
1014
+ },
1015
+ ]
1016
+
1017
+ return edges
1018
+
1019
+
1020
+ def get_bbox_overlap(a, b):
1021
+ a_left, a_top, a_right, a_bottom = a
1022
+ b_left, b_top, b_right, b_bottom = b
1023
+ o_left = max(a_left, b_left)
1024
+ o_right = min(a_right, b_right)
1025
+ o_bottom = min(a_bottom, b_bottom)
1026
+ o_top = max(a_top, b_top)
1027
+ o_width = o_right - o_left
1028
+ o_height = o_bottom - o_top
1029
+ if o_height >= 0 and o_width >= 0 and o_height + o_width > 0:
1030
+ return (o_left, o_top, o_right, o_bottom)
1031
+ else:
1032
+ return None
1033
+
1034
+
1035
+ def words_to_edges_v(words, word_threshold: int = DEFAULT_MIN_WORDS_VERTICAL):
1036
+ """
1037
+ Find (imaginary) vertical lines that connect the left, right, or
1038
+ center of at least `word_threshold` words.
1039
+ """
1040
+ # Find words that share the same left, right, or centerpoints
1041
+ by_x0 = cluster_objects(words, itemgetter("x0"), 1)
1042
+ by_x1 = cluster_objects(words, itemgetter("x1"), 1)
1043
+
1044
+ def get_center(word):
1045
+ return float(word["x0"] + word["x1"]) / 2
1046
+
1047
+ by_center = cluster_objects(words, get_center, 1)
1048
+ clusters = by_x0 + by_x1 + by_center
1049
+
1050
+ # Find the points that align with the most words
1051
+ sorted_clusters = sorted(clusters, key=lambda x: -len(x))
1052
+ large_clusters = filter(lambda x: len(x) >= word_threshold, sorted_clusters)
1053
+
1054
+ # For each of those points, find the bboxes fitting all matching words
1055
+ bboxes = list(map(objects_to_bbox, large_clusters))
1056
+
1057
+ # Iterate through those bboxes, condensing overlapping bboxes
1058
+ condensed_bboxes = []
1059
+ for bbox in bboxes:
1060
+ overlap = any(get_bbox_overlap(bbox, c) for c in condensed_bboxes)
1061
+ if not overlap:
1062
+ condensed_bboxes.append(bbox)
1063
+
1064
+ if len(condensed_bboxes) == 0:
1065
+ return []
1066
+
1067
+ condensed_rects = map(bbox_to_rect, condensed_bboxes)
1068
+ sorted_rects = list(sorted(condensed_rects, key=itemgetter("x0")))
1069
+
1070
+ max_x1 = max(map(itemgetter("x1"), sorted_rects))
1071
+ min_top = min(map(itemgetter("top"), sorted_rects))
1072
+ max_bottom = max(map(itemgetter("bottom"), sorted_rects))
1073
+
1074
+ return [
1075
+ {
1076
+ "x0": b["x0"],
1077
+ "x1": b["x0"],
1078
+ "top": min_top,
1079
+ "bottom": max_bottom,
1080
+ "height": max_bottom - min_top,
1081
+ "orientation": "v",
1082
+ }
1083
+ for b in sorted_rects
1084
+ ] + [
1085
+ {
1086
+ "x0": max_x1,
1087
+ "x1": max_x1,
1088
+ "top": min_top,
1089
+ "bottom": max_bottom,
1090
+ "height": max_bottom - min_top,
1091
+ "orientation": "v",
1092
+ }
1093
+ ]
1094
+
1095
+
1096
+ def edges_to_intersections(edges, x_tolerance=1, y_tolerance=1) -> dict:
1097
+ """
1098
+ Given a list of edges, return the points at which they intersect
1099
+ within `tolerance` pixels.
1100
+ """
1101
+ intersections = {}
1102
+ v_edges, h_edges = [
1103
+ list(filter(lambda x: x["orientation"] == o, edges)) for o in ("v", "h")
1104
+ ]
1105
+ for v in sorted(v_edges, key=itemgetter("x0", "top")):
1106
+ for h in sorted(h_edges, key=itemgetter("top", "x0")):
1107
+ if (
1108
+ (v["top"] <= (h["top"] + y_tolerance))
1109
+ and (v["bottom"] >= (h["top"] - y_tolerance))
1110
+ and (v["x0"] >= (h["x0"] - x_tolerance))
1111
+ and (v["x0"] <= (h["x1"] + x_tolerance))
1112
+ ):
1113
+ vertex = (v["x0"], h["top"])
1114
+ if vertex not in intersections:
1115
+ intersections[vertex] = {"v": [], "h": []}
1116
+ intersections[vertex]["v"].append(v)
1117
+ intersections[vertex]["h"].append(h)
1118
+ return intersections
1119
+
1120
+
1121
+ def obj_to_bbox(obj):
1122
+ """
1123
+ Return the bounding box for an object.
1124
+ """
1125
+ return bbox_getter(obj)
1126
+
1127
+
1128
+ def intersections_to_cells(intersections):
1129
+ """
1130
+ Given a list of points (`intersections`), return all rectangular "cells"
1131
+ that those points describe.
1132
+
1133
+ `intersections` should be a dictionary with (x0, top) tuples as keys,
1134
+ and a list of edge objects as values. The edge objects should correspond
1135
+ to the edges that touch the intersection.
1136
+ """
1137
+
1138
+ def edge_connects(p1, p2) -> bool:
1139
+ def edges_to_set(edges):
1140
+ return set(map(obj_to_bbox, edges))
1141
+
1142
+ if p1[0] == p2[0]:
1143
+ common = edges_to_set(intersections[p1]["v"]).intersection(
1144
+ edges_to_set(intersections[p2]["v"])
1145
+ )
1146
+ if len(common):
1147
+ return True
1148
+
1149
+ if p1[1] == p2[1]:
1150
+ common = edges_to_set(intersections[p1]["h"]).intersection(
1151
+ edges_to_set(intersections[p2]["h"])
1152
+ )
1153
+ if len(common):
1154
+ return True
1155
+ return False
1156
+
1157
+ points = list(sorted(intersections.keys()))
1158
+ n_points = len(points)
1159
+
1160
+ def find_smallest_cell(points, i: int):
1161
+ if i == n_points - 1:
1162
+ return None
1163
+ pt = points[i]
1164
+ rest = points[i + 1 :]
1165
+ # Get all the points directly below and directly right
1166
+ below = [x for x in rest if x[0] == pt[0]]
1167
+ right = [x for x in rest if x[1] == pt[1]]
1168
+ for below_pt in below:
1169
+ if not edge_connects(pt, below_pt):
1170
+ continue
1171
+
1172
+ for right_pt in right:
1173
+ if not edge_connects(pt, right_pt):
1174
+ continue
1175
+
1176
+ bottom_right = (right_pt[0], below_pt[1])
1177
+
1178
+ if (
1179
+ (bottom_right in intersections)
1180
+ and edge_connects(bottom_right, right_pt)
1181
+ and edge_connects(bottom_right, below_pt)
1182
+ ):
1183
+ return (pt[0], pt[1], bottom_right[0], bottom_right[1])
1184
+ return None
1185
+
1186
+ cell_gen = (find_smallest_cell(points, i) for i in range(len(points)))
1187
+ return list(filter(None, cell_gen))
1188
+
1189
+
1190
+ def cells_to_tables(page, cells) -> list:
1191
+ """
1192
+ Given a list of bounding boxes (`cells`), return a list of tables that
1193
+ hold those cells most simply (and contiguously).
1194
+ """
1195
+
1196
+ def bbox_to_corners(bbox) -> tuple:
1197
+ x0, top, x1, bottom = bbox
1198
+ return ((x0, top), (x0, bottom), (x1, top), (x1, bottom))
1199
+
1200
+ remaining_cells = list(cells)
1201
+
1202
+ # Iterate through the cells found above, and assign them
1203
+ # to contiguous tables
1204
+
1205
+ current_corners = set()
1206
+ current_cells = []
1207
+
1208
+ tables = []
1209
+ while len(remaining_cells):
1210
+ initial_cell_count = len(current_cells)
1211
+ for cell in list(remaining_cells):
1212
+ cell_corners = bbox_to_corners(cell)
1213
+ # If we're just starting a table ...
1214
+ if len(current_cells) == 0:
1215
+ # ... immediately assign it to the empty group
1216
+ current_corners |= set(cell_corners)
1217
+ current_cells.append(cell)
1218
+ remaining_cells.remove(cell)
1219
+ else:
1220
+ # How many corners does this table share with the current group?
1221
+ corner_count = sum(c in current_corners for c in cell_corners)
1222
+
1223
+ # If touching on at least one corner...
1224
+ if corner_count > 0:
1225
+ # ... assign it to the current group
1226
+ current_corners |= set(cell_corners)
1227
+ current_cells.append(cell)
1228
+ remaining_cells.remove(cell)
1229
+
1230
+ # If this iteration did not find any more cells to append...
1231
+ if len(current_cells) == initial_cell_count:
1232
+ # ... start a new cell group
1233
+ tables.append(list(current_cells))
1234
+ current_corners.clear()
1235
+ current_cells.clear()
1236
+
1237
+ # Once we have exhausting the list of cells ...
1238
+
1239
+ # ... and we have a cell group that has not been stored
1240
+ if len(current_cells):
1241
+ # ... store it.
1242
+ tables.append(list(current_cells))
1243
+
1244
+ # PyMuPDF modification:
1245
+ # Remove tables without text or having only 1 column
1246
+ for i in range(len(tables) - 1, -1, -1):
1247
+ r = EMPTY_RECT()
1248
+ x1_vals = set()
1249
+ x0_vals = set()
1250
+ for c in tables[i]:
1251
+ r |= c
1252
+ x1_vals.add(c[2])
1253
+ x0_vals.add(c[0])
1254
+ if (
1255
+ len(x1_vals) < 2
1256
+ or len(x0_vals) < 2
1257
+ or white_spaces.issuperset(
1258
+ page.get_textbox(
1259
+ r,
1260
+ textpage=TEXTPAGE,
1261
+ )
1262
+ )
1263
+ ):
1264
+ del tables[i]
1265
+
1266
+ # Sort the tables top-to-bottom-left-to-right based on the value of the
1267
+ # topmost-and-then-leftmost coordinate of a table.
1268
+ _sorted = sorted(tables, key=lambda t: min((c[1], c[0]) for c in t))
1269
+ return _sorted
1270
+
1271
+
1272
+ class CellGroup:
1273
+ def __init__(self, cells):
1274
+ self.cells = cells
1275
+ self.bbox = (
1276
+ min(map(itemgetter(0), filter(None, cells))),
1277
+ min(map(itemgetter(1), filter(None, cells))),
1278
+ max(map(itemgetter(2), filter(None, cells))),
1279
+ max(map(itemgetter(3), filter(None, cells))),
1280
+ )
1281
+
1282
+
1283
+ class TableRow(CellGroup):
1284
+ pass
1285
+
1286
+
1287
+ class TableHeader:
1288
+ """PyMuPDF extension containing the identified table header."""
1289
+
1290
+ def __init__(self, bbox, cells, names, above):
1291
+ self.bbox = bbox
1292
+ self.cells = cells
1293
+ self.names = names
1294
+ self.external = above
1295
+
1296
+
1297
+ class Table:
1298
+ def __init__(self, page, cells):
1299
+ self.page = page
1300
+ self.cells = cells
1301
+ self.header = self._get_header() # PyMuPDF extension
1302
+
1303
+ @property
1304
+ def bbox(self):
1305
+ c = self.cells
1306
+ return (
1307
+ min(map(itemgetter(0), c)),
1308
+ min(map(itemgetter(1), c)),
1309
+ max(map(itemgetter(2), c)),
1310
+ max(map(itemgetter(3), c)),
1311
+ )
1312
+
1313
+ @property
1314
+ def rows(self) -> list:
1315
+ _sorted = sorted(self.cells, key=itemgetter(1, 0))
1316
+ xs = list(sorted(set(map(itemgetter(0), self.cells))))
1317
+ rows = []
1318
+ for y, row_cells in itertools.groupby(_sorted, itemgetter(1)):
1319
+ xdict = {cell[0]: cell for cell in row_cells}
1320
+ row = TableRow([xdict.get(x) for x in xs])
1321
+ rows.append(row)
1322
+ return rows
1323
+
1324
+ @property
1325
+ def row_count(self) -> int: # PyMuPDF extension
1326
+ return len(self.rows)
1327
+
1328
+ @property
1329
+ def col_count(self) -> int: # PyMuPDF extension
1330
+ return max([len(r.cells) for r in self.rows])
1331
+
1332
+ def extract(self, **kwargs) -> list:
1333
+ chars = CHARS
1334
+ table_arr = []
1335
+
1336
+ def char_in_bbox(char, bbox) -> bool:
1337
+ v_mid = (char["top"] + char["bottom"]) / 2
1338
+ h_mid = (char["x0"] + char["x1"]) / 2
1339
+ x0, top, x1, bottom = bbox
1340
+ return bool(
1341
+ (h_mid >= x0) and (h_mid < x1) and (v_mid >= top) and (v_mid < bottom)
1342
+ )
1343
+
1344
+ for row in self.rows:
1345
+ arr = []
1346
+ row_chars = [char for char in chars if char_in_bbox(char, row.bbox)]
1347
+
1348
+ for cell in row.cells:
1349
+ if cell is None:
1350
+ cell_text = None
1351
+ else:
1352
+ cell_chars = [
1353
+ char for char in row_chars if char_in_bbox(char, cell)
1354
+ ]
1355
+
1356
+ if len(cell_chars):
1357
+ kwargs["x_shift"] = cell[0]
1358
+ kwargs["y_shift"] = cell[1]
1359
+ if "layout" in kwargs:
1360
+ kwargs["layout_width"] = cell[2] - cell[0]
1361
+ kwargs["layout_height"] = cell[3] - cell[1]
1362
+ cell_text = extract_text(cell_chars, **kwargs)
1363
+ else:
1364
+ cell_text = ""
1365
+ arr.append(cell_text)
1366
+ table_arr.append(arr)
1367
+
1368
+ return table_arr
1369
+
1370
+ def to_markdown(self, clean=True):
1371
+ """Output table content as a string in Github-markdown format.
1372
+
1373
+ If clean is true, markdown syntax is removed from cell content."""
1374
+ output = "|"
1375
+
1376
+ # generate header string and MD underline
1377
+ for i, name in enumerate(self.header.names):
1378
+ if name is None or name == "": # generate a name if empty
1379
+ name = f"Col{i+1}"
1380
+ name = name.replace("\n", " ") # remove any line breaks
1381
+ if clean: # remove sensitive syntax
1382
+ name = html.escape(name.replace("-", "&#45;"))
1383
+ output += name + "|"
1384
+
1385
+ output += "\n"
1386
+ output += "|" + "|".join("---" for i in range(self.col_count)) + "|\n"
1387
+
1388
+ # skip first row in details if header is part of the table
1389
+ j = 0 if self.header.external else 1
1390
+
1391
+ # iterate over detail rows
1392
+ for row in self.extract()[j:]:
1393
+ line = "|"
1394
+ for i, cell in enumerate(row):
1395
+ # output None cells with empty string
1396
+ cell = "" if cell is None else cell.replace("\n", " ")
1397
+ if clean: # remove sensitive syntax
1398
+ cell = html.escape(cell.replace("-", "&#45;"))
1399
+ line += cell + "|"
1400
+ line += "\n"
1401
+ output += line
1402
+ return output + "\n"
1403
+
1404
+ def to_pandas(self, **kwargs):
1405
+ """Return a pandas DataFrame version of the table."""
1406
+ try:
1407
+ import pandas as pd
1408
+ except ModuleNotFoundError:
1409
+ message("Package 'pandas' is not installed")
1410
+ raise
1411
+
1412
+ pd_dict = {}
1413
+ extract = self.extract()
1414
+ hdr = self.header
1415
+ names = self.header.names
1416
+ hdr_len = len(names)
1417
+ # ensure uniqueness of column names
1418
+ for i in range(hdr_len):
1419
+ name = names[i]
1420
+ if not name:
1421
+ names[i] = f"Col{i}"
1422
+ if hdr_len != len(set(names)):
1423
+ for i in range(hdr_len):
1424
+ name = names[i]
1425
+ if name != f"Col{i}":
1426
+ names[i] = f"{i}-{name}"
1427
+
1428
+ if not hdr.external: # header is part of 'extract'
1429
+ extract = extract[1:]
1430
+
1431
+ for i in range(hdr_len):
1432
+ key = names[i]
1433
+ value = []
1434
+ for j in range(len(extract)):
1435
+ value.append(extract[j][i])
1436
+ pd_dict[key] = value
1437
+
1438
+ return pd.DataFrame(pd_dict)
1439
+
1440
+ def _get_header(self, y_tolerance=3):
1441
+ """Identify the table header.
1442
+
1443
+ *** PyMuPDF extension. ***
1444
+
1445
+ Starting from the first line above the table upwards, check if it
1446
+ qualifies to be part of the table header.
1447
+
1448
+ Criteria include:
1449
+ * A one-line table never has an extra header.
1450
+ * Column borders must not intersect any word. If this happens, all
1451
+ text of this line and above of it is ignored.
1452
+ * No excess inter-line distance: If a line further up has a distance
1453
+ of more than 1.5 times of its font size, it will be ignored and
1454
+ all lines above of it.
1455
+ * Must have same text properties.
1456
+ * Starting with the top table line, a bold text property cannot change
1457
+ back to non-bold.
1458
+
1459
+ If not all criteria are met (or there is no text above the table),
1460
+ the first table row is assumed to be the header.
1461
+ """
1462
+ page = self.page
1463
+ y_delta = y_tolerance
1464
+
1465
+ def top_row_is_bold(bbox):
1466
+ """Check if row 0 has bold text anywhere.
1467
+
1468
+ If this is true, then any non-bold text in lines above disqualify
1469
+ these lines as header.
1470
+
1471
+ bbox is the (potentially repaired) row 0 bbox.
1472
+
1473
+ Returns True or False
1474
+ """
1475
+ for b in page.get_text("dict", flags=TEXTFLAGS_TEXT, clip=bbox)["blocks"]:
1476
+ for l in b["lines"]:
1477
+ for s in l["spans"]:
1478
+ if s["flags"] & 16:
1479
+ return True
1480
+ return False
1481
+
1482
+ try:
1483
+ row = self.rows[0]
1484
+ cells = row.cells
1485
+ bbox = Rect(row.bbox)
1486
+ except IndexError: # this table has no rows
1487
+ return None
1488
+
1489
+ # return this if we determine that the top row is the header
1490
+ header_top_row = TableHeader(bbox, cells, self.extract()[0], False)
1491
+
1492
+ # one-line tables have no extra header
1493
+ if len(self.rows) < 2:
1494
+ return header_top_row
1495
+
1496
+ # x-ccordinates of columns between x0 and x1 of the table
1497
+ if len(cells) < 2:
1498
+ return header_top_row
1499
+
1500
+ col_x = [
1501
+ c[2] if c is not None else None for c in cells[:-1]
1502
+ ] # column (x) coordinates
1503
+
1504
+ # Special check: is top row bold?
1505
+ # If first line above table is not bold, but top-left table cell is bold,
1506
+ # we take first table row as header
1507
+ top_row_bold = top_row_is_bold(bbox)
1508
+
1509
+ # clip = area above table
1510
+ # We will inspect this area for text qualifying as column header.
1511
+ clip = +bbox # take row 0 bbox
1512
+ clip.y0 = 0 # start at top of page
1513
+ clip.y1 = bbox.y0 # end at top of table
1514
+
1515
+ spans = [] # the text spans inside clip
1516
+ for b in page.get_text("dict", clip=clip, flags=TEXTFLAGS_TEXT)["blocks"]:
1517
+ for l in b["lines"]:
1518
+ for s in l["spans"]:
1519
+ if (
1520
+ not s["flags"] & 1 and s["text"].strip()
1521
+ ): # ignore superscripts and empty text
1522
+ spans.append(s)
1523
+
1524
+ select = [] # y1 coordinates above, sorted descending
1525
+ line_heights = [] # line heights above, sorted descending
1526
+ line_bolds = [] # bold indicator per line above, same sorting
1527
+
1528
+ # spans sorted descending
1529
+ spans.sort(key=lambda s: s["bbox"][3], reverse=True)
1530
+ # walk through the spans and fill above 3 lists
1531
+ for i in range(len(spans)):
1532
+ s = spans[i]
1533
+ y1 = s["bbox"][3] # span bottom
1534
+ h = y1 - s["bbox"][1] # span bbox height
1535
+ bold = s["flags"] & 16
1536
+
1537
+ # use first item to start the lists
1538
+ if i == 0:
1539
+ select.append(y1)
1540
+ line_heights.append(h)
1541
+ line_bolds.append(bold)
1542
+ continue
1543
+
1544
+ # get last items from the 3 lists
1545
+ y0 = select[-1]
1546
+ h0 = line_heights[-1]
1547
+ bold0 = line_bolds[-1]
1548
+
1549
+ if bold0 and not bold:
1550
+ break # stop if switching from bold to non-bold
1551
+
1552
+ # if fitting in height of previous span, modify bbox
1553
+ if y0 - y1 <= y_delta or abs((y0 - h0) - s["bbox"][1]) <= y_delta:
1554
+ s["bbox"] = (s["bbox"][0], y0 - h0, s["bbox"][2], y0)
1555
+ spans[i] = s
1556
+ if bold:
1557
+ line_bolds[-1] = bold
1558
+ continue
1559
+ elif y0 - y1 > 1.5 * h0:
1560
+ break # stop if distance to previous line too large
1561
+ select.append(y1)
1562
+ line_heights.append(h)
1563
+ line_bolds.append(bold)
1564
+
1565
+ if select == []: # nothing above the table?
1566
+ return header_top_row
1567
+
1568
+ select = select[:5] # only accept up to 5 lines in any header
1569
+
1570
+ # take top row as header if text above table is too far apart
1571
+ if bbox.y0 - select[0] >= line_heights[0]:
1572
+ return header_top_row
1573
+
1574
+ # if top table row is bold, but line above is not:
1575
+ if top_row_bold and not line_bolds[0]:
1576
+ return header_top_row
1577
+
1578
+ if spans == []: # nothing left above the table, return top row
1579
+ return header_top_row
1580
+
1581
+ # re-compute clip above table
1582
+ nclip = EMPTY_RECT()
1583
+ for s in [s for s in spans if s["bbox"][3] >= select[-1]]:
1584
+ nclip |= s["bbox"]
1585
+ if not nclip.is_empty:
1586
+ clip = nclip
1587
+
1588
+ clip.y1 = bbox.y0 # make sure we still include every word above
1589
+
1590
+ # Confirm that no word in clip is intersecting a column separator
1591
+ word_rects = [Rect(w[:4]) for w in page.get_text("words", clip=clip)]
1592
+ word_tops = sorted(list(set([r[1] for r in word_rects])), reverse=True)
1593
+
1594
+ select = []
1595
+
1596
+ # exclude lines with words that intersect a column border
1597
+ for top in word_tops:
1598
+ intersecting = [
1599
+ (x, r)
1600
+ for x in col_x
1601
+ if x is not None
1602
+ for r in word_rects
1603
+ if r[1] == top and r[0] < x and r[2] > x
1604
+ ]
1605
+ if intersecting == []:
1606
+ select.append(top)
1607
+ else: # detected a word crossing a column border
1608
+ break
1609
+
1610
+ if select == []: # nothing left over: return first row
1611
+ return header_top_row
1612
+
1613
+ hdr_bbox = +clip # compute the header cells
1614
+ hdr_bbox.y0 = select[-1] # hdr_bbox top is smallest top coord of words
1615
+ hdr_cells = [
1616
+ (c[0], hdr_bbox.y0, c[2], hdr_bbox.y1) if c is not None else None
1617
+ for c in cells
1618
+ ]
1619
+
1620
+ # adjust left/right of header bbox
1621
+ hdr_bbox.x0 = self.bbox[0]
1622
+ hdr_bbox.x1 = self.bbox[2]
1623
+
1624
+ # column names: no line breaks, no excess spaces
1625
+ hdr_names = [
1626
+ (
1627
+ page.get_textbox(c).replace("\n", " ").replace(" ", " ").strip()
1628
+ if c is not None
1629
+ else ""
1630
+ )
1631
+ for c in hdr_cells
1632
+ ]
1633
+ return TableHeader(tuple(hdr_bbox), hdr_cells, hdr_names, True)
1634
+
1635
+
1636
+ @dataclass
1637
+ class TableSettings:
1638
+ vertical_strategy: str = "lines"
1639
+ horizontal_strategy: str = "lines"
1640
+ explicit_vertical_lines: list = None
1641
+ explicit_horizontal_lines: list = None
1642
+ snap_tolerance: float = DEFAULT_SNAP_TOLERANCE
1643
+ snap_x_tolerance: float = UNSET
1644
+ snap_y_tolerance: float = UNSET
1645
+ join_tolerance: float = DEFAULT_JOIN_TOLERANCE
1646
+ join_x_tolerance: float = UNSET
1647
+ join_y_tolerance: float = UNSET
1648
+ edge_min_length: float = 3
1649
+ min_words_vertical: float = DEFAULT_MIN_WORDS_VERTICAL
1650
+ min_words_horizontal: float = DEFAULT_MIN_WORDS_HORIZONTAL
1651
+ intersection_tolerance: float = 3
1652
+ intersection_x_tolerance: float = UNSET
1653
+ intersection_y_tolerance: float = UNSET
1654
+ text_settings: dict = None
1655
+
1656
+ def __post_init__(self) -> "TableSettings":
1657
+ """Clean up user-provided table settings.
1658
+
1659
+ Validates that the table settings provided consists of acceptable values and
1660
+ returns a cleaned up version. The cleaned up version fills out the missing
1661
+ values with the default values in the provided settings.
1662
+
1663
+ TODO: Can be further used to validate that the values are of the correct
1664
+ type. For example, raising a value error when a non-boolean input is
1665
+ provided for the key ``keep_blank_chars``.
1666
+
1667
+ :param table_settings: User-provided table settings.
1668
+ :returns: A cleaned up version of the user-provided table settings.
1669
+ :raises ValueError: When an unrecognised key is provided.
1670
+ """
1671
+
1672
+ for setting in NON_NEGATIVE_SETTINGS:
1673
+ if (getattr(self, setting) or 0) < 0:
1674
+ raise ValueError(f"Table setting '{setting}' cannot be negative")
1675
+
1676
+ for orientation in ["horizontal", "vertical"]:
1677
+ strategy = getattr(self, orientation + "_strategy")
1678
+ if strategy not in TABLE_STRATEGIES:
1679
+ raise ValueError(
1680
+ f"{orientation}_strategy must be one of"
1681
+ f'{{{",".join(TABLE_STRATEGIES)}}}'
1682
+ )
1683
+
1684
+ if self.text_settings is None:
1685
+ self.text_settings = {}
1686
+
1687
+ # This next section is for backwards compatibility
1688
+ for attr in ["x_tolerance", "y_tolerance"]:
1689
+ if attr not in self.text_settings:
1690
+ self.text_settings[attr] = self.text_settings.get("tolerance", 3)
1691
+
1692
+ if "tolerance" in self.text_settings:
1693
+ del self.text_settings["tolerance"]
1694
+ # End of that section
1695
+
1696
+ for attr, fallback in [
1697
+ ("snap_x_tolerance", "snap_tolerance"),
1698
+ ("snap_y_tolerance", "snap_tolerance"),
1699
+ ("join_x_tolerance", "join_tolerance"),
1700
+ ("join_y_tolerance", "join_tolerance"),
1701
+ ("intersection_x_tolerance", "intersection_tolerance"),
1702
+ ("intersection_y_tolerance", "intersection_tolerance"),
1703
+ ]:
1704
+ if getattr(self, attr) is UNSET:
1705
+ setattr(self, attr, getattr(self, fallback))
1706
+
1707
+ return self
1708
+
1709
+ @classmethod
1710
+ def resolve(cls, settings=None):
1711
+ if settings is None:
1712
+ return cls()
1713
+ elif isinstance(settings, cls):
1714
+ return settings
1715
+ elif isinstance(settings, dict):
1716
+ core_settings = {}
1717
+ text_settings = {}
1718
+ for k, v in settings.items():
1719
+ if k[:5] == "text_":
1720
+ text_settings[k[5:]] = v
1721
+ else:
1722
+ core_settings[k] = v
1723
+ core_settings["text_settings"] = text_settings
1724
+ return cls(**core_settings)
1725
+ else:
1726
+ raise ValueError(f"Cannot resolve settings: {settings}")
1727
+
1728
+
1729
+ class TableFinder:
1730
+ """
1731
+ Given a PDF page, find plausible table structures.
1732
+
1733
+ Largely borrowed from Anssi Nurminen's master's thesis:
1734
+ http://dspace.cc.tut.fi/dpub/bitstream/handle/123456789/21520/Nurminen.pdf?sequence=3
1735
+
1736
+ ... and inspired by Tabula:
1737
+ https://github.com/tabulapdf/tabula-extractor/issues/16
1738
+ """
1739
+
1740
+ def __init__(self, page, settings=None):
1741
+ self.page = page
1742
+ self.settings = TableSettings.resolve(settings)
1743
+ self.edges = self.get_edges()
1744
+ self.intersections = edges_to_intersections(
1745
+ self.edges,
1746
+ self.settings.intersection_x_tolerance,
1747
+ self.settings.intersection_y_tolerance,
1748
+ )
1749
+ self.cells = intersections_to_cells(self.intersections)
1750
+ self.tables = [
1751
+ Table(self.page, cell_group)
1752
+ for cell_group in cells_to_tables(self.page, self.cells)
1753
+ ]
1754
+
1755
+ def get_edges(self) -> list:
1756
+ settings = self.settings
1757
+
1758
+ for orientation in ["vertical", "horizontal"]:
1759
+ strategy = getattr(settings, orientation + "_strategy")
1760
+ if strategy == "explicit":
1761
+ lines = getattr(settings, "explicit_" + orientation + "_lines")
1762
+ if len(lines) < 2:
1763
+ raise ValueError(
1764
+ f"If {orientation}_strategy == 'explicit', "
1765
+ f"explicit_{orientation}_lines "
1766
+ f"must be specified as a list/tuple of two or more "
1767
+ f"floats/ints."
1768
+ )
1769
+
1770
+ v_strat = settings.vertical_strategy
1771
+ h_strat = settings.horizontal_strategy
1772
+
1773
+ if v_strat == "text" or h_strat == "text":
1774
+ words = extract_words(CHARS, **(settings.text_settings or {}))
1775
+ else:
1776
+ words = []
1777
+
1778
+ v_explicit = []
1779
+ for desc in settings.explicit_vertical_lines or []:
1780
+ if isinstance(desc, dict):
1781
+ for e in obj_to_edges(desc):
1782
+ if e["orientation"] == "v":
1783
+ v_explicit.append(e)
1784
+ else:
1785
+ v_explicit.append(
1786
+ {
1787
+ "x0": desc,
1788
+ "x1": desc,
1789
+ "top": self.page.rect[1],
1790
+ "bottom": self.page.rect[3],
1791
+ "height": self.page.rect[3] - self.page.rect[1],
1792
+ "orientation": "v",
1793
+ }
1794
+ )
1795
+
1796
+ if v_strat == "lines":
1797
+ v_base = filter_edges(EDGES, "v")
1798
+ elif v_strat == "lines_strict":
1799
+ v_base = filter_edges(EDGES, "v", edge_type="line")
1800
+ elif v_strat == "text":
1801
+ v_base = words_to_edges_v(words, word_threshold=settings.min_words_vertical)
1802
+ elif v_strat == "explicit":
1803
+ v_base = []
1804
+ else:
1805
+ v_base = []
1806
+
1807
+ v = v_base + v_explicit
1808
+
1809
+ h_explicit = []
1810
+ for desc in settings.explicit_horizontal_lines or []:
1811
+ if isinstance(desc, dict):
1812
+ for e in obj_to_edges(desc):
1813
+ if e["orientation"] == "h":
1814
+ h_explicit.append(e)
1815
+ else:
1816
+ h_explicit.append(
1817
+ {
1818
+ "x0": self.page.rect[0],
1819
+ "x1": self.page.rect[2],
1820
+ "width": self.page.rect[2] - self.page.rect[0],
1821
+ "top": desc,
1822
+ "bottom": desc,
1823
+ "orientation": "h",
1824
+ }
1825
+ )
1826
+
1827
+ if h_strat == "lines":
1828
+ h_base = filter_edges(EDGES, "h")
1829
+ elif h_strat == "lines_strict":
1830
+ h_base = filter_edges(EDGES, "h", edge_type="line")
1831
+ elif h_strat == "text":
1832
+ h_base = words_to_edges_h(
1833
+ words, word_threshold=settings.min_words_horizontal
1834
+ )
1835
+ elif h_strat == "explicit":
1836
+ h_base = []
1837
+ else:
1838
+ h_base = []
1839
+
1840
+ h = h_base + h_explicit
1841
+
1842
+ edges = list(v) + list(h)
1843
+
1844
+ edges = merge_edges(
1845
+ edges,
1846
+ snap_x_tolerance=settings.snap_x_tolerance,
1847
+ snap_y_tolerance=settings.snap_y_tolerance,
1848
+ join_x_tolerance=settings.join_x_tolerance,
1849
+ join_y_tolerance=settings.join_y_tolerance,
1850
+ )
1851
+
1852
+ return filter_edges(edges, min_length=settings.edge_min_length)
1853
+
1854
+ def __getitem__(self, i):
1855
+ tcount = len(self.tables)
1856
+ if i >= tcount:
1857
+ raise IndexError("table not on page")
1858
+ while i < 0:
1859
+ i += tcount
1860
+ return self.tables[i]
1861
+
1862
+
1863
+ """
1864
+ Start of PyMuPDF interface code.
1865
+ The following functions are executed when "page.find_tables()" is called.
1866
+
1867
+ * make_chars: Fills the CHARS list with text character information extracted
1868
+ via "rawdict" text extraction. Items in CHARS are formatted
1869
+ as expected by the table code.
1870
+ * make_edges: Fills the EDGES list with vector graphic information extracted
1871
+ via "get_drawings". Items in EDGES are formatted as expected
1872
+ by the table code.
1873
+
1874
+ The lists CHARS and EDGES are used to replace respective document access
1875
+ of pdfplumber or, respectively pdfminer.
1876
+ The table code has been modified to use these lists instead of accessing
1877
+ page information themselves.
1878
+ """
1879
+
1880
+
1881
+ # -----------------------------------------------------------------------------
1882
+ # Extract all page characters to fill the CHARS list
1883
+ # -----------------------------------------------------------------------------
1884
+ def make_chars(page, clip=None):
1885
+ """Extract text as "rawdict" to fill CHARS."""
1886
+ global CHARS, TEXTPAGE
1887
+ page_number = page.number + 1
1888
+ page_height = page.rect.height
1889
+ ctm = page.transformation_matrix
1890
+ TEXTPAGE = page.get_textpage(clip=clip, flags=TEXTFLAGS_TEXT)
1891
+ blocks = page.get_text("rawdict", textpage=TEXTPAGE)["blocks"]
1892
+ doctop_base = page_height * page.number
1893
+ for block in blocks:
1894
+ for line in block["lines"]:
1895
+ ldir = line["dir"] # = (cosine, sine) of angle
1896
+ ldir = (round(ldir[0], 4), round(ldir[1], 4))
1897
+ matrix = Matrix(ldir[0], -ldir[1], ldir[1], ldir[0], 0, 0)
1898
+ if ldir[1] == 0:
1899
+ upright = True
1900
+ else:
1901
+ upright = False
1902
+ for span in sorted(line["spans"], key=lambda s: s["bbox"][0]):
1903
+ fontname = span["font"]
1904
+ fontsize = span["size"]
1905
+ color = sRGB_to_pdf(span["color"])
1906
+ for char in sorted(span["chars"], key=lambda c: c["bbox"][0]):
1907
+ bbox = Rect(char["bbox"])
1908
+ bbox_ctm = bbox * ctm
1909
+ origin = Point(char["origin"]) * ctm
1910
+ matrix.e = origin.x
1911
+ matrix.f = origin.y
1912
+ text = char["c"]
1913
+ char_dict = {
1914
+ "adv": bbox.x1 - bbox.x0 if upright else bbox.y1 - bbox.y0,
1915
+ "bottom": bbox.y1,
1916
+ "doctop": bbox.y0 + doctop_base,
1917
+ "fontname": fontname,
1918
+ "height": bbox.y1 - bbox.y0,
1919
+ "matrix": tuple(matrix),
1920
+ "ncs": "DeviceRGB",
1921
+ "non_stroking_color": color,
1922
+ "non_stroking_pattern": None,
1923
+ "object_type": "char",
1924
+ "page_number": page_number,
1925
+ "size": fontsize if upright else bbox.y1 - bbox.y0,
1926
+ "stroking_color": color,
1927
+ "stroking_pattern": None,
1928
+ "text": text,
1929
+ "top": bbox.y0,
1930
+ "upright": upright,
1931
+ "width": bbox.x1 - bbox.x0,
1932
+ "x0": bbox.x0,
1933
+ "x1": bbox.x1,
1934
+ "y0": bbox_ctm.y0,
1935
+ "y1": bbox_ctm.y1,
1936
+ }
1937
+ CHARS.append(char_dict)
1938
+
1939
+
1940
+ # ------------------------------------------------------------------------
1941
+ # Extract all page vector graphics to fill the EDGES list.
1942
+ # We are ignoring Bézier curves completely and are converting everything
1943
+ # else to lines.
1944
+ # ------------------------------------------------------------------------
1945
+ def make_edges(page, clip=None, tset=None, add_lines=None):
1946
+ global EDGES
1947
+ snap_x = tset.snap_x_tolerance
1948
+ snap_y = tset.snap_y_tolerance
1949
+ min_length = tset.edge_min_length
1950
+ lines_strict = (
1951
+ tset.vertical_strategy == "lines_strict"
1952
+ or tset.horizontal_strategy == "lines_strict"
1953
+ )
1954
+ page_height = page.rect.height
1955
+ doctop_basis = page.number * page_height
1956
+ page_number = page.number + 1
1957
+ prect = page.rect
1958
+ if page.rotation in (90, 270):
1959
+ w, h = prect.br
1960
+ prect = Rect(0, 0, h, w)
1961
+ if clip is not None:
1962
+ clip = Rect(clip)
1963
+ else:
1964
+ clip = prect
1965
+
1966
+ def are_neighbors(r1, r2):
1967
+ """Detect whether r1, r2 are neighbors.
1968
+
1969
+ Defined as:
1970
+ The minimum distance between points of r1 and points of r2 is not
1971
+ larger than some delta.
1972
+
1973
+ This check supports empty rect-likes and thus also lines.
1974
+
1975
+ Note:
1976
+ This type of check is MUCH faster than native Rect containment checks.
1977
+ """
1978
+ if ( # check if x-coordinates of r1 are within those of r2
1979
+ r2.x0 - snap_x <= r1.x0 <= r2.x1 + snap_x
1980
+ or r2.x0 - snap_x <= r1.x1 <= r2.x1 + snap_x
1981
+ ) and ( # ... same for y-coordinates
1982
+ r2.y0 - snap_y <= r1.y0 <= r2.y1 + snap_y
1983
+ or r2.y0 - snap_y <= r1.y1 <= r2.y1 + snap_y
1984
+ ):
1985
+ return True
1986
+
1987
+ # same check with r1 / r2 exchanging their roles (this is necessary!)
1988
+ if (
1989
+ r1.x0 - snap_x <= r2.x0 <= r1.x1 + snap_x
1990
+ or r1.x0 - snap_x <= r2.x1 <= r1.x1 + snap_x
1991
+ ) and (
1992
+ r1.y0 - snap_y <= r2.y0 <= r1.y1 + snap_y
1993
+ or r1.y0 - snap_y <= r2.y1 <= r1.y1 + snap_y
1994
+ ):
1995
+ return True
1996
+ return False
1997
+
1998
+ def clean_graphics():
1999
+ """Detect and join rectangles of "connected" vector graphics."""
2000
+
2001
+ paths = [] # paths relevant for table detection
2002
+ for p in page.get_drawings():
2003
+ # ignore fill-only graphics if they do not simulate lines,
2004
+ # which means one of width or height are small.
2005
+ if (
2006
+ p["type"] == "f"
2007
+ and lines_strict
2008
+ and p["rect"].width > snap_x
2009
+ and p["rect"].height > snap_y
2010
+ ):
2011
+ continue
2012
+ paths.append(p)
2013
+
2014
+ # start with all vector graphics rectangles
2015
+ prects = sorted(set([p["rect"] for p in paths]), key=lambda r: (r.y1, r.x0))
2016
+ new_rects = [] # the final list of joined rectangles
2017
+ # ----------------------------------------------------------------
2018
+ # Strategy: Join rectangles that "almost touch" each other.
2019
+ # Extend first rectangle with any other that is a "neighbor".
2020
+ # Then move it to the final list and continue with the rest.
2021
+ # ----------------------------------------------------------------
2022
+ while prects: # the algorithm will empty this list
2023
+ prect0 = prects[0] # copy of first rectangle (performance reasons!)
2024
+ repeat = True
2025
+ while repeat: # this loop extends first rect in list
2026
+ repeat = False # set to true again if some other rect touches
2027
+ for i in range(len(prects) - 1, 0, -1): # run backwards
2028
+ if are_neighbors(prect0, prects[i]): # close enough to rect 0?
2029
+ prect0 |= prects[i].tl # extend rect 0
2030
+ prect0 |= prects[i].br # extend rect 0
2031
+ del prects[i] # delete this rect
2032
+ repeat = True # keep checking the rest
2033
+
2034
+ # move rect 0 over to result list if there is some text in it
2035
+ if not white_spaces.issuperset(page.get_textbox(prect0, textpage=TEXTPAGE)):
2036
+ # contains text, so accept it as a table bbox candidate
2037
+ new_rects.append(prect0)
2038
+ del prects[0] # remove from rect list
2039
+
2040
+ return new_rects, paths
2041
+
2042
+ bboxes, paths = clean_graphics()
2043
+
2044
+ def is_parallel(p1, p2):
2045
+ """Check if line is roughly axis-parallel."""
2046
+ if abs(p1.x - p2.x) <= snap_x or abs(p1.y - p2.y) <= snap_y:
2047
+ return True
2048
+ return False
2049
+
2050
+ def make_line(p, p1, p2, clip):
2051
+ """Given 2 points, make a line dictionary for table detection."""
2052
+ if not is_parallel(p1, p2): # only accepting axis-parallel lines
2053
+ return {}
2054
+ # compute the extremal values
2055
+ x0 = min(p1.x, p2.x)
2056
+ x1 = max(p1.x, p2.x)
2057
+ y0 = min(p1.y, p2.y)
2058
+ y1 = max(p1.y, p2.y)
2059
+
2060
+ # check for outside clip
2061
+ if x0 > clip.x1 or x1 < clip.x0 or y0 > clip.y1 or y1 < clip.y0:
2062
+ return {}
2063
+
2064
+ if x0 < clip.x0:
2065
+ x0 = clip.x0 # adjust to clip boundary
2066
+
2067
+ if x1 > clip.x1:
2068
+ x1 = clip.x1 # adjust to clip boundary
2069
+
2070
+ if y0 < clip.y0:
2071
+ y0 = clip.y0 # adjust to clip boundary
2072
+
2073
+ if y1 > clip.y1:
2074
+ y1 = clip.y1 # adjust to clip boundary
2075
+
2076
+ width = x1 - x0 # from adjusted values
2077
+ height = y1 - y0 # from adjusted values
2078
+ if width == height == 0:
2079
+ return {} # nothing left to deal with
2080
+ line_dict = {
2081
+ "x0": x0,
2082
+ "y0": page_height - y0,
2083
+ "x1": x1,
2084
+ "y1": page_height - y1,
2085
+ "width": width,
2086
+ "height": height,
2087
+ "pts": [(x0, y0), (x1, y1)],
2088
+ "linewidth": p["width"],
2089
+ "stroke": True,
2090
+ "fill": False,
2091
+ "evenodd": False,
2092
+ "stroking_color": p["color"] if p["color"] else p["fill"],
2093
+ "non_stroking_color": None,
2094
+ "object_type": "line",
2095
+ "page_number": page_number,
2096
+ "stroking_pattern": None,
2097
+ "non_stroking_pattern": None,
2098
+ "top": y0,
2099
+ "bottom": y1,
2100
+ "doctop": y0 + doctop_basis,
2101
+ }
2102
+ return line_dict
2103
+
2104
+ for p in paths:
2105
+ items = p["items"] # items in this path
2106
+
2107
+ # if 'closePath', add a line from last to first point
2108
+ if p["closePath"] and items[0][0] == "l" and items[-1][0] == "l":
2109
+ items.append(("l", items[-1][2], items[0][1]))
2110
+
2111
+ for i in items:
2112
+ if i[0] not in ("l", "re", "qu"):
2113
+ continue # ignore anything else
2114
+
2115
+ if i[0] == "l": # a line
2116
+ p1, p2 = i[1:]
2117
+ line_dict = make_line(p, p1, p2, clip)
2118
+ if line_dict:
2119
+ EDGES.append(line_to_edge(line_dict))
2120
+
2121
+ elif i[0] == "re":
2122
+ # A rectangle: decompose into 4 lines, but filter out
2123
+ # the ones that simulate a line
2124
+ rect = i[1].normalize() # normalize the rectangle
2125
+
2126
+ if (
2127
+ rect.width <= min_length and rect.width < rect.height
2128
+ ): # simulates a vertical line
2129
+ x = abs(rect.x1 + rect.x0) / 2 # take middle value for x
2130
+ p1 = Point(x, rect.y0)
2131
+ p2 = Point(x, rect.y1)
2132
+ line_dict = make_line(p, p1, p2, clip)
2133
+ if line_dict:
2134
+ EDGES.append(line_to_edge(line_dict))
2135
+ continue
2136
+
2137
+ if (
2138
+ rect.height <= min_length and rect.height < rect.width
2139
+ ): # simulates a horizontal line
2140
+ y = abs(rect.y1 + rect.y0) / 2 # take middle value for y
2141
+ p1 = Point(rect.x0, y)
2142
+ p2 = Point(rect.x1, y)
2143
+ line_dict = make_line(p, p1, p2, clip)
2144
+ if line_dict:
2145
+ EDGES.append(line_to_edge(line_dict))
2146
+ continue
2147
+
2148
+ line_dict = make_line(p, rect.tl, rect.bl, clip)
2149
+ if line_dict:
2150
+ EDGES.append(line_to_edge(line_dict))
2151
+
2152
+ line_dict = make_line(p, rect.bl, rect.br, clip)
2153
+ if line_dict:
2154
+ EDGES.append(line_to_edge(line_dict))
2155
+
2156
+ line_dict = make_line(p, rect.br, rect.tr, clip)
2157
+ if line_dict:
2158
+ EDGES.append(line_to_edge(line_dict))
2159
+
2160
+ line_dict = make_line(p, rect.tr, rect.tl, clip)
2161
+ if line_dict:
2162
+ EDGES.append(line_to_edge(line_dict))
2163
+
2164
+ else: # must be a quad
2165
+ # we convert it into (up to) 4 lines
2166
+ ul, ur, ll, lr = i[1]
2167
+
2168
+ line_dict = make_line(p, ul, ll, clip)
2169
+ if line_dict:
2170
+ EDGES.append(line_to_edge(line_dict))
2171
+
2172
+ line_dict = make_line(p, ll, lr, clip)
2173
+ if line_dict:
2174
+ EDGES.append(line_to_edge(line_dict))
2175
+
2176
+ line_dict = make_line(p, lr, ur, clip)
2177
+ if line_dict:
2178
+ EDGES.append(line_to_edge(line_dict))
2179
+
2180
+ line_dict = make_line(p, ur, ul, clip)
2181
+ if line_dict:
2182
+ EDGES.append(line_to_edge(line_dict))
2183
+
2184
+ path = {"color": (0, 0, 0), "fill": None, "width": 1}
2185
+ for bbox in bboxes: # add the border lines for all enveloping bboxes
2186
+ line_dict = make_line(path, bbox.tl, bbox.tr, clip)
2187
+ if line_dict:
2188
+ EDGES.append(line_to_edge(line_dict))
2189
+
2190
+ line_dict = make_line(path, bbox.bl, bbox.br, clip)
2191
+ if line_dict:
2192
+ EDGES.append(line_to_edge(line_dict))
2193
+
2194
+ line_dict = make_line(path, bbox.tl, bbox.bl, clip)
2195
+ if line_dict:
2196
+ EDGES.append(line_to_edge(line_dict))
2197
+
2198
+ line_dict = make_line(path, bbox.tr, bbox.br, clip)
2199
+ if line_dict:
2200
+ EDGES.append(line_to_edge(line_dict))
2201
+
2202
+ if add_lines is not None: # add user-specified lines
2203
+ assert isinstance(add_lines, (tuple, list))
2204
+ else:
2205
+ add_lines = []
2206
+ for p1, p2 in add_lines:
2207
+ p1 = Point(p1)
2208
+ p2 = Point(p2)
2209
+ line_dict = make_line(path, p1, p2, clip)
2210
+ if line_dict:
2211
+ EDGES.append(line_to_edge(line_dict))
2212
+
2213
+
2214
+ def page_rotation_set0(page):
2215
+ """Nullify page rotation.
2216
+
2217
+ To correctly detect tables, page rotation must be zero.
2218
+ This function performs the necessary adjustments and returns information
2219
+ for reverting this changes.
2220
+ """
2221
+ mediabox = page.mediabox
2222
+ rot = page.rotation # contains normalized rotation value
2223
+ # need to derotate the page's content
2224
+ mb = page.mediabox # current mediabox
2225
+
2226
+ if rot == 90:
2227
+ # before derotation, shift content horizontally
2228
+ mat0 = Matrix(1, 0, 0, 1, mb.y1 - mb.x1 - mb.x0 - mb.y0, 0)
2229
+ elif rot == 270:
2230
+ # before derotation, shift content vertically
2231
+ mat0 = Matrix(1, 0, 0, 1, 0, mb.x1 - mb.y1 - mb.y0 - mb.x0)
2232
+ else:
2233
+ mat0 = Matrix(1, 0, 0, 1, -2 * mb.x0, -2 * mb.y0)
2234
+
2235
+ # prefix with derotation matrix
2236
+ mat = mat0 * page.derotation_matrix
2237
+ cmd = b"%g %g %g %g %g %g cm " % tuple(mat)
2238
+ xref = TOOLS._insert_contents(page, cmd, 0)
2239
+
2240
+ # swap x- and y-coordinates
2241
+ if rot in (90, 270):
2242
+ x0, y0, x1, y1 = mb
2243
+ mb.x0 = y0
2244
+ mb.y0 = x0
2245
+ mb.x1 = y1
2246
+ mb.y1 = x1
2247
+ page.set_mediabox(mb)
2248
+
2249
+ page.set_rotation(0)
2250
+
2251
+ # refresh the page to apply these changes
2252
+ doc = page.parent
2253
+ pno = page.number
2254
+ page = doc[pno]
2255
+ return page, xref, rot, mediabox
2256
+
2257
+
2258
+ def page_rotation_reset(page, xref, rot, mediabox):
2259
+ """Reset page rotation to original values.
2260
+
2261
+ To be used before we return tables."""
2262
+ doc = page.parent # document of the page
2263
+ doc.update_stream(xref, b" ") # remove de-rotation matrix
2264
+ page.set_mediabox(mediabox) # set mediabox to old value
2265
+ page.set_rotation(rot) # set rotation to old value
2266
+ pno = page.number
2267
+ page = doc[pno] # update page info
2268
+ return page
2269
+
2270
+
2271
+ def find_tables(
2272
+ page,
2273
+ clip=None,
2274
+ vertical_strategy: str = "lines",
2275
+ horizontal_strategy: str = "lines",
2276
+ vertical_lines: list = None,
2277
+ horizontal_lines: list = None,
2278
+ snap_tolerance: float = DEFAULT_SNAP_TOLERANCE,
2279
+ snap_x_tolerance: float = None,
2280
+ snap_y_tolerance: float = None,
2281
+ join_tolerance: float = DEFAULT_JOIN_TOLERANCE,
2282
+ join_x_tolerance: float = None,
2283
+ join_y_tolerance: float = None,
2284
+ edge_min_length: float = 3,
2285
+ min_words_vertical: float = DEFAULT_MIN_WORDS_VERTICAL,
2286
+ min_words_horizontal: float = DEFAULT_MIN_WORDS_HORIZONTAL,
2287
+ intersection_tolerance: float = 3,
2288
+ intersection_x_tolerance: float = None,
2289
+ intersection_y_tolerance: float = None,
2290
+ text_tolerance=3,
2291
+ text_x_tolerance=3,
2292
+ text_y_tolerance=3,
2293
+ strategy=None, # offer abbreviation
2294
+ add_lines=None, # optional user-specified lines
2295
+ ):
2296
+ global CHARS, EDGES
2297
+ CHARS = []
2298
+ EDGES = []
2299
+ old_small = bool(TOOLS.set_small_glyph_heights()) # save old value
2300
+ TOOLS.set_small_glyph_heights(True) # we need minimum bboxes
2301
+ if page.rotation != 0:
2302
+ page, old_xref, old_rot, old_mediabox = page_rotation_set0(page)
2303
+ else:
2304
+ old_xref, old_rot, old_mediabox = None, None, None
2305
+
2306
+ if snap_x_tolerance is None:
2307
+ snap_x_tolerance = UNSET
2308
+ if snap_y_tolerance is None:
2309
+ snap_y_tolerance = UNSET
2310
+ if join_x_tolerance is None:
2311
+ join_x_tolerance = UNSET
2312
+ if join_y_tolerance is None:
2313
+ join_y_tolerance = UNSET
2314
+ if intersection_x_tolerance is None:
2315
+ intersection_x_tolerance = UNSET
2316
+ if intersection_y_tolerance is None:
2317
+ intersection_y_tolerance = UNSET
2318
+ if strategy is not None:
2319
+ vertical_strategy = strategy
2320
+ horizontal_strategy = strategy
2321
+
2322
+ settings = {
2323
+ "vertical_strategy": vertical_strategy,
2324
+ "horizontal_strategy": horizontal_strategy,
2325
+ "explicit_vertical_lines": vertical_lines,
2326
+ "explicit_horizontal_lines": horizontal_lines,
2327
+ "snap_tolerance": snap_tolerance,
2328
+ "snap_x_tolerance": snap_x_tolerance,
2329
+ "snap_y_tolerance": snap_y_tolerance,
2330
+ "join_tolerance": join_tolerance,
2331
+ "join_x_tolerance": join_x_tolerance,
2332
+ "join_y_tolerance": join_y_tolerance,
2333
+ "edge_min_length": edge_min_length,
2334
+ "min_words_vertical": min_words_vertical,
2335
+ "min_words_horizontal": min_words_horizontal,
2336
+ "intersection_tolerance": intersection_tolerance,
2337
+ "intersection_x_tolerance": intersection_x_tolerance,
2338
+ "intersection_y_tolerance": intersection_y_tolerance,
2339
+ "text_tolerance": text_tolerance,
2340
+ "text_x_tolerance": text_x_tolerance,
2341
+ "text_y_tolerance": text_y_tolerance,
2342
+ }
2343
+ tset = TableSettings.resolve(settings=settings)
2344
+ page.table_settings = tset
2345
+
2346
+ make_chars(page, clip=clip) # create character list of page
2347
+ make_edges(
2348
+ page, clip=clip, tset=tset, add_lines=add_lines
2349
+ ) # create lines and curves
2350
+ tables = TableFinder(page, settings=tset)
2351
+
2352
+ TOOLS.set_small_glyph_heights(old_small)
2353
+ if old_xref is not None:
2354
+ page = page_rotation_reset(page, old_xref, old_rot, old_mediabox)
2355
+ return tables