magic-pdf 0.5.8__py3-none-any.whl → 0.5.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- magic_pdf/cli/magicpdf.py +9 -10
- magic_pdf/libs/version.py +1 -1
- magic_pdf/para/para_split_v2.py +43 -28
- magic_pdf/pdf_parse_by_ocr.py +18 -219
- magic_pdf/pdf_parse_by_txt.py +10 -401
- magic_pdf/pre_proc/equations_replace.py +1 -1
- magic_pdf/user_api.py +3 -41
- magic_pdf-0.5.10.dist-info/METADATA +133 -0
- {magic_pdf-0.5.8.dist-info → magic_pdf-0.5.10.dist-info}/RECORD +13 -14
- magic_pdf-0.5.10.dist-info/entry_points.txt +2 -0
- magic_pdf/pdf_parse_by_ocr_v2.py +0 -17
- magic_pdf/pdf_parse_by_txt_v2.py +0 -56
- magic_pdf-0.5.8.dist-info/METADATA +0 -28
- {magic_pdf-0.5.8.dist-info → magic_pdf-0.5.10.dist-info}/LICENSE.md +0 -0
- {magic_pdf-0.5.8.dist-info → magic_pdf-0.5.10.dist-info}/WHEEL +0 -0
- {magic_pdf-0.5.8.dist-info → magic_pdf-0.5.10.dist-info}/top_level.txt +0 -0
magic_pdf/cli/magicpdf.py
CHANGED
@@ -17,8 +17,8 @@
|
|
17
17
|
|
18
18
|
|
19
19
|
效果:
|
20
|
-
python magicpdf.py --json s3://llm-pdf-text/scihub/xxxx.json?bytes=0,81350
|
21
|
-
python magicpdf.py --pdf /home/llm/Downloads/xxxx.pdf --model /home/llm/Downloads/xxxx.json 或者 python magicpdf.py --pdf /home/llm/Downloads/xxxx.pdf
|
20
|
+
python magicpdf.py json-command --json s3://llm-pdf-text/scihub/xxxx.json?bytes=0,81350
|
21
|
+
python magicpdf.py pdf-command --pdf /home/llm/Downloads/xxxx.pdf --model /home/llm/Downloads/xxxx.json 或者 python magicpdf.py --pdf /home/llm/Downloads/xxxx.pdf
|
22
22
|
"""
|
23
23
|
|
24
24
|
import os
|
@@ -45,6 +45,7 @@ from magic_pdf.rw.S3ReaderWriter import S3ReaderWriter
|
|
45
45
|
from magic_pdf.rw.DiskReaderWriter import DiskReaderWriter
|
46
46
|
from magic_pdf.rw.AbsReaderWriter import AbsReaderWriter
|
47
47
|
import csv
|
48
|
+
import copy
|
48
49
|
|
49
50
|
parse_pdf_methods = click.Choice(["ocr", "txt", "auto"])
|
50
51
|
|
@@ -81,10 +82,11 @@ def do_parse(
|
|
81
82
|
f_dump_orig_pdf=True,
|
82
83
|
f_dump_content_list=True,
|
83
84
|
):
|
85
|
+
orig_model_list = copy.deepcopy(model_list)
|
84
86
|
|
85
87
|
local_image_dir, local_md_dir = prepare_env(pdf_file_name, parse_method)
|
86
88
|
image_writer, md_writer = DiskReaderWriter(local_image_dir), DiskReaderWriter(local_md_dir)
|
87
|
-
image_dir = (os.path.basename(local_image_dir)
|
89
|
+
image_dir = str(os.path.basename(local_image_dir))
|
88
90
|
|
89
91
|
if parse_method == "auto":
|
90
92
|
jso_useful_key = {"_pdf_type": "", "model_list": model_list}
|
@@ -110,10 +112,7 @@ def do_parse(
|
|
110
112
|
if f_draw_span_bbox:
|
111
113
|
draw_span_bbox(pdf_info, pdf_bytes, local_md_dir)
|
112
114
|
|
113
|
-
|
114
|
-
# [pdf_file_name, pipe.pdf_mid_data['not_common_character_rate'], pipe.pdf_mid_data['not_printable_rate']])
|
115
|
-
|
116
|
-
md_content = pipe.pipe_mk_markdown(str(image_dir), drop_mode=DropMode.NONE)
|
115
|
+
md_content = pipe.pipe_mk_markdown(image_dir, drop_mode=DropMode.NONE)
|
117
116
|
if f_dump_md:
|
118
117
|
"""写markdown"""
|
119
118
|
md_writer.write(
|
@@ -133,7 +132,7 @@ def do_parse(
|
|
133
132
|
if f_dump_model_json:
|
134
133
|
"""写model_json"""
|
135
134
|
md_writer.write(
|
136
|
-
content=json_parse.dumps(
|
135
|
+
content=json_parse.dumps(orig_model_list, ensure_ascii=False, indent=4),
|
137
136
|
path=f"{pdf_file_name}_model.json",
|
138
137
|
mode=AbsReaderWriter.MODE_TXT,
|
139
138
|
)
|
@@ -146,7 +145,7 @@ def do_parse(
|
|
146
145
|
mode=AbsReaderWriter.MODE_BIN,
|
147
146
|
)
|
148
147
|
|
149
|
-
content_list = pipe.pipe_mk_uni_format(
|
148
|
+
content_list = pipe.pipe_mk_uni_format(image_dir, drop_mode=DropMode.NONE)
|
150
149
|
if f_dump_content_list:
|
151
150
|
"""写content_list"""
|
152
151
|
md_writer.write(
|
@@ -281,7 +280,7 @@ def pdf_command(pdf, model, method):
|
|
281
280
|
model_path = pdf.replace(".pdf", ".json")
|
282
281
|
if not os.path.exists(model_path):
|
283
282
|
logger.warning(
|
284
|
-
f"not found json {model_path} existed
|
283
|
+
f"not found json {model_path} existed"
|
285
284
|
)
|
286
285
|
# 本地无模型数据则调用内置paddle分析,先传空list,在内部识别到空list再调用paddle
|
287
286
|
model_json = "[]"
|
magic_pdf/libs/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.5.
|
1
|
+
__version__ = "0.5.10"
|
magic_pdf/para/para_split_v2.py
CHANGED
@@ -11,6 +11,7 @@ LINE_STOP_FLAG = ['.', '!', '?', '。', '!', '?', ":", ":", ")", ")", ";
|
|
11
11
|
INLINE_EQUATION = ContentType.InlineEquation
|
12
12
|
INTERLINE_EQUATION = ContentType.InterlineEquation
|
13
13
|
TEXT = ContentType.Text
|
14
|
+
debug_able = False
|
14
15
|
|
15
16
|
|
16
17
|
def __get_span_text(span):
|
@@ -22,6 +23,7 @@ def __get_span_text(span):
|
|
22
23
|
|
23
24
|
|
24
25
|
def __detect_list_lines(lines, new_layout_bboxes, lang):
|
26
|
+
global debug_able
|
25
27
|
"""
|
26
28
|
探测是否包含了列表,并且把列表的行分开.
|
27
29
|
这样的段落特点是,顶格字母大写/数字,紧跟着几行缩进的。缩进的行首字母含小写的。
|
@@ -41,12 +43,14 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
|
|
41
43
|
if lst[i] == 1:
|
42
44
|
ones_in_this_interval.append(i)
|
43
45
|
i += 1
|
44
|
-
if len(ones_in_this_interval) > 1 or (
|
46
|
+
if len(ones_in_this_interval) > 1 or (
|
47
|
+
start < len(lst) - 1 and ones_in_this_interval and lst[start + 1] in [2, 3]):
|
45
48
|
indices.append((start, i - 1))
|
46
49
|
ones_indices.append(ones_in_this_interval)
|
47
50
|
else:
|
48
51
|
i += 1
|
49
52
|
return indices, ones_indices
|
53
|
+
|
50
54
|
def find_repeating_patterns(lst):
|
51
55
|
indices = []
|
52
56
|
ones_indices = []
|
@@ -132,7 +136,8 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
|
|
132
136
|
|
133
137
|
list_indice, list_start_idx = find_repeating_patterns2(line_fea_encode)
|
134
138
|
if len(list_indice) > 0:
|
135
|
-
|
139
|
+
if debug_able:
|
140
|
+
logger.info(f"发现了列表,列表行数:{list_indice}, {list_start_idx}")
|
136
141
|
|
137
142
|
# TODO check一下这个特列表里缩进的行左侧是不是对齐的。
|
138
143
|
segments = []
|
@@ -140,13 +145,16 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
|
|
140
145
|
for i in range(start, end + 1):
|
141
146
|
if i > 0:
|
142
147
|
if line_fea_encode[i] == 4:
|
143
|
-
|
148
|
+
if debug_able:
|
149
|
+
logger.info(f"列表行的第{i}行不是顶格的")
|
144
150
|
break
|
145
151
|
else:
|
146
|
-
|
152
|
+
if debug_able:
|
153
|
+
logger.info(f"列表行的第{start}到第{end}行是列表")
|
147
154
|
|
148
155
|
return split_indices(total_lines, list_indice), list_start_idx
|
149
156
|
|
157
|
+
|
150
158
|
def cluster_line_x(lines: list) -> dict:
|
151
159
|
"""
|
152
160
|
对一个block内所有lines的bbox的x0聚类
|
@@ -170,6 +178,7 @@ def cluster_line_x(lines: list) -> dict:
|
|
170
178
|
min_x0 = x0_new_val
|
171
179
|
return x0_2_new_val, min_x0
|
172
180
|
|
181
|
+
|
173
182
|
def if_match_reference_list(text: str) -> bool:
|
174
183
|
pattern = re.compile(r'^\d+\..*')
|
175
184
|
if pattern.match(text):
|
@@ -190,7 +199,8 @@ def __valign_lines(blocks, layout_bboxes):
|
|
190
199
|
new_layout_bboxes = []
|
191
200
|
|
192
201
|
for layout_box in layout_bboxes:
|
193
|
-
blocks_in_layoutbox = [b for b in blocks if
|
202
|
+
blocks_in_layoutbox = [b for b in blocks if
|
203
|
+
b["type"] == BlockType.Text and is_in_layout(b['bbox'], layout_box['layout_bbox'])]
|
194
204
|
if len(blocks_in_layoutbox) == 0 or len(blocks_in_layoutbox[0]["lines"]) == 0:
|
195
205
|
new_layout_bboxes.append(layout_box['layout_bbox'])
|
196
206
|
continue
|
@@ -253,7 +263,8 @@ def __align_text_in_layout(blocks, layout_bboxes):
|
|
253
263
|
"""
|
254
264
|
for layout in layout_bboxes:
|
255
265
|
lb = layout['layout_bbox']
|
256
|
-
blocks_in_layoutbox = [block for block in blocks if
|
266
|
+
blocks_in_layoutbox = [block for block in blocks if
|
267
|
+
block["type"] == BlockType.Text and is_in_layout(block['bbox'], lb)]
|
257
268
|
if len(blocks_in_layoutbox) == 0:
|
258
269
|
continue
|
259
270
|
|
@@ -383,7 +394,6 @@ def __split_para_lines(lines: list, text_blocks: list) -> list:
|
|
383
394
|
continue
|
384
395
|
text_lines.append(line)
|
385
396
|
|
386
|
-
|
387
397
|
for block in text_blocks:
|
388
398
|
block_bbox = block["bbox"]
|
389
399
|
para = []
|
@@ -394,14 +404,12 @@ def __split_para_lines(lines: list, text_blocks: list) -> list:
|
|
394
404
|
if len(para) > 0:
|
395
405
|
text_paras.append(para)
|
396
406
|
paras = other_paras.extend(text_paras)
|
397
|
-
paras_sorted = sorted(paras, key
|
407
|
+
paras_sorted = sorted(paras, key=lambda x: x[0]["bbox"][1])
|
398
408
|
return paras_sorted
|
399
409
|
|
400
410
|
|
401
|
-
|
402
|
-
|
403
|
-
|
404
411
|
def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info, page_num, lang):
|
412
|
+
global debug_able
|
405
413
|
"""
|
406
414
|
如果上个layout的最后一个段落是列表,下一个layout的第一个段落也是列表,那么将他们连接起来。 TODO 因为没有区分列表和段落,所以这个方法暂时不实现。
|
407
415
|
根据layout_list_info判断是不是列表。,下个layout的第一个段如果不是列表,那么看他们是否有几行都有相同的缩进。
|
@@ -410,7 +418,7 @@ def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
|
|
410
418
|
return blocks_group, [False, False]
|
411
419
|
|
412
420
|
for i in range(1, len(blocks_group)):
|
413
|
-
if len(blocks_group[i]) == 0 or len(blocks_group[i-1]) == 0:
|
421
|
+
if len(blocks_group[i]) == 0 or len(blocks_group[i - 1]) == 0:
|
414
422
|
continue
|
415
423
|
pre_layout_list_info = layout_list_info[i - 1]
|
416
424
|
next_layout_list_info = layout_list_info[i]
|
@@ -418,8 +426,10 @@ def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
|
|
418
426
|
next_paras = blocks_group[i]
|
419
427
|
next_first_para = next_paras[0]
|
420
428
|
|
421
|
-
if pre_layout_list_info[1] and not next_layout_list_info[0] and next_first_para[
|
422
|
-
|
429
|
+
if pre_layout_list_info[1] and not next_layout_list_info[0] and next_first_para[
|
430
|
+
"type"] == BlockType.Text: # 前一个是列表结尾,后一个是非列表开头,此时检测是否有相同的缩进
|
431
|
+
if debug_able:
|
432
|
+
logger.info(f"连接page {page_num} 内的list")
|
423
433
|
# 向layout_paras[i] 寻找开头具有相同缩进的连续的行
|
424
434
|
may_list_lines = []
|
425
435
|
lines = next_first_para.get("lines", [])
|
@@ -450,7 +460,8 @@ def __connect_list_inter_page(pre_page_paras, next_page_paras, pre_page_layout_b
|
|
450
460
|
if pre_page_paras[-1][-1]["type"] != BlockType.Text or next_page_paras[0][0]["type"] != BlockType.Text:
|
451
461
|
return False
|
452
462
|
if pre_page_list_info[1] and not next_page_list_info[0]: # 前一个是列表结尾,后一个是非列表开头,此时检测是否有相同的缩进
|
453
|
-
|
463
|
+
if debug_able:
|
464
|
+
logger.info(f"连接page {page_num} 内的list")
|
454
465
|
# 向layout_paras[i] 寻找开头具有相同缩进的连续的行
|
455
466
|
may_list_lines = []
|
456
467
|
next_page_first_para = next_page_paras[0][0]
|
@@ -503,7 +514,7 @@ def __connect_para_inter_layoutbox(blocks_group, new_layout_bbox):
|
|
503
514
|
try:
|
504
515
|
if len(blocks_group[i]) == 0:
|
505
516
|
continue
|
506
|
-
if
|
517
|
+
if len(blocks_group[i - 1]) == 0: # TODO 考虑连接问题,
|
507
518
|
connected_layout_blocks.append(blocks_group[i])
|
508
519
|
continue
|
509
520
|
# text类型的段才需要考虑layout间的合并
|
@@ -534,13 +545,14 @@ def __connect_para_inter_layoutbox(blocks_group, new_layout_bbox):
|
|
534
545
|
|
535
546
|
pre_last_line_text = pre_last_line_text.strip()
|
536
547
|
next_first_line_text = next_first_line_text.strip()
|
537
|
-
if pre_last_line['bbox'][2] == pre_x2_max and pre_last_line_text and pre_last_line_text[
|
548
|
+
if pre_last_line['bbox'][2] == pre_x2_max and pre_last_line_text and pre_last_line_text[
|
549
|
+
-1] not in LINE_STOP_FLAG and \
|
538
550
|
next_first_line['bbox'][0] == next_x0_min: # 前面一行沾满了整个行,并且没有结尾符号.下一行没有空白开头。
|
539
551
|
"""连接段落条件成立,将前一个layout的段落和后一个layout的段落连接。"""
|
540
552
|
#connected_layout_paras[-1][-1].extend(layout_paras[i][0])
|
541
553
|
connected_layout_blocks[-1][-1]["lines"].extend(blocks_group[i][0]["lines"])
|
542
554
|
#layout_paras[i].pop(0) # 删除后一个layout的第一个段落, 因为他已经被合并到前一个layout的最后一个段落了。
|
543
|
-
blocks_group[i][0]["lines"] = []
|
555
|
+
blocks_group[i][0]["lines"] = [] #删除后一个layout第一个段落中的lines,因为他已经被合并到前一个layout的最后一个段落了
|
544
556
|
blocks_group[i][0][LINES_DELETED] = True
|
545
557
|
# if len(layout_paras[i]) == 0:
|
546
558
|
# layout_paras.pop(i)
|
@@ -638,7 +650,8 @@ def find_consecutive_true_regions(input_array):
|
|
638
650
|
return regions
|
639
651
|
|
640
652
|
|
641
|
-
def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang
|
653
|
+
def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang):
|
654
|
+
global debug_able
|
642
655
|
"""
|
643
656
|
找出来中间对齐的连续单行文本,如果连续行高度相同,那么合并为一个段落。
|
644
657
|
一个line居中的条件是:
|
@@ -660,11 +673,12 @@ def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, deb
|
|
660
673
|
for start, end in consecutive_single_line_indices:
|
661
674
|
#start += index_offset
|
662
675
|
#end += index_offset
|
663
|
-
line_hi = np.array([block["lines"][0]['bbox'][3] - block["lines"][0]['bbox'][1] for block in
|
676
|
+
line_hi = np.array([block["lines"][0]['bbox'][3] - block["lines"][0]['bbox'][1] for block in
|
677
|
+
layout_para[start:end + 1]])
|
664
678
|
first_line_text = ''.join([__get_span_text(span) for span in layout_para[start]["lines"][0]['spans']])
|
665
679
|
if "Table" in first_line_text or "Figure" in first_line_text:
|
666
680
|
pass
|
667
|
-
if
|
681
|
+
if debug_able:
|
668
682
|
logger.info(line_hi.std())
|
669
683
|
|
670
684
|
if line_hi.std() < 2:
|
@@ -677,10 +691,10 @@ def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, deb
|
|
677
691
|
and not all([x1 == layout_box[2] for x1 in all_right_x1]):
|
678
692
|
merge_para = [block["lines"][0] for block in layout_para[start:end + 1]]
|
679
693
|
para_text = ''.join([__get_span_text(span) for line in merge_para for span in line['spans']])
|
680
|
-
if
|
694
|
+
if debug_able:
|
681
695
|
logger.info(para_text)
|
682
696
|
layout_para[start]["lines"] = merge_para
|
683
|
-
for i_para in range(start+1, end+1):
|
697
|
+
for i_para in range(start + 1, end + 1):
|
684
698
|
layout_para[i_para]["lines"] = []
|
685
699
|
layout_para[i_para][LINES_DELETED] = True
|
686
700
|
#layout_para[start:end + 1] = [merge_para]
|
@@ -713,14 +727,15 @@ def __do_split_page(blocks, layout_bboxes, new_layout_bbox, page_num, lang):
|
|
713
727
|
blocks_group = __group_line_by_layout(blocks, layout_bboxes) # block内分段
|
714
728
|
layout_list_info = __split_para_in_layoutbox(blocks_group, new_layout_bbox, lang) # layout内分段
|
715
729
|
blocks_group, page_list_info = __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
|
716
|
-
|
730
|
+
page_num, lang) # layout之间连接列表段落
|
717
731
|
connected_layout_blocks = __connect_para_inter_layoutbox(blocks_group, new_layout_bbox) # layout间链接段落
|
718
732
|
|
719
733
|
return connected_layout_blocks, page_list_info
|
720
734
|
|
721
735
|
|
722
|
-
|
723
736
|
def para_split(pdf_info_dict, debug_mode, lang="en"):
|
737
|
+
global debug_able
|
738
|
+
debug_able = debug_mode
|
724
739
|
new_layout_of_pages = [] # 数组的数组,每个元素是一个页面的layoutS
|
725
740
|
all_page_list_info = [] # 保存每个页面开头和结尾是否是列表
|
726
741
|
for page_num, page in pdf_info_dict.items():
|
@@ -744,14 +759,14 @@ def para_split(pdf_info_dict, debug_mode, lang="en"):
|
|
744
759
|
|
745
760
|
is_conn = __connect_para_inter_page(pre_page_paras, next_page_paras, pre_page_layout_bbox,
|
746
761
|
next_page_layout_bbox, page_num, lang)
|
747
|
-
if
|
762
|
+
if debug_able:
|
748
763
|
if is_conn:
|
749
764
|
logger.info(f"连接了第{page_num - 1}页和第{page_num}页的段落")
|
750
765
|
|
751
766
|
is_list_conn = __connect_list_inter_page(pre_page_paras, next_page_paras, pre_page_layout_bbox,
|
752
767
|
next_page_layout_bbox, all_page_list_info[page_num - 1],
|
753
768
|
all_page_list_info[page_num], page_num, lang)
|
754
|
-
if
|
769
|
+
if debug_able:
|
755
770
|
if is_list_conn:
|
756
771
|
logger.info(f"连接了第{page_num - 1}页和第{page_num}页的列表段落")
|
757
772
|
|
@@ -762,7 +777,7 @@ def para_split(pdf_info_dict, debug_mode, lang="en"):
|
|
762
777
|
for page_num, page in enumerate(pdf_info_dict.values()):
|
763
778
|
page_paras = page['para_blocks']
|
764
779
|
new_layout_bbox = new_layout_of_pages[page_num]
|
765
|
-
__connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang
|
780
|
+
__connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang)
|
766
781
|
__merge_signle_list_text(page_paras, new_layout_bbox, page_num, lang)
|
767
782
|
|
768
783
|
# layout展平
|
magic_pdf/pdf_parse_by_ocr.py
CHANGED
@@ -1,219 +1,18 @@
|
|
1
|
-
import
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
from magic_pdf.pre_proc.cut_image import ocr_cut_image_and_table
|
20
|
-
from magic_pdf.pre_proc.ocr_detect_layout import layout_detect
|
21
|
-
from magic_pdf.pre_proc.ocr_dict_merge import (
|
22
|
-
merge_spans_to_line_by_layout, merge_lines_to_block,
|
23
|
-
)
|
24
|
-
from magic_pdf.pre_proc.ocr_span_list_modify import remove_spans_by_bboxes, remove_overlaps_min_spans, \
|
25
|
-
adjust_bbox_for_standalone_block, modify_y_axis, modify_inline_equation, get_qa_need_list, \
|
26
|
-
remove_spans_by_bboxes_dict
|
27
|
-
from magic_pdf.pre_proc.remove_bbox_overlap import remove_overlap_between_bbox_for_span
|
28
|
-
|
29
|
-
|
30
|
-
def parse_pdf_by_ocr(
|
31
|
-
pdf_bytes,
|
32
|
-
pdf_model_output,
|
33
|
-
imageWriter,
|
34
|
-
start_page_id=0,
|
35
|
-
end_page_id=None,
|
36
|
-
debug_mode=False,
|
37
|
-
):
|
38
|
-
pdf_bytes_md5 = compute_md5(pdf_bytes)
|
39
|
-
|
40
|
-
pdf_docs = fitz.open("pdf", pdf_bytes)
|
41
|
-
# 初始化空的pdf_info_dict
|
42
|
-
pdf_info_dict = {}
|
43
|
-
|
44
|
-
start_time = time.time()
|
45
|
-
|
46
|
-
end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
|
47
|
-
for page_id in range(start_page_id, end_page_id + 1):
|
48
|
-
|
49
|
-
# 获取当前页的page对象
|
50
|
-
page = pdf_docs[page_id]
|
51
|
-
# 获取当前页的宽高
|
52
|
-
page_w = page.rect.width
|
53
|
-
page_h = page.rect.height
|
54
|
-
|
55
|
-
if debug_mode:
|
56
|
-
time_now = time.time()
|
57
|
-
logger.info(
|
58
|
-
f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}"
|
59
|
-
)
|
60
|
-
start_time = time_now
|
61
|
-
|
62
|
-
# 获取当前页的模型数据
|
63
|
-
ocr_page_info = get_docx_model_output(
|
64
|
-
pdf_model_output, page_id
|
65
|
-
)
|
66
|
-
|
67
|
-
"""从json中获取每页的页码、页眉、页脚的bbox"""
|
68
|
-
page_no_bboxes = parse_pageNos(page_id, page, ocr_page_info)
|
69
|
-
header_bboxes = parse_headers(page_id, page, ocr_page_info)
|
70
|
-
footer_bboxes = parse_footers(page_id, page, ocr_page_info)
|
71
|
-
footnote_bboxes = parse_footnotes_by_model(page_id, page, ocr_page_info, debug_mode=debug_mode)
|
72
|
-
|
73
|
-
# 构建需要remove的bbox字典
|
74
|
-
need_remove_spans_bboxes_dict = {
|
75
|
-
DropTag.PAGE_NUMBER: page_no_bboxes,
|
76
|
-
DropTag.HEADER: header_bboxes,
|
77
|
-
DropTag.FOOTER: footer_bboxes,
|
78
|
-
DropTag.FOOTNOTE: footnote_bboxes,
|
79
|
-
}
|
80
|
-
|
81
|
-
layout_dets = ocr_page_info["layout_dets"]
|
82
|
-
spans = []
|
83
|
-
|
84
|
-
# 计算模型坐标和pymu坐标的缩放比例
|
85
|
-
horizontal_scale_ratio, vertical_scale_ratio = get_scale_ratio(
|
86
|
-
ocr_page_info, page
|
87
|
-
)
|
88
|
-
|
89
|
-
for layout_det in layout_dets:
|
90
|
-
category_id = layout_det["category_id"]
|
91
|
-
allow_category_id_list = [1, 7, 13, 14, 15]
|
92
|
-
if category_id in allow_category_id_list:
|
93
|
-
x0, y0, _, _, x1, y1, _, _ = layout_det["poly"]
|
94
|
-
bbox = [
|
95
|
-
int(x0 / horizontal_scale_ratio),
|
96
|
-
int(y0 / vertical_scale_ratio),
|
97
|
-
int(x1 / horizontal_scale_ratio),
|
98
|
-
int(y1 / vertical_scale_ratio),
|
99
|
-
]
|
100
|
-
# 删除高度或者宽度为0的spans
|
101
|
-
if bbox[2] - bbox[0] == 0 or bbox[3] - bbox[1] == 0:
|
102
|
-
continue
|
103
|
-
"""要删除的"""
|
104
|
-
# 3: 'header', # 页眉
|
105
|
-
# 4: 'page number', # 页码
|
106
|
-
# 5: 'footnote', # 脚注
|
107
|
-
# 6: 'footer', # 页脚
|
108
|
-
"""当成span拼接的"""
|
109
|
-
# 1: 'image', # 图片
|
110
|
-
# 7: 'table', # 表格
|
111
|
-
# 13: 'inline_equation', # 行内公式
|
112
|
-
# 14: 'interline_equation', # 行间公式
|
113
|
-
# 15: 'text', # ocr识别文本
|
114
|
-
"""layout信息"""
|
115
|
-
# 11: 'full column', # 单栏
|
116
|
-
# 12: 'sub column', # 多栏
|
117
|
-
span = {
|
118
|
-
"bbox": bbox,
|
119
|
-
}
|
120
|
-
if category_id == 1:
|
121
|
-
span["type"] = ContentType.Image
|
122
|
-
|
123
|
-
elif category_id == 7:
|
124
|
-
span["type"] = ContentType.Table
|
125
|
-
|
126
|
-
elif category_id == 13:
|
127
|
-
span["content"] = layout_det["latex"]
|
128
|
-
span["type"] = ContentType.InlineEquation
|
129
|
-
elif category_id == 14:
|
130
|
-
span["content"] = layout_det["latex"]
|
131
|
-
span["type"] = ContentType.InterlineEquation
|
132
|
-
elif category_id == 15:
|
133
|
-
span["content"] = layout_det["text"]
|
134
|
-
span["type"] = ContentType.Text
|
135
|
-
# print(span)
|
136
|
-
spans.append(span)
|
137
|
-
else:
|
138
|
-
continue
|
139
|
-
|
140
|
-
'''删除重叠spans中较小的那些'''
|
141
|
-
spans, dropped_spans_by_span_overlap = remove_overlaps_min_spans(spans)
|
142
|
-
|
143
|
-
'''
|
144
|
-
删除remove_span_block_bboxes中的bbox
|
145
|
-
并增加drop相关数据
|
146
|
-
'''
|
147
|
-
spans, dropped_spans_by_removed_bboxes = remove_spans_by_bboxes_dict(spans, need_remove_spans_bboxes_dict)
|
148
|
-
|
149
|
-
'''对image和table截图'''
|
150
|
-
spans = ocr_cut_image_and_table(spans, page, page_id, pdf_bytes_md5, imageWriter)
|
151
|
-
|
152
|
-
'''行内公式调整, 高度调整至与同行文字高度一致(优先左侧, 其次右侧)'''
|
153
|
-
displayed_list = []
|
154
|
-
text_inline_lines = []
|
155
|
-
modify_y_axis(spans, displayed_list, text_inline_lines)
|
156
|
-
|
157
|
-
'''模型识别错误的行间公式, type类型转换成行内公式'''
|
158
|
-
spans = modify_inline_equation(spans, displayed_list, text_inline_lines)
|
159
|
-
|
160
|
-
'''bbox去除粘连'''
|
161
|
-
spans = remove_overlap_between_bbox_for_span(spans)
|
162
|
-
'''
|
163
|
-
对tpye=["interline_equation", "image", "table"]进行额外处理,
|
164
|
-
如果左边有字的话,将该span的bbox中y0调整至不高于文字的y0
|
165
|
-
'''
|
166
|
-
spans = adjust_bbox_for_standalone_block(spans)
|
167
|
-
|
168
|
-
'''从ocr_page_info中解析layout信息(按自然阅读方向排序,并修复重叠和交错的bad case)'''
|
169
|
-
layout_bboxes, layout_tree = layout_detect(ocr_page_info['subfield_dets'], page, ocr_page_info)
|
170
|
-
|
171
|
-
'''将spans合并成line(在layout内,从上到下,从左到右)'''
|
172
|
-
lines, dropped_spans_by_layout = merge_spans_to_line_by_layout(spans, layout_bboxes)
|
173
|
-
|
174
|
-
'''将lines合并成block'''
|
175
|
-
blocks = merge_lines_to_block(lines)
|
176
|
-
|
177
|
-
'''获取QA需要外置的list'''
|
178
|
-
images, tables, interline_equations, inline_equations = get_qa_need_list(blocks)
|
179
|
-
|
180
|
-
'''drop的span_list合并'''
|
181
|
-
dropped_spans = []
|
182
|
-
dropped_spans.extend(dropped_spans_by_span_overlap)
|
183
|
-
dropped_spans.extend(dropped_spans_by_removed_bboxes)
|
184
|
-
dropped_spans.extend(dropped_spans_by_layout)
|
185
|
-
|
186
|
-
dropped_text_block = []
|
187
|
-
dropped_image_block = []
|
188
|
-
dropped_table_block = []
|
189
|
-
dropped_equation_block = []
|
190
|
-
for span in dropped_spans:
|
191
|
-
# drop出的spans进行分类
|
192
|
-
if span['type'] == ContentType.Text:
|
193
|
-
dropped_text_block.append(span)
|
194
|
-
elif span['type'] == ContentType.Image:
|
195
|
-
dropped_image_block.append(span)
|
196
|
-
elif span['type'] == ContentType.Table:
|
197
|
-
dropped_table_block.append(span)
|
198
|
-
elif span['type'] in [ContentType.InlineEquation, ContentType.InterlineEquation]:
|
199
|
-
dropped_equation_block.append(span)
|
200
|
-
|
201
|
-
'''构造pdf_info_dict'''
|
202
|
-
page_info = ocr_construct_page_component(blocks, layout_bboxes, page_id, page_w, page_h, layout_tree,
|
203
|
-
images, tables, interline_equations, inline_equations,
|
204
|
-
dropped_text_block, dropped_image_block, dropped_table_block,
|
205
|
-
dropped_equation_block,
|
206
|
-
need_remove_spans_bboxes_dict)
|
207
|
-
pdf_info_dict[f"page_{page_id}"] = page_info
|
208
|
-
|
209
|
-
"""分段"""
|
210
|
-
|
211
|
-
para_split(pdf_info_dict, debug_mode=debug_mode)
|
212
|
-
|
213
|
-
"""dict转list"""
|
214
|
-
pdf_info_list = dict_to_list(pdf_info_dict)
|
215
|
-
new_pdf_info_dict = {
|
216
|
-
"pdf_info": pdf_info_list,
|
217
|
-
}
|
218
|
-
|
219
|
-
return new_pdf_info_dict
|
1
|
+
from magic_pdf.pdf_parse_union_core import pdf_parse_union
|
2
|
+
|
3
|
+
|
4
|
+
def parse_pdf_by_ocr(pdf_bytes,
|
5
|
+
model_list,
|
6
|
+
imageWriter,
|
7
|
+
start_page_id=0,
|
8
|
+
end_page_id=None,
|
9
|
+
debug_mode=False,
|
10
|
+
):
|
11
|
+
return pdf_parse_union(pdf_bytes,
|
12
|
+
model_list,
|
13
|
+
imageWriter,
|
14
|
+
"ocr",
|
15
|
+
start_page_id=start_page_id,
|
16
|
+
end_page_id=end_page_id,
|
17
|
+
debug_mode=debug_mode,
|
18
|
+
)
|