magic-pdf 0.5.8__py3-none-any.whl → 0.5.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
magic_pdf/cli/magicpdf.py CHANGED
@@ -17,8 +17,8 @@
17
17
 
18
18
 
19
19
  效果:
20
- python magicpdf.py --json s3://llm-pdf-text/scihub/xxxx.json?bytes=0,81350
21
- python magicpdf.py --pdf /home/llm/Downloads/xxxx.pdf --model /home/llm/Downloads/xxxx.json 或者 python magicpdf.py --pdf /home/llm/Downloads/xxxx.pdf
20
+ python magicpdf.py json-command --json s3://llm-pdf-text/scihub/xxxx.json?bytes=0,81350
21
+ python magicpdf.py pdf-command --pdf /home/llm/Downloads/xxxx.pdf --model /home/llm/Downloads/xxxx.json 或者 python magicpdf.py --pdf /home/llm/Downloads/xxxx.pdf
22
22
  """
23
23
 
24
24
  import os
@@ -45,6 +45,7 @@ from magic_pdf.rw.S3ReaderWriter import S3ReaderWriter
45
45
  from magic_pdf.rw.DiskReaderWriter import DiskReaderWriter
46
46
  from magic_pdf.rw.AbsReaderWriter import AbsReaderWriter
47
47
  import csv
48
+ import copy
48
49
 
49
50
  parse_pdf_methods = click.Choice(["ocr", "txt", "auto"])
50
51
 
@@ -81,10 +82,11 @@ def do_parse(
81
82
  f_dump_orig_pdf=True,
82
83
  f_dump_content_list=True,
83
84
  ):
85
+ orig_model_list = copy.deepcopy(model_list)
84
86
 
85
87
  local_image_dir, local_md_dir = prepare_env(pdf_file_name, parse_method)
86
88
  image_writer, md_writer = DiskReaderWriter(local_image_dir), DiskReaderWriter(local_md_dir)
87
- image_dir = (os.path.basename(local_image_dir),)
89
+ image_dir = str(os.path.basename(local_image_dir))
88
90
 
89
91
  if parse_method == "auto":
90
92
  jso_useful_key = {"_pdf_type": "", "model_list": model_list}
@@ -110,10 +112,7 @@ def do_parse(
110
112
  if f_draw_span_bbox:
111
113
  draw_span_bbox(pdf_info, pdf_bytes, local_md_dir)
112
114
 
113
- # write_to_csv(r"D:\project\20231108code-clean\linshixuqiu\pdf_dev\新模型\新建文件夹\luanma.csv",
114
- # [pdf_file_name, pipe.pdf_mid_data['not_common_character_rate'], pipe.pdf_mid_data['not_printable_rate']])
115
-
116
- md_content = pipe.pipe_mk_markdown(str(image_dir), drop_mode=DropMode.NONE)
115
+ md_content = pipe.pipe_mk_markdown(image_dir, drop_mode=DropMode.NONE)
117
116
  if f_dump_md:
118
117
  """写markdown"""
119
118
  md_writer.write(
@@ -133,7 +132,7 @@ def do_parse(
133
132
  if f_dump_model_json:
134
133
  """写model_json"""
135
134
  md_writer.write(
136
- content=json_parse.dumps(pipe.model_list, ensure_ascii=False, indent=4),
135
+ content=json_parse.dumps(orig_model_list, ensure_ascii=False, indent=4),
137
136
  path=f"{pdf_file_name}_model.json",
138
137
  mode=AbsReaderWriter.MODE_TXT,
139
138
  )
@@ -146,7 +145,7 @@ def do_parse(
146
145
  mode=AbsReaderWriter.MODE_BIN,
147
146
  )
148
147
 
149
- content_list = pipe.pipe_mk_uni_format(str(image_dir), drop_mode=DropMode.NONE)
148
+ content_list = pipe.pipe_mk_uni_format(image_dir, drop_mode=DropMode.NONE)
150
149
  if f_dump_content_list:
151
150
  """写content_list"""
152
151
  md_writer.write(
@@ -281,7 +280,7 @@ def pdf_command(pdf, model, method):
281
280
  model_path = pdf.replace(".pdf", ".json")
282
281
  if not os.path.exists(model_path):
283
282
  logger.warning(
284
- f"not found json {model_path} existed, use paddle analyze"
283
+ f"not found json {model_path} existed"
285
284
  )
286
285
  # 本地无模型数据则调用内置paddle分析,先传空list,在内部识别到空list再调用paddle
287
286
  model_json = "[]"
magic_pdf/libs/version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.5.8"
1
+ __version__ = "0.5.10"
@@ -11,6 +11,7 @@ LINE_STOP_FLAG = ['.', '!', '?', '。', '!', '?', ":", ":", ")", ")", ";
11
11
  INLINE_EQUATION = ContentType.InlineEquation
12
12
  INTERLINE_EQUATION = ContentType.InterlineEquation
13
13
  TEXT = ContentType.Text
14
+ debug_able = False
14
15
 
15
16
 
16
17
  def __get_span_text(span):
@@ -22,6 +23,7 @@ def __get_span_text(span):
22
23
 
23
24
 
24
25
  def __detect_list_lines(lines, new_layout_bboxes, lang):
26
+ global debug_able
25
27
  """
26
28
  探测是否包含了列表,并且把列表的行分开.
27
29
  这样的段落特点是,顶格字母大写/数字,紧跟着几行缩进的。缩进的行首字母含小写的。
@@ -41,12 +43,14 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
41
43
  if lst[i] == 1:
42
44
  ones_in_this_interval.append(i)
43
45
  i += 1
44
- if len(ones_in_this_interval) > 1 or (start < len(lst) - 1 and ones_in_this_interval and lst[start + 1] in [2, 3]):
46
+ if len(ones_in_this_interval) > 1 or (
47
+ start < len(lst) - 1 and ones_in_this_interval and lst[start + 1] in [2, 3]):
45
48
  indices.append((start, i - 1))
46
49
  ones_indices.append(ones_in_this_interval)
47
50
  else:
48
51
  i += 1
49
52
  return indices, ones_indices
53
+
50
54
  def find_repeating_patterns(lst):
51
55
  indices = []
52
56
  ones_indices = []
@@ -132,7 +136,8 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
132
136
 
133
137
  list_indice, list_start_idx = find_repeating_patterns2(line_fea_encode)
134
138
  if len(list_indice) > 0:
135
- logger.info(f"发现了列表,列表行数:{list_indice}, {list_start_idx}")
139
+ if debug_able:
140
+ logger.info(f"发现了列表,列表行数:{list_indice}, {list_start_idx}")
136
141
 
137
142
  # TODO check一下这个特列表里缩进的行左侧是不是对齐的。
138
143
  segments = []
@@ -140,13 +145,16 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
140
145
  for i in range(start, end + 1):
141
146
  if i > 0:
142
147
  if line_fea_encode[i] == 4:
143
- logger.info(f"列表行的第{i}行不是顶格的")
148
+ if debug_able:
149
+ logger.info(f"列表行的第{i}行不是顶格的")
144
150
  break
145
151
  else:
146
- logger.info(f"列表行的第{start}到第{end}行是列表")
152
+ if debug_able:
153
+ logger.info(f"列表行的第{start}到第{end}行是列表")
147
154
 
148
155
  return split_indices(total_lines, list_indice), list_start_idx
149
156
 
157
+
150
158
  def cluster_line_x(lines: list) -> dict:
151
159
  """
152
160
  对一个block内所有lines的bbox的x0聚类
@@ -170,6 +178,7 @@ def cluster_line_x(lines: list) -> dict:
170
178
  min_x0 = x0_new_val
171
179
  return x0_2_new_val, min_x0
172
180
 
181
+
173
182
  def if_match_reference_list(text: str) -> bool:
174
183
  pattern = re.compile(r'^\d+\..*')
175
184
  if pattern.match(text):
@@ -190,7 +199,8 @@ def __valign_lines(blocks, layout_bboxes):
190
199
  new_layout_bboxes = []
191
200
 
192
201
  for layout_box in layout_bboxes:
193
- blocks_in_layoutbox = [b for b in blocks if b["type"] == BlockType.Text and is_in_layout(b['bbox'], layout_box['layout_bbox'])]
202
+ blocks_in_layoutbox = [b for b in blocks if
203
+ b["type"] == BlockType.Text and is_in_layout(b['bbox'], layout_box['layout_bbox'])]
194
204
  if len(blocks_in_layoutbox) == 0 or len(blocks_in_layoutbox[0]["lines"]) == 0:
195
205
  new_layout_bboxes.append(layout_box['layout_bbox'])
196
206
  continue
@@ -253,7 +263,8 @@ def __align_text_in_layout(blocks, layout_bboxes):
253
263
  """
254
264
  for layout in layout_bboxes:
255
265
  lb = layout['layout_bbox']
256
- blocks_in_layoutbox = [block for block in blocks if block["type"] == BlockType.Text and is_in_layout(block['bbox'], lb)]
266
+ blocks_in_layoutbox = [block for block in blocks if
267
+ block["type"] == BlockType.Text and is_in_layout(block['bbox'], lb)]
257
268
  if len(blocks_in_layoutbox) == 0:
258
269
  continue
259
270
 
@@ -383,7 +394,6 @@ def __split_para_lines(lines: list, text_blocks: list) -> list:
383
394
  continue
384
395
  text_lines.append(line)
385
396
 
386
-
387
397
  for block in text_blocks:
388
398
  block_bbox = block["bbox"]
389
399
  para = []
@@ -394,14 +404,12 @@ def __split_para_lines(lines: list, text_blocks: list) -> list:
394
404
  if len(para) > 0:
395
405
  text_paras.append(para)
396
406
  paras = other_paras.extend(text_paras)
397
- paras_sorted = sorted(paras, key = lambda x: x[0]["bbox"][1])
407
+ paras_sorted = sorted(paras, key=lambda x: x[0]["bbox"][1])
398
408
  return paras_sorted
399
409
 
400
410
 
401
-
402
-
403
-
404
411
  def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info, page_num, lang):
412
+ global debug_able
405
413
  """
406
414
  如果上个layout的最后一个段落是列表,下一个layout的第一个段落也是列表,那么将他们连接起来。 TODO 因为没有区分列表和段落,所以这个方法暂时不实现。
407
415
  根据layout_list_info判断是不是列表。,下个layout的第一个段如果不是列表,那么看他们是否有几行都有相同的缩进。
@@ -410,7 +418,7 @@ def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
410
418
  return blocks_group, [False, False]
411
419
 
412
420
  for i in range(1, len(blocks_group)):
413
- if len(blocks_group[i]) == 0 or len(blocks_group[i-1]) == 0:
421
+ if len(blocks_group[i]) == 0 or len(blocks_group[i - 1]) == 0:
414
422
  continue
415
423
  pre_layout_list_info = layout_list_info[i - 1]
416
424
  next_layout_list_info = layout_list_info[i]
@@ -418,8 +426,10 @@ def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
418
426
  next_paras = blocks_group[i]
419
427
  next_first_para = next_paras[0]
420
428
 
421
- if pre_layout_list_info[1] and not next_layout_list_info[0] and next_first_para["type"] == BlockType.Text: # 前一个是列表结尾,后一个是非列表开头,此时检测是否有相同的缩进
422
- logger.info(f"连接page {page_num} 内的list")
429
+ if pre_layout_list_info[1] and not next_layout_list_info[0] and next_first_para[
430
+ "type"] == BlockType.Text: # 前一个是列表结尾,后一个是非列表开头,此时检测是否有相同的缩进
431
+ if debug_able:
432
+ logger.info(f"连接page {page_num} 内的list")
423
433
  # 向layout_paras[i] 寻找开头具有相同缩进的连续的行
424
434
  may_list_lines = []
425
435
  lines = next_first_para.get("lines", [])
@@ -450,7 +460,8 @@ def __connect_list_inter_page(pre_page_paras, next_page_paras, pre_page_layout_b
450
460
  if pre_page_paras[-1][-1]["type"] != BlockType.Text or next_page_paras[0][0]["type"] != BlockType.Text:
451
461
  return False
452
462
  if pre_page_list_info[1] and not next_page_list_info[0]: # 前一个是列表结尾,后一个是非列表开头,此时检测是否有相同的缩进
453
- logger.info(f"连接page {page_num} 内的list")
463
+ if debug_able:
464
+ logger.info(f"连接page {page_num} 内的list")
454
465
  # 向layout_paras[i] 寻找开头具有相同缩进的连续的行
455
466
  may_list_lines = []
456
467
  next_page_first_para = next_page_paras[0][0]
@@ -503,7 +514,7 @@ def __connect_para_inter_layoutbox(blocks_group, new_layout_bbox):
503
514
  try:
504
515
  if len(blocks_group[i]) == 0:
505
516
  continue
506
- if len(blocks_group[i - 1]) == 0: # TODO 考虑连接问题,
517
+ if len(blocks_group[i - 1]) == 0: # TODO 考虑连接问题,
507
518
  connected_layout_blocks.append(blocks_group[i])
508
519
  continue
509
520
  # text类型的段才需要考虑layout间的合并
@@ -534,13 +545,14 @@ def __connect_para_inter_layoutbox(blocks_group, new_layout_bbox):
534
545
 
535
546
  pre_last_line_text = pre_last_line_text.strip()
536
547
  next_first_line_text = next_first_line_text.strip()
537
- if pre_last_line['bbox'][2] == pre_x2_max and pre_last_line_text and pre_last_line_text[-1] not in LINE_STOP_FLAG and \
548
+ if pre_last_line['bbox'][2] == pre_x2_max and pre_last_line_text and pre_last_line_text[
549
+ -1] not in LINE_STOP_FLAG and \
538
550
  next_first_line['bbox'][0] == next_x0_min: # 前面一行沾满了整个行,并且没有结尾符号.下一行没有空白开头。
539
551
  """连接段落条件成立,将前一个layout的段落和后一个layout的段落连接。"""
540
552
  #connected_layout_paras[-1][-1].extend(layout_paras[i][0])
541
553
  connected_layout_blocks[-1][-1]["lines"].extend(blocks_group[i][0]["lines"])
542
554
  #layout_paras[i].pop(0) # 删除后一个layout的第一个段落, 因为他已经被合并到前一个layout的最后一个段落了。
543
- blocks_group[i][0]["lines"] = [] #删除后一个layout第一个段落中的lines,因为他已经被合并到前一个layout的最后一个段落了
555
+ blocks_group[i][0]["lines"] = [] #删除后一个layout第一个段落中的lines,因为他已经被合并到前一个layout的最后一个段落了
544
556
  blocks_group[i][0][LINES_DELETED] = True
545
557
  # if len(layout_paras[i]) == 0:
546
558
  # layout_paras.pop(i)
@@ -638,7 +650,8 @@ def find_consecutive_true_regions(input_array):
638
650
  return regions
639
651
 
640
652
 
641
- def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, debug_mode):
653
+ def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang):
654
+ global debug_able
642
655
  """
643
656
  找出来中间对齐的连续单行文本,如果连续行高度相同,那么合并为一个段落。
644
657
  一个line居中的条件是:
@@ -660,11 +673,12 @@ def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, deb
660
673
  for start, end in consecutive_single_line_indices:
661
674
  #start += index_offset
662
675
  #end += index_offset
663
- line_hi = np.array([block["lines"][0]['bbox'][3] - block["lines"][0]['bbox'][1] for block in layout_para[start:end + 1]])
676
+ line_hi = np.array([block["lines"][0]['bbox'][3] - block["lines"][0]['bbox'][1] for block in
677
+ layout_para[start:end + 1]])
664
678
  first_line_text = ''.join([__get_span_text(span) for span in layout_para[start]["lines"][0]['spans']])
665
679
  if "Table" in first_line_text or "Figure" in first_line_text:
666
680
  pass
667
- if debug_mode:
681
+ if debug_able:
668
682
  logger.info(line_hi.std())
669
683
 
670
684
  if line_hi.std() < 2:
@@ -677,10 +691,10 @@ def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, deb
677
691
  and not all([x1 == layout_box[2] for x1 in all_right_x1]):
678
692
  merge_para = [block["lines"][0] for block in layout_para[start:end + 1]]
679
693
  para_text = ''.join([__get_span_text(span) for line in merge_para for span in line['spans']])
680
- if debug_mode:
694
+ if debug_able:
681
695
  logger.info(para_text)
682
696
  layout_para[start]["lines"] = merge_para
683
- for i_para in range(start+1, end+1):
697
+ for i_para in range(start + 1, end + 1):
684
698
  layout_para[i_para]["lines"] = []
685
699
  layout_para[i_para][LINES_DELETED] = True
686
700
  #layout_para[start:end + 1] = [merge_para]
@@ -713,14 +727,15 @@ def __do_split_page(blocks, layout_bboxes, new_layout_bbox, page_num, lang):
713
727
  blocks_group = __group_line_by_layout(blocks, layout_bboxes) # block内分段
714
728
  layout_list_info = __split_para_in_layoutbox(blocks_group, new_layout_bbox, lang) # layout内分段
715
729
  blocks_group, page_list_info = __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
716
- page_num, lang) # layout之间连接列表段落
730
+ page_num, lang) # layout之间连接列表段落
717
731
  connected_layout_blocks = __connect_para_inter_layoutbox(blocks_group, new_layout_bbox) # layout间链接段落
718
732
 
719
733
  return connected_layout_blocks, page_list_info
720
734
 
721
735
 
722
-
723
736
  def para_split(pdf_info_dict, debug_mode, lang="en"):
737
+ global debug_able
738
+ debug_able = debug_mode
724
739
  new_layout_of_pages = [] # 数组的数组,每个元素是一个页面的layoutS
725
740
  all_page_list_info = [] # 保存每个页面开头和结尾是否是列表
726
741
  for page_num, page in pdf_info_dict.items():
@@ -744,14 +759,14 @@ def para_split(pdf_info_dict, debug_mode, lang="en"):
744
759
 
745
760
  is_conn = __connect_para_inter_page(pre_page_paras, next_page_paras, pre_page_layout_bbox,
746
761
  next_page_layout_bbox, page_num, lang)
747
- if debug_mode:
762
+ if debug_able:
748
763
  if is_conn:
749
764
  logger.info(f"连接了第{page_num - 1}页和第{page_num}页的段落")
750
765
 
751
766
  is_list_conn = __connect_list_inter_page(pre_page_paras, next_page_paras, pre_page_layout_bbox,
752
767
  next_page_layout_bbox, all_page_list_info[page_num - 1],
753
768
  all_page_list_info[page_num], page_num, lang)
754
- if debug_mode:
769
+ if debug_able:
755
770
  if is_list_conn:
756
771
  logger.info(f"连接了第{page_num - 1}页和第{page_num}页的列表段落")
757
772
 
@@ -762,7 +777,7 @@ def para_split(pdf_info_dict, debug_mode, lang="en"):
762
777
  for page_num, page in enumerate(pdf_info_dict.values()):
763
778
  page_paras = page['para_blocks']
764
779
  new_layout_bbox = new_layout_of_pages[page_num]
765
- __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, debug_mode=debug_mode)
780
+ __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang)
766
781
  __merge_signle_list_text(page_paras, new_layout_bbox, page_num, lang)
767
782
 
768
783
  # layout展平
@@ -1,219 +1,18 @@
1
- import time
2
- from loguru import logger
3
- from magic_pdf.libs.commons import (
4
- fitz,
5
- get_delta_time,
6
- get_docx_model_output,
7
- )
8
- from magic_pdf.libs.convert_utils import dict_to_list
9
- from magic_pdf.libs.coordinate_transform import get_scale_ratio
10
- from magic_pdf.libs.drop_tag import DropTag
11
- from magic_pdf.libs.hash_utils import compute_md5
12
- from magic_pdf.libs.ocr_content_type import ContentType
13
- from magic_pdf.para.para_split import para_split
14
- from magic_pdf.pre_proc.construct_page_dict import ocr_construct_page_component
15
- from magic_pdf.pre_proc.detect_footer_by_model import parse_footers
16
- from magic_pdf.pre_proc.detect_footnote import parse_footnotes_by_model
17
- from magic_pdf.pre_proc.detect_header import parse_headers
18
- from magic_pdf.pre_proc.detect_page_number import parse_pageNos
19
- from magic_pdf.pre_proc.cut_image import ocr_cut_image_and_table
20
- from magic_pdf.pre_proc.ocr_detect_layout import layout_detect
21
- from magic_pdf.pre_proc.ocr_dict_merge import (
22
- merge_spans_to_line_by_layout, merge_lines_to_block,
23
- )
24
- from magic_pdf.pre_proc.ocr_span_list_modify import remove_spans_by_bboxes, remove_overlaps_min_spans, \
25
- adjust_bbox_for_standalone_block, modify_y_axis, modify_inline_equation, get_qa_need_list, \
26
- remove_spans_by_bboxes_dict
27
- from magic_pdf.pre_proc.remove_bbox_overlap import remove_overlap_between_bbox_for_span
28
-
29
-
30
- def parse_pdf_by_ocr(
31
- pdf_bytes,
32
- pdf_model_output,
33
- imageWriter,
34
- start_page_id=0,
35
- end_page_id=None,
36
- debug_mode=False,
37
- ):
38
- pdf_bytes_md5 = compute_md5(pdf_bytes)
39
-
40
- pdf_docs = fitz.open("pdf", pdf_bytes)
41
- # 初始化空的pdf_info_dict
42
- pdf_info_dict = {}
43
-
44
- start_time = time.time()
45
-
46
- end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
47
- for page_id in range(start_page_id, end_page_id + 1):
48
-
49
- # 获取当前页的page对象
50
- page = pdf_docs[page_id]
51
- # 获取当前页的宽高
52
- page_w = page.rect.width
53
- page_h = page.rect.height
54
-
55
- if debug_mode:
56
- time_now = time.time()
57
- logger.info(
58
- f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}"
59
- )
60
- start_time = time_now
61
-
62
- # 获取当前页的模型数据
63
- ocr_page_info = get_docx_model_output(
64
- pdf_model_output, page_id
65
- )
66
-
67
- """从json中获取每页的页码、页眉、页脚的bbox"""
68
- page_no_bboxes = parse_pageNos(page_id, page, ocr_page_info)
69
- header_bboxes = parse_headers(page_id, page, ocr_page_info)
70
- footer_bboxes = parse_footers(page_id, page, ocr_page_info)
71
- footnote_bboxes = parse_footnotes_by_model(page_id, page, ocr_page_info, debug_mode=debug_mode)
72
-
73
- # 构建需要remove的bbox字典
74
- need_remove_spans_bboxes_dict = {
75
- DropTag.PAGE_NUMBER: page_no_bboxes,
76
- DropTag.HEADER: header_bboxes,
77
- DropTag.FOOTER: footer_bboxes,
78
- DropTag.FOOTNOTE: footnote_bboxes,
79
- }
80
-
81
- layout_dets = ocr_page_info["layout_dets"]
82
- spans = []
83
-
84
- # 计算模型坐标和pymu坐标的缩放比例
85
- horizontal_scale_ratio, vertical_scale_ratio = get_scale_ratio(
86
- ocr_page_info, page
87
- )
88
-
89
- for layout_det in layout_dets:
90
- category_id = layout_det["category_id"]
91
- allow_category_id_list = [1, 7, 13, 14, 15]
92
- if category_id in allow_category_id_list:
93
- x0, y0, _, _, x1, y1, _, _ = layout_det["poly"]
94
- bbox = [
95
- int(x0 / horizontal_scale_ratio),
96
- int(y0 / vertical_scale_ratio),
97
- int(x1 / horizontal_scale_ratio),
98
- int(y1 / vertical_scale_ratio),
99
- ]
100
- # 删除高度或者宽度为0的spans
101
- if bbox[2] - bbox[0] == 0 or bbox[3] - bbox[1] == 0:
102
- continue
103
- """要删除的"""
104
- # 3: 'header', # 页眉
105
- # 4: 'page number', # 页码
106
- # 5: 'footnote', # 脚注
107
- # 6: 'footer', # 页脚
108
- """当成span拼接的"""
109
- # 1: 'image', # 图片
110
- # 7: 'table', # 表格
111
- # 13: 'inline_equation', # 行内公式
112
- # 14: 'interline_equation', # 行间公式
113
- # 15: 'text', # ocr识别文本
114
- """layout信息"""
115
- # 11: 'full column', # 单栏
116
- # 12: 'sub column', # 多栏
117
- span = {
118
- "bbox": bbox,
119
- }
120
- if category_id == 1:
121
- span["type"] = ContentType.Image
122
-
123
- elif category_id == 7:
124
- span["type"] = ContentType.Table
125
-
126
- elif category_id == 13:
127
- span["content"] = layout_det["latex"]
128
- span["type"] = ContentType.InlineEquation
129
- elif category_id == 14:
130
- span["content"] = layout_det["latex"]
131
- span["type"] = ContentType.InterlineEquation
132
- elif category_id == 15:
133
- span["content"] = layout_det["text"]
134
- span["type"] = ContentType.Text
135
- # print(span)
136
- spans.append(span)
137
- else:
138
- continue
139
-
140
- '''删除重叠spans中较小的那些'''
141
- spans, dropped_spans_by_span_overlap = remove_overlaps_min_spans(spans)
142
-
143
- '''
144
- 删除remove_span_block_bboxes中的bbox
145
- 并增加drop相关数据
146
- '''
147
- spans, dropped_spans_by_removed_bboxes = remove_spans_by_bboxes_dict(spans, need_remove_spans_bboxes_dict)
148
-
149
- '''对image和table截图'''
150
- spans = ocr_cut_image_and_table(spans, page, page_id, pdf_bytes_md5, imageWriter)
151
-
152
- '''行内公式调整, 高度调整至与同行文字高度一致(优先左侧, 其次右侧)'''
153
- displayed_list = []
154
- text_inline_lines = []
155
- modify_y_axis(spans, displayed_list, text_inline_lines)
156
-
157
- '''模型识别错误的行间公式, type类型转换成行内公式'''
158
- spans = modify_inline_equation(spans, displayed_list, text_inline_lines)
159
-
160
- '''bbox去除粘连'''
161
- spans = remove_overlap_between_bbox_for_span(spans)
162
- '''
163
- 对tpye=["interline_equation", "image", "table"]进行额外处理,
164
- 如果左边有字的话,将该span的bbox中y0调整至不高于文字的y0
165
- '''
166
- spans = adjust_bbox_for_standalone_block(spans)
167
-
168
- '''从ocr_page_info中解析layout信息(按自然阅读方向排序,并修复重叠和交错的bad case)'''
169
- layout_bboxes, layout_tree = layout_detect(ocr_page_info['subfield_dets'], page, ocr_page_info)
170
-
171
- '''将spans合并成line(在layout内,从上到下,从左到右)'''
172
- lines, dropped_spans_by_layout = merge_spans_to_line_by_layout(spans, layout_bboxes)
173
-
174
- '''将lines合并成block'''
175
- blocks = merge_lines_to_block(lines)
176
-
177
- '''获取QA需要外置的list'''
178
- images, tables, interline_equations, inline_equations = get_qa_need_list(blocks)
179
-
180
- '''drop的span_list合并'''
181
- dropped_spans = []
182
- dropped_spans.extend(dropped_spans_by_span_overlap)
183
- dropped_spans.extend(dropped_spans_by_removed_bboxes)
184
- dropped_spans.extend(dropped_spans_by_layout)
185
-
186
- dropped_text_block = []
187
- dropped_image_block = []
188
- dropped_table_block = []
189
- dropped_equation_block = []
190
- for span in dropped_spans:
191
- # drop出的spans进行分类
192
- if span['type'] == ContentType.Text:
193
- dropped_text_block.append(span)
194
- elif span['type'] == ContentType.Image:
195
- dropped_image_block.append(span)
196
- elif span['type'] == ContentType.Table:
197
- dropped_table_block.append(span)
198
- elif span['type'] in [ContentType.InlineEquation, ContentType.InterlineEquation]:
199
- dropped_equation_block.append(span)
200
-
201
- '''构造pdf_info_dict'''
202
- page_info = ocr_construct_page_component(blocks, layout_bboxes, page_id, page_w, page_h, layout_tree,
203
- images, tables, interline_equations, inline_equations,
204
- dropped_text_block, dropped_image_block, dropped_table_block,
205
- dropped_equation_block,
206
- need_remove_spans_bboxes_dict)
207
- pdf_info_dict[f"page_{page_id}"] = page_info
208
-
209
- """分段"""
210
-
211
- para_split(pdf_info_dict, debug_mode=debug_mode)
212
-
213
- """dict转list"""
214
- pdf_info_list = dict_to_list(pdf_info_dict)
215
- new_pdf_info_dict = {
216
- "pdf_info": pdf_info_list,
217
- }
218
-
219
- return new_pdf_info_dict
1
+ from magic_pdf.pdf_parse_union_core import pdf_parse_union
2
+
3
+
4
+ def parse_pdf_by_ocr(pdf_bytes,
5
+ model_list,
6
+ imageWriter,
7
+ start_page_id=0,
8
+ end_page_id=None,
9
+ debug_mode=False,
10
+ ):
11
+ return pdf_parse_union(pdf_bytes,
12
+ model_list,
13
+ imageWriter,
14
+ "ocr",
15
+ start_page_id=start_page_id,
16
+ end_page_id=end_page_id,
17
+ debug_mode=debug_mode,
18
+ )