magic-pdf 0.5.9__py3-none-any.whl → 0.5.10__py3-none-any.whl

magic_pdf/cli/magicpdf.py

@@ -17,8 +17,8 @@
     
 
 效果：
-python magicpdf.py --json  s3://llm-pdf-text/scihub/xxxx.json?bytes=0,81350 
-python magicpdf.py --pdf  /home/llm/Downloads/xxxx.pdf --model /home/llm/Downloads/xxxx.json  或者 python magicpdf.py --pdf  /home/llm/Downloads/xxxx.pdf
+python magicpdf.py json-command --json  s3://llm-pdf-text/scihub/xxxx.json?bytes=0,81350
+python magicpdf.py pdf-command --pdf  /home/llm/Downloads/xxxx.pdf --model /home/llm/Downloads/xxxx.json  或者 python magicpdf.py --pdf  /home/llm/Downloads/xxxx.pdf
 """
 
 import os
@@ -45,6 +45,7 @@ from magic_pdf.rw.S3ReaderWriter import S3ReaderWriter
 from magic_pdf.rw.DiskReaderWriter import DiskReaderWriter
 from magic_pdf.rw.AbsReaderWriter import AbsReaderWriter
 import csv
+import copy
 
 parse_pdf_methods = click.Choice(["ocr", "txt", "auto"])
 
@@ -81,10 +82,11 @@ def do_parse(
     f_dump_orig_pdf=True,
     f_dump_content_list=True,
 ):
+    orig_model_list = copy.deepcopy(model_list)
 
     local_image_dir, local_md_dir = prepare_env(pdf_file_name, parse_method)
     image_writer, md_writer = DiskReaderWriter(local_image_dir), DiskReaderWriter(local_md_dir)
-    image_dir = (os.path.basename(local_image_dir),)
+    image_dir = str(os.path.basename(local_image_dir))
 
     if parse_method == "auto":
         jso_useful_key = {"_pdf_type": "", "model_list": model_list}
@@ -110,10 +112,7 @@ def do_parse(
     if f_draw_span_bbox:
         draw_span_bbox(pdf_info, pdf_bytes, local_md_dir)
 
-    # write_to_csv(r"D:\project\20231108code-clean\linshixuqiu\pdf_dev\新模型\新建文件夹\luanma.csv",
-    #              [pdf_file_name, pipe.pdf_mid_data['not_common_character_rate'], pipe.pdf_mid_data['not_printable_rate']])
-
-    md_content = pipe.pipe_mk_markdown(str(image_dir), drop_mode=DropMode.NONE)
+    md_content = pipe.pipe_mk_markdown(image_dir, drop_mode=DropMode.NONE)
     if f_dump_md:
         """写markdown"""
         md_writer.write(
@@ -133,7 +132,7 @@ def do_parse(
     if f_dump_model_json:
         """写model_json"""
         md_writer.write(
-            content=json_parse.dumps(pipe.model_list, ensure_ascii=False, indent=4),
+            content=json_parse.dumps(orig_model_list, ensure_ascii=False, indent=4),
             path=f"{pdf_file_name}_model.json",
             mode=AbsReaderWriter.MODE_TXT,
         )
@@ -146,7 +145,7 @@ def do_parse(
             mode=AbsReaderWriter.MODE_BIN,
         )
 
-    content_list = pipe.pipe_mk_uni_format(str(image_dir), drop_mode=DropMode.NONE)
+    content_list = pipe.pipe_mk_uni_format(image_dir, drop_mode=DropMode.NONE)
     if f_dump_content_list:
         """写content_list"""
         md_writer.write(
@@ -281,7 +280,7 @@ def pdf_command(pdf, model, method):
             model_path = pdf.replace(".pdf", ".json")
             if not os.path.exists(model_path):
                 logger.warning(
-                    f"not found json {model_path} existed, use paddle analyze"
+                    f"not found json {model_path} existed"
                 )
                 # 本地无模型数据则调用内置paddle分析，先传空list，在内部识别到空list再调用paddle
                 model_json = "[]"

magic_pdf/libs/version.py

@@ -1 +1 @@
-__version__ = "0.5.9"
+__version__ = "0.5.10"

magic_pdf/para/para_split_v2.py

@@ -11,6 +11,7 @@ LINE_STOP_FLAG = ['.', '!', '?', '。', '！', '？', "：", ":", ")", "）", ";
 INLINE_EQUATION = ContentType.InlineEquation
 INTERLINE_EQUATION = ContentType.InterlineEquation
 TEXT = ContentType.Text
+debug_able = False
 
 
 def __get_span_text(span):
@@ -22,6 +23,7 @@ def __get_span_text(span):
 
 
 def __detect_list_lines(lines, new_layout_bboxes, lang):
+    global debug_able
     """
     探测是否包含了列表，并且把列表的行分开.
     这样的段落特点是，顶格字母大写/数字，紧跟着几行缩进的。缩进的行首字母含小写的。
@@ -41,12 +43,14 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
                     if lst[i] == 1:
                         ones_in_this_interval.append(i)
                     i += 1
-                if len(ones_in_this_interval) > 1 or (start < len(lst) - 1 and ones_in_this_interval and lst[start + 1] in [2, 3]):
+                if len(ones_in_this_interval) > 1 or (
+                        start < len(lst) - 1 and ones_in_this_interval and lst[start + 1] in [2, 3]):
                     indices.append((start, i - 1))
                     ones_indices.append(ones_in_this_interval)
             else:
                 i += 1
         return indices, ones_indices
+
     def find_repeating_patterns(lst):
         indices = []
         ones_indices = []
@@ -132,7 +136,8 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
 
         list_indice, list_start_idx = find_repeating_patterns2(line_fea_encode)
         if len(list_indice) > 0:
-            logger.info(f"发现了列表，列表行数：{list_indice}， {list_start_idx}")
+            if debug_able:
+                logger.info(f"发现了列表，列表行数：{list_indice}， {list_start_idx}")
 
         # TODO check一下这个特列表里缩进的行左侧是不是对齐的。
         segments = []
@@ -140,13 +145,16 @@ def __detect_list_lines(lines, new_layout_bboxes, lang):
             for i in range(start, end + 1):
                 if i > 0:
                     if line_fea_encode[i] == 4:
-                        logger.info(f"列表行的第{i}行不是顶格的")
+                        if debug_able:
+                            logger.info(f"列表行的第{i}行不是顶格的")
                         break
             else:
-                logger.info(f"列表行的第{start}到第{end}行是列表")
+                if debug_able:
+                    logger.info(f"列表行的第{start}到第{end}行是列表")
 
         return split_indices(total_lines, list_indice), list_start_idx
 
+
 def cluster_line_x(lines: list) -> dict:
     """
     对一个block内所有lines的bbox的x0聚类
@@ -170,6 +178,7 @@ def cluster_line_x(lines: list) -> dict:
             min_x0 = x0_new_val
     return x0_2_new_val, min_x0
 
+
 def if_match_reference_list(text: str) -> bool:
     pattern = re.compile(r'^\d+\..*')
     if pattern.match(text):
@@ -190,7 +199,8 @@ def __valign_lines(blocks, layout_bboxes):
     new_layout_bboxes = []
 
     for layout_box in layout_bboxes:
-        blocks_in_layoutbox = [b for b in blocks if b["type"] == BlockType.Text and is_in_layout(b['bbox'], layout_box['layout_bbox'])]
+        blocks_in_layoutbox = [b for b in blocks if
+                               b["type"] == BlockType.Text and is_in_layout(b['bbox'], layout_box['layout_bbox'])]
         if len(blocks_in_layoutbox) == 0 or len(blocks_in_layoutbox[0]["lines"]) == 0:
             new_layout_bboxes.append(layout_box['layout_bbox'])
             continue
@@ -253,7 +263,8 @@ def __align_text_in_layout(blocks, layout_bboxes):
     """
     for layout in layout_bboxes:
         lb = layout['layout_bbox']
-        blocks_in_layoutbox = [block for block in blocks if block["type"] == BlockType.Text and is_in_layout(block['bbox'], lb)]
+        blocks_in_layoutbox = [block for block in blocks if
+                               block["type"] == BlockType.Text and is_in_layout(block['bbox'], lb)]
         if len(blocks_in_layoutbox) == 0:
             continue
 
@@ -383,7 +394,6 @@ def __split_para_lines(lines: list, text_blocks: list) -> list:
             continue
         text_lines.append(line)
 
-
     for block in text_blocks:
         block_bbox = block["bbox"]
         para = []
@@ -394,14 +404,12 @@ def __split_para_lines(lines: list, text_blocks: list) -> list:
         if len(para) > 0:
             text_paras.append(para)
     paras = other_paras.extend(text_paras)
-    paras_sorted = sorted(paras, key = lambda x: x[0]["bbox"][1])
+    paras_sorted = sorted(paras, key=lambda x: x[0]["bbox"][1])
     return paras_sorted
 
 
-
-
-
 def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info, page_num, lang):
+    global debug_able
     """
     如果上个layout的最后一个段落是列表，下一个layout的第一个段落也是列表，那么将他们连接起来。 TODO 因为没有区分列表和段落，所以这个方法暂时不实现。
     根据layout_list_info判断是不是列表。，下个layout的第一个段如果不是列表，那么看他们是否有几行都有相同的缩进。
@@ -410,7 +418,7 @@ def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
         return blocks_group, [False, False]
 
     for i in range(1, len(blocks_group)):
-        if len(blocks_group[i]) == 0 or len(blocks_group[i-1]) == 0:
+        if len(blocks_group[i]) == 0 or len(blocks_group[i - 1]) == 0:
             continue
         pre_layout_list_info = layout_list_info[i - 1]
         next_layout_list_info = layout_list_info[i]
@@ -418,8 +426,10 @@ def __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
         next_paras = blocks_group[i]
         next_first_para = next_paras[0]
 
-        if pre_layout_list_info[1] and not next_layout_list_info[0] and next_first_para["type"] == BlockType.Text:  # 前一个是列表结尾，后一个是非列表开头，此时检测是否有相同的缩进
-            logger.info(f"连接page {page_num} 内的list")
+        if pre_layout_list_info[1] and not next_layout_list_info[0] and next_first_para[
+            "type"] == BlockType.Text:  # 前一个是列表结尾，后一个是非列表开头，此时检测是否有相同的缩进
+            if debug_able:
+                logger.info(f"连接page {page_num} 内的list")
             # 向layout_paras[i] 寻找开头具有相同缩进的连续的行
             may_list_lines = []
             lines = next_first_para.get("lines", [])
@@ -450,7 +460,8 @@ def __connect_list_inter_page(pre_page_paras, next_page_paras, pre_page_layout_b
     if pre_page_paras[-1][-1]["type"] != BlockType.Text or next_page_paras[0][0]["type"] != BlockType.Text:
         return False
     if pre_page_list_info[1] and not next_page_list_info[0]:  # 前一个是列表结尾，后一个是非列表开头，此时检测是否有相同的缩进
-        logger.info(f"连接page {page_num} 内的list")
+        if debug_able:
+            logger.info(f"连接page {page_num} 内的list")
         # 向layout_paras[i] 寻找开头具有相同缩进的连续的行
         may_list_lines = []
         next_page_first_para = next_page_paras[0][0]
@@ -503,7 +514,7 @@ def __connect_para_inter_layoutbox(blocks_group, new_layout_bbox):
         try:
             if len(blocks_group[i]) == 0:
                 continue
-            if  len(blocks_group[i - 1]) == 0:  # TODO 考虑连接问题，
+            if len(blocks_group[i - 1]) == 0:  # TODO 考虑连接问题，
                 connected_layout_blocks.append(blocks_group[i])
                 continue
             # text类型的段才需要考虑layout间的合并
@@ -534,13 +545,14 @@ def __connect_para_inter_layoutbox(blocks_group, new_layout_bbox):
 
         pre_last_line_text = pre_last_line_text.strip()
         next_first_line_text = next_first_line_text.strip()
-        if pre_last_line['bbox'][2] == pre_x2_max and pre_last_line_text and pre_last_line_text[-1] not in LINE_STOP_FLAG and \
+        if pre_last_line['bbox'][2] == pre_x2_max and pre_last_line_text and pre_last_line_text[
+            -1] not in LINE_STOP_FLAG and \
                 next_first_line['bbox'][0] == next_x0_min:  # 前面一行沾满了整个行，并且没有结尾符号.下一行没有空白开头。
             """连接段落条件成立，将前一个layout的段落和后一个layout的段落连接。"""
             #connected_layout_paras[-1][-1].extend(layout_paras[i][0])
             connected_layout_blocks[-1][-1]["lines"].extend(blocks_group[i][0]["lines"])
             #layout_paras[i].pop(0)  # 删除后一个layout的第一个段落， 因为他已经被合并到前一个layout的最后一个段落了。
-            blocks_group[i][0]["lines"] = [] #删除后一个layout第一个段落中的lines，因为他已经被合并到前一个layout的最后一个段落了
+            blocks_group[i][0]["lines"] = []  #删除后一个layout第一个段落中的lines，因为他已经被合并到前一个layout的最后一个段落了
             blocks_group[i][0][LINES_DELETED] = True
             # if len(layout_paras[i]) == 0:
             #     layout_paras.pop(i)
@@ -638,7 +650,8 @@ def find_consecutive_true_regions(input_array):
     return regions
 
 
-def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, debug_mode):
+def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang):
+    global debug_able
     """
     找出来中间对齐的连续单行文本，如果连续行高度相同，那么合并为一个段落。
     一个line居中的条件是：
@@ -660,11 +673,12 @@ def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, deb
             for start, end in consecutive_single_line_indices:
                 #start += index_offset
                 #end += index_offset
-                line_hi = np.array([block["lines"][0]['bbox'][3] - block["lines"][0]['bbox'][1] for block in layout_para[start:end + 1]])
+                line_hi = np.array([block["lines"][0]['bbox'][3] - block["lines"][0]['bbox'][1] for block in
+                                    layout_para[start:end + 1]])
                 first_line_text = ''.join([__get_span_text(span) for span in layout_para[start]["lines"][0]['spans']])
                 if "Table" in first_line_text or "Figure" in first_line_text:
                     pass
-                if debug_mode:
+                if debug_able:
                     logger.info(line_hi.std())
 
                 if line_hi.std() < 2:
@@ -677,10 +691,10 @@ def __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, deb
                             and not all([x1 == layout_box[2] for x1 in all_right_x1]):
                         merge_para = [block["lines"][0] for block in layout_para[start:end + 1]]
                         para_text = ''.join([__get_span_text(span) for line in merge_para for span in line['spans']])
-                        if debug_mode:
+                        if debug_able:
                             logger.info(para_text)
                         layout_para[start]["lines"] = merge_para
-                        for i_para in range(start+1, end+1):
+                        for i_para in range(start + 1, end + 1):
                             layout_para[i_para]["lines"] = []
                             layout_para[i_para][LINES_DELETED] = True
                         #layout_para[start:end + 1] = [merge_para]
@@ -713,14 +727,15 @@ def __do_split_page(blocks, layout_bboxes, new_layout_bbox, page_num, lang):
     blocks_group = __group_line_by_layout(blocks, layout_bboxes)  # block内分段
     layout_list_info = __split_para_in_layoutbox(blocks_group, new_layout_bbox, lang)  # layout内分段
     blocks_group, page_list_info = __connect_list_inter_layout(blocks_group, new_layout_bbox, layout_list_info,
-                                                                page_num, lang)  # layout之间连接列表段落
+                                                               page_num, lang)  # layout之间连接列表段落
     connected_layout_blocks = __connect_para_inter_layoutbox(blocks_group, new_layout_bbox)  # layout间链接段落
 
     return connected_layout_blocks, page_list_info
 
 
-
 def para_split(pdf_info_dict, debug_mode, lang="en"):
+    global debug_able
+    debug_able = debug_mode
     new_layout_of_pages = []  # 数组的数组，每个元素是一个页面的layoutS
     all_page_list_info = []  # 保存每个页面开头和结尾是否是列表
     for page_num, page in pdf_info_dict.items():
@@ -744,14 +759,14 @@ def para_split(pdf_info_dict, debug_mode, lang="en"):
 
         is_conn = __connect_para_inter_page(pre_page_paras, next_page_paras, pre_page_layout_bbox,
                                             next_page_layout_bbox, page_num, lang)
-        if debug_mode:
+        if debug_able:
             if is_conn:
                 logger.info(f"连接了第{page_num - 1}页和第{page_num}页的段落")
 
         is_list_conn = __connect_list_inter_page(pre_page_paras, next_page_paras, pre_page_layout_bbox,
                                                  next_page_layout_bbox, all_page_list_info[page_num - 1],
                                                  all_page_list_info[page_num], page_num, lang)
-        if debug_mode:
+        if debug_able:
             if is_list_conn:
                 logger.info(f"连接了第{page_num - 1}页和第{page_num}页的列表段落")
 
@@ -762,7 +777,7 @@ def para_split(pdf_info_dict, debug_mode, lang="en"):
     for page_num, page in enumerate(pdf_info_dict.values()):
         page_paras = page['para_blocks']
         new_layout_bbox = new_layout_of_pages[page_num]
-        __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang, debug_mode=debug_mode)
+        __connect_middle_align_text(page_paras, new_layout_bbox, page_num, lang)
         __merge_signle_list_text(page_paras, new_layout_bbox, page_num, lang)
 
     # layout展平

magic_pdf/pdf_parse_by_ocr.py

@@ -1,219 +1,18 @@
-import time
-from loguru import logger
-from magic_pdf.libs.commons import (
-    fitz,
-    get_delta_time,
-    get_docx_model_output,
-)
-from magic_pdf.libs.convert_utils import dict_to_list
-from magic_pdf.libs.coordinate_transform import get_scale_ratio
-from magic_pdf.libs.drop_tag import DropTag
-from magic_pdf.libs.hash_utils import compute_md5
-from magic_pdf.libs.ocr_content_type import ContentType
-from magic_pdf.para.para_split import para_split
-from magic_pdf.pre_proc.construct_page_dict import ocr_construct_page_component
-from magic_pdf.pre_proc.detect_footer_by_model import parse_footers
-from magic_pdf.pre_proc.detect_footnote import parse_footnotes_by_model
-from magic_pdf.pre_proc.detect_header import parse_headers
-from magic_pdf.pre_proc.detect_page_number import parse_pageNos
-from magic_pdf.pre_proc.cut_image import ocr_cut_image_and_table
-from magic_pdf.pre_proc.ocr_detect_layout import layout_detect
-from magic_pdf.pre_proc.ocr_dict_merge import (
-    merge_spans_to_line_by_layout, merge_lines_to_block,
-)
-from magic_pdf.pre_proc.ocr_span_list_modify import remove_spans_by_bboxes, remove_overlaps_min_spans, \
-    adjust_bbox_for_standalone_block, modify_y_axis, modify_inline_equation, get_qa_need_list, \
-    remove_spans_by_bboxes_dict
-from magic_pdf.pre_proc.remove_bbox_overlap import remove_overlap_between_bbox_for_span
-
-
-def parse_pdf_by_ocr(
-        pdf_bytes,
-        pdf_model_output,
-        imageWriter,
-        start_page_id=0,
-        end_page_id=None,
-        debug_mode=False,
-):
-    pdf_bytes_md5 = compute_md5(pdf_bytes)
-
-    pdf_docs = fitz.open("pdf", pdf_bytes)
-    # 初始化空的pdf_info_dict
-    pdf_info_dict = {}
-
-    start_time = time.time()
-
-    end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
-    for page_id in range(start_page_id, end_page_id + 1):
-
-        # 获取当前页的page对象
-        page = pdf_docs[page_id]
-        # 获取当前页的宽高
-        page_w = page.rect.width
-        page_h = page.rect.height
-
-        if debug_mode:
-            time_now = time.time()
-            logger.info(
-                f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}"
-            )
-            start_time = time_now
-
-        # 获取当前页的模型数据
-        ocr_page_info = get_docx_model_output(
-            pdf_model_output, page_id
-        )
-
-        """从json中获取每页的页码、页眉、页脚的bbox"""
-        page_no_bboxes = parse_pageNos(page_id, page, ocr_page_info)
-        header_bboxes = parse_headers(page_id, page, ocr_page_info)
-        footer_bboxes = parse_footers(page_id, page, ocr_page_info)
-        footnote_bboxes = parse_footnotes_by_model(page_id, page, ocr_page_info, debug_mode=debug_mode)
-
-        # 构建需要remove的bbox字典
-        need_remove_spans_bboxes_dict = {
-            DropTag.PAGE_NUMBER: page_no_bboxes,
-            DropTag.HEADER: header_bboxes,
-            DropTag.FOOTER: footer_bboxes,
-            DropTag.FOOTNOTE: footnote_bboxes,
-        }
-
-        layout_dets = ocr_page_info["layout_dets"]
-        spans = []
-
-        # 计算模型坐标和pymu坐标的缩放比例
-        horizontal_scale_ratio, vertical_scale_ratio = get_scale_ratio(
-            ocr_page_info, page
-        )
-
-        for layout_det in layout_dets:
-            category_id = layout_det["category_id"]
-            allow_category_id_list = [1, 7, 13, 14, 15]
-            if category_id in allow_category_id_list:
-                x0, y0, _, _, x1, y1, _, _ = layout_det["poly"]
-                bbox = [
-                    int(x0 / horizontal_scale_ratio),
-                    int(y0 / vertical_scale_ratio),
-                    int(x1 / horizontal_scale_ratio),
-                    int(y1 / vertical_scale_ratio),
-                ]
-                # 删除高度或者宽度为0的spans
-                if bbox[2] - bbox[0] == 0 or bbox[3] - bbox[1] == 0:
-                    continue
-                """要删除的"""
-                #  3: 'header',      # 页眉
-                #  4: 'page number', # 页码
-                #  5: 'footnote',    # 脚注
-                #  6: 'footer',      # 页脚
-                """当成span拼接的"""
-                #  1: 'image', # 图片
-                #  7: 'table',       # 表格
-                #  13: 'inline_equation',     # 行内公式
-                #  14: 'interline_equation',      # 行间公式
-                #  15: 'text',      # ocr识别文本
-                """layout信息"""
-                #  11: 'full column',   # 单栏
-                #  12: 'sub column',    # 多栏
-                span = {
-                    "bbox": bbox,
-                }
-                if category_id == 1:
-                    span["type"] = ContentType.Image
-
-                elif category_id == 7:
-                    span["type"] = ContentType.Table
-
-                elif category_id == 13:
-                    span["content"] = layout_det["latex"]
-                    span["type"] = ContentType.InlineEquation
-                elif category_id == 14:
-                    span["content"] = layout_det["latex"]
-                    span["type"] = ContentType.InterlineEquation
-                elif category_id == 15:
-                    span["content"] = layout_det["text"]
-                    span["type"] = ContentType.Text
-                # print(span)
-                spans.append(span)
-            else:
-                continue
-
-        '''删除重叠spans中较小的那些'''
-        spans, dropped_spans_by_span_overlap = remove_overlaps_min_spans(spans)
-
-        '''
-        删除remove_span_block_bboxes中的bbox
-        并增加drop相关数据
-        '''
-        spans, dropped_spans_by_removed_bboxes = remove_spans_by_bboxes_dict(spans, need_remove_spans_bboxes_dict)
-
-        '''对image和table截图'''
-        spans = ocr_cut_image_and_table(spans, page, page_id, pdf_bytes_md5, imageWriter)
-
-        '''行内公式调整, 高度调整至与同行文字高度一致(优先左侧, 其次右侧)'''
-        displayed_list = []
-        text_inline_lines = []
-        modify_y_axis(spans, displayed_list, text_inline_lines)
-
-        '''模型识别错误的行间公式, type类型转换成行内公式'''
-        spans = modify_inline_equation(spans, displayed_list, text_inline_lines)
-
-        '''bbox去除粘连'''
-        spans = remove_overlap_between_bbox_for_span(spans)
-        '''
-        对tpye=["interline_equation", "image", "table"]进行额外处理,
-        如果左边有字的话,将该span的bbox中y0调整至不高于文字的y0
-        '''
-        spans = adjust_bbox_for_standalone_block(spans)
-
-        '''从ocr_page_info中解析layout信息(按自然阅读方向排序,并修复重叠和交错的bad case)'''
-        layout_bboxes, layout_tree = layout_detect(ocr_page_info['subfield_dets'], page, ocr_page_info)
-
-        '''将spans合并成line(在layout内,从上到下,从左到右)'''
-        lines, dropped_spans_by_layout = merge_spans_to_line_by_layout(spans, layout_bboxes)
-
-        '''将lines合并成block'''
-        blocks = merge_lines_to_block(lines)
-
-        '''获取QA需要外置的list'''
-        images, tables, interline_equations, inline_equations = get_qa_need_list(blocks)
-
-        '''drop的span_list合并'''
-        dropped_spans = []
-        dropped_spans.extend(dropped_spans_by_span_overlap)
-        dropped_spans.extend(dropped_spans_by_removed_bboxes)
-        dropped_spans.extend(dropped_spans_by_layout)
-
-        dropped_text_block = []
-        dropped_image_block = []
-        dropped_table_block = []
-        dropped_equation_block = []
-        for span in dropped_spans:
-            # drop出的spans进行分类
-            if span['type'] == ContentType.Text:
-                dropped_text_block.append(span)
-            elif span['type'] == ContentType.Image:
-                dropped_image_block.append(span)
-            elif span['type'] == ContentType.Table:
-                dropped_table_block.append(span)
-            elif span['type'] in [ContentType.InlineEquation, ContentType.InterlineEquation]:
-                dropped_equation_block.append(span)
-
-        '''构造pdf_info_dict'''
-        page_info = ocr_construct_page_component(blocks, layout_bboxes, page_id, page_w, page_h, layout_tree,
-                                                 images, tables, interline_equations, inline_equations,
-                                                 dropped_text_block, dropped_image_block, dropped_table_block,
-                                                 dropped_equation_block,
-                                                 need_remove_spans_bboxes_dict)
-        pdf_info_dict[f"page_{page_id}"] = page_info
-
-    """分段"""
-
-    para_split(pdf_info_dict, debug_mode=debug_mode)
-
-    """dict转list"""
-    pdf_info_list = dict_to_list(pdf_info_dict)
-    new_pdf_info_dict = {
-        "pdf_info": pdf_info_list,
-    }
-
-    return new_pdf_info_dict
+from magic_pdf.pdf_parse_union_core import pdf_parse_union
+
+
+def parse_pdf_by_ocr(pdf_bytes,
+                     model_list,
+                     imageWriter,
+                     start_page_id=0,
+                     end_page_id=None,
+                     debug_mode=False,
+                     ):
+    return pdf_parse_union(pdf_bytes,
+                           model_list,
+                           imageWriter,
+                           "ocr",
+                           start_page_id=start_page_id,
+                           end_page_id=end_page_id,
+                           debug_mode=debug_mode,
+                           )

magic_pdf/pdf_parse_by_txt.py

@@ -1,410 +1,19 @@
-import time
-
-# from anyio import Path
-
-from magic_pdf.libs.commons import fitz, get_delta_time, get_img_s3_client, get_docx_model_output
-import json
-import os
-import math
-from loguru import logger
-from magic_pdf.layout.bbox_sort import (
-    prepare_bboxes_for_layout_split,
-)
-from magic_pdf.layout.layout_sort import LAYOUT_UNPROC, get_bboxes_layout, get_columns_cnt_of_layout, sort_text_block
-from magic_pdf.libs.convert_utils import dict_to_list
-from magic_pdf.libs.drop_reason import DropReason
-from magic_pdf.libs.hash_utils import compute_md5
-from magic_pdf.libs.markdown_utils import escape_special_markdown_char
-from magic_pdf.libs.safe_filename import sanitize_filename
-from magic_pdf.libs.vis_utils import draw_bbox_on_page, draw_layout_bbox_on_page
-from magic_pdf.pre_proc.cut_image import txt_save_images_by_bboxes
-from magic_pdf.pre_proc.detect_images import parse_images
-from magic_pdf.pre_proc.detect_tables import parse_tables  # 获取tables的bbox
-from magic_pdf.pre_proc.detect_equation import parse_equations  # 获取equations的bbox
-from magic_pdf.pre_proc.detect_header import parse_headers  # 获取headers的bbox
-from magic_pdf.pre_proc.detect_page_number import parse_pageNos  # 获取pageNos的bbox
-from magic_pdf.pre_proc.detect_footnote import parse_footnotes_by_model, parse_footnotes_by_rule  # 获取footnotes的bbox
-from magic_pdf.pre_proc.detect_footer_by_model import parse_footers  # 获取footers的bbox
-
-from magic_pdf.post_proc.detect_para import (
-    ParaProcessPipeline,
-    TitleDetectionException,
-    TitleLevelException,
-    ParaSplitException,
-    ParaMergeException,
-    DenseSingleLineBlockException,
-)
-from magic_pdf.pre_proc.main_text_font import get_main_text_font
-from magic_pdf.pre_proc.remove_colored_strip_bbox import remove_colored_strip_textblock
-from magic_pdf.pre_proc.remove_footer_header import remove_headder_footer_one_page
-
-'''
-from para.para_pipeline import ParaProcessPipeline
-from para.exceptions import (
-    TitleDetectionException,
-    TitleLevelException,
-    ParaSplitException,
-    ParaMergeException,
-    DenseSingleLineBlockException,
-)
-'''
-
-from magic_pdf.post_proc.remove_footnote import merge_footnote_blocks, remove_footnote_blocks
-from magic_pdf.pre_proc.citationmarker_remove import remove_citation_marker
-from magic_pdf.pre_proc.equations_replace import combine_chars_to_pymudict, remove_chars_in_text_blocks, replace_equations_in_textblock
-from magic_pdf.pre_proc.pdf_pre_filter import pdf_filter
-from magic_pdf.pre_proc.detect_footer_header_by_statistics import drop_footer_header
-from magic_pdf.pre_proc.construct_page_dict import construct_page_component
-from magic_pdf.pre_proc.fix_image import combine_images, fix_image_vertical, fix_seperated_image, include_img_title
-from magic_pdf.post_proc.pdf_post_filter import pdf_post_filter
-from magic_pdf.pre_proc.remove_rotate_bbox import get_side_boundry, remove_rotate_side_textblock, remove_side_blank_block
-from magic_pdf.pre_proc.resolve_bbox_conflict import check_text_block_horizontal_overlap, resolve_bbox_overlap_conflict
-from magic_pdf.pre_proc.fix_table import fix_table_text_block, fix_tables, include_table_title
-from magic_pdf.pre_proc.solve_line_alien import solve_inline_too_large_interval
-
-denseSingleLineBlockException_msg = DenseSingleLineBlockException().message
-titleDetectionException_msg = TitleDetectionException().message
-titleLevelException_msg = TitleLevelException().message
-paraSplitException_msg = ParaSplitException().message
-paraMergeException_msg = ParaMergeException().message
-
-
+from magic_pdf.pdf_parse_union_core import pdf_parse_union
 
 
 def parse_pdf_by_txt(
     pdf_bytes,
-    pdf_model_output,
+    model_list,
     imageWriter,
     start_page_id=0,
     end_page_id=None,
     debug_mode=False,
 ):
-    pdf_bytes_md5 = compute_md5(pdf_bytes)
-
-    pdf_docs = fitz.open("pdf", pdf_bytes)
-    pdf_info_dict = {}
-    start_time = time.time()
-
-    """通过统计pdf全篇文字,识别正文字体"""
-    main_text_font = get_main_text_font(pdf_docs)
-
-    end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
-    for page_id in range(start_page_id, end_page_id + 1):
-        page = pdf_docs[page_id]
-        page_width = page.rect.width
-        page_height = page.rect.height
-
-        if debug_mode:
-            time_now = time.time()
-            logger.info(f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}")
-            start_time = time_now
-        """
-        # 通过一个规则，过滤掉单页超过1500非junkimg的pdf
-        # 对单页面非重复id的img数量做统计,如果当前页超过1500则直接return need_drop
-        """
-        page_imgs = page.get_images()
-
-        # 去除对junkimg的依赖，简化逻辑
-        if len(page_imgs) > 1500:  # 如果当前页超过1500张图片，直接跳过
-            logger.warning(f"page_id: {page_id}, img_counts: {len(page_imgs)}, drop this pdf")
-            result = {"_need_drop": True, "_drop_reason": DropReason.HIGH_COMPUTATIONAL_lOAD_BY_IMGS}
-            if not debug_mode:
-                return result
-
-        """
-        ==================================================================================================================================
-        首先获取基本的block数据，对pdf进行分解，获取图片、表格、公式、text的bbox
-        """
-        # 解析pdf原始文本block
-        text_raw_blocks = page.get_text(
-            "dict",
-            flags=fitz.TEXTFLAGS_TEXT,
-        )["blocks"]
-        model_output_json = get_docx_model_output(pdf_model_output, page_id)
-
-        # 解析图片
-        image_bboxes = parse_images(page_id, page, model_output_json)
-        image_bboxes = fix_image_vertical(image_bboxes, text_raw_blocks)  # 修正图片的位置
-        image_bboxes = fix_seperated_image(image_bboxes)  # 合并有边重合的图片
-        image_bboxes = include_img_title(text_raw_blocks, image_bboxes)  # 向图片上方和下方寻找title，使用规则进行匹配，暂时只支持英文规则
-        """此时image_bboxes中可能出现这种情况，水平并列的2个图片，下方分别有各自的子标题，2个子标题下方又有大标题（形如Figxxx)，会出现2个图片的bbox都包含了这个大标题，这种情况需要把图片合并"""
-        image_bboxes = combine_images(image_bboxes)  # 合并图片
-
-        # 解析表格并对table_bboxes进行位置的微调,防止表格周围的文字被截断
-        table_bboxes = parse_tables(page_id, page, model_output_json)
-        table_bboxes = fix_tables(page, table_bboxes, include_table_title=True, scan_line_num=2)  # 修正
-        table_bboxes = fix_table_text_block(text_raw_blocks, table_bboxes)  # 修正与text block的关系,某些table修正与pymupdf获取到的table内textblock没有完全包含，因此要进行一次修正。
-        #debug_show_bbox(pdf_docs, page_id, table_bboxes, [], [b['bbox'] for b in text_raw_blocks], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
-        table_bboxes = include_table_title(text_raw_blocks, table_bboxes)  # 向table上方和下方寻找title，使用规则进行匹配，暂时只支持英文规则
-
-        # 解析公式
-        equations_inline_bboxes, equations_interline_bboxes = parse_equations(page_id, page, model_output_json)
-
-        """
-        ==================================================================================================================================
-        进入预处理-1阶段
-        -------------------
-        # # 解析标题
-        # title_bboxs = parse_titles(page_id, page, model_output_json)
-        # # 评估Layout是否规整、简单
-        # isSimpleLayout_flag, fullColumn_cnt, subColumn_cnt, curPage_loss = evaluate_pdf_layout(page_id, page, model_output_json)
-        接下来开始进行预处理过程
-        """
-        
-        """去掉每页的页码、页眉、页脚"""
-        page_no_bboxs = parse_pageNos(page_id, page, model_output_json)
-        header_bboxs = parse_headers(page_id, page, model_output_json)
-        footer_bboxs = parse_footers(page_id, page, model_output_json)
-        image_bboxes, table_bboxes, remain_text_blocks, removed_hdr_foot_txt_block, removed_hdr_foot_img_block, removed_hdr_foot_table = remove_headder_footer_one_page(text_raw_blocks, image_bboxes, table_bboxes, header_bboxs, footer_bboxs, page_no_bboxs, page_width, page_height)
-
-        """去除页面上半部分长条色块内的文本块"""
-        remain_text_blocks, removed_colored_narrow_strip_background_text_block = remove_colored_strip_textblock(remain_text_blocks, page)
-
-        #debug_show_bbox(pdf_docs, page_id, footnote_bboxes_by_model, [b['bbox'] for b in remain_text_blocks], header_bboxs, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
-
-        """去掉旋转的文字：水印、垂直排列的文字"""
-        remain_text_blocks, removed_non_horz_text_block = remove_rotate_side_textblock(
-            remain_text_blocks, page_width, page_height
-        )  # 去掉水印，非水平文字
-        remain_text_blocks, removed_empty_side_block = remove_side_blank_block(remain_text_blocks, page_width, page_height) # 删除页面四周可能会留下的完全空白的textblock，这种block形成原因未知
-
-        """出现在图片、表格上的文字块去掉，把层叠的图片单独分离出来，不参与layout的计算"""
-        (
-            image_bboxes,
-            table_bboxes,
-            equations_interline_bboxes,
-            equations_inline_bboxes,
-            remain_text_blocks,
-            text_block_on_image_removed,
-            images_overlap_backup,
-            interline_eq_temp_text_block
-        ) = resolve_bbox_overlap_conflict(
-            image_bboxes, table_bboxes, equations_interline_bboxes, equations_inline_bboxes, remain_text_blocks
-        )
-
-        # """去掉footnote, 从文字和图片中"""
-        # # 通过模型识别到的footnote
-        # footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, md_bookname_save_path,
-        #                                                     debug_mode=debug_mode)
-        # # 通过规则识别到的footnote
-        # footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id)
-        """
-        ==================================================================================================================================
-        """
-
-        # 把图、表、公式都进行截图，保存到存储上，返回图片路径作为内容
-        image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info = txt_save_images_by_bboxes(
-            page_id,
-            page,
-            pdf_bytes_md5,
-            image_bboxes,
-            images_overlap_backup,
-            table_bboxes,
-            equations_inline_bboxes,
-            equations_interline_bboxes,
-            imageWriter
-        )  # 只要表格和图片的截图
-        
-        """"以下进入到公式替换环节 """
-        char_level_text_blocks = page.get_text("rawdict", flags=fitz.TEXTFLAGS_TEXT)['blocks']
-        remain_text_blocks = combine_chars_to_pymudict(remain_text_blocks, char_level_text_blocks)# 合并chars
-        remain_text_blocks = replace_equations_in_textblock(remain_text_blocks, inline_eq_info, interline_eq_info)
-        remain_text_blocks = remove_citation_marker(remain_text_blocks) # 公式替换之后去角标，防止公式无法替换成功。但是这样也会带来个问题就是把角标当公式。各有优劣。
-        remain_text_blocks = remove_chars_in_text_blocks(remain_text_blocks) # 减少中间态数据体积
-        #debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in inline_eq_info], [b['bbox'] for b in interline_eq_info], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 3)
-
-        """去掉footnote, 从文字和图片中(先去角标再去footnote试试)"""
-        # 通过模型识别到的footnote
-        footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, debug_mode=debug_mode)
-        # 通过规则识别到的footnote
-        footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id, main_text_font)
-        """进入pdf过滤器，去掉一些不合理的pdf"""
-        is_good_pdf, err = pdf_filter(page, remain_text_blocks, table_bboxes, image_bboxes)
-        if not is_good_pdf:
-            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {err}")
-            if not debug_mode:
-                return err
-
-        """
-        ==================================================================================================================================
-        进行版面布局切分和过滤
-        """
-        """在切分之前，先检查一下bbox是否有左右重叠的情况，如果有，那么就认为这个pdf暂时没有能力处理好，这种左右重叠的情况大概率是由于pdf里的行间公式、表格没有被正确识别出来造成的 """
-
-        is_text_block_horz_overlap = check_text_block_horizontal_overlap(remain_text_blocks, header_bboxs, footer_bboxs)
-
-        if is_text_block_horz_overlap:
-            # debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in remain_text_blocks], [], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 0)
-            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TEXT_BLCOK_HOR_OVERLAP}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.TEXT_BLCOK_HOR_OVERLAP}
-            if not debug_mode:
-                return result
-
-        """统一格式化成一个数据结构用于计算layout"""
-        page_y0 = 0 if len(header_bboxs) == 0 else max([b[3] for b in header_bboxs])
-        page_y1 = page_height if len(footer_bboxs) == 0 else min([b[1] for b in footer_bboxs])
-        left_x, right_x = get_side_boundry(removed_non_horz_text_block, page_width, page_height)
-        page_boundry = [math.floor(left_x), page_y0 + 1, math.ceil(right_x), page_y1 - 1]
-        # 返回的是一个数组，每个元素[x0, y0, x1, y1, block_content, idx_x, idx_y], 初始时候idx_x, idx_y都是None. 对于图片、公式来说，block_content是图片的地址， 对于段落来说，block_content是段落的内容
-        
-        all_bboxes = prepare_bboxes_for_layout_split(
-            image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info, remain_text_blocks, page_boundry, page)
-        #debug_show_bbox(pdf_docs, page_id, [], [], all_bboxes, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 1)
-        """page_y0, page_y1能够过滤掉页眉和页脚，不会算作layout内"""
-        layout_bboxes, layout_tree = get_bboxes_layout(all_bboxes, page_boundry, page_id)
-        
-        if len(remain_text_blocks)>0 and len(all_bboxes)>0 and len(layout_bboxes)==0:
-            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}
-            if not debug_mode:
-                return result
-
-        """以下去掉复杂的布局和超过2列的布局"""
-        if any([lay["layout_label"] == LAYOUT_UNPROC for lay in layout_bboxes]):  # 复杂的布局
-            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.COMPLICATED_LAYOUT}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.COMPLICATED_LAYOUT}
-            if not debug_mode:
-                return result
-
-        layout_column_width = get_columns_cnt_of_layout(layout_tree)
-        if layout_column_width > 2:  # 去掉超过2列的布局pdf
-            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TOO_MANY_LAYOUT_COLUMNS}")
-            result = {
-                "_need_drop": True,
-                "_drop_reason": DropReason.TOO_MANY_LAYOUT_COLUMNS,
-                "extra_info": {"column_cnt": layout_column_width},
-            }
-            if not debug_mode:
-                return result
-
-
-        """
-        ==================================================================================================================================
-        构造出下游需要的数据结构
-        """
-        remain_text_blocks = remain_text_blocks + interline_eq_temp_text_block # 把计算layout时候临时删除的行间公式再放回去，防止行间公式替换的时候丢失。
-        removed_text_blocks = []
-        removed_text_blocks.extend(removed_hdr_foot_txt_block)
-        # removed_text_blocks.extend(removed_footnote_text_block)
-        removed_text_blocks.extend(text_block_on_image_removed)
-        removed_text_blocks.extend(removed_non_horz_text_block)
-        removed_text_blocks.extend(removed_colored_narrow_strip_background_text_block)
-
-        removed_images = []
-        # removed_images.extend(footnote_imgs)
-        removed_images.extend(removed_hdr_foot_img_block)
-
-        images_backup = []
-        images_backup.extend(image_backup_info)
-        remain_text_blocks = escape_special_markdown_char(remain_text_blocks) # 转义span里的text
-        sorted_text_remain_text_block = sort_text_block(remain_text_blocks, layout_bboxes)
-
-        footnote_bboxes_tmp = []
-        footnote_bboxes_tmp.extend(footnote_bboxes_by_model)
-        footnote_bboxes_tmp.extend(footnote_bboxes_by_rule)
-        
-
-        page_info = construct_page_component(
-            page_id,
-            image_info,
-            table_info,
-            sorted_text_remain_text_block,
-            layout_bboxes,
-            inline_eq_info,
-            interline_eq_info,
-            page.get_text("dict", flags=fitz.TEXTFLAGS_TEXT)["blocks"],
-            removed_text_blocks=removed_text_blocks,
-            removed_image_blocks=removed_images,
-            images_backup=images_backup,
-            droped_table_block=[],
-            table_backup=[],
-            layout_tree=layout_tree,
-            page_w=page.rect.width,
-            page_h=page.rect.height,
-            footnote_bboxes_tmp=footnote_bboxes_tmp
-        )
-        pdf_info_dict[f"page_{page_id}"] = page_info
-
-    # end page for
-
-    '''计算后处理阶段耗时'''
-    start_time = time.time()
-
-    """
-    ==================================================================================================================================
-    去掉页眉和页脚，这里需要用到一定的统计量，所以放到最后
-    页眉和页脚主要从文本box和图片box中去除，位于页面的四周。
-    下面函数会直接修改pdf_info_dict,从文字块中、图片中删除属于页眉页脚的内容，删除内容做相对应记录
-    """
-    # 去页眉页脚
-    header, footer = drop_footer_header(pdf_info_dict)
-
-    """对单个layout内footnote和他下面的所有textbbox合并"""
-
-    for page_key, page_info in pdf_info_dict.items():
-        page_info = merge_footnote_blocks(page_info, main_text_font)
-        page_info = remove_footnote_blocks(page_info)
-        pdf_info_dict[page_key] = page_info
-
-    """进入pdf后置过滤器，去掉一些不合理的pdf"""
-
-    i = 0
-    for page_info in pdf_info_dict.values():
-        is_good_pdf, err = pdf_post_filter(page_info)
-        if not is_good_pdf:
-            logger.warning(f"page_id: {i}, drop this pdf: {pdf_bytes_md5}, reason: {err}")
-            if not debug_mode:
-                return err
-        i += 1
-
-    if debug_mode:
-        # 打印后处理阶段耗时
-        logger.info(f"post_processing_time: {get_delta_time(start_time)}")
-
-    """
-    ==================================================================================================================================
-    进入段落处理-2阶段
-    """
-
-    # 处理行内文字间距较大问题
-    pdf_info_dict = solve_inline_too_large_interval(pdf_info_dict)
-    
-    start_time = time.time()
-
-    para_process_pipeline = ParaProcessPipeline()
-
-    def _deal_with_text_exception(error_info):
-        logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {error_info}")
-        if error_info == denseSingleLineBlockException_msg:
-            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.DENSE_SINGLE_LINE_BLOCK}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.DENSE_SINGLE_LINE_BLOCK}
-            return result
-        if error_info == titleDetectionException_msg:
-            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TITLE_DETECTION_FAILED}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.TITLE_DETECTION_FAILED}
-            return result
-        elif error_info == titleLevelException_msg:
-            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TITLE_LEVEL_FAILED}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.TITLE_LEVEL_FAILED}
-            return result
-        elif error_info == paraSplitException_msg:
-            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.PARA_SPLIT_FAILED}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.PARA_SPLIT_FAILED}
-            return result
-        elif error_info == paraMergeException_msg:
-            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.PARA_MERGE_FAILED}")
-            result = {"_need_drop": True, "_drop_reason": DropReason.PARA_MERGE_FAILED}
-            return result
-
-    pdf_info_dict, error_info = para_process_pipeline.para_process_pipeline(pdf_info_dict)
-    if error_info is not None:
-        return _deal_with_text_exception(error_info)
-
-
-    """dict转list"""
-    pdf_info_list = dict_to_list(pdf_info_dict)
-    new_pdf_info_dict = {
-        "pdf_info": pdf_info_list,
-    }
-
-    return new_pdf_info_dict
+    return pdf_parse_union(pdf_bytes,
+                           model_list,
+                           imageWriter,
+                           "txt",
+                           start_page_id=start_page_id,
+                           end_page_id=end_page_id,
+                           debug_mode=debug_mode,
+                           )

magic_pdf/pre_proc/equations_replace.py

@@ -299,7 +299,7 @@ def replace_line_v2(eqinfo, line):
             [b["bbox"][2] for b in delete_chars]
         )
     else:
-        logger.debug(f"行内公式替换没有发生，尝试下一行匹配, eqinfo={eqinfo}")
+        # logger.debug(f"行内公式替换没有发生，尝试下一行匹配, eqinfo={eqinfo}")
         return False
 
     # 删除位于x0, x1这两个中间的span

magic_pdf/user_api.py

@@ -18,8 +18,8 @@ from loguru import logger
 from magic_pdf.libs.version import __version__
 from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze
 from magic_pdf.rw import AbsReaderWriter
-from magic_pdf.pdf_parse_by_ocr_v2 import parse_pdf_by_ocr
-from magic_pdf.pdf_parse_by_txt_v2 import parse_pdf_by_txt
+from magic_pdf.pdf_parse_by_ocr import parse_pdf_by_ocr
+from magic_pdf.pdf_parse_by_txt import parse_pdf_by_txt
 
 PARSE_TYPE_TXT = "txt"
 PARSE_TYPE_OCR = "ocr"
@@ -86,45 +86,7 @@ def parse_union_pdf(pdf_bytes: bytes, pdf_models: list, imageWriter: AbsReaderWr
             return None
 
     pdf_info_dict = parse_pdf(parse_pdf_by_txt)
-    # text_all = ""
-    # for page_dict in pdf_info_dict['pdf_info']:
-    #     for para_block in page_dict['para_blocks']:
-    #         if para_block['type'] in ['title', 'text']:
-    #             for line in para_block['lines']:
-    #                 for span in line['spans']:
-    #                     text_all += span['content']
-
-    # def calculate_not_common_character_rate(text):
-    #     garbage_regex = re.compile(r'[^\u4e00-\u9fa5\u0030-\u0039\u0041-\u005a\u0061-\u007a\u3000-\u303f\uff00-\uffef]')
-    #     # 计算乱码字符的数量
-    #     garbage_count = len(garbage_regex.findall(text))
-    #     total = len(text)
-    #     if total == 0:
-    #         return 0  # 避免除以零的错误
-    #     return garbage_count / total
-    #
-    # def calculate_not_printable_rate(text):
-    #     printable_text = ""
-    #     for c in text:
-    #         if c.isprintable():
-    #             printable_text += c
-    #     printable_total = len(printable_text)
-    #     total = len(text)
-    #     if total == 0:
-    #         return 0  # 避免除以零的错误
-    #     return (total - printable_total) / total
-    #
-    # not_common_character_rate = calculate_not_common_character_rate(text_all)
-    # not_printable_rate = calculate_not_printable_rate(text_all)
-    # pdf_info_dict["_not_common_character_rate"] = not_common_character_rate
-    # pdf_info_dict["_not_printable_rate"] = not_printable_rate
-    # logger.info(f"not_common_character_rate: {not_common_character_rate}, not_printable_rate: {not_printable_rate}")
-    '''新逻辑使用pdfminer识别乱码pdf,准确率高且不会误伤,已在解析流程之前进行处理'''
-    # not_common_character_rate对小语种可能会有误伤，not_printable_rate对小语种较为友好
-    if (pdf_info_dict is None
-            or pdf_info_dict.get("_need_drop", False)
-            # or not_printable_rate > 0.02  # 参考一些正常的pdf，这个值没有超过0.01的，阈值设为0.02
-    ):
+    if pdf_info_dict is None or pdf_info_dict.get("_need_drop", False):
         logger.warning(f"parse_pdf_by_txt drop or error, switch to parse_pdf_by_ocr")
         if input_model_is_empty:
             pdf_models = doc_analyze(pdf_bytes, ocr=True)

{magic_pdf-0.5.9.dist-info → magic_pdf-0.5.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: magic-pdf
-Version: 0.5.9
+Version: 0.5.10
 Summary: A practical tool for converting PDF to Markdown
 Home-page: https://github.com/magicpdf/Magic-PDF
 Requires-Python: >=3.9
@@ -72,20 +72,57 @@ Key features include:
 
 ### Usage Instructions
 
-1. **Install Magic-PDF**
-
+#### 1. Install Magic-PDF
 ```bash
-pip install magic-pdf[cpu] # Install the CPU version 
-or
-pip install magic-pdf[gpu] # Install the GPU version
+pip install magic-pdf
 ```
 
-2. **Usage via Command Line**
+#### 2. Usage via Command Line
 
+###### simple
+```bash
+cp magic-pdf.template.json to ~/magic-pdf.json
+magic-pdf pdf-command --pdf "pdf_path" --model "model_json_path"
+```
+###### more 
 ```bash
 magic-pdf --help
 ```
 
+#### 3. Usage via Api
+
+###### Local
+```python
+image_writer = DiskReaderWriter(local_image_dir)
+image_dir = str(os.path.basename(local_image_dir))
+jso_useful_key = {"_pdf_type": "", "model_list": model_json}
+pipe = UNIPipe(pdf_bytes, jso_useful_key, image_writer)
+pipe.pipe_classify()
+pipe.pipe_parse()
+md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
+```
+
+###### Object Storage
+```python
+s3pdf_cli = S3ReaderWriter(pdf_ak, pdf_sk, pdf_endpoint)
+image_dir = "s3://img_bucket/"
+s3image_cli = S3ReaderWriter(img_ak, img_sk, img_endpoint, parent_path=image_dir)
+pdf_bytes = s3pdf_cli.read(s3_pdf_path, mode=s3pdf_cli.MODE_BIN)
+jso_useful_key = {"_pdf_type": "", "model_list": model_json}
+pipe = UNIPipe(pdf_bytes, jso_useful_key, s3image_cli)
+pipe.pipe_classify()
+pipe.pipe_parse()
+md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
+```
+
+Demo can be referred to [demo.py](https://github.com/magicpdf/Magic-PDF/blob/master/demo/demo.py)
+
+## All Thanks To Our Contributors
+
+<a href="https://github.com/magicpdf/Magic-PDF/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=magicpdf/Magic-PDF" />
+</a>
+
 ## License Information
 
 See [LICENSE.md](https://github.com/magicpdf/Magic-PDF/blob/master/LICENSE.md) for details.

{magic_pdf-0.5.9.dist-info → magic_pdf-0.5.10.dist-info}/RECORD

@@ -1,13 +1,11 @@
 magic_pdf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/pdf_parse_by_ocr.py,sha256=jYEfc4P6o4f7mZqfPFITs3NWlBxe4v1gwLh1yu2VDIQ,8950
-magic_pdf/pdf_parse_by_ocr_v2.py,sha256=1XVRwmcGRtxMMRYgqkJcctD4tLjCprgpGQsHpeVQktM,637
-magic_pdf/pdf_parse_by_txt.py,sha256=5_kdfvDkv_XwDove2AW7SopGysYLJ1-tsOQy2yuII1Y,21932
-magic_pdf/pdf_parse_by_txt_v2.py,sha256=mGadyYamoCNGNsKOQM1uXQR65zMUKyL24yURGHADmVs,1908
+magic_pdf/pdf_parse_by_ocr.py,sha256=IWnSWt1Z-d35xRqspzdLR2iUtma_SAu4W7K4kEk8SHc,638
+magic_pdf/pdf_parse_by_txt.py,sha256=KUSH7Gh83CZmdyWw59pqDskwyJ2Kg-jU-9fnQGJQEs4,537
 magic_pdf/pdf_parse_for_train.py,sha256=Oby61DMjJ716Jj_ri7lwXfv2Chus0pbBR2RPXrmBW08,28661
 magic_pdf/pdf_parse_union_core.py,sha256=a67iQuEfuslAEF-wQplGZKXUuz5mT3HiCyvuR52E6Gw,10584
-magic_pdf/user_api.py,sha256=R4onPBXlA8GARwlqBh5wmxUtTxwQb-PUeFK7eTAWzoU,4971
+magic_pdf/user_api.py,sha256=CVQH-VSiZpz0bSkyMT4czk1epZriIPSJsLsPbluPa9Q,3054
 magic_pdf/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/cli/magicpdf.py,sha256=aMmY_J83_8IdkkxyZUHg8WzIbCA_oW4cyjlUUZE0Wvc,11117
+magic_pdf/cli/magicpdf.py,sha256=FF6flO6wUcKG9Qx_FG6-xhHfmQzQWLjwrkMa5kowDgs,10937
 magic_pdf/dict2md/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/dict2md/mkcontent.py,sha256=rWUY-2opd0jeowEUEVOV_uWcKum1Q7ng4nOoT6-ka_s,17459
 magic_pdf/dict2md/ocr_mkcontent.py,sha256=RyxebPtvFfNce_HCa-_YGxwFx_srzL-BfMKc85V9JG0,15442
@@ -46,7 +44,7 @@ magic_pdf/libs/pdf_check.py,sha256=MAe8wzwT0qvPf_I72wEZG7k1g4haNHS7oUtLqkB5rlE,2
 magic_pdf/libs/pdf_image_tools.py,sha256=CAd01giTKr_UJz1_QtDOARG9G9z69GFpzRZwcWSfLtE,1282
 magic_pdf/libs/safe_filename.py,sha256=ckwcM_eqoysTb5id8czp-tXq2G9da0-l3pshZDCHQtE,236
 magic_pdf/libs/textbase.py,sha256=SC1Frhz3Fb7V7n2SFRBsl7Bmg0JZdlvZskq0lfW1vIk,732
-magic_pdf/libs/version.py,sha256=JXLyhF5WmLgRZBfWGz9zWe2g5ISKSLpn2jp8yLaC-s4,22
+magic_pdf/libs/version.py,sha256=1nlPInsRzDbcDPveZ3ghSJ6v6KveN9n6gnj-twW4DkI,23
 magic_pdf/libs/vis_utils.py,sha256=hTOTEakKV0pGMbk0tbRkVI_tku7A3dGc96ynObZ4kwI,10207
 magic_pdf/model/360_layout_analysis.py,sha256=GbchKPJRVcrxvwNXMnR4vt8lOLPauTWMl-43ayyhX7U,221
 magic_pdf/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -64,7 +62,7 @@ magic_pdf/para/exceptions.py,sha256=kpjGxrSZ-drNmoKlmuQ0asTjI8cKKKWsdDDBoDHQP9M,
 magic_pdf/para/layout_match_processor.py,sha256=yr4FEO7GJ502udShqGRqIJQ_FQxoa0aG_mhmWd8nLwI,1554
 magic_pdf/para/para_pipeline.py,sha256=zLaCHI9jLi1UPzh0lHP44mUjpKVTHS0gE_5YrkjVqEY,11796
 magic_pdf/para/para_split.py,sha256=-UJM2jREW_2h3ZlJAU7dRD8bK3CMGKuhJrfgqv3Auvk,31310
-magic_pdf/para/para_split_v2.py,sha256=a04dsUFE3JD4DA9e2DULJgbKrcqWuCfK58de1p-T3Io,36610
+magic_pdf/para/para_split_v2.py,sha256=jGOhsubdh_CEgSv9WMNmp1loq1YNlpcAj3yh3g0gPhw,37027
 magic_pdf/para/raw_processor.py,sha256=mHxD9FrdOSXH7NqM41s55URyCyuyACvm9kKtowkIb3k,6317
 magic_pdf/para/stats.py,sha256=-6Pf9Y8jkP1uJOYWiHUjw9Lb-Fb9GY7MHr_ok7x2GX0,9731
 magic_pdf/para/title_processor.py,sha256=pYZv9vEkIjAtCz8jIUtl9AVUy_ib5SdAZmMVoZtsMRI,38593
@@ -89,7 +87,7 @@ magic_pdf/pre_proc/detect_header.py,sha256=KOmRehgKMuMqNa_2weXkdNSiRVWMFgLMQE4e1
 magic_pdf/pre_proc/detect_images.py,sha256=8DwGGTb5IjxqADZDTc_ngwJrTYXxK2qpRqI2FBoPr00,30432
 magic_pdf/pre_proc/detect_page_number.py,sha256=qvYrBbCtBbREvw-MySL_p7byCRvcm1fkLJ5ZB4TP8OM,2848
 magic_pdf/pre_proc/detect_tables.py,sha256=srJzgLVeVuOsqnESqfdJfVukTF84K8qmI5mgFX_BZGs,2800
-magic_pdf/pre_proc/equations_replace.py,sha256=rpEHD5IvavRgUSus0tLjKcIv2faStp_Ii9vf09HVUtI,20384
+magic_pdf/pre_proc/equations_replace.py,sha256=fXj7ZV7F3YtkDYrAhE9g5tHk4_3pVUyLbhDtMjbxjWU,20386
 magic_pdf/pre_proc/fix_image.py,sha256=5MOfkXc8abfIp49g-68vll40wwTUZ5tcQ2gtsJuFmvs,11486
 magic_pdf/pre_proc/fix_table.py,sha256=20sqJe27fAXcL7_C0qQ9mpsggmH37WuX-wPYWyRgACA,13227
 magic_pdf/pre_proc/main_text_font.py,sha256=1gkjvPuBdKC4oVFkLvnRm2zghsLtVlfAEMKXouyVonM,1048
@@ -117,9 +115,9 @@ magic_pdf/train_utils/convert_to_train_format.py,sha256=ifo2FAoBMa_etCvz0O4v03xO
 magic_pdf/train_utils/extract_caption.py,sha256=gommEqIEWLplSDEJWD7_66daqlOBsWhpRBW1DHpkny4,1825
 magic_pdf/train_utils/remove_footer_header.py,sha256=pyeNNdJ-th3wl5Xwb10ZLYNaFN4-6BmahoMFE8VTNNs,5978
 magic_pdf/train_utils/vis_utils.py,sha256=MV9N9cT3ifJ35u7LFKGF9I_bOIQrtU1zcsxu2hj3aqM,10111
-magic_pdf-0.5.9.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-magic_pdf-0.5.9.dist-info/METADATA,sha256=6Y0tWpKEWrjYaNVrBWddqU9mn4EKR8cSbka47hUSmog,2971
-magic_pdf-0.5.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-magic_pdf-0.5.9.dist-info/entry_points.txt,sha256=NbSkSmE08UuTwdoJD8Uofq8iyufySA4x7jmIIk4YCzI,57
-magic_pdf-0.5.9.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
-magic_pdf-0.5.9.dist-info/RECORD,,
+magic_pdf-0.5.10.dist-info/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+magic_pdf-0.5.10.dist-info/METADATA,sha256=B3e0sVOyFhk47EfHPuLFRUNxzdasWYx3XuYR53LSJX8,4175
+magic_pdf-0.5.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+magic_pdf-0.5.10.dist-info/entry_points.txt,sha256=NbSkSmE08UuTwdoJD8Uofq8iyufySA4x7jmIIk4YCzI,57
+magic_pdf-0.5.10.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
+magic_pdf-0.5.10.dist-info/RECORD,,

magic_pdf/pdf_parse_by_ocr_v2.py

@@ -1,17 +0,0 @@
-from magic_pdf.pdf_parse_union_core import pdf_parse_union
-
-def parse_pdf_by_ocr(pdf_bytes,
-                     model_list,
-                     imageWriter,
-                     start_page_id=0,
-                     end_page_id=None,
-                     debug_mode=False,
-                     ):
-    return pdf_parse_union(pdf_bytes,
-                           model_list,
-                           imageWriter,
-                           "ocr",
-                           start_page_id=start_page_id,
-                           end_page_id=end_page_id,
-                           debug_mode=debug_mode,
-                           )

magic_pdf/pdf_parse_by_txt_v2.py

@@ -1,56 +0,0 @@
-from magic_pdf.pdf_parse_union_core import pdf_parse_union
-
-
-def parse_pdf_by_txt(
-    pdf_bytes,
-    model_list,
-    imageWriter,
-    start_page_id=0,
-    end_page_id=None,
-    debug_mode=False,
-):
-    return pdf_parse_union(pdf_bytes,
-                           model_list,
-                           imageWriter,
-                           "txt",
-                           start_page_id=start_page_id,
-                           end_page_id=end_page_id,
-                           debug_mode=debug_mode,
-                           )
-
-
-if __name__ == "__main__":
-    pass
-    # if 1:
-    #     import fitz
-    #     import json
-    #
-    #     with open("/opt/data/pdf/20240418/25536-00.pdf", "rb") as f:
-    #         pdf_bytes = f.read()
-    #     pdf_docs = fitz.open("pdf", pdf_bytes)
-    #
-    #     with open("/opt/data/pdf/20240418/25536-00.json") as f:
-    #         model_list = json.loads(f.readline())
-    #
-    #     magic_model = MagicModel(model_list, pdf_docs)
-    #     for i in range(7):
-    #         print(magic_model.get_imgs(i))
-    #
-    #     for page_no, page in enumerate(pdf_docs):
-    #         inline_equations, interline_equations, interline_equation_blocks = (
-    #             magic_model.get_equations(page_no)
-    #         )
-    #
-    #         text_raw_blocks = page.get_text("dict", flags=fitz.TEXTFLAGS_TEXT)["blocks"]
-    #         char_level_text_blocks = page.get_text(
-    #             "rawdict", flags=fitz.TEXTFLAGS_TEXT
-    #         )["blocks"]
-    #         text_blocks = combine_chars_to_pymudict(
-    #             text_raw_blocks, char_level_text_blocks
-    #         )
-    #         text_blocks = replace_equations_in_textblock(
-    #             text_blocks, inline_equations, interline_equations
-    #         )
-    #         text_blocks = remove_citation_marker(text_blocks)
-    #
-    #         text_blocks = remove_chars_in_text_blocks(text_blocks)