magic-pdf 0.5.4__py3-none-any.whl

magic_pdf/pdf_parse_by_ocr.py

@@ -0,0 +1,219 @@
+import time
+from loguru import logger
+from magic_pdf.libs.commons import (
+    fitz,
+    get_delta_time,
+    get_docx_model_output,
+)
+from magic_pdf.libs.convert_utils import dict_to_list
+from magic_pdf.libs.coordinate_transform import get_scale_ratio
+from magic_pdf.libs.drop_tag import DropTag
+from magic_pdf.libs.hash_utils import compute_md5
+from magic_pdf.libs.ocr_content_type import ContentType
+from magic_pdf.para.para_split import para_split
+from magic_pdf.pre_proc.construct_page_dict import ocr_construct_page_component
+from magic_pdf.pre_proc.detect_footer_by_model import parse_footers
+from magic_pdf.pre_proc.detect_footnote import parse_footnotes_by_model
+from magic_pdf.pre_proc.detect_header import parse_headers
+from magic_pdf.pre_proc.detect_page_number import parse_pageNos
+from magic_pdf.pre_proc.cut_image import ocr_cut_image_and_table
+from magic_pdf.pre_proc.ocr_detect_layout import layout_detect
+from magic_pdf.pre_proc.ocr_dict_merge import (
+    merge_spans_to_line_by_layout, merge_lines_to_block,
+)
+from magic_pdf.pre_proc.ocr_span_list_modify import remove_spans_by_bboxes, remove_overlaps_min_spans, \
+    adjust_bbox_for_standalone_block, modify_y_axis, modify_inline_equation, get_qa_need_list, \
+    remove_spans_by_bboxes_dict
+from magic_pdf.pre_proc.remove_bbox_overlap import remove_overlap_between_bbox_for_span
+
+
+def parse_pdf_by_ocr(
+        pdf_bytes,
+        pdf_model_output,
+        imageWriter,
+        start_page_id=0,
+        end_page_id=None,
+        debug_mode=False,
+):
+    pdf_bytes_md5 = compute_md5(pdf_bytes)
+
+    pdf_docs = fitz.open("pdf", pdf_bytes)
+    # 初始化空的pdf_info_dict
+    pdf_info_dict = {}
+
+    start_time = time.time()
+
+    end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
+    for page_id in range(start_page_id, end_page_id + 1):
+
+        # 获取当前页的page对象
+        page = pdf_docs[page_id]
+        # 获取当前页的宽高
+        page_w = page.rect.width
+        page_h = page.rect.height
+
+        if debug_mode:
+            time_now = time.time()
+            logger.info(
+                f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}"
+            )
+            start_time = time_now
+
+        # 获取当前页的模型数据
+        ocr_page_info = get_docx_model_output(
+            pdf_model_output, page_id
+        )
+
+        """从json中获取每页的页码、页眉、页脚的bbox"""
+        page_no_bboxes = parse_pageNos(page_id, page, ocr_page_info)
+        header_bboxes = parse_headers(page_id, page, ocr_page_info)
+        footer_bboxes = parse_footers(page_id, page, ocr_page_info)
+        footnote_bboxes = parse_footnotes_by_model(page_id, page, ocr_page_info, debug_mode=debug_mode)
+
+        # 构建需要remove的bbox字典
+        need_remove_spans_bboxes_dict = {
+            DropTag.PAGE_NUMBER: page_no_bboxes,
+            DropTag.HEADER: header_bboxes,
+            DropTag.FOOTER: footer_bboxes,
+            DropTag.FOOTNOTE: footnote_bboxes,
+        }
+
+        layout_dets = ocr_page_info["layout_dets"]
+        spans = []
+
+        # 计算模型坐标和pymu坐标的缩放比例
+        horizontal_scale_ratio, vertical_scale_ratio = get_scale_ratio(
+            ocr_page_info, page
+        )
+
+        for layout_det in layout_dets:
+            category_id = layout_det["category_id"]
+            allow_category_id_list = [1, 7, 13, 14, 15]
+            if category_id in allow_category_id_list:
+                x0, y0, _, _, x1, y1, _, _ = layout_det["poly"]
+                bbox = [
+                    int(x0 / horizontal_scale_ratio),
+                    int(y0 / vertical_scale_ratio),
+                    int(x1 / horizontal_scale_ratio),
+                    int(y1 / vertical_scale_ratio),
+                ]
+                # 删除高度或者宽度为0的spans
+                if bbox[2] - bbox[0] == 0 or bbox[3] - bbox[1] == 0:
+                    continue
+                """要删除的"""
+                #  3: 'header',      # 页眉
+                #  4: 'page number', # 页码
+                #  5: 'footnote',    # 脚注
+                #  6: 'footer',      # 页脚
+                """当成span拼接的"""
+                #  1: 'image', # 图片
+                #  7: 'table',       # 表格
+                #  13: 'inline_equation',     # 行内公式
+                #  14: 'interline_equation',      # 行间公式
+                #  15: 'text',      # ocr识别文本
+                """layout信息"""
+                #  11: 'full column',   # 单栏
+                #  12: 'sub column',    # 多栏
+                span = {
+                    "bbox": bbox,
+                }
+                if category_id == 1:
+                    span["type"] = ContentType.Image
+
+                elif category_id == 7:
+                    span["type"] = ContentType.Table
+
+                elif category_id == 13:
+                    span["content"] = layout_det["latex"]
+                    span["type"] = ContentType.InlineEquation
+                elif category_id == 14:
+                    span["content"] = layout_det["latex"]
+                    span["type"] = ContentType.InterlineEquation
+                elif category_id == 15:
+                    span["content"] = layout_det["text"]
+                    span["type"] = ContentType.Text
+                # print(span)
+                spans.append(span)
+            else:
+                continue
+
+        '''删除重叠spans中较小的那些'''
+        spans, dropped_spans_by_span_overlap = remove_overlaps_min_spans(spans)
+
+        '''
+        删除remove_span_block_bboxes中的bbox
+        并增加drop相关数据
+        '''
+        spans, dropped_spans_by_removed_bboxes = remove_spans_by_bboxes_dict(spans, need_remove_spans_bboxes_dict)
+
+        '''对image和table截图'''
+        spans = ocr_cut_image_and_table(spans, page, page_id, pdf_bytes_md5, imageWriter)
+
+        '''行内公式调整, 高度调整至与同行文字高度一致(优先左侧, 其次右侧)'''
+        displayed_list = []
+        text_inline_lines = []
+        modify_y_axis(spans, displayed_list, text_inline_lines)
+
+        '''模型识别错误的行间公式, type类型转换成行内公式'''
+        spans = modify_inline_equation(spans, displayed_list, text_inline_lines)
+
+        '''bbox去除粘连'''
+        spans = remove_overlap_between_bbox_for_span(spans)
+        '''
+        对tpye=["interline_equation", "image", "table"]进行额外处理,
+        如果左边有字的话,将该span的bbox中y0调整至不高于文字的y0
+        '''
+        spans = adjust_bbox_for_standalone_block(spans)
+
+        '''从ocr_page_info中解析layout信息(按自然阅读方向排序,并修复重叠和交错的bad case)'''
+        layout_bboxes, layout_tree = layout_detect(ocr_page_info['subfield_dets'], page, ocr_page_info)
+
+        '''将spans合并成line(在layout内,从上到下,从左到右)'''
+        lines, dropped_spans_by_layout = merge_spans_to_line_by_layout(spans, layout_bboxes)
+
+        '''将lines合并成block'''
+        blocks = merge_lines_to_block(lines)
+
+        '''获取QA需要外置的list'''
+        images, tables, interline_equations, inline_equations = get_qa_need_list(blocks)
+
+        '''drop的span_list合并'''
+        dropped_spans = []
+        dropped_spans.extend(dropped_spans_by_span_overlap)
+        dropped_spans.extend(dropped_spans_by_removed_bboxes)
+        dropped_spans.extend(dropped_spans_by_layout)
+
+        dropped_text_block = []
+        dropped_image_block = []
+        dropped_table_block = []
+        dropped_equation_block = []
+        for span in dropped_spans:
+            # drop出的spans进行分类
+            if span['type'] == ContentType.Text:
+                dropped_text_block.append(span)
+            elif span['type'] == ContentType.Image:
+                dropped_image_block.append(span)
+            elif span['type'] == ContentType.Table:
+                dropped_table_block.append(span)
+            elif span['type'] in [ContentType.InlineEquation, ContentType.InterlineEquation]:
+                dropped_equation_block.append(span)
+
+        '''构造pdf_info_dict'''
+        page_info = ocr_construct_page_component(blocks, layout_bboxes, page_id, page_w, page_h, layout_tree,
+                                                 images, tables, interline_equations, inline_equations,
+                                                 dropped_text_block, dropped_image_block, dropped_table_block,
+                                                 dropped_equation_block,
+                                                 need_remove_spans_bboxes_dict)
+        pdf_info_dict[f"page_{page_id}"] = page_info
+
+    """分段"""
+
+    para_split(pdf_info_dict, debug_mode=debug_mode)
+
+    """dict转list"""
+    pdf_info_list = dict_to_list(pdf_info_dict)
+    new_pdf_info_dict = {
+        "pdf_info": pdf_info_list,
+    }
+
+    return new_pdf_info_dict

magic_pdf/pdf_parse_by_ocr_v2.py

@@ -0,0 +1,17 @@
+from magic_pdf.pdf_parse_union_core import pdf_parse_union
+
+def parse_pdf_by_ocr(pdf_bytes,
+                     model_list,
+                     imageWriter,
+                     start_page_id=0,
+                     end_page_id=None,
+                     debug_mode=False,
+                     ):
+    return pdf_parse_union(pdf_bytes,
+                           model_list,
+                           imageWriter,
+                           "ocr",
+                           start_page_id=start_page_id,
+                           end_page_id=end_page_id,
+                           debug_mode=debug_mode,
+                           )

magic_pdf/pdf_parse_by_txt.py

@@ -0,0 +1,410 @@
+import time
+
+# from anyio import Path
+
+from magic_pdf.libs.commons import fitz, get_delta_time, get_img_s3_client, get_docx_model_output
+import json
+import os
+import math
+from loguru import logger
+from magic_pdf.layout.bbox_sort import (
+    prepare_bboxes_for_layout_split,
+)
+from magic_pdf.layout.layout_sort import LAYOUT_UNPROC, get_bboxes_layout, get_columns_cnt_of_layout, sort_text_block
+from magic_pdf.libs.convert_utils import dict_to_list
+from magic_pdf.libs.drop_reason import DropReason
+from magic_pdf.libs.hash_utils import compute_md5
+from magic_pdf.libs.markdown_utils import escape_special_markdown_char
+from magic_pdf.libs.safe_filename import sanitize_filename
+from magic_pdf.libs.vis_utils import draw_bbox_on_page, draw_layout_bbox_on_page
+from magic_pdf.pre_proc.cut_image import txt_save_images_by_bboxes
+from magic_pdf.pre_proc.detect_images import parse_images
+from magic_pdf.pre_proc.detect_tables import parse_tables  # 获取tables的bbox
+from magic_pdf.pre_proc.detect_equation import parse_equations  # 获取equations的bbox
+from magic_pdf.pre_proc.detect_header import parse_headers  # 获取headers的bbox
+from magic_pdf.pre_proc.detect_page_number import parse_pageNos  # 获取pageNos的bbox
+from magic_pdf.pre_proc.detect_footnote import parse_footnotes_by_model, parse_footnotes_by_rule  # 获取footnotes的bbox
+from magic_pdf.pre_proc.detect_footer_by_model import parse_footers  # 获取footers的bbox
+
+from magic_pdf.post_proc.detect_para import (
+    ParaProcessPipeline,
+    TitleDetectionException,
+    TitleLevelException,
+    ParaSplitException,
+    ParaMergeException,
+    DenseSingleLineBlockException,
+)
+from magic_pdf.pre_proc.main_text_font import get_main_text_font
+from magic_pdf.pre_proc.remove_colored_strip_bbox import remove_colored_strip_textblock
+from magic_pdf.pre_proc.remove_footer_header import remove_headder_footer_one_page
+
+'''
+from para.para_pipeline import ParaProcessPipeline
+from para.exceptions import (
+    TitleDetectionException,
+    TitleLevelException,
+    ParaSplitException,
+    ParaMergeException,
+    DenseSingleLineBlockException,
+)
+'''
+
+from magic_pdf.post_proc.remove_footnote import merge_footnote_blocks, remove_footnote_blocks
+from magic_pdf.pre_proc.citationmarker_remove import remove_citation_marker
+from magic_pdf.pre_proc.equations_replace import combine_chars_to_pymudict, remove_chars_in_text_blocks, replace_equations_in_textblock
+from magic_pdf.pre_proc.pdf_pre_filter import pdf_filter
+from magic_pdf.pre_proc.detect_footer_header_by_statistics import drop_footer_header
+from magic_pdf.pre_proc.construct_page_dict import construct_page_component
+from magic_pdf.pre_proc.fix_image import combine_images, fix_image_vertical, fix_seperated_image, include_img_title
+from magic_pdf.post_proc.pdf_post_filter import pdf_post_filter
+from magic_pdf.pre_proc.remove_rotate_bbox import get_side_boundry, remove_rotate_side_textblock, remove_side_blank_block
+from magic_pdf.pre_proc.resolve_bbox_conflict import check_text_block_horizontal_overlap, resolve_bbox_overlap_conflict
+from magic_pdf.pre_proc.fix_table import fix_table_text_block, fix_tables, include_table_title
+from magic_pdf.pre_proc.solve_line_alien import solve_inline_too_large_interval
+
+denseSingleLineBlockException_msg = DenseSingleLineBlockException().message
+titleDetectionException_msg = TitleDetectionException().message
+titleLevelException_msg = TitleLevelException().message
+paraSplitException_msg = ParaSplitException().message
+paraMergeException_msg = ParaMergeException().message
+
+
+
+
+def parse_pdf_by_txt(
+    pdf_bytes,
+    pdf_model_output,
+    imageWriter,
+    start_page_id=0,
+    end_page_id=None,
+    debug_mode=False,
+):
+    pdf_bytes_md5 = compute_md5(pdf_bytes)
+
+    pdf_docs = fitz.open("pdf", pdf_bytes)
+    pdf_info_dict = {}
+    start_time = time.time()
+
+    """通过统计pdf全篇文字,识别正文字体"""
+    main_text_font = get_main_text_font(pdf_docs)
+
+    end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
+    for page_id in range(start_page_id, end_page_id + 1):
+        page = pdf_docs[page_id]
+        page_width = page.rect.width
+        page_height = page.rect.height
+
+        if debug_mode:
+            time_now = time.time()
+            logger.info(f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}")
+            start_time = time_now
+        """
+        # 通过一个规则，过滤掉单页超过1500非junkimg的pdf
+        # 对单页面非重复id的img数量做统计,如果当前页超过1500则直接return need_drop
+        """
+        page_imgs = page.get_images()
+
+        # 去除对junkimg的依赖，简化逻辑
+        if len(page_imgs) > 1500:  # 如果当前页超过1500张图片，直接跳过
+            logger.warning(f"page_id: {page_id}, img_counts: {len(page_imgs)}, drop this pdf")
+            result = {"_need_drop": True, "_drop_reason": DropReason.HIGH_COMPUTATIONAL_lOAD_BY_IMGS}
+            if not debug_mode:
+                return result
+
+        """
+        ==================================================================================================================================
+        首先获取基本的block数据，对pdf进行分解，获取图片、表格、公式、text的bbox
+        """
+        # 解析pdf原始文本block
+        text_raw_blocks = page.get_text(
+            "dict",
+            flags=fitz.TEXTFLAGS_TEXT,
+        )["blocks"]
+        model_output_json = get_docx_model_output(pdf_model_output, page_id)
+
+        # 解析图片
+        image_bboxes = parse_images(page_id, page, model_output_json)
+        image_bboxes = fix_image_vertical(image_bboxes, text_raw_blocks)  # 修正图片的位置
+        image_bboxes = fix_seperated_image(image_bboxes)  # 合并有边重合的图片
+        image_bboxes = include_img_title(text_raw_blocks, image_bboxes)  # 向图片上方和下方寻找title，使用规则进行匹配，暂时只支持英文规则
+        """此时image_bboxes中可能出现这种情况，水平并列的2个图片，下方分别有各自的子标题，2个子标题下方又有大标题（形如Figxxx)，会出现2个图片的bbox都包含了这个大标题，这种情况需要把图片合并"""
+        image_bboxes = combine_images(image_bboxes)  # 合并图片
+
+        # 解析表格并对table_bboxes进行位置的微调,防止表格周围的文字被截断
+        table_bboxes = parse_tables(page_id, page, model_output_json)
+        table_bboxes = fix_tables(page, table_bboxes, include_table_title=True, scan_line_num=2)  # 修正
+        table_bboxes = fix_table_text_block(text_raw_blocks, table_bboxes)  # 修正与text block的关系,某些table修正与pymupdf获取到的table内textblock没有完全包含，因此要进行一次修正。
+        #debug_show_bbox(pdf_docs, page_id, table_bboxes, [], [b['bbox'] for b in text_raw_blocks], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
+        table_bboxes = include_table_title(text_raw_blocks, table_bboxes)  # 向table上方和下方寻找title，使用规则进行匹配，暂时只支持英文规则
+
+        # 解析公式
+        equations_inline_bboxes, equations_interline_bboxes = parse_equations(page_id, page, model_output_json)
+
+        """
+        ==================================================================================================================================
+        进入预处理-1阶段
+        -------------------
+        # # 解析标题
+        # title_bboxs = parse_titles(page_id, page, model_output_json)
+        # # 评估Layout是否规整、简单
+        # isSimpleLayout_flag, fullColumn_cnt, subColumn_cnt, curPage_loss = evaluate_pdf_layout(page_id, page, model_output_json)
+        接下来开始进行预处理过程
+        """
+        
+        """去掉每页的页码、页眉、页脚"""
+        page_no_bboxs = parse_pageNos(page_id, page, model_output_json)
+        header_bboxs = parse_headers(page_id, page, model_output_json)
+        footer_bboxs = parse_footers(page_id, page, model_output_json)
+        image_bboxes, table_bboxes, remain_text_blocks, removed_hdr_foot_txt_block, removed_hdr_foot_img_block, removed_hdr_foot_table = remove_headder_footer_one_page(text_raw_blocks, image_bboxes, table_bboxes, header_bboxs, footer_bboxs, page_no_bboxs, page_width, page_height)
+
+        """去除页面上半部分长条色块内的文本块"""
+        remain_text_blocks, removed_colored_narrow_strip_background_text_block = remove_colored_strip_textblock(remain_text_blocks, page)
+
+        #debug_show_bbox(pdf_docs, page_id, footnote_bboxes_by_model, [b['bbox'] for b in remain_text_blocks], header_bboxs, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
+
+        """去掉旋转的文字：水印、垂直排列的文字"""
+        remain_text_blocks, removed_non_horz_text_block = remove_rotate_side_textblock(
+            remain_text_blocks, page_width, page_height
+        )  # 去掉水印，非水平文字
+        remain_text_blocks, removed_empty_side_block = remove_side_blank_block(remain_text_blocks, page_width, page_height) # 删除页面四周可能会留下的完全空白的textblock，这种block形成原因未知
+
+        """出现在图片、表格上的文字块去掉，把层叠的图片单独分离出来，不参与layout的计算"""
+        (
+            image_bboxes,
+            table_bboxes,
+            equations_interline_bboxes,
+            equations_inline_bboxes,
+            remain_text_blocks,
+            text_block_on_image_removed,
+            images_overlap_backup,
+            interline_eq_temp_text_block
+        ) = resolve_bbox_overlap_conflict(
+            image_bboxes, table_bboxes, equations_interline_bboxes, equations_inline_bboxes, remain_text_blocks
+        )
+
+        # """去掉footnote, 从文字和图片中"""
+        # # 通过模型识别到的footnote
+        # footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, md_bookname_save_path,
+        #                                                     debug_mode=debug_mode)
+        # # 通过规则识别到的footnote
+        # footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id)
+        """
+        ==================================================================================================================================
+        """
+
+        # 把图、表、公式都进行截图，保存到存储上，返回图片路径作为内容
+        image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info = txt_save_images_by_bboxes(
+            page_id,
+            page,
+            pdf_bytes_md5,
+            image_bboxes,
+            images_overlap_backup,
+            table_bboxes,
+            equations_inline_bboxes,
+            equations_interline_bboxes,
+            imageWriter
+        )  # 只要表格和图片的截图
+        
+        """"以下进入到公式替换环节 """
+        char_level_text_blocks = page.get_text("rawdict", flags=fitz.TEXTFLAGS_TEXT)['blocks']
+        remain_text_blocks = combine_chars_to_pymudict(remain_text_blocks, char_level_text_blocks)# 合并chars
+        remain_text_blocks = replace_equations_in_textblock(remain_text_blocks, inline_eq_info, interline_eq_info)
+        remain_text_blocks = remove_citation_marker(remain_text_blocks) # 公式替换之后去角标，防止公式无法替换成功。但是这样也会带来个问题就是把角标当公式。各有优劣。
+        remain_text_blocks = remove_chars_in_text_blocks(remain_text_blocks) # 减少中间态数据体积
+        #debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in inline_eq_info], [b['bbox'] for b in interline_eq_info], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 3)
+
+        """去掉footnote, 从文字和图片中(先去角标再去footnote试试)"""
+        # 通过模型识别到的footnote
+        footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, debug_mode=debug_mode)
+        # 通过规则识别到的footnote
+        footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id, main_text_font)
+        """进入pdf过滤器，去掉一些不合理的pdf"""
+        is_good_pdf, err = pdf_filter(page, remain_text_blocks, table_bboxes, image_bboxes)
+        if not is_good_pdf:
+            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {err}")
+            if not debug_mode:
+                return err
+
+        """
+        ==================================================================================================================================
+        进行版面布局切分和过滤
+        """
+        """在切分之前，先检查一下bbox是否有左右重叠的情况，如果有，那么就认为这个pdf暂时没有能力处理好，这种左右重叠的情况大概率是由于pdf里的行间公式、表格没有被正确识别出来造成的 """
+
+        is_text_block_horz_overlap = check_text_block_horizontal_overlap(remain_text_blocks, header_bboxs, footer_bboxs)
+
+        if is_text_block_horz_overlap:
+            # debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in remain_text_blocks], [], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 0)
+            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TEXT_BLCOK_HOR_OVERLAP}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.TEXT_BLCOK_HOR_OVERLAP}
+            if not debug_mode:
+                return result
+
+        """统一格式化成一个数据结构用于计算layout"""
+        page_y0 = 0 if len(header_bboxs) == 0 else max([b[3] for b in header_bboxs])
+        page_y1 = page_height if len(footer_bboxs) == 0 else min([b[1] for b in footer_bboxs])
+        left_x, right_x = get_side_boundry(removed_non_horz_text_block, page_width, page_height)
+        page_boundry = [math.floor(left_x), page_y0 + 1, math.ceil(right_x), page_y1 - 1]
+        # 返回的是一个数组，每个元素[x0, y0, x1, y1, block_content, idx_x, idx_y], 初始时候idx_x, idx_y都是None. 对于图片、公式来说，block_content是图片的地址， 对于段落来说，block_content是段落的内容
+        
+        all_bboxes = prepare_bboxes_for_layout_split(
+            image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info, remain_text_blocks, page_boundry, page)
+        #debug_show_bbox(pdf_docs, page_id, [], [], all_bboxes, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 1)
+        """page_y0, page_y1能够过滤掉页眉和页脚，不会算作layout内"""
+        layout_bboxes, layout_tree = get_bboxes_layout(all_bboxes, page_boundry, page_id)
+        
+        if len(remain_text_blocks)>0 and len(all_bboxes)>0 and len(layout_bboxes)==0:
+            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}
+            if not debug_mode:
+                return result
+
+        """以下去掉复杂的布局和超过2列的布局"""
+        if any([lay["layout_label"] == LAYOUT_UNPROC for lay in layout_bboxes]):  # 复杂的布局
+            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.COMPLICATED_LAYOUT}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.COMPLICATED_LAYOUT}
+            if not debug_mode:
+                return result
+
+        layout_column_width = get_columns_cnt_of_layout(layout_tree)
+        if layout_column_width > 2:  # 去掉超过2列的布局pdf
+            logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TOO_MANY_LAYOUT_COLUMNS}")
+            result = {
+                "_need_drop": True,
+                "_drop_reason": DropReason.TOO_MANY_LAYOUT_COLUMNS,
+                "extra_info": {"column_cnt": layout_column_width},
+            }
+            if not debug_mode:
+                return result
+
+
+        """
+        ==================================================================================================================================
+        构造出下游需要的数据结构
+        """
+        remain_text_blocks = remain_text_blocks + interline_eq_temp_text_block # 把计算layout时候临时删除的行间公式再放回去，防止行间公式替换的时候丢失。
+        removed_text_blocks = []
+        removed_text_blocks.extend(removed_hdr_foot_txt_block)
+        # removed_text_blocks.extend(removed_footnote_text_block)
+        removed_text_blocks.extend(text_block_on_image_removed)
+        removed_text_blocks.extend(removed_non_horz_text_block)
+        removed_text_blocks.extend(removed_colored_narrow_strip_background_text_block)
+
+        removed_images = []
+        # removed_images.extend(footnote_imgs)
+        removed_images.extend(removed_hdr_foot_img_block)
+
+        images_backup = []
+        images_backup.extend(image_backup_info)
+        remain_text_blocks = escape_special_markdown_char(remain_text_blocks) # 转义span里的text
+        sorted_text_remain_text_block = sort_text_block(remain_text_blocks, layout_bboxes)
+
+        footnote_bboxes_tmp = []
+        footnote_bboxes_tmp.extend(footnote_bboxes_by_model)
+        footnote_bboxes_tmp.extend(footnote_bboxes_by_rule)
+        
+
+        page_info = construct_page_component(
+            page_id,
+            image_info,
+            table_info,
+            sorted_text_remain_text_block,
+            layout_bboxes,
+            inline_eq_info,
+            interline_eq_info,
+            page.get_text("dict", flags=fitz.TEXTFLAGS_TEXT)["blocks"],
+            removed_text_blocks=removed_text_blocks,
+            removed_image_blocks=removed_images,
+            images_backup=images_backup,
+            droped_table_block=[],
+            table_backup=[],
+            layout_tree=layout_tree,
+            page_w=page.rect.width,
+            page_h=page.rect.height,
+            footnote_bboxes_tmp=footnote_bboxes_tmp
+        )
+        pdf_info_dict[f"page_{page_id}"] = page_info
+
+    # end page for
+
+    '''计算后处理阶段耗时'''
+    start_time = time.time()
+
+    """
+    ==================================================================================================================================
+    去掉页眉和页脚，这里需要用到一定的统计量，所以放到最后
+    页眉和页脚主要从文本box和图片box中去除，位于页面的四周。
+    下面函数会直接修改pdf_info_dict,从文字块中、图片中删除属于页眉页脚的内容，删除内容做相对应记录
+    """
+    # 去页眉页脚
+    header, footer = drop_footer_header(pdf_info_dict)
+
+    """对单个layout内footnote和他下面的所有textbbox合并"""
+
+    for page_key, page_info in pdf_info_dict.items():
+        page_info = merge_footnote_blocks(page_info, main_text_font)
+        page_info = remove_footnote_blocks(page_info)
+        pdf_info_dict[page_key] = page_info
+
+    """进入pdf后置过滤器，去掉一些不合理的pdf"""
+
+    i = 0
+    for page_info in pdf_info_dict.values():
+        is_good_pdf, err = pdf_post_filter(page_info)
+        if not is_good_pdf:
+            logger.warning(f"page_id: {i}, drop this pdf: {pdf_bytes_md5}, reason: {err}")
+            if not debug_mode:
+                return err
+        i += 1
+
+    if debug_mode:
+        # 打印后处理阶段耗时
+        logger.info(f"post_processing_time: {get_delta_time(start_time)}")
+
+    """
+    ==================================================================================================================================
+    进入段落处理-2阶段
+    """
+
+    # 处理行内文字间距较大问题
+    pdf_info_dict = solve_inline_too_large_interval(pdf_info_dict)
+    
+    start_time = time.time()
+
+    para_process_pipeline = ParaProcessPipeline()
+
+    def _deal_with_text_exception(error_info):
+        logger.warning(f"page_id: {page_id}, drop this pdf: {pdf_bytes_md5}, reason: {error_info}")
+        if error_info == denseSingleLineBlockException_msg:
+            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.DENSE_SINGLE_LINE_BLOCK}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.DENSE_SINGLE_LINE_BLOCK}
+            return result
+        if error_info == titleDetectionException_msg:
+            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TITLE_DETECTION_FAILED}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.TITLE_DETECTION_FAILED}
+            return result
+        elif error_info == titleLevelException_msg:
+            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.TITLE_LEVEL_FAILED}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.TITLE_LEVEL_FAILED}
+            return result
+        elif error_info == paraSplitException_msg:
+            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.PARA_SPLIT_FAILED}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.PARA_SPLIT_FAILED}
+            return result
+        elif error_info == paraMergeException_msg:
+            logger.warning(f"Drop this pdf: {pdf_bytes_md5}, reason: {DropReason.PARA_MERGE_FAILED}")
+            result = {"_need_drop": True, "_drop_reason": DropReason.PARA_MERGE_FAILED}
+            return result
+
+    pdf_info_dict, error_info = para_process_pipeline.para_process_pipeline(pdf_info_dict)
+    if error_info is not None:
+        return _deal_with_text_exception(error_info)
+
+
+    """dict转list"""
+    pdf_info_list = dict_to_list(pdf_info_dict)
+    new_pdf_info_dict = {
+        "pdf_info": pdf_info_list,
+    }
+
+    return new_pdf_info_dict