magic-pdf 0.10.1__py3-none-any.whl → 0.10.3__py3-none-any.whl

magic_pdf/pre_proc/ocr_detect_all_bboxes.py

@@ -1,184 +1,11 @@
-
 from magic_pdf.config.ocr_content_type import BlockType
 from magic_pdf.libs.boxbase import (
-    calculate_iou, calculate_overlap_area_in_bbox1_area_ratio,
+    calculate_iou,
+    calculate_overlap_area_in_bbox1_area_ratio,
     calculate_vertical_projection_overlap_ratio,
-    get_minbox_if_overlap_by_ratio)
-from magic_pdf.pre_proc.remove_bbox_overlap import \
-    remove_overlap_between_bbox_for_block
-
-
-def ocr_prepare_bboxes_for_layout_split(
-    img_blocks,
-    table_blocks,
-    discarded_blocks,
-    text_blocks,
-    title_blocks,
-    interline_equation_blocks,
-    page_w,
-    page_h,
-):
-    all_bboxes = []
-    all_discarded_blocks = []
-    for image in img_blocks:
-        x0, y0, x1, y1 = image['bbox']
-        all_bboxes.append(
-            [
-                x0,
-                y0,
-                x1,
-                y1,
-                None,
-                None,
-                None,
-                BlockType.Image,
-                None,
-                None,
-                None,
-                None,
-                image['score'],
-            ]
-        )
-
-    for table in table_blocks:
-        x0, y0, x1, y1 = table['bbox']
-        all_bboxes.append(
-            [
-                x0,
-                y0,
-                x1,
-                y1,
-                None,
-                None,
-                None,
-                BlockType.Table,
-                None,
-                None,
-                None,
-                None,
-                table['score'],
-            ]
-        )
-
-    for text in text_blocks:
-        x0, y0, x1, y1 = text['bbox']
-        all_bboxes.append(
-            [
-                x0,
-                y0,
-                x1,
-                y1,
-                None,
-                None,
-                None,
-                BlockType.Text,
-                None,
-                None,
-                None,
-                None,
-                text['score'],
-            ]
-        )
-
-    for title in title_blocks:
-        x0, y0, x1, y1 = title['bbox']
-        all_bboxes.append(
-            [
-                x0,
-                y0,
-                x1,
-                y1,
-                None,
-                None,
-                None,
-                BlockType.Title,
-                None,
-                None,
-                None,
-                None,
-                title['score'],
-            ]
-        )
-
-    for interline_equation in interline_equation_blocks:
-        x0, y0, x1, y1 = interline_equation['bbox']
-        all_bboxes.append(
-            [
-                x0,
-                y0,
-                x1,
-                y1,
-                None,
-                None,
-                None,
-                BlockType.InterlineEquation,
-                None,
-                None,
-                None,
-                None,
-                interline_equation['score'],
-            ]
-        )
-
-    """block嵌套问题解决"""
-    """文本框与标题框重叠，优先信任文本框"""
-    all_bboxes = fix_text_overlap_title_blocks(all_bboxes)
-    """任何框体与舍弃框重叠，优先信任舍弃框"""
-    all_bboxes = remove_need_drop_blocks(all_bboxes, discarded_blocks)
-
-    # interline_equation 与title或text框冲突的情况，分两种情况处理
-    """interline_equation框与文本类型框iou比较接近1的时候，信任行间公式框"""
-    all_bboxes = fix_interline_equation_overlap_text_blocks_with_hi_iou(all_bboxes)
-    """interline_equation框被包含在文本类型框内，且interline_equation比文本区块小很多时信任文本框，这时需要舍弃公式框"""
-    # 通过后续大框套小框逻辑删除
-
-    """discarded_blocks中只保留宽度超过1/3页面宽度的，高度超过10的，处于页面下半50%区域的（限定footnote）"""
-    for discarded in discarded_blocks:
-        x0, y0, x1, y1 = discarded['bbox']
-        all_discarded_blocks.append(
-            [
-                x0,
-                y0,
-                x1,
-                y1,
-                None,
-                None,
-                None,
-                BlockType.Discarded,
-                None,
-                None,
-                None,
-                None,
-                discarded['score'],
-            ]
-        )
-        # 将footnote加入到all_bboxes中，用来计算layout
-        if (x1 - x0) > (page_w / 3) and (y1 - y0) > 10 and y0 > (page_h / 2):
-            all_bboxes.append(
-                [
-                    x0,
-                    y0,
-                    x1,
-                    y1,
-                    None,
-                    None,
-                    None,
-                    BlockType.Footnote,
-                    None,
-                    None,
-                    None,
-                    None,
-                    discarded['score'],
-                ]
-            )
-
-    """经过以上处理后，还存在大框套小框的情况，则删除小框"""
-    all_bboxes = remove_overlaps_min_blocks(all_bboxes)
-    all_discarded_blocks = remove_overlaps_min_blocks(all_discarded_blocks)
-    """将剩余的bbox做分离处理，防止后面分layout时出错"""
-    all_bboxes, drop_reasons = remove_overlap_between_bbox_for_block(all_bboxes)
-
-    return all_bboxes, all_discarded_blocks, drop_reasons
+    get_minbox_if_overlap_by_ratio
+)
+from magic_pdf.pre_proc.remove_bbox_overlap import remove_overlap_between_bbox_for_block
 
 
 def add_bboxes(blocks, block_type, bboxes):

magic_pdf/pre_proc/ocr_dict_merge.py

@@ -1,8 +1,5 @@
-from magic_pdf.config.drop_tag import DropTag
 from magic_pdf.config.ocr_content_type import BlockType, ContentType
-from magic_pdf.libs.boxbase import (__is_overlaps_y_exceeds_threshold,
-                                    _is_in_or_part_overlap_with_area_ratio,
-                                    calculate_overlap_area_in_bbox1_area_ratio)
+from magic_pdf.libs.boxbase import __is_overlaps_y_exceeds_threshold, calculate_overlap_area_in_bbox1_area_ratio
 
 
 # 将每一个line中的span从左到右排序
@@ -63,86 +60,6 @@ def merge_spans_to_line(spans, threshold=0.6):
         return lines
 
 
-def merge_spans_to_line_by_layout(spans, layout_bboxes):
-    lines = []
-    new_spans = []
-    dropped_spans = []
-    for item in layout_bboxes:
-        layout_bbox = item['layout_bbox']
-        # 遍历spans,将每个span放入对应的layout中
-        layout_sapns = []
-        for span in spans:
-            if calculate_overlap_area_in_bbox1_area_ratio(
-                    span['bbox'], layout_bbox) > 0.6:
-                layout_sapns.append(span)
-        # 如果layout_sapns不为空，则放入new_spans中
-        if len(layout_sapns) > 0:
-            new_spans.append(layout_sapns)
-            # 从spans删除已经放入layout_sapns中的span
-            for layout_sapn in layout_sapns:
-                spans.remove(layout_sapn)
-
-    if len(new_spans) > 0:
-        for layout_sapns in new_spans:
-            layout_lines = merge_spans_to_line(layout_sapns)
-            lines.extend(layout_lines)
-
-    # 对line中的span进行排序
-    lines = line_sort_spans_by_left_to_right(lines)
-
-    for span in spans:
-        span['tag'] = DropTag.NOT_IN_LAYOUT
-        dropped_spans.append(span)
-
-    return lines, dropped_spans
-
-
-def merge_lines_to_block(lines):
-    # 目前不做block拼接,先做个结构,每个block中只有一个line,block的bbox就是line的bbox
-    blocks = []
-    for line in lines:
-        blocks.append({
-            'bbox': line['bbox'],
-            'lines': [line],
-        })
-    return blocks
-
-
-def sort_blocks_by_layout(all_bboxes, layout_bboxes):
-    new_blocks = []
-    sort_blocks = []
-    for item in layout_bboxes:
-        layout_bbox = item['layout_bbox']
-
-        # 遍历blocks,将每个blocks放入对应的layout中
-        layout_blocks = []
-        for block in all_bboxes:
-            # 如果是footnote则跳过
-            if block[7] == BlockType.Footnote:
-                continue
-            block_bbox = block[:4]
-            if calculate_overlap_area_in_bbox1_area_ratio(
-                    block_bbox, layout_bbox) > 0.8:
-                layout_blocks.append(block)
-
-        # 如果layout_blocks不为空，则放入new_blocks中
-        if len(layout_blocks) > 0:
-            new_blocks.append(layout_blocks)
-            # 从all_bboxes删除已经放入layout_blocks中的block
-            for layout_block in layout_blocks:
-                all_bboxes.remove(layout_block)
-
-    # 如果new_blocks不为空，则对new_blocks中每个block进行排序
-    if len(new_blocks) > 0:
-        for bboxes_in_layout_block in new_blocks:
-            bboxes_in_layout_block.sort(
-                key=lambda x: x[1])  # 一个layout内部的box，按照y0自上而下排序
-            sort_blocks.extend(bboxes_in_layout_block)
-
-    # sort_blocks中已经包含了当前页面所有最终留下的block，且已经排好了顺序
-    return sort_blocks
-
-
 def fill_spans_in_blocks(blocks, spans, radio):
     """将allspans中的span按位置关系，放入blocks中."""
     block_with_spans = []
@@ -164,14 +81,6 @@ def fill_spans_in_blocks(blocks, spans, radio):
             if calculate_overlap_area_in_bbox1_area_ratio(
                     span_bbox, block_bbox) > radio:
                 block_spans.append(span)
-        '''行内公式调整, 高度调整至与同行文字高度一致(优先左侧, 其次右侧)'''
-        # displayed_list = []
-        # text_inline_lines = []
-        # modify_y_axis(block_spans, displayed_list, text_inline_lines)
-        '''模型识别错误的行间公式, type类型转换成行内公式'''
-        # block_spans = modify_inline_equation(block_spans, displayed_list, text_inline_lines)
-        '''bbox去除粘连'''  # 去粘连会影响span的bbox，导致后续fill的时候出错
-        # block_spans = remove_overlap_between_bbox_for_span(block_spans)
 
         block_dict['spans'] = block_spans
         block_with_spans.append(block_dict)
@@ -184,32 +93,7 @@ def fill_spans_in_blocks(blocks, spans, radio):
     return block_with_spans, spans
 
 
-def fix_block_spans(block_with_spans, img_blocks, table_blocks):
-    """1、img_block和table_block因为包含caption和footnote的关系，存在block的嵌套关系
-    需要将caption和footnote的text_span放入相应img_block和table_block内的
-    caption_block和footnote_block中 2、同时需要删除block中的spans字段."""
-    fix_blocks = []
-    for block in block_with_spans:
-        block_type = block['type']
-
-        if block_type == BlockType.Image:
-            block = fix_image_block(block, img_blocks)
-        elif block_type == BlockType.Table:
-            block = fix_table_block(block, table_blocks)
-        elif block_type in [BlockType.Text, BlockType.Title]:
-            block = fix_text_block(block)
-        elif block_type == BlockType.InterlineEquation:
-            block = fix_interline_block(block)
-        else:
-            continue
-        fix_blocks.append(block)
-    return fix_blocks
-
-
 def fix_block_spans_v2(block_with_spans):
-    """1、img_block和table_block因为包含caption和footnote的关系，存在block的嵌套关系
-    需要将caption和footnote的text_span放入相应img_block和table_block内的
-    caption_block和footnote_block中 2、同时需要删除block中的spans字段."""
     fix_blocks = []
     for block in block_with_spans:
         block_type = block['type']
@@ -235,113 +119,6 @@ def fix_discarded_block(discarded_block_with_spans):
     return fix_discarded_blocks
 
 
-def merge_spans_to_block(spans: list, block_bbox: list, block_type: str):
-    block_spans = []
-    # 如果有img_caption，则将img_block中的text_spans放入img_caption_block中
-    for span in spans:
-        if calculate_overlap_area_in_bbox1_area_ratio(span['bbox'],
-                                                      block_bbox) > 0.6:
-            block_spans.append(span)
-    block_lines = merge_spans_to_line(block_spans)
-    # 对line中的span进行排序
-    sort_block_lines = line_sort_spans_by_left_to_right(block_lines)
-    block = {'bbox': block_bbox, 'type': block_type, 'lines': sort_block_lines}
-    return block, block_spans
-
-
-def make_body_block(span: dict, block_bbox: list, block_type: str):
-    # 创建body_block
-    body_line = {
-        'bbox': block_bbox,
-        'spans': [span],
-    }
-    body_block = {'bbox': block_bbox, 'type': block_type, 'lines': [body_line]}
-    return body_block
-
-
-def fix_image_block(block, img_blocks):
-    block['blocks'] = []
-    # 遍历img_blocks,找到与当前block匹配的img_block
-    for img_block in img_blocks:
-        if _is_in_or_part_overlap_with_area_ratio(block['bbox'],
-                                                  img_block['bbox'], 0.95):
-
-            # 创建img_body_block
-            for span in block['spans']:
-                if span['type'] == ContentType.Image and img_block[
-                        'img_body_bbox'] == span['bbox']:
-                    # 创建img_body_block
-                    img_body_block = make_body_block(
-                        span, img_block['img_body_bbox'], BlockType.ImageBody)
-                    block['blocks'].append(img_body_block)
-
-                    # 从spans中移除img_body_block中已经放入的span
-                    block['spans'].remove(span)
-                    break
-
-            # 根据list长度，判断img_block中是否有img_caption
-            if img_block['img_caption_bbox'] is not None:
-                img_caption_block, img_caption_spans = merge_spans_to_block(
-                    block['spans'], img_block['img_caption_bbox'],
-                    BlockType.ImageCaption)
-                block['blocks'].append(img_caption_block)
-
-            if img_block['img_footnote_bbox'] is not None:
-                img_footnote_block, img_footnote_spans = merge_spans_to_block(
-                    block['spans'], img_block['img_footnote_bbox'],
-                    BlockType.ImageFootnote)
-                block['blocks'].append(img_footnote_block)
-            break
-    del block['spans']
-    return block
-
-
-def fix_table_block(block, table_blocks):
-    block['blocks'] = []
-    # 遍历table_blocks,找到与当前block匹配的table_block
-    for table_block in table_blocks:
-        if _is_in_or_part_overlap_with_area_ratio(block['bbox'],
-                                                  table_block['bbox'], 0.95):
-
-            # 创建table_body_block
-            for span in block['spans']:
-                if span['type'] == ContentType.Table and table_block[
-                        'table_body_bbox'] == span['bbox']:
-                    # 创建table_body_block
-                    table_body_block = make_body_block(
-                        span, table_block['table_body_bbox'],
-                        BlockType.TableBody)
-                    block['blocks'].append(table_body_block)
-
-                    # 从spans中移除img_body_block中已经放入的span
-                    block['spans'].remove(span)
-                    break
-
-            # 根据list长度，判断table_block中是否有caption
-            if table_block['table_caption_bbox'] is not None:
-                table_caption_block, table_caption_spans = merge_spans_to_block(
-                    block['spans'], table_block['table_caption_bbox'],
-                    BlockType.TableCaption)
-                block['blocks'].append(table_caption_block)
-
-                # 如果table_caption_block_spans不为空
-                if len(table_caption_spans) > 0:
-                    #  一些span已经放入了caption_block中，需要从block['spans']中删除
-                    for span in table_caption_spans:
-                        block['spans'].remove(span)
-
-            # 根据list长度，判断table_block中是否有table_note
-            if table_block['table_footnote_bbox'] is not None:
-                table_footnote_block, table_footnote_spans = merge_spans_to_block(
-                    block['spans'], table_block['table_footnote_bbox'],
-                    BlockType.TableFootnote)
-                block['blocks'].append(table_footnote_block)
-
-            break
-    del block['spans']
-    return block
-
-
 def fix_text_block(block):
     # 文本block中的公式span都应该转换成行内type
     for span in block['spans']: