magic-pdf 0.10.0__py3-none-any.whl → 0.10.2__py3-none-any.whl

magic_pdf/pre_proc/pdf_pre_filter.py

@@ -1,78 +0,0 @@
-from magic_pdf.config.drop_reason import DropReason
-from magic_pdf.libs.boxbase import _is_in, _is_in_or_part_overlap
-from magic_pdf.libs.commons import fitz
-
-
-def __area(box):
-    return (box[2] - box[0]) * (box[3] - box[1])
-
-
-def __is_contain_color_background_rect(
-    page: fitz.Page, text_blocks, image_bboxes
-) -> bool:
-    """检查page是包含有颜色背景的矩形."""
-    color_bg_rect = []
-    p_width, p_height = page.rect.width, page.rect.height
-
-    # 先找到最大的带背景矩形
-    blocks = page.get_cdrawings()
-    for block in blocks:
-        if 'fill' in block and block['fill']:  # 过滤掉透明的
-            fill = list(block['fill'])
-            fill[0], fill[1], fill[2] = int(fill[0]), int(fill[1]), int(fill[2])
-            if fill == (1.0, 1.0, 1.0):
-                continue
-            rect = block['rect']
-            # 过滤掉特别小的矩形
-            if __area(rect) < 10 * 10:
-                continue
-            # 为了防止是svg图片上的色块，这里过滤掉这类
-
-            if any(
-                [_is_in_or_part_overlap(rect, img_bbox) for img_bbox in image_bboxes]
-            ):
-                continue
-            color_bg_rect.append(rect)
-
-    # 找到最大的背景矩形
-    if len(color_bg_rect) > 0:
-        max_rect = max(color_bg_rect, key=lambda x: __area(x))
-        max_rect_int = (
-            int(max_rect[0]),
-            int(max_rect[1]),
-            int(max_rect[2]),
-            int(max_rect[3]),
-        )
-        # 判断最大的背景矩形是否包含超过3行文字，或者50个字 TODO
-        if (
-            max_rect[2] - max_rect[0] > 0.2 * p_width
-            and max_rect[3] - max_rect[1] > 0.1 * p_height
-        ):  # 宽度符合
-            # 看是否有文本块落入到这个矩形中
-            for text_block in text_blocks:
-                box = text_block['bbox']
-                box_int = (int(box[0]), int(box[1]), int(box[2]), int(box[3]))
-                if _is_in(box_int, max_rect_int):
-                    return True
-
-    return False
-
-
-def __is_table_overlap_text_block(text_blocks, table_bbox):
-    """检查table_bbox是否覆盖了text_blocks里的文本块 TODO."""
-    for text_block in text_blocks:
-        box = text_block['bbox']
-        if _is_in_or_part_overlap(table_bbox, box):
-            return True
-    return False
-
-
-def pdf_filter(page: fitz.Page, text_blocks, table_bboxes, image_bboxes) -> tuple:
-    """return:(True|False, err_msg) True, 如果pdf符合要求 False, 如果pdf不符合要求."""
-    if __is_contain_color_background_rect(page, text_blocks, image_bboxes):
-        return False, {
-            '_need_drop': True,
-            '_drop_reason': DropReason.COLOR_BACKGROUND_TEXT_BOX,
-        }
-
-    return True, None

magic_pdf/pre_proc/remove_colored_strip_bbox.py

@@ -1,101 +0,0 @@
-from loguru import logger
-
-from magic_pdf.config.drop_tag import COLOR_BG_HEADER_TXT_BLOCK
-from magic_pdf.libs.boxbase import (_is_in, _is_in_or_part_overlap,
-                                    calculate_overlap_area_2_minbox_area_ratio)
-
-
-def __area(box):
-    return (box[2] - box[0]) * (box[3] - box[1])
-
-
-def rectangle_position_determination(rect, p_width):
-    """判断矩形是否在页面中轴线附近。
-
-    Args:
-        rect (list): 矩形坐标，格式为[x1, y1, x2, y2]。
-        p_width (int): 页面宽度。
-
-    Returns:
-        bool: 若矩形在页面中轴线附近则返回True，否则返回False。
-    """
-    # 页面中轴线x坐标
-    x_axis = p_width / 2
-    # 矩形是否跨越中轴线
-    is_span = rect[0] < x_axis and rect[2] > x_axis
-    if is_span:
-        return True
-    else:
-        # 矩形与中轴线的距离，只算近的那一边
-        distance = rect[0] - x_axis if rect[0] > x_axis else x_axis - rect[2]
-        # 判断矩形与中轴线的距离是否小于页面宽度的20%
-        if distance < p_width * 0.2:
-            return True
-        else:
-            return False
-
-
-def remove_colored_strip_textblock(remain_text_blocks, page):
-    """根据页面中特定颜色和大小过滤文本块，将符合条件的文本块从remain_text_blocks中移除，并返回移除的文本块列表colored_str
-    ip_textblock。
-
-    Args:
-        remain_text_blocks (list): 剩余文本块列表。
-        page (Page): 页面对象。
-
-    Returns:
-        tuple: 剩余文本块列表和移除的文本块列表。
-    """
-    colored_strip_textblocks = []  # 先构造一个空的返回
-    if len(remain_text_blocks) > 0:
-        p_width, p_height = page.rect.width, page.rect.height
-        blocks = page.get_cdrawings()
-        colored_strip_bg_rect = []
-        for block in blocks:
-            is_filled = (
-                'fill' in block and block['fill'] and block['fill'] != (1.0, 1.0, 1.0)
-            )  # 过滤掉透明的
-            rect = block['rect']
-            area_is_large_enough = __area(rect) > 100  # 过滤掉特别小的矩形
-            rectangle_position_determination_result = rectangle_position_determination(
-                rect, p_width
-            )
-            in_upper_half_page = (
-                rect[3] < p_height * 0.3
-            )  # 找到位于页面上半部分的矩形，下边界小于页面高度的30%
-            aspect_ratio_exceeds_4 = (rect[2] - rect[0]) > (
-                rect[3] - rect[1]
-            ) * 4  # 找到长宽比超过4的矩形
-
-            if (
-                is_filled
-                and area_is_large_enough
-                and rectangle_position_determination_result
-                and in_upper_half_page
-                and aspect_ratio_exceeds_4
-            ):
-                colored_strip_bg_rect.append(rect)
-
-        if len(colored_strip_bg_rect) > 0:
-            for colored_strip_block_bbox in colored_strip_bg_rect:
-                for text_block in remain_text_blocks:
-                    text_bbox = text_block['bbox']
-                    if _is_in(text_bbox, colored_strip_block_bbox) or (
-                        _is_in_or_part_overlap(text_bbox, colored_strip_block_bbox)
-                        and calculate_overlap_area_2_minbox_area_ratio(
-                            text_bbox, colored_strip_block_bbox
-                        )
-                        > 0.6
-                    ):
-                        logger.info(
-                            f'remove_colored_strip_textblock: {text_bbox}, {colored_strip_block_bbox}'
-                        )
-                        text_block['tag'] = COLOR_BG_HEADER_TXT_BLOCK
-                        colored_strip_textblocks.append(text_block)
-
-                if len(colored_strip_textblocks) > 0:
-                    for colored_strip_textblock in colored_strip_textblocks:
-                        if colored_strip_textblock in remain_text_blocks:
-                            remain_text_blocks.remove(colored_strip_textblock)
-
-    return remain_text_blocks, colored_strip_textblocks

magic_pdf/pre_proc/remove_footer_header.py

@@ -1,114 +0,0 @@
-import re
-
-from magic_pdf.config.drop_tag import CONTENT_IN_FOOT_OR_HEADER, PAGE_NO
-from magic_pdf.libs.boxbase import _is_in_or_part_overlap
-
-
-def remove_headder_footer_one_page(text_raw_blocks, image_bboxes, table_bboxes, header_bboxs, footer_bboxs,
-                                   page_no_bboxs, page_w, page_h):
-    """删除页眉页脚，页码 从line级别进行删除，删除之后观察这个text-block是否是空的，如果是空的，则移动到remove_list中."""
-    header = []
-    footer = []
-    if len(header) == 0:
-        model_header = header_bboxs
-        if model_header:
-            x0 = min([x for x, _, _, _ in model_header])
-            y0 = min([y for _, y, _, _ in model_header])
-            x1 = max([x1 for _, _, x1, _ in model_header])
-            y1 = max([y1 for _, _, _, y1 in model_header])
-            header = [x0, y0, x1, y1]
-    if len(footer) == 0:
-        model_footer = footer_bboxs
-        if model_footer:
-            x0 = min([x for x, _, _, _ in model_footer])
-            y0 = min([y for _, y, _, _ in model_footer])
-            x1 = max([x1 for _, _, x1, _ in model_footer])
-            y1 = max([y1 for _, _, _, y1 in model_footer])
-            footer = [x0, y0, x1, y1]
-
-    header_y0 = 0 if len(header) == 0 else header[3]
-    footer_y0 = page_h if len(footer) == 0 else footer[1]
-    if page_no_bboxs:
-        top_part = [b for b in page_no_bboxs if b[3] < page_h / 2]
-        btn_part = [b for b in page_no_bboxs if b[1] > page_h / 2]
-
-        top_max_y0 = max([b[1] for b in top_part]) if top_part else 0
-        btn_min_y1 = min([b[3] for b in btn_part]) if btn_part else page_h
-
-        header_y0 = max(header_y0, top_max_y0)
-        footer_y0 = min(footer_y0, btn_min_y1)
-
-    content_boundry = [0, header_y0, page_w, footer_y0]
-
-    header = [0, 0, page_w, header_y0]
-    footer = [0, footer_y0, page_w, page_h]
-
-    """以上计算出来了页眉页脚的边界，下面开始进行删除"""
-    text_block_to_remove = []
-    # 首先检查每个textblock
-    for blk in text_raw_blocks:
-        if len(blk['lines']) > 0:
-            for line in blk['lines']:
-                line_del = []
-                for span in line['spans']:
-                    span_del = []
-                    if span['bbox'][3] < header_y0:
-                        span_del.append(span)
-                    elif _is_in_or_part_overlap(span['bbox'], header) or _is_in_or_part_overlap(span['bbox'], footer):
-                        span_del.append(span)
-                for span in span_del:
-                    line['spans'].remove(span)
-                if not line['spans']:
-                    line_del.append(line)
-
-            for line in line_del:
-                blk['lines'].remove(line)
-        else:
-            # if not blk['lines']:
-            blk['tag'] = CONTENT_IN_FOOT_OR_HEADER
-            text_block_to_remove.append(blk)
-
-    """有的时候由于pageNo太小了，总是会有一点和content_boundry重叠一点，被放入正文，因此对于pageNo，进行span粒度的删除"""
-    page_no_block_2_remove = []
-    if page_no_bboxs:
-        for pagenobox in page_no_bboxs:
-            for block in text_raw_blocks:
-                if _is_in_or_part_overlap(pagenobox, block['bbox']):  # 在span级别删除页码
-                    for line in block['lines']:
-                        for span in line['spans']:
-                            if _is_in_or_part_overlap(pagenobox, span['bbox']):
-                                # span['text'] = ''
-                                span['tag'] = PAGE_NO
-                                # 检查这个block是否只有这一个span，如果是，那么就把这个block也删除
-                                if len(line['spans']) == 1 and len(block['lines']) == 1:
-                                    page_no_block_2_remove.append(block)
-    else:
-        # 测试最后一个是不是页码：规则是，最后一个block仅有1个line,一个span,且text是数字，空格，符号组成，不含字母,并且包含数字
-        if len(text_raw_blocks) > 0:
-            text_raw_blocks.sort(key=lambda x: x['bbox'][1], reverse=True)
-            last_block = text_raw_blocks[0]
-            if len(last_block['lines']) == 1:
-                last_line = last_block['lines'][0]
-                if len(last_line['spans']) == 1:
-                    last_span = last_line['spans'][0]
-                    if last_span['text'].strip() and not re.search('[a-zA-Z]', last_span['text']) and re.search('[0-9]',
-                                                                                                                last_span[
-                                                                                                                    'text']):
-                        last_span['tag'] = PAGE_NO
-                        page_no_block_2_remove.append(last_block)
-
-    for b in page_no_block_2_remove:
-        text_block_to_remove.append(b)
-
-    for blk in text_block_to_remove:
-        if blk in text_raw_blocks:
-            text_raw_blocks.remove(blk)
-
-    text_block_remain = text_raw_blocks
-    image_bbox_to_remove = [bbox for bbox in image_bboxes if not _is_in_or_part_overlap(bbox, content_boundry)]
-
-    image_bbox_remain = [bbox for bbox in image_bboxes if _is_in_or_part_overlap(bbox, content_boundry)]
-    table_bbox_to_remove = [bbox for bbox in table_bboxes if not _is_in_or_part_overlap(bbox, content_boundry)]
-    table_bbox_remain = [bbox for bbox in table_bboxes if _is_in_or_part_overlap(bbox, content_boundry)]
-
-    return image_bbox_remain, table_bbox_remain, text_block_remain, text_block_to_remove, image_bbox_to_remove, table_bbox_to_remove

magic_pdf/pre_proc/remove_rotate_bbox.py

@@ -1,236 +0,0 @@
-import math
-import re
-
-from magic_pdf.config.drop_tag import (EMPTY_SIDE_BLOCK, ROTATE_TEXT,
-                                       VERTICAL_TEXT)
-from magic_pdf.libs.boxbase import is_vbox_on_side
-
-
-def detect_non_horizontal_texts(result_dict):
-    """This function detects watermarks and vertical margin notes in the
-    document.
-
-    Watermarks are identified by finding blocks with the same coordinates and frequently occurring identical texts across multiple pages.
-    If these conditions are met, the blocks are highly likely to be watermarks, as opposed to headers or footers, which can change from page to page.
-    If the direction of these blocks is not horizontal, they are definitely considered to be watermarks.
-
-    Vertical margin notes are identified by finding blocks with the same coordinates and frequently occurring identical texts across multiple pages.
-    If these conditions are met, the blocks are highly likely to be vertical margin notes, which typically appear on the left and right sides of the page. # noqa: E501
-    If the direction of these blocks is vertical, they are definitely considered to be vertical margin notes.
-
-
-    Parameters
-    ----------
-    result_dict : dict
-        The result dictionary.
-
-    Returns
-    -------
-    result_dict : dict
-        The updated result dictionary.
-    """
-    # Dictionary to store information about potential watermarks
-    potential_watermarks = {}
-    potential_margin_notes = {}
-
-    for page_id, page_content in result_dict.items():
-        if page_id.startswith('page_'):
-            for block_id, block_data in page_content.items():
-                if block_id.startswith('block_'):
-                    if 'dir' in block_data:
-                        coordinates_text = (
-                            block_data['bbox'],
-                            block_data['text'],
-                        )  # Tuple of coordinates and text
-
-                        angle = math.atan2(block_data['dir'][1], block_data['dir'][0])
-                        angle = abs(math.degrees(angle))
-
-                        if angle > 5 and angle < 85:  # Check if direction is watermarks
-                            if coordinates_text in potential_watermarks:
-                                potential_watermarks[coordinates_text] += 1
-                            else:
-                                potential_watermarks[coordinates_text] = 1
-
-                        if angle > 85 and angle < 105:  # Check if direction is vertical
-                            if coordinates_text in potential_margin_notes:
-                                potential_margin_notes[coordinates_text] += (
-                                    1  # Increment count
-                                )
-                            else:
-                                potential_margin_notes[coordinates_text] = (
-                                    1  # Initialize count
-                                )
-
-    # Identify watermarks by finding entries with counts higher than a threshold (e.g., appearing on more than half of the pages)
-    watermark_threshold = len(result_dict) // 2
-    watermarks = {
-        k: v for k, v in potential_watermarks.items() if v > watermark_threshold
-    }
-
-    # Identify margin notes by finding entries with counts higher than a threshold (e.g., appearing on more than half of the pages)
-    margin_note_threshold = len(result_dict) // 2
-    margin_notes = {
-        k: v for k, v in potential_margin_notes.items() if v > margin_note_threshold
-    }
-
-    # Add watermark information to the result dictionary
-    for page_id, blocks in result_dict.items():
-        if page_id.startswith('page_'):
-            for block_id, block_data in blocks.items():
-                coordinates_text = (block_data['bbox'], block_data['text'])
-                if coordinates_text in watermarks:
-                    block_data['is_watermark'] = 1
-                else:
-                    block_data['is_watermark'] = 0
-
-                if coordinates_text in margin_notes:
-                    block_data['is_vertical_margin_note'] = 1
-                else:
-                    block_data['is_vertical_margin_note'] = 0
-
-    return result_dict
-
-
-"""
-1. 当一个block里全部文字都不是dir=(1,0)，这个block整体去掉
-2. 当一个block里全部文字都是dir=(1,0)，但是每行只有一个字，这个block整体去掉。这个block必须出现在页面的四周，否则不去掉
-"""
-
-
-def __is_a_word(sentence):
-    # 如果输入是中文并且长度为1，则返回True
-    if re.fullmatch(r'[\u4e00-\u9fa5]', sentence):
-        return True
-    # 判断是否为单个英文单词或字符（包括ASCII标点）
-    elif re.fullmatch(r'[a-zA-Z0-9]+', sentence) and len(sentence) <= 2:
-        return True
-    else:
-        return False
-
-
-def __get_text_color(num):
-    """获取字体的颜色RGB值."""
-    blue = num & 255
-    green = (num >> 8) & 255
-    red = (num >> 16) & 255
-    return red, green, blue
-
-
-def __is_empty_side_box(text_block):
-    """是否是边缘上的空白没有任何内容的block."""
-    for line in text_block['lines']:
-        for span in line['spans']:
-            font_color = span['color']
-            r, g, b = __get_text_color(font_color)
-            if len(span['text'].strip()) > 0 and (r, g, b) != (255, 255, 255):
-                return False
-
-    return True
-
-
-def remove_rotate_side_textblock(pymu_text_block, page_width, page_height):
-    """返回删除了垂直，水印，旋转的textblock 删除的内容打上tag返回."""
-    removed_text_block = []
-
-    for i, block in enumerate(
-        pymu_text_block
-    ):  # 格式参考test/assets/papre/pymu_textblocks.json
-        lines = block['lines']
-        block_bbox = block['bbox']
-        if not is_vbox_on_side(
-            block_bbox, page_width, page_height, 0.2
-        ):  # 保证这些box必须在页面的两边
-            continue
-
-        if (
-            all(
-                [
-                    __is_a_word(line['spans'][0]['text'])
-                    for line in lines
-                    if len(line['spans']) > 0
-                ]
-            )
-            and len(lines) > 1
-            and all([len(line['spans']) == 1 for line in lines])
-        ):
-            is_box_valign = (
-                (
-                    len(
-                        set(
-                            [
-                                int(line['spans'][0]['bbox'][0])
-                                for line in lines
-                                if len(line['spans']) > 0
-                            ]
-                        )
-                    )
-                    == 1
-                )
-                and (
-                    len(
-                        [
-                            int(line['spans'][0]['bbox'][0])
-                            for line in lines
-                            if len(line['spans']) > 0
-                        ]
-                    )
-                    > 1
-                )
-            )  # 测试bbox在垂直方向是不是x0都相等，也就是在垂直方向排列.同时必须大于等于2个字
-
-            if is_box_valign:
-                block['tag'] = VERTICAL_TEXT
-                removed_text_block.append(block)
-                continue
-
-        for line in lines:
-            if line['dir'] != (1, 0):
-                block['tag'] = ROTATE_TEXT
-                removed_text_block.append(
-                    block
-                )  # 只要有一个line不是dir=(1,0)，就把整个block都删掉
-                break
-
-    for block in removed_text_block:
-        pymu_text_block.remove(block)
-
-    return pymu_text_block, removed_text_block
-
-
-def get_side_boundry(rotate_bbox, page_width, page_height):
-    """根据rotate_bbox，返回页面的左右正文边界."""
-    left_x = 0
-    right_x = page_width
-    for x in rotate_bbox:
-        box = x['bbox']
-        if box[2] < page_width / 2:
-            left_x = max(left_x, box[2])
-        else:
-            right_x = min(right_x, box[0])
-
-    return left_x + 1, right_x - 1
-
-
-def remove_side_blank_block(pymu_text_block, page_width, page_height):
-    """删除页面两侧的空白block."""
-    removed_text_block = []
-
-    for i, block in enumerate(
-        pymu_text_block
-    ):  # 格式参考test/assets/papre/pymu_textblocks.json
-        block_bbox = block['bbox']
-        if not is_vbox_on_side(
-            block_bbox, page_width, page_height, 0.2
-        ):  # 保证这些box必须在页面的两边
-            continue
-
-        if __is_empty_side_box(block):
-            block['tag'] = EMPTY_SIDE_BLOCK
-            removed_text_block.append(block)
-            continue
-
-    for block in removed_text_block:
-        pymu_text_block.remove(block)
-
-    return pymu_text_block, removed_text_block