magic-pdf 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

magic_pdf/libs/version.py

@@ -1 +1 @@
-__version__ = "1.3.3"
+__version__ = "1.3.4"

magic_pdf/model/sub_modules/model_utils.py

@@ -2,6 +2,8 @@ import time
 import torch
 from loguru import logger
 import numpy as np
+
+from magic_pdf.libs.boxbase import get_minbox_if_overlap_by_ratio
 from magic_pdf.libs.clean_memory import clean_memory
 
 
@@ -188,9 +190,46 @@ def filter_nested_tables(table_res_list, overlap_threshold=0.8, area_threshold=0
     return [table for i, table in enumerate(table_res_list) if i not in big_tables_idx]
 
 
+def remove_overlaps_min_blocks(res_list):
+    #  重叠block，小的不能直接删除，需要和大的那个合并成一个更大的。
+    #  删除重叠blocks中较小的那些
+    need_remove = []
+    for res1 in res_list:
+        for res2 in res_list:
+            if res1 != res2:
+                overlap_box = get_minbox_if_overlap_by_ratio(
+                    res1['bbox'], res2['bbox'], 0.8
+                )
+                if overlap_box is not None:
+                    res_to_remove = next(
+                        (res for res in res_list if res['bbox'] == overlap_box),
+                        None,
+                    )
+                    if (
+                        res_to_remove is not None
+                        and res_to_remove not in need_remove
+                    ):
+                        large_res = res1 if res1 != res_to_remove else res2
+                        x1, y1, x2, y2 = large_res['bbox']
+                        sx1, sy1, sx2, sy2 = res_to_remove['bbox']
+                        x1 = min(x1, sx1)
+                        y1 = min(y1, sy1)
+                        x2 = max(x2, sx2)
+                        y2 = max(y2, sy2)
+                        large_res['bbox'] = [x1, y1, x2, y2]
+                        need_remove.append(res_to_remove)
+
+    if len(need_remove) > 0:
+        for res in need_remove:
+            res_list.remove(res)
+
+    return res_list, need_remove
+
+
 def get_res_list_from_layout_res(layout_res, iou_threshold=0.7, overlap_threshold=0.8, area_threshold=0.8):
     """Extract OCR, table and other regions from layout results."""
     ocr_res_list = []
+    text_res_list = []
     table_res_list = []
     table_indices = []
     single_page_mfdetrec_res = []
@@ -204,11 +243,14 @@ def get_res_list_from_layout_res(layout_res, iou_threshold=0.7, overlap_threshol
                 "bbox": [int(res['poly'][0]), int(res['poly'][1]),
                          int(res['poly'][4]), int(res['poly'][5])],
             })
-        elif category_id in [0, 1, 2, 4, 6, 7]:  # OCR regions
+        elif category_id in [0, 2, 4, 6, 7]:  # OCR regions
             ocr_res_list.append(res)
         elif category_id == 5:  # Table regions
             table_res_list.append(res)
             table_indices.append(i)
+        elif category_id in [1]:  # Text regions
+            res['bbox'] = [int(res['poly'][0]), int(res['poly'][1]), int(res['poly'][4]), int(res['poly'][5])]
+            text_res_list.append(res)
 
     # Process tables: merge high IoU tables first, then filter nested tables
     table_res_list, table_indices = merge_high_iou_tables(
@@ -226,6 +268,22 @@ def get_res_list_from_layout_res(layout_res, iou_threshold=0.7, overlap_threshol
         for idx in sorted(to_remove, reverse=True):
             del layout_res[idx]
 
+    # Remove overlaps in OCR and text regions
+    text_res_list, need_remove = remove_overlaps_min_blocks(text_res_list)
+    for res in text_res_list:
+        # 将res的poly使用bbox重构
+        res['poly'] = [res['bbox'][0], res['bbox'][1], res['bbox'][2], res['bbox'][1],
+                       res['bbox'][2], res['bbox'][3], res['bbox'][0], res['bbox'][3]]
+        # 删除res的bbox
+        del res['bbox']
+
+    ocr_res_list.extend(text_res_list)
+
+    if len(need_remove) > 0:
+        for res in need_remove:
+            del res['bbox']
+            layout_res.remove(res)
+
     return ocr_res_list, filtered_table_res_list, single_page_mfdetrec_res
 
 

magic_pdf/pdf_parse_union_core_v2.py

@@ -490,7 +490,7 @@ def insert_lines_into_block(block_bbox, line_height, page_w, page_h):
         return [[x0, y0, x1, y1]]
 
 
-def sort_lines_by_model(fix_blocks, page_w, page_h, line_height):
+def sort_lines_by_model(fix_blocks, page_w, page_h, line_height, footnote_blocks):
     page_line_list = []
 
     def add_lines_to_block(b):
@@ -519,6 +519,10 @@ def sort_lines_by_model(fix_blocks, page_w, page_h, line_height):
             block['real_lines'] = copy.deepcopy(block['lines'])
             add_lines_to_block(block)
 
+    for block in footnote_blocks:
+        footnote_block = {'bbox': block[:4]}
+        add_lines_to_block(footnote_block)
+
     if len(page_line_list) > 200:  # layoutreader最高支持512line
         return None
 
@@ -779,7 +783,7 @@ def parse_page_core(
     # interline_equation_blocks参数不够准，后面切换到interline_equations上
     interline_equation_blocks = []
     if len(interline_equation_blocks) > 0:
-        all_bboxes, all_discarded_blocks = ocr_prepare_bboxes_for_layout_split_v2(
+        all_bboxes, all_discarded_blocks, footnote_blocks = ocr_prepare_bboxes_for_layout_split_v2(
             img_body_blocks, img_caption_blocks, img_footnote_blocks,
             table_body_blocks, table_caption_blocks, table_footnote_blocks,
             discarded_blocks,
@@ -790,7 +794,7 @@ def parse_page_core(
             page_h,
         )
     else:
-        all_bboxes, all_discarded_blocks = ocr_prepare_bboxes_for_layout_split_v2(
+        all_bboxes, all_discarded_blocks, footnote_blocks = ocr_prepare_bboxes_for_layout_split_v2(
             img_body_blocks, img_caption_blocks, img_footnote_blocks,
             table_body_blocks, table_caption_blocks, table_footnote_blocks,
             discarded_blocks,
@@ -866,7 +870,7 @@ def parse_page_core(
     line_height = get_line_height(fix_blocks)
 
     """获取所有line并对line排序"""
-    sorted_bboxes = sort_lines_by_model(fix_blocks, page_w, page_h, line_height)
+    sorted_bboxes = sort_lines_by_model(fix_blocks, page_w, page_h, line_height, footnote_blocks)
 
     """根据line的中位数算block的序列关系"""
     fix_blocks = cal_block_index(fix_blocks, sorted_bboxes)

magic_pdf/pre_proc/ocr_detect_all_bboxes.py

@@ -119,7 +119,7 @@ def ocr_prepare_bboxes_for_layout_split_v2(
     """将剩余的bbox做分离处理，防止后面分layout时出错"""
     # all_bboxes, drop_reasons = remove_overlap_between_bbox_for_block(all_bboxes)
     all_bboxes.sort(key=lambda x: x[0]+x[1])
-    return all_bboxes, all_discarded_blocks
+    return all_bboxes, all_discarded_blocks, footnote_blocks
 
 
 def find_blocks_under_footnote(all_bboxes, footnote_blocks):

magic_pdf/utils/office_to_pdf.py

@@ -1,6 +1,8 @@
 import os
 import subprocess
+import platform
 from pathlib import Path
+import shutil
 
 
 class ConvertToPdfError(Exception):
@@ -9,21 +11,114 @@ class ConvertToPdfError(Exception):
         super().__init__(self.msg)
 
 
+# Chinese font list
+REQUIRED_CHS_FONTS = ['SimSun', 'Microsoft YaHei', 'Noto Sans CJK SC']
+
+
+def check_fonts_installed():
+    """Check if required Chinese fonts are installed."""
+    system_type = platform.system()
+
+    if system_type == 'Windows':
+        # Windows: check fonts via registry or system font folder
+        font_dir = Path("C:/Windows/Fonts")
+        installed_fonts = [f.name for f in font_dir.glob("*.ttf")]
+        if any(font for font in REQUIRED_CHS_FONTS if any(font in f for f in installed_fonts)):
+            return True
+        raise EnvironmentError(
+            f"Missing Chinese font. Please install at least one of: {', '.join(REQUIRED_CHS_FONTS)}"
+        )
+    else:
+        # Linux/macOS: use fc-list
+        try:
+            output = subprocess.check_output(['fc-list', ':lang=zh'], encoding='utf-8')
+            for font in REQUIRED_CHS_FONTS:
+                if font in output:
+                    return True
+            raise EnvironmentError(
+                f"Missing Chinese font. Please install at least one of: {', '.join(REQUIRED_CHS_FONTS)}"
+            )
+        except Exception as e:
+            raise EnvironmentError(f"Font detection failed. Please install 'fontconfig' and fonts: {str(e)}")
+
+
+def get_soffice_command():
+    """Return the path to LibreOffice's soffice executable depending on the platform."""
+    system_type = platform.system()
+
+    # First check if soffice is in PATH
+    soffice_path = shutil.which('soffice')
+    if soffice_path:
+        return soffice_path
+
+    if system_type == 'Windows':
+        # Check common installation paths
+        possible_paths = [
+            Path(os.environ.get('PROGRAMFILES', 'C:/Program Files')) / 'LibreOffice/program/soffice.exe',
+            Path(os.environ.get('PROGRAMFILES(X86)', 'C:/Program Files (x86)')) / 'LibreOffice/program/soffice.exe',
+            Path('C:/Program Files/LibreOffice/program/soffice.exe'),
+            Path('C:/Program Files (x86)/LibreOffice/program/soffice.exe')
+        ]
+
+        # Check other drives for windows
+        for drive in ['C:', 'D:', 'E:', 'F:', 'G:', 'H:']:
+            possible_paths.append(Path(f"{drive}/LibreOffice/program/soffice.exe"))
+
+        for path in possible_paths:
+            if path.exists():
+                return str(path)
+
+        raise ConvertToPdfError(
+            "LibreOffice not found. Please install LibreOffice from https://www.libreoffice.org/ "
+            "or ensure soffice.exe is in your PATH environment variable."
+        )
+    else:
+        # For Linux/macOS, provide installation instructions if not found
+        try:
+            # Try to find soffice in standard locations
+            possible_paths = [
+                '/usr/bin/soffice',
+                '/usr/local/bin/soffice',
+                '/opt/libreoffice/program/soffice',
+                '/Applications/LibreOffice.app/Contents/MacOS/soffice'
+            ]
+            for path in possible_paths:
+                if os.path.exists(path):
+                    return path
+
+            raise ConvertToPdfError(
+                "LibreOffice not found. Please install it:\n"
+                "  - Ubuntu/Debian: sudo apt-get install libreoffice\n"
+                "  - CentOS/RHEL: sudo yum install libreoffice\n"
+                "  - macOS: brew install libreoffice or download from https://www.libreoffice.org/\n"
+                "  - Or ensure soffice is in your PATH environment variable."
+            )
+        except Exception as e:
+            raise ConvertToPdfError(f"Error locating LibreOffice: {str(e)}")
+
+
 def convert_file_to_pdf(input_path, output_dir):
+    """Convert a single document (ppt, doc, etc.) to PDF."""
     if not os.path.isfile(input_path):
         raise FileNotFoundError(f"The input file {input_path} does not exist.")
 
     os.makedirs(output_dir, exist_ok=True)
-    
+
+    check_fonts_installed()
+
+    soffice_cmd = get_soffice_command()
+
     cmd = [
-        'soffice',
+        soffice_cmd,
         '--headless',
+        '--norestore',
+        '--invisible',
         '--convert-to', 'pdf',
         '--outdir', str(output_dir),
         str(input_path)
     ]
-    
+
     process = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    
+
     if process.returncode != 0:
-        raise ConvertToPdfError(process.stderr.decode())
+        raise ConvertToPdfError(f"LibreOffice convert failed: {process.stderr.decode()}")

{magic_pdf-1.3.3.dist-info → magic_pdf-1.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: magic-pdf
-Version: 1.3.3
+Version: 1.3.4
 Summary: A practical tool for converting PDF to Markdown
 License: AGPL-3.0
 Project-URL: Home, https://mineru.net/
@@ -107,6 +107,9 @@ Easier to use: Just grab MinerU Desktop. No coding, no login, just a simple inte
 </div>
 
 # Changelog
+- 2025/04/16 1.3.4 Released
+  - Slightly improved the speed of OCR detection by removing some unused blocks.
+  - Fixed page-level sorting errors caused by footnotes in certain cases.
 - 2025/04/12 1.3.2 released
   - Fixed the issue of incompatible dependency package versions when installing in Python 3.13 environment on Windows systems.
   - Optimized memory usage during batch inference.

{magic_pdf-1.3.3.dist-info → magic_pdf-1.3.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 magic_pdf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-magic_pdf/pdf_parse_union_core_v2.py,sha256=6b7EHzNJrKGBIIRnK8zqeWn8tcnNZpP-7hYGP9DA82I,40384
+magic_pdf/pdf_parse_union_core_v2.py,sha256=2dW8Y2FTuSsy9hun1kmKpJFcanijHfPUVd1jKrrvqXU,40575
 magic_pdf/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/config/constants.py,sha256=fXGzANULnJWLPxwYp3BEFWx-rnorzpySMx63ffyEyq4,1272
 magic_pdf/config/drop_reason.py,sha256=CqjMzBE96Qo8OeFvhhhItY8WhyqsKhE3DmyJLoQZNCc,2248
@@ -52,7 +52,7 @@ magic_pdf/libs/pdf_check.py,sha256=7GWWvDR6g_rj_fE6XJlbTq5AFVX11ngRIzT0N18F214,3
 magic_pdf/libs/pdf_image_tools.py,sha256=_au7plmKKctpPKozBumSKgP8689q4vH1mU8VMLO0IbM,2260
 magic_pdf/libs/performance_stats.py,sha256=DW-c6nUTUnWKGTONRKfpucsYZm1ake016F9K7jJwbik,2136
 magic_pdf/libs/safe_filename.py,sha256=ckwcM_eqoysTb5id8czp-tXq2G9da0-l3pshZDCHQtE,236
-magic_pdf/libs/version.py,sha256=Vi6om3KImlKsS_Wg5CjUgYffoi2zx7T-SRPnnGL0G7M,22
+magic_pdf/libs/version.py,sha256=U6E-HRsrit7kgSGgIeTI2eMUeyUCny5DH8LDV4I1o0g,22
 magic_pdf/model/__init__.py,sha256=sa-dO2k-TLy25I2gRrzjm_cQeYfzMf-pLwBJHkIxGo0,51
 magic_pdf/model/batch_analyze.py,sha256=yKhKQuZTh9GG83p61bw2BRqKMbnsjsmX73gfuTRk8xE,11272
 magic_pdf/model/doc_analyze_by_custom_model.py,sha256=-cjn7DQi6kZCqVZ0IxbXuL2kmeGhSVLzLaezIHPFzMU,10317
@@ -62,7 +62,7 @@ magic_pdf/model/pdf_extract_kit.py,sha256=C3sKqRkoD20Ldmo-cqGn1zRldEL-l5NYqcFvd0
 magic_pdf/model/pp_structure_v2.py,sha256=NcqFWL4nUtjl82MFak8HX_8V3i4Aw_fK4dATrIp5uGs,3840
 magic_pdf/model/sub_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/model/sub_modules/model_init.py,sha256=e2t95kxiuU47luOHByDokbQ2uob6oQhEA4b6UGVfMjY,8303
-magic_pdf/model/sub_modules/model_utils.py,sha256=iNC-zuDLWkwUAwMZ0YcGxAwHn5SAAFRdZBQgTy9nmgY,9880
+magic_pdf/model/sub_modules/model_utils.py,sha256=HKRC9ubCs6O0nNqaztrZO0YKuFpRhs0LKWOaeZfDrTw,12166
 magic_pdf/model/sub_modules/language_detection/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
 magic_pdf/model/sub_modules/language_detection/utils.py,sha256=Q__v6DdNJztt8GhVSuSB0txahVq-aj8RLhWn2VScx4w,3047
 magic_pdf/model/sub_modules/language_detection/yolov11/YOLOv11.py,sha256=7T8eFl8zlZe6F0j0jSB3jrwOapDft320JQ1fuWxpvAY,5230
@@ -178,7 +178,7 @@ magic_pdf/post_proc/para_split_v3.py,sha256=SPN_VVGvFX5KpFMGw9OzgoE-kTZq-FF036i0
 magic_pdf/pre_proc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/pre_proc/construct_page_dict.py,sha256=OFmq5XRKi6fYIo-lmGlL-NB16Sf0egzsfEx-fT2uYrc,660
 magic_pdf/pre_proc/cut_image.py,sha256=NDzbxwD7z7Tb4uAxL4KR6LzURFdN1Tzr4nPvj-VmEqc,1225
-magic_pdf/pre_proc/ocr_detect_all_bboxes.py,sha256=3_bEbZC_BDwbuaBLPdCIbkxz93-g9oCtvjuXD8qbklo,9330
+magic_pdf/pre_proc/ocr_detect_all_bboxes.py,sha256=o1VgMgP0KB-0pKyrkQfANYkhf7cGtzsSOXZR6DdxYZ4,9347
 magic_pdf/pre_proc/ocr_dict_merge.py,sha256=PscKGF0uJIjMxZRM69FLUs1SZO_wOswDQQV1f0M2xAo,5627
 magic_pdf/pre_proc/ocr_span_list_modify.py,sha256=bs5RLvk4kIyx9_Hqq0FU3AGPPxE8Sxs97Uwlf1sBryM,4725
 magic_pdf/pre_proc/remove_bbox_overlap.py,sha256=mcdxAh4P56NZ3Ij8h3vW8qC_SrszfXflVWuWUuUiTNg,3089
@@ -194,10 +194,10 @@ magic_pdf/tools/cli_dev.py,sha256=3RbubfTIagWoFYdu8wSDanr-BJDjFGeDet55jTy7He0,39
 magic_pdf/tools/common.py,sha256=-x0RSFr7SNbdYq7DntaLYmQmaxyF-xKSf4xMpSUTzA0,12623
 magic_pdf/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 magic_pdf/utils/annotations.py,sha256=82ou3uELNbQWa9hOFFkVt0gsIskAKf5msCv5J2IJ5V0,211
-magic_pdf/utils/office_to_pdf.py,sha256=7aj-Ls2v8saD-Rgu_t3FIc-J3Ka9wnmiEH5zY-H1Vxs,729
-magic_pdf-1.3.3.dist-info/LICENSE.md,sha256=jVa0BUaKrRH4erV2P5AeJ24I2WRv9chIGxditreJ6e0,34524
-magic_pdf-1.3.3.dist-info/METADATA,sha256=1Y-a4UouLQRhsldrhz6UZLlx4KUFOdjSk5R1gK_oYjs,45615
-magic_pdf-1.3.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-magic_pdf-1.3.3.dist-info/entry_points.txt,sha256=wXwYke3j8fqDQTocUspL-CqDUEv3Tfcwp09fM8dZAhA,98
-magic_pdf-1.3.3.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
-magic_pdf-1.3.3.dist-info/RECORD,,
+magic_pdf/utils/office_to_pdf.py,sha256=uYiKYPcJtxl_VmQYuQiQq3JJsfh5XXAHwmH5KyV6VhM,4525
+magic_pdf-1.3.4.dist-info/LICENSE.md,sha256=jVa0BUaKrRH4erV2P5AeJ24I2WRv9chIGxditreJ6e0,34524
+magic_pdf-1.3.4.dist-info/METADATA,sha256=t4InKDeOdm6_4ptByThj-XFEmTmpFSEMhl62SELBXrk,45798
+magic_pdf-1.3.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+magic_pdf-1.3.4.dist-info/entry_points.txt,sha256=wXwYke3j8fqDQTocUspL-CqDUEv3Tfcwp09fM8dZAhA,98
+magic_pdf-1.3.4.dist-info/top_level.txt,sha256=J9I0AzmHWGkp9c6DL8Oe4mEx3yYphLzkRn4H25Lg1rE,10
+magic_pdf-1.3.4.dist-info/RECORD,,