mineru 2.7.4__py3-none-any.whl → 2.7.5__py3-none-any.whl

mineru/utils/os_env_config.py

@@ -11,6 +11,11 @@ def get_load_images_timeout() -> int:
     return get_value_from_string(env_value, 300)
 
 
+def get_load_images_threads() -> int:
+    env_value = os.getenv('MINERU_PDF_RENDER_THREADS', None)
+    return get_value_from_string(env_value, 4)
+
+
 def get_value_from_string(env_value: str, default_value: int) -> int:
     if env_value is not None:
         try:

mineru/utils/pdf_image_tools.py

@@ -1,5 +1,7 @@
 # Copyright (c) Opendatalab. All rights reserved.
 import os
+import signal
+import time
 from io import BytesIO
 
 import numpy as np
@@ -9,13 +11,13 @@ from PIL import Image, ImageOps
 
 from mineru.data.data_reader_writer import FileBasedDataWriter
 from mineru.utils.check_sys_env import is_windows_environment
-from mineru.utils.os_env_config import get_load_images_timeout
+from mineru.utils.os_env_config import get_load_images_timeout, get_load_images_threads
 from mineru.utils.pdf_reader import image_to_b64str, image_to_bytes, page_to_image
 from mineru.utils.enum_class import ImageType
 from mineru.utils.hash_utils import str_sha256
 from mineru.utils.pdf_page_id import get_end_page_id
 
-from concurrent.futures import ProcessPoolExecutor, TimeoutError as FuturesTimeoutError
+from concurrent.futures import ProcessPoolExecutor, wait, ALL_COMPLETED
 
 
 def pdf_page_to_image(page: pdfium.PdfPage, dpi=200, image_type=ImageType.PIL) -> dict:
@@ -57,7 +59,7 @@ def load_images_from_pdf(
     end_page_id=None,
     image_type=ImageType.PIL,
     timeout=None,
-    threads=4,
+    threads=None,
 ):
     """带超时控制的 PDF 转图片函数,支持多进程加速
 
@@ -67,8 +69,8 @@ def load_images_from_pdf(
         start_page_id (int, optional): 起始页码. Defaults to 0.
         end_page_id (int | None, optional): 结束页码. Defaults to None.
         image_type (ImageType, optional): 图片类型. Defaults to ImageType.PIL.
-        timeout (int | None, optional): 超时时间(秒)。如果为 None，则从环境变量 MINERU_PDF_LOAD_IMAGES_TIMEOUT 读取，若未设置则默认为 300 秒。
-        threads (int): 进程数,默认 4
+        timeout (int | None, optional): 超时时间(秒)。如果为 None，则从环境变量 MINERU_PDF_RENDER_TIMEOUT 读取，若未设置则默认为 300 秒。
+        threads (int): 进程数, 如果为 None，则从环境变量 MINERU_PDF_RENDER_THREADS 读取，若未设置则默认为 4.
 
     Raises:
         TimeoutError: 当转换超时时抛出
@@ -86,6 +88,9 @@ def load_images_from_pdf(
     else:
         if timeout is None:
             timeout = get_load_images_timeout()
+        if threads is None:
+            threads = get_load_images_threads()
+
         end_page_id = get_end_page_id(end_page_id, len(pdf_doc))
 
         # 计算总页数
@@ -108,11 +113,13 @@ def load_images_from_pdf(
 
             page_ranges.append((range_start, range_end))
 
-        # logger.debug(f"PDF to images using {actual_threads} processes, page ranges: {page_ranges}")
+        logger.debug(f"PDF to images using {actual_threads} processes, page ranges: {page_ranges}")
 
-        with ProcessPoolExecutor(max_workers=actual_threads) as executor:
+        executor = ProcessPoolExecutor(max_workers=actual_threads)
+        try:
             # 提交所有任务
             futures = []
+            future_to_range = {}
             for range_start, range_end in page_ranges:
                 future = executor.submit(
                     _load_images_from_pdf_worker,
@@ -122,27 +129,68 @@ def load_images_from_pdf(
                     range_end,
                     image_type,
                 )
-                futures.append((range_start, future))
-
-            try:
-                # 收集结果并按页码排序
-                all_results = []
-                for range_start, future in futures:
-                    images_list = future.result(timeout=timeout)
-                    all_results.append((range_start, images_list))
-
-                # 按起始页码排序并合并结果
-                all_results.sort(key=lambda x: x[0])
-                images_list = []
-                for _, imgs in all_results:
-                    images_list.extend(imgs)
-
-                return images_list, pdf_doc
-            except FuturesTimeoutError:
+                futures.append(future)
+                future_to_range[future] = range_start
+
+            # 使用 wait() 设置单一全局超时
+            done, not_done = wait(futures, timeout=timeout, return_when=ALL_COMPLETED)
+
+            # 检查是否有未完成的任务（超时情况）
+            if not_done:
+                # 超时：强制终止所有子进程
+                _terminate_executor_processes(executor)
                 pdf_doc.close()
-                executor.shutdown(wait=False, cancel_futures=True)
                 raise TimeoutError(f"PDF to images conversion timeout after {timeout}s")
 
+            # 所有任务完成，收集结果
+            all_results = []
+            for future in futures:
+                range_start = future_to_range[future]
+                # 这里不需要 timeout，因为任务已完成
+                images_list = future.result()
+                all_results.append((range_start, images_list))
+
+            # 按起始页码排序并合并结果
+            all_results.sort(key=lambda x: x[0])
+            images_list = []
+            for _, imgs in all_results:
+                images_list.extend(imgs)
+
+            return images_list, pdf_doc
+
+        except Exception as e:
+            # 发生任何异常时，确保清理子进程
+            _terminate_executor_processes(executor)
+            pdf_doc.close()
+            if isinstance(e, TimeoutError):
+                raise
+            raise
+        finally:
+            executor.shutdown(wait=False, cancel_futures=True)
+
+
+def _terminate_executor_processes(executor):
+    """强制终止 ProcessPoolExecutor 中的所有子进程"""
+    if hasattr(executor, '_processes'):
+        for pid, process in executor._processes.items():
+            if process.is_alive():
+                try:
+                    # 先发送 SIGTERM 允许优雅退出
+                    os.kill(pid, signal.SIGTERM)
+                except (ProcessLookupError, OSError):
+                    pass
+
+        # 给子进程一点时间响应 SIGTERM
+        time.sleep(0.1)
+
+        # 对仍然存活的进程发送 SIGKILL 强制终止
+        for pid, process in executor._processes.items():
+            if process.is_alive():
+                try:
+                    os.kill(pid, signal.SIGKILL)
+                except (ProcessLookupError, OSError):
+                    pass
+
 
 def load_images_from_pdf_core(
     pdf_bytes: bytes,

mineru/version.py

@@ -1 +1 @@
-__version__ = "2.7.4"
+__version__ = "2.7.5"

{mineru-2.7.4.dist-info → mineru-2.7.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mineru
-Version: 2.7.4
+Version: 2.7.5
 Summary: A practical tool for converting PDF to Markdown
 License: AGPL-3.0
 Project-URL: homepage, https://mineru.net/

{mineru-2.7.4.dist-info → mineru-2.7.5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mineru/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
-mineru/version.py,sha256=yLdxKZXyzrDqew_33G4dvZoqgGxRCyEx9vhYW3y2Je4,22
+mineru/version.py,sha256=lBcjVwt4I0-VUeE_7gM1gQBbtKOi9jGT3DavJbzcYnQ,22
 mineru/backend/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
 mineru/backend/utils.py,sha256=GLJU3IznDmhE1_qNmkU1UOtsuskIHBezgsEVO6Uar-Y,698
 mineru/backend/hybrid/__init__.py,sha256=IFgr2C8NfSAj8q7JF7QOqMvCiJ6Fc8TIuU3Uh2DaFZU,51
@@ -175,9 +175,9 @@ mineru/utils/magic_model_utils.py,sha256=8Hv-BDk9Ez4TUx6hrVJ_675yZZggPj6Uib81lSp
 mineru/utils/model_utils.py,sha256=xlw5hUYKa6o1NiM8PoXO1HFvHfrgY5e4Ut_upGEY9yI,19909
 mineru/utils/models_download_utils.py,sha256=UfjvwhxO6BkJHa5JSpEVNZ71GoLMPMmJpym3THET2T4,2957
 mineru/utils/ocr_utils.py,sha256=lPIrwNUib5mrzUkponRYHuUCdjV2qvETNLSzOLyflrU,15990
-mineru/utils/os_env_config.py,sha256=ZNtkR4KrJW72CeIoTNzGDL6tMKv_hL8nzvWIssGWbqY,842
+mineru/utils/os_env_config.py,sha256=VHK9lS3QFJhrwWa9FOFU1Swm7oXnby4SaNNjTyonTTg,990
 mineru/utils/pdf_classify.py,sha256=6DF5pH_9Uq83fsFtp7n4i-OdYQGzoNOV9L0VBUhgBMQ,8078
-mineru/utils/pdf_image_tools.py,sha256=L2kHKoFaQo4CGjS1d68JACrlBycx6gyCnnFlbBFRKuw,8273
+mineru/utils/pdf_image_tools.py,sha256=tTSk39fgJKLEshwPAuJGLl_pVSrmEKjWA55F6dGcr4g,9987
 mineru/utils/pdf_page_id.py,sha256=em966k12CRW4Rj49RGiLB_8ILwkXPBnWRetApax3eTs,400
 mineru/utils/pdf_reader.py,sha256=WeINm5SyWBUXT0wP9lzIbeHs8P6WUIkN6nVL5X4LzG4,3267
 mineru/utils/pdf_text_tool.py,sha256=KEztjfdqsIHHuiTEAMAL7Lr1OS3R7Ur-uTqGiCRjReQ,1364
@@ -185,9 +185,9 @@ mineru/utils/run_async.py,sha256=rPeP4BCZerR8VByRDhiYzfZiahLVqoZEBVAS54dAjNg,128
 mineru/utils/span_block_fix.py,sha256=0eVQjJCrT03woRt9hoh6Uu42Tp1dacfGTv2x3B9qq94,8797
 mineru/utils/span_pre_proc.py,sha256=nu6Bh5TWPKFzHuFfbEs0Asr04M4xOL5IONz_8GJHn44,13862
 mineru/utils/table_merge.py,sha256=LORxz0THemCqH746FMViqEuLzM088M4HgIkEuwDIfNU,21393
-mineru-2.7.4.dist-info/licenses/LICENSE.md,sha256=jVa0BUaKrRH4erV2P5AeJ24I2WRv9chIGxditreJ6e0,34524
-mineru-2.7.4.dist-info/METADATA,sha256=lNxDREB_s7eDnknMUeBn5FCgtDc8qPQS-hEs4Wb6WTg,36928
-mineru-2.7.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-mineru-2.7.4.dist-info/entry_points.txt,sha256=a9AHBIiYe3dpT3oofVQJC8fI0WjDhQASCUlhdMOK120,376
-mineru-2.7.4.dist-info/top_level.txt,sha256=zuGQfZcbsHv4I4oKI9gaKPqEWBFm6xJroKuug2LnKP8,7
-mineru-2.7.4.dist-info/RECORD,,
+mineru-2.7.5.dist-info/licenses/LICENSE.md,sha256=jVa0BUaKrRH4erV2P5AeJ24I2WRv9chIGxditreJ6e0,34524
+mineru-2.7.5.dist-info/METADATA,sha256=MvPv4AgyLwaHz3hspAPrZ0wEeSE0wnu0MkMwfAJ5hTs,36928
+mineru-2.7.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+mineru-2.7.5.dist-info/entry_points.txt,sha256=a9AHBIiYe3dpT3oofVQJC8fI0WjDhQASCUlhdMOK120,376
+mineru-2.7.5.dist-info/top_level.txt,sha256=zuGQfZcbsHv4I4oKI9gaKPqEWBFm6xJroKuug2LnKP8,7
+mineru-2.7.5.dist-info/RECORD,,