mineru 2.6.5__py3-none-any.whl → 2.6.7__py3-none-any.whl

mineru/backend/pipeline/pipeline_analyze.py

@@ -159,7 +159,6 @@ def batch_image_analyze(
 
     model_manager = ModelSingleton()
 
-    batch_ratio = 1
     device = get_device()
 
     if str(device).startswith('npu'):
@@ -173,25 +172,21 @@ def batch_image_analyze(
                 "Please ensure that the torch_npu package is installed correctly."
             ) from e
 
-    if str(device).startswith('npu') or str(device).startswith('cuda'):
-        vram = get_vram(device)
-        if vram is not None:
-            gpu_memory = int(os.getenv('MINERU_VIRTUAL_VRAM_SIZE', round(vram)))
-            if gpu_memory >= 16:
-                batch_ratio = 16
-            elif gpu_memory >= 12:
-                batch_ratio = 8
-            elif gpu_memory >= 8:
-                batch_ratio = 4
-            elif gpu_memory >= 6:
-                batch_ratio = 2
-            else:
-                batch_ratio = 1
-            logger.info(f'gpu_memory: {gpu_memory} GB, batch_ratio: {batch_ratio}')
-        else:
-            # Default batch_ratio when VRAM can't be determined
-            batch_ratio = 1
-            logger.info(f'Could not determine GPU memory, using default batch_ratio: {batch_ratio}')
+    gpu_memory = get_vram(device)
+    if gpu_memory >= 16:
+        batch_ratio = 16
+    elif gpu_memory >= 12:
+        batch_ratio = 8
+    elif gpu_memory >= 8:
+        batch_ratio = 4
+    elif gpu_memory >= 6:
+        batch_ratio = 2
+    else:
+        batch_ratio = 1
+    logger.info(
+            f'GPU Memory: {gpu_memory} GB, Batch Ratio: {batch_ratio}. '
+            f'You can set MINERU_VIRTUAL_VRAM_SIZE environment variable to adjust GPU memory allocation.'
+    )
 
     # 检测torch的版本号
     import torch

mineru/backend/vlm/utils.py

@@ -81,20 +81,16 @@ def set_default_gpu_memory_utilization() -> float:
 def set_default_batch_size() -> int:
     try:
         device = get_device()
-        vram = get_vram(device)
-        if vram is not None:
-            gpu_memory = int(os.getenv('MINERU_VIRTUAL_VRAM_SIZE', round(vram)))
-            if gpu_memory >= 16:
-                batch_size = 8
-            elif gpu_memory >= 8:
-                batch_size = 4
-            else:
-                batch_size = 1
-            logger.info(f'gpu_memory: {gpu_memory} GB, batch_size: {batch_size}')
+        gpu_memory = get_vram(device)
+
+        if gpu_memory >= 16:
+            batch_size = 8
+        elif gpu_memory >= 8:
+            batch_size = 4
         else:
-            # Default batch_ratio when VRAM can't be determined
             batch_size = 1
-            logger.info(f'Could not determine GPU memory, using default batch_ratio: {batch_size}')
+        logger.info(f'gpu_memory: {gpu_memory} GB, batch_size: {batch_size}')
+
     except Exception as e:
         logger.warning(f'Error determining VRAM: {e}, using default batch_ratio: 1')
         batch_size = 1

mineru/backend/vlm/vlm_magic_model.py

@@ -45,7 +45,7 @@ class MagicModel:
                 continue
 
             span_type = "unknown"
-            line_type = None
+            code_block_sub_type = None
             guess_lang = None
 
             if block_type in [
@@ -74,7 +74,7 @@ class MagicModel:
                 span_type = ContentType.TABLE
             elif block_type in ["code", "algorithm"]:
                 block_content = code_content_clean(block_content)
-                line_type = block_type
+                code_block_sub_type = block_type
                 block_type = BlockType.CODE_BODY
                 span_type = ContentType.TEXT
                 guess_lang = guess_language_by_text(block_content)
@@ -82,6 +82,9 @@ class MagicModel:
                 block_type = BlockType.INTERLINE_EQUATION
                 span_type = ContentType.INTERLINE_EQUATION
 
+            #  code 和 algorithm 类型的块，如果内容中包含行内公式，则需要将块类型切换为algorithm
+            switch_code_to_algorithm = False
+
             if span_type in ["image", "table"]:
                 span = {
                     "bbox": block_bbox,
@@ -102,6 +105,8 @@ class MagicModel:
 
                 if block_content and block_content.count("\\(") == block_content.count("\\)") and block_content.count("\\(") > 0:
 
+                    switch_code_to_algorithm = True
+
                     # 生成包含文本和公式的span列表
                     spans = []
                     last_end = 0
@@ -160,7 +165,9 @@ class MagicModel:
 
             # 构造line对象
             if block_type in [BlockType.CODE_BODY]:
-                line = {"bbox": block_bbox, "spans": spans, "extra": {"type": line_type, "guess_lang": guess_lang}}
+                if switch_code_to_algorithm and code_block_sub_type == "code":
+                    code_block_sub_type = "algorithm"
+                line = {"bbox": block_bbox, "spans": spans, "extra": {"type": code_block_sub_type, "guess_lang": guess_lang}}
             else:
                 line = {"bbox": block_bbox, "spans": spans}
 

mineru/backend/vlm/vlm_middle_json_mkcontent.py

@@ -1,7 +1,8 @@
 import os
 
+from loguru import logger
 from mineru.utils.config_reader import get_latex_delimiter_config, get_formula_enable, get_table_enable
-from mineru.utils.enum_class import MakeMode, BlockType, ContentType
+from mineru.utils.enum_class import MakeMode, BlockType, ContentType, ContentTypeV2
 
 latex_delimiters_config = get_latex_delimiter_config()
 
@@ -234,6 +235,277 @@ def make_blocks_to_content_list(para_block, img_buket_path, page_idx, page_size)
 
     return para_content
 
+
+def make_blocks_to_content_list_v2(para_block, img_buket_path, page_size):
+    para_type = para_block['type']
+    para_content = {}
+    if para_type in [
+        BlockType.HEADER,
+        BlockType.FOOTER,
+        BlockType.ASIDE_TEXT,
+        BlockType.PAGE_NUMBER,
+        BlockType.PAGE_FOOTNOTE,
+    ]:
+        if para_type == BlockType.HEADER:
+            content_type = ContentTypeV2.PAGE_HEADER
+        elif para_type == BlockType.FOOTER:
+            content_type = ContentTypeV2.PAGE_FOOTER
+        elif para_type == BlockType.ASIDE_TEXT:
+            content_type = ContentTypeV2.PAGE_ASIDE_TEXT
+        elif para_type == BlockType.PAGE_NUMBER:
+            content_type = ContentTypeV2.PAGE_NUMBER
+        elif para_type == BlockType.PAGE_FOOTNOTE:
+            content_type = ContentTypeV2.PAGE_FOOTNOTE
+        else:
+            raise ValueError(f"Unknown para_type: {para_type}")
+        para_content = {
+            'type': content_type,
+            'content': {
+                f"{content_type}_content": merge_para_with_text_v2(para_block),
+            }
+        }
+    elif para_type == BlockType.TITLE:
+        title_level = get_title_level(para_block)
+        if title_level != 0:
+            para_content = {
+                'type': ContentTypeV2.TITLE,
+                'content': {
+                    "title_content": merge_para_with_text_v2(para_block),
+                    "level": title_level
+                }
+            }
+        else:
+            para_content = {
+                'type': ContentTypeV2.PARAGRAPH,
+                'content': {
+                    "paragraph_content": merge_para_with_text_v2(para_block),
+                }
+            }
+    elif para_type in [
+        BlockType.TEXT,
+        BlockType.PHONETIC
+    ]:
+        para_content = {
+            'type': ContentTypeV2.PARAGRAPH,
+            'content': {
+                'paragraph_content': merge_para_with_text_v2(para_block),
+            }
+        }
+    elif para_type == BlockType.INTERLINE_EQUATION:
+        image_path, math_content = get_body_data(para_block)
+        para_content = {
+            'type': ContentTypeV2.EQUATION_INTERLINE,
+            'content': {
+                'math_content': math_content,
+                'math_type': 'latex',
+                'image_source': {'path': f"{img_buket_path}/{image_path}"},
+            }
+        }
+    elif para_type == BlockType.IMAGE:
+        image_caption = []
+        image_footnote = []
+        image_path, _ = get_body_data(para_block)
+        image_source = {
+            'path': f"{img_buket_path}/{image_path}",
+        }
+        for block in para_block['blocks']:
+            if block['type'] == BlockType.IMAGE_CAPTION:
+                image_caption.extend(merge_para_with_text_v2(block))
+            if block['type'] == BlockType.IMAGE_FOOTNOTE:
+                image_footnote.extend(merge_para_with_text_v2(block))
+        para_content = {
+            'type': ContentTypeV2.IMAGE,
+            'content': {
+                'image_source': image_source,
+                'image_caption': image_caption,
+                'image_footnote': image_footnote,
+            }
+        }
+    elif para_type == BlockType.TABLE:
+        table_caption = []
+        table_footnote = []
+        image_path, html = get_body_data(para_block)
+        image_source = {
+            'path': f"{img_buket_path}/{image_path}",
+        }
+        if html.count("<table") > 1:
+            table_nest_level = 2
+        else:
+            table_nest_level = 1
+        if (
+                "colspan" in html or
+                "rowspan" in html or
+                table_nest_level > 1
+        ):
+            table_type = ContentTypeV2.TABLE_COMPLEX
+        else:
+            table_type = ContentTypeV2.TABLE_SIMPLE
+
+        for block in para_block['blocks']:
+            if block['type'] == BlockType.TABLE_CAPTION:
+                table_caption.extend(merge_para_with_text_v2(block))
+            if block['type'] == BlockType.TABLE_FOOTNOTE:
+                table_footnote.extend(merge_para_with_text_v2(block))
+        para_content = {
+            'type': ContentTypeV2.TABLE,
+            'content': {
+                'image_source': image_source,
+                'table_caption': table_caption,
+                'table_footnote': table_footnote,
+                'html': html,
+                'table_type': table_type,
+                'table_nest_level': table_nest_level,
+            }
+        }
+    elif para_type == BlockType.CODE:
+        code_caption = []
+        code_content = []
+        for block in para_block['blocks']:
+            if block['type'] == BlockType.CODE_CAPTION:
+                code_caption.extend(merge_para_with_text_v2(block))
+            if block['type'] == BlockType.CODE_BODY:
+                code_content = merge_para_with_text_v2(block)
+        sub_type = para_block["sub_type"]
+        if sub_type == BlockType.CODE:
+            para_content = {
+                'type': ContentTypeV2.CODE,
+                'content': {
+                    'code_caption': code_caption,
+                    'code_content': code_content,
+                    'code_language': para_block.get('guess_lang', 'txt'),
+                }
+            }
+        elif sub_type == BlockType.ALGORITHM:
+            para_content = {
+                'type': ContentTypeV2.ALGORITHM,
+                'content': {
+                    'algorithm_caption': code_caption,
+                    'algorithm_content': code_content,
+                }
+            }
+        else:
+            raise ValueError(f"Unknown code sub_type: {sub_type}")
+    elif para_type == BlockType.REF_TEXT:
+        para_content = {
+            'type': ContentTypeV2.LIST,
+            'content': {
+                'list_type': ContentTypeV2.LIST_REF,
+                'list_items': [
+                    {
+                        'item_type': 'text',
+                        'item_content': merge_para_with_text_v2(para_block),
+                    }
+                ],
+            }
+        }
+    elif para_type == BlockType.LIST:
+        if 'sub_type' in para_block:
+            if para_block['sub_type'] == BlockType.REF_TEXT:
+                list_type = ContentTypeV2.LIST_REF
+            elif para_block['sub_type'] == BlockType.TEXT:
+                list_type = ContentTypeV2.LIST_TEXT
+            else:
+                raise ValueError(f"Unknown list sub_type: {para_block['sub_type']}")
+        else:
+            list_type = ContentTypeV2.LIST_TEXT
+        list_items = []
+        for block in para_block['blocks']:
+            item_content = merge_para_with_text_v2(block)
+            if item_content:
+                list_items.append({
+                    'item_type': 'text',
+                    'item_content': item_content,
+                })
+        para_content = {
+            'type': ContentTypeV2.LIST,
+            'content': {
+                'list_type': list_type,
+                'list_items': list_items,
+            }
+        }
+
+    page_width, page_height = page_size
+    para_bbox = para_block.get('bbox')
+    if para_bbox:
+        x0, y0, x1, y1 = para_bbox
+        para_content['bbox'] = [
+            int(x0 * 1000 / page_width),
+            int(y0 * 1000 / page_height),
+            int(x1 * 1000 / page_width),
+            int(y1 * 1000 / page_height),
+        ]
+
+    return para_content
+
+
+
+
+
+def get_body_data(para_block):
+    """
+    Extract image_path and html from para_block
+    Returns:
+        - For IMAGE/INTERLINE_EQUATION: (image_path, '')
+        - For TABLE: (image_path, html)
+        - Default: ('', '')
+    """
+
+    def get_data_from_spans(lines):
+        for line in lines:
+            for span in line.get('spans', []):
+                span_type = span.get('type')
+                if span_type == ContentType.TABLE:
+                    return span.get('image_path', ''), span.get('html', '')
+                elif span_type == ContentType.IMAGE:
+                    return span.get('image_path', ''), ''
+                elif span_type == ContentType.INTERLINE_EQUATION:
+                    return span.get('image_path', ''), span.get('content', '')
+                elif span_type == ContentType.TEXT:
+                    return '', span.get('content', '')
+        return '', ''
+
+    # 处理嵌套的 blocks 结构
+    if 'blocks' in para_block:
+        for block in para_block['blocks']:
+            block_type = block.get('type')
+            if block_type in [BlockType.IMAGE_BODY, BlockType.TABLE_BODY, BlockType.CODE_BODY]:
+                result = get_data_from_spans(block.get('lines', []))
+                if result != ('', ''):
+                    return result
+        return '', ''
+
+    # 处理直接包含 lines 的结构
+    return get_data_from_spans(para_block.get('lines', []))
+
+
+def merge_para_with_text_v2(para_block):
+    para_content = []
+    para_type = para_block['type']
+    for line in para_block['lines']:
+        for span in line['spans']:
+            span_type = span['type']
+            if span['content']:
+                if para_type == BlockType.PHONETIC and span_type == ContentTypeV2.SPAN_TEXT:
+                    span_type = ContentTypeV2.SPAN_PHONETIC
+                if span_type == ContentType.INLINE_EQUATION:
+                    span_type = ContentTypeV2.SPAN_EQUATION_INLINE
+                if span_type in [
+                    ContentTypeV2.SPAN_TEXT,
+                    ContentTypeV2.SPAN_PHONETIC,
+                    ContentTypeV2.SPAN_EQUATION_INLINE,
+                    ContentTypeV2.SPAN_MD,
+                    ContentTypeV2.SPAN_CODE_INLINE,
+                ]:
+                    span_content = {
+                        'type': span_type,
+                        'content': span['content'],
+                    }
+                    para_content.append(span_content)
+                else:
+                    logger.warning(f"Unknown span type in merge_para_with_text_v2: {span_type}")
+    return para_content
+
+
 def union_make(pdf_info_dict: list,
                make_mode: str,
                img_buket_path: str = '',
@@ -260,10 +532,20 @@ def union_make(pdf_info_dict: list,
             for para_block in para_blocks:
                 para_content = make_blocks_to_content_list(para_block, img_buket_path, page_idx, page_size)
                 output_content.append(para_content)
+        elif make_mode == MakeMode.CONTENT_LIST_V2:
+            # https://github.com/drunkpig/llm-webkit-mirror/blob/dev6/docs/specification/output_format/content_list_spec.md
+            page_contents = []
+            para_blocks = (paras_of_layout or []) + (paras_of_discarded or [])
+            if not para_blocks:
+                continue
+            for para_block in para_blocks:
+                para_content = make_blocks_to_content_list_v2(para_block, img_buket_path, page_size)
+                page_contents.append(para_content)
+            output_content.append(page_contents)
 
     if make_mode in [MakeMode.MM_MD, MakeMode.NLP_MD]:
         return '\n\n'.join(output_content)
-    elif make_mode == MakeMode.CONTENT_LIST:
+    elif make_mode in [MakeMode.CONTENT_LIST, MakeMode.CONTENT_LIST_V2]:
         return output_content
     return None
 

mineru/cli/client.py

@@ -113,7 +113,7 @@ if is_mac_os_version_supported():
     '--formula',
     'formula_enable',
     type=bool,
-    help='Enable formula parsing. Default is True. Adapted only for the case where the backend is set to "pipeline".',
+    help='Enable formula parsing. Default is True. ',
     default=True,
 )
 @click.option(
@@ -121,7 +121,7 @@ if is_mac_os_version_supported():
     '--table',
     'table_enable',
     type=bool,
-    help='Enable table parsing. Default is True. Adapted only for the case where the backend is set to "pipeline".',
+    help='Enable table parsing. Default is True. ',
     default=True,
 )
 @click.option(
@@ -172,9 +172,8 @@ def main(
         def get_virtual_vram_size() -> int:
             if virtual_vram is not None:
                 return virtual_vram
-            if get_device_mode().startswith("cuda") or get_device_mode().startswith("npu"):
-                return round(get_vram(get_device_mode()))
-            return 1
+            else:
+                return get_vram(get_device_mode())
         if os.getenv('MINERU_VIRTUAL_VRAM_SIZE', None) is None:
             os.environ['MINERU_VIRTUAL_VRAM_SIZE']= str(get_virtual_vram_size())
 

mineru/cli/common.py

@@ -144,6 +144,13 @@ def _process_output(
             f"{pdf_file_name}_content_list.json",
             json.dumps(content_list, ensure_ascii=False, indent=4),
         )
+        if not is_pipeline:
+            content_list_v2 = make_func(pdf_info, MakeMode.CONTENT_LIST_V2, image_dir)
+            md_writer.write_string(
+                f"{pdf_file_name}_content_list_v2.json",
+                json.dumps(content_list_v2, ensure_ascii=False, indent=4),
+            )
+
 
     if f_dump_middle_json:
         md_writer.write_string(

mineru/cli/fast_api.py

@@ -8,7 +8,7 @@ import click
 import zipfile
 from pathlib import Path
 import glob
-from fastapi import FastAPI, UploadFile, File, Form
+from fastapi import Depends, FastAPI, HTTPException, UploadFile, File, Form
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import JSONResponse, FileResponse
 from starlette.background import BackgroundTask
@@ -21,14 +21,53 @@ from mineru.utils.cli_parser import arg_parse
 from mineru.utils.guess_suffix_or_lang import guess_suffix_by_path
 from mineru.version import __version__
 
-app = FastAPI()
-app.add_middleware(GZipMiddleware, minimum_size=1000)
+# 并发控制器
+_request_semaphore: Optional[asyncio.Semaphore] = None
+
+# 并发控制依赖函数
+async def limit_concurrency():
+    if _request_semaphore is not None:
+        if _request_semaphore.locked():
+            raise HTTPException(
+                status_code=503,
+                detail=f"Server is at maximum capacity: {os.getenv('MINERU_API_MAX_CONCURRENT_REQUESTS', 'unset')}. Please try again later."
+            )
+        async with _request_semaphore:
+            yield
+    else:
+        yield
+
+def create_app():
+    # By default, the OpenAPI documentation endpoints (openapi_url, docs_url, redoc_url) are enabled.
+    # To disable the FastAPI docs and schema endpoints, set the environment variable MINERU_API_ENABLE_FASTAPI_DOCS=0.
+    enable_docs = str(os.getenv("MINERU_API_ENABLE_FASTAPI_DOCS", "1")).lower() in ("1", "true", "yes")
+    app = FastAPI(
+        openapi_url="/openapi.json" if enable_docs else None,
+        docs_url="/docs" if enable_docs else None,
+        redoc_url="/redoc" if enable_docs else None,
+    )
+
+    # 初始化并发控制器：从环境变量MINERU_API_MAX_CONCURRENT_REQUESTS读取
+    global _request_semaphore
+    try:
+        max_concurrent_requests = int(os.getenv("MINERU_API_MAX_CONCURRENT_REQUESTS", "0"))
+    except ValueError:
+        max_concurrent_requests = 0
+
+    if max_concurrent_requests > 0:
+        _request_semaphore = asyncio.Semaphore(max_concurrent_requests)
+        logger.info(f"Request concurrency limited to {max_concurrent_requests}")
+
+    app.add_middleware(GZipMiddleware, minimum_size=1000)
+    return app
+
+app = create_app()
 
 
 def sanitize_filename(filename: str) -> str:
     """
     格式化压缩文件的文件名
-    移除路径遍历字符, 保留 Unicode 字母、数字、._- 
+    移除路径遍历字符, 保留 Unicode 字母、数字、._-
     禁止隐藏文件
     """
     sanitized = re.sub(r'[/\\\.]{2,}|[/\\]', '', filename)
@@ -60,24 +99,48 @@ def get_infer_result(file_suffix_identifier: str, pdf_name: str, parse_dir: str)
     return None
 
 
-@app.post(path="/file_parse",)
+@app.post(path="/file_parse", dependencies=[Depends(limit_concurrency)])
 async def parse_pdf(
-        files: List[UploadFile] = File(...),
-        output_dir: str = Form("./output"),
-        lang_list: List[str] = Form(["ch"]),
-        backend: str = Form("pipeline"),
-        parse_method: str = Form("auto"),
-        formula_enable: bool = Form(True),
-        table_enable: bool = Form(True),
-        server_url: Optional[str] = Form(None),
-        return_md: bool = Form(True),
-        return_middle_json: bool = Form(False),
-        return_model_output: bool = Form(False),
-        return_content_list: bool = Form(False),
-        return_images: bool = Form(False),
-        response_format_zip: bool = Form(False),
-        start_page_id: int = Form(0),
-        end_page_id: int = Form(99999),
+        files: List[UploadFile] = File(..., description="Upload pdf or image files for parsing"),
+        output_dir: str = Form("./output", description="Output local directory"),
+        lang_list: List[str] = Form(
+            ["ch"],
+            description="""(Adapted only for pipeline backend)Input the languages in the pdf to improve OCR accuracy.
+Options: ch, ch_server, ch_lite, en, korean, japan, chinese_cht, ta, te, ka, th, el, latin, arabic, east_slavic, cyrillic, devanagari.
+"""
+        ),
+        backend: str = Form(
+            "pipeline",
+            description="""The backend for parsing:
+- pipeline: More general
+- vlm-transformers: More general, but slower
+- vlm-mlx-engine: Faster than transformers (need apple silicon and macOS 13.5+)
+- vlm-vllm-async-engine: Faster (vllm-engine, need vllm installed)
+- vlm-lmdeploy-engine: Faster (lmdeploy-engine, need lmdeploy installed)
+- vlm-http-client: Faster (client suitable for openai-compatible servers)"""
+        ),
+        parse_method: str = Form(
+            "auto",
+            description="""(Adapted only for pipeline backend)The method for parsing PDF:
+- auto: Automatically determine the method based on the file type
+- txt: Use text extraction method
+- ocr: Use OCR method for image-based PDFs
+"""
+        ),
+        formula_enable: bool = Form(True, description="Enable formula parsing."),
+        table_enable: bool = Form(True, description="Enable table parsing."),
+        server_url: Optional[str] = Form(
+            None,
+            description="(Adapted only for vlm-http-client backend)openai compatible server url, e.g., http://127.0.0.1:30000"
+        ),
+        return_md: bool = Form(True, description="Return markdown content in response"),
+        return_middle_json: bool = Form(False, description="Return middle JSON in response"),
+        return_model_output: bool = Form(False, description="Return model output JSON in response"),
+        return_content_list: bool = Form(False, description="Return content list JSON in response"),
+        return_images: bool = Form(False, description="Return extracted images in response"),
+        response_format_zip: bool = Form(False, description="Return results as a ZIP file instead of JSON"),
+        start_page_id: int = Form(0, description="The starting page for PDF parsing, beginning from 0"),
+        end_page_id: int = Form(99999, description="The ending page for PDF parsing, beginning from 0"),
 ):
 
     # 获取命令行配置参数
@@ -153,7 +216,7 @@ async def parse_pdf(
         # 根据 response_format_zip 决定返回类型
         if response_format_zip:
             zip_fd, zip_path = tempfile.mkstemp(suffix=".zip", prefix="mineru_results_")
-            os.close(zip_fd) 
+            os.close(zip_fd)
             with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
                 for pdf_name in pdf_file_names:
                     safe_pdf_name = sanitize_filename(pdf_name)
@@ -178,7 +241,7 @@ async def parse_pdf(
 
                     if return_model_output:
                         path = os.path.join(parse_dir, f"{pdf_name}_model.json")
-                        if os.path.exists(path): 
+                        if os.path.exists(path):
                             zf.write(path, arcname=os.path.join(safe_pdf_name, os.path.basename(path)))
 
                     if return_content_list:
@@ -259,11 +322,16 @@ def main(ctx, host, port, reload, **kwargs):
     # 将配置参数存储到应用状态中
     app.state.config = kwargs
 
+    # 将 CLI 的并发参数同步到环境变量，确保 uvicorn 重载子进程可见
+    try:
+        mcr = int(kwargs.get("mineru_api_max_concurrent_requests", 0) or 0)
+    except ValueError:
+        mcr = 0
+    os.environ["MINERU_API_MAX_CONCURRENT_REQUESTS"] = str(mcr)
+
     """启动MinerU FastAPI服务器的命令行入口"""
     print(f"Start MinerU FastAPI Service: http://{host}:{port}")
-    print("The API documentation can be accessed at the following address:")
-    print(f"- Swagger UI: http://{host}:{port}/docs")
-    print(f"- ReDoc: http://{host}:{port}/redoc")
+    print(f"API documentation: http://{host}:{port}/docs")
 
     uvicorn.run(
         "mineru.cli.fast_api:app",
@@ -274,4 +342,4 @@ def main(ctx, host, port, reload, **kwargs):
 
 
 if __name__ == "__main__":
-    main()
+    main()

mineru/utils/enum_class.py

@@ -38,6 +38,31 @@ class ContentType:
     CODE = 'code'
 
 
+class ContentTypeV2:
+    CODE = 'code'
+    ALGORITHM = "algorithm"
+    EQUATION_INTERLINE = 'equation_interline'
+    IMAGE = 'image'
+    TABLE = 'table'
+    TABLE_SIMPLE = 'simple_table'
+    TABLE_COMPLEX = 'complex_table'
+    LIST = 'list'
+    LIST_TEXT = 'text_list'
+    LIST_REF = 'reference_list'
+    TITLE = 'title'
+    PARAGRAPH = 'paragraph'
+    SPAN_TEXT = 'text'
+    SPAN_EQUATION_INLINE = 'equation_inline'
+    SPAN_PHONETIC = 'phonetic'
+    SPAN_MD = 'md'
+    SPAN_CODE_INLINE = 'code_inline'
+    PAGE_HEADER = "page_header"
+    PAGE_FOOTER = "page_footer"
+    PAGE_NUMBER = "page_number"
+    PAGE_ASIDE_TEXT = "page_aside_text"
+    PAGE_FOOTNOTE = "page_footnote"
+
+
 class CategoryId:
     Title = 0
     Text = 1
@@ -60,6 +85,7 @@ class MakeMode:
     MM_MD = 'mm_markdown'
     NLP_MD = 'nlp_markdown'
     CONTENT_LIST = 'content_list'
+    CONTENT_LIST_V2 = 'content_list_v2'
 
 
 class ModelPath:

mineru/utils/guess_suffix_or_lang.py

@@ -1,9 +1,11 @@
 from pathlib import Path
 
+from loguru import logger
 from magika import Magika
 
 
 DEFAULT_LANG = "txt"
+PDF_SIG_BYTES = b'%PDF'
 magika = Magika()
 
 def guess_language_by_text(code):
@@ -14,7 +16,7 @@ def guess_language_by_text(code):
 
 def guess_suffix_by_bytes(file_bytes, file_path=None) -> str:
     suffix = magika.identify_bytes(file_bytes).prediction.output.label
-    if file_path and suffix in ["ai"] and Path(file_path).suffix.lower() in [".pdf"]:
+    if file_path and suffix in ["ai", "html"] and Path(file_path).suffix.lower() in [".pdf"] and file_bytes[:4] == PDF_SIG_BYTES:
         suffix = "pdf"
     return suffix
 
@@ -23,6 +25,11 @@ def guess_suffix_by_path(file_path) -> str:
     if not isinstance(file_path, Path):
         file_path = Path(file_path)
     suffix = magika.identify_path(file_path).prediction.output.label
-    if suffix in ["ai"] and file_path.suffix.lower() in [".pdf"]:
-        suffix = "pdf"
+    if suffix in ["ai", "html"] and file_path.suffix.lower() in [".pdf"]:
+        try:
+            with open(file_path, 'rb') as f:
+                if f.read(4) == PDF_SIG_BYTES:
+                    suffix = "pdf"
+        except Exception as e:
+            logger.warning(f"Failed to read file {file_path} for PDF signature check: {e}")
     return suffix

mineru/utils/model_utils.py

@@ -428,8 +428,6 @@ def clean_memory(device='cuda'):
 
 def clean_vram(device, vram_threshold=8):
     total_memory = get_vram(device)
-    if total_memory is not None:
-        total_memory = int(os.getenv('MINERU_VIRTUAL_VRAM_SIZE', round(total_memory)))
     if total_memory and total_memory <= vram_threshold:
         gc_start = time.time()
         clean_memory(device)
@@ -437,13 +435,28 @@ def clean_vram(device, vram_threshold=8):
         # logger.info(f"gc time: {gc_time}")
 
 
-def get_vram(device):
+def get_vram(device) -> int:
+    env_vram = os.getenv("MINERU_VIRTUAL_VRAM_SIZE")
+
+    # 如果环境变量已配置,尝试解析并返回
+    if env_vram is not None:
+        try:
+            total_memory = int(env_vram)
+            if total_memory > 0:
+                return total_memory
+            else:
+                logger.warning(
+                    f"MINERU_VIRTUAL_VRAM_SIZE value '{env_vram}' is not positive, falling back to auto-detection")
+        except ValueError:
+            logger.warning(
+                f"MINERU_VIRTUAL_VRAM_SIZE value '{env_vram}' is not a valid integer, falling back to auto-detection")
+
+    # 环境变量未配置或配置错误,根据device自动获取
+    total_memory = 1
     if torch.cuda.is_available() and str(device).startswith("cuda"):
-        total_memory = torch.cuda.get_device_properties(device).total_memory / (1024 ** 3)  # 将字节转换为 GB
-        return total_memory
+        total_memory = round(torch.cuda.get_device_properties(device).total_memory / (1024 ** 3))  # 将字节转换为 GB
     elif str(device).startswith("npu"):
         if torch_npu.npu.is_available():
-            total_memory = torch_npu.npu.get_device_properties(device).total_memory / (1024 ** 3)  # 转为 GB
-            return total_memory
-    else:
-        return None
+            total_memory = round(torch_npu.npu.get_device_properties(device).total_memory / (1024 ** 3))  # 转为 GB
+
+    return total_memory

mineru/utils/table_merge.py

@@ -7,6 +7,9 @@ from mineru.backend.vlm.vlm_middle_json_mkcontent import merge_para_with_text
 from mineru.utils.enum_class import BlockType, SplitFlag
 
 
+CONTINUATION_MARKERS = ["(续)", "(续表)", "(continued)", "(cont.)"]
+
+
 def full_to_half(text: str) -> str:
     """Convert full-width characters to half-width characters using code point manipulation.
 
@@ -174,8 +177,13 @@ def can_merge_tables(current_table_block, previous_table_block):
     # 如果有TABLE_CAPTION类型的块,检查是否至少有一个以"(续)"结尾
     caption_blocks = [block for block in current_table_block["blocks"] if block["type"] == BlockType.TABLE_CAPTION]
     if caption_blocks:
-        # 如果所有caption都不以"(续)"结尾,则不合并
-        if not any(full_to_half(merge_para_with_text(block).strip()).endswith("(续)") for block in caption_blocks):
+        # 如果所有caption都不以"(续)"、"(续表)"、"(continued)"或"(cont.)"结尾,则不合并
+
+        if not any(
+                any(full_to_half(merge_para_with_text(block).strip()).lower().endswith(marker.lower())
+                    for marker in CONTINUATION_MARKERS)
+                for block in caption_blocks
+        ):
             return False, None, None, None, None
 
     if any(block["type"] == BlockType.TABLE_FOOTNOTE for block in previous_table_block["blocks"]):

mineru/version.py

@@ -1 +1 @@
-__version__ = "2.6.5"
+__version__ = "2.6.7"

{mineru-2.6.5.dist-info → mineru-2.6.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mineru
-Version: 2.6.5
+Version: 2.6.7
 Summary: A practical tool for converting PDF to Markdown
 License: AGPL-3.0
 Project-URL: homepage, https://mineru.net/
@@ -74,8 +74,8 @@ Requires-Dist: fastapi; extra == "api"
 Requires-Dist: python-multipart; extra == "api"
 Requires-Dist: uvicorn; extra == "api"
 Provides-Extra: gradio
-Requires-Dist: gradio<6,>=5.34; extra == "gradio"
-Requires-Dist: gradio-pdf>=0.0.22; extra == "gradio"
+Requires-Dist: gradio==5.49.1; extra == "gradio"
+Requires-Dist: gradio-pdf==0.0.22; extra == "gradio"
 Provides-Extra: core
 Requires-Dist: mineru[vlm]; extra == "core"
 Requires-Dist: mineru[pipeline]; extra == "core"
@@ -134,6 +134,16 @@ Dynamic: license-file
 </div>
 
 # Changelog
+
+- 2025/12/12 2.6.7 Release
+  - Bug fix: #4168
+
+- 2025/12/02 2.6.6 Release
+  - `mineru-api` tool optimizations
+    - Added descriptive text to `mineru-api` interface parameters to improve API documentation readability.
+    - You can use the environment variable `MINERU_API_ENABLE_FASTAPI_DOCS` to control whether the auto-generated interface documentation page is enabled (enabled by default).
+    - Added concurrency configuration options for the `vlm-vllm-async-engine`, `vlm-lmdeploy-engine`, and `vlm-http-client` backends. Users can use the environment variable `MINERU_API_MAX_CONCURRENT_REQUESTS` to set the maximum number of concurrent API requests (unlimited by default).
+
 - 2025/11/26 2.6.5 Release
   - Added support for a new backend vlm-lmdeploy-engine. Its usage is similar to vlm-vllm-(async)engine, but it uses lmdeploy as the inference engine and additionally supports native inference acceleration on Windows platforms compared to vllm.
 
@@ -887,6 +897,8 @@ Currently, some models in this project are trained based on YOLO. However, since
 - [pdfminer.six](https://github.com/pdfminer/pdfminer.six)
 - [pypdf](https://github.com/py-pdf/pypdf)
 - [magika](https://github.com/google/magika)
+- [vLLM](https://github.com/vllm-project/vllm)
+- [LMDeploy](https://github.com/InternLM/lmdeploy)
 
 # Citation
 

{mineru-2.6.5.dist-info → mineru-2.6.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mineru/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
-mineru/version.py,sha256=b8L3dijps7oaMPmOpJzOuXwvOcbIuro9wWmuPwiL87o,22
+mineru/version.py,sha256=Kc2cyTvfIdfaMxraNhj61Hw9Mr3C-eTCdQPLK5KH1AQ,22
 mineru/backend/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
 mineru/backend/utils.py,sha256=GLJU3IznDmhE1_qNmkU1UOtsuskIHBezgsEVO6Uar-Y,698
 mineru/backend/pipeline/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
@@ -8,19 +8,19 @@ mineru/backend/pipeline/model_init.py,sha256=OAylOcQD9gu5TBcX7nMt7X5NpJMtQICI5Iv
 mineru/backend/pipeline/model_json_to_middle_json.py,sha256=reXkUR_wKmJD64d7vRNXMxFviwkzDlGjRshpdwsVquI,10951
 mineru/backend/pipeline/model_list.py,sha256=7cXMBfZrP0K6qWueg1D_-WoUANeSINzkn_ic9E7YQLs,222
 mineru/backend/pipeline/para_split.py,sha256=Kq95MmvkPm7rKxlCSGiTvVKyF7CErHI2eGGAs5sLl0Q,17119
-mineru/backend/pipeline/pipeline_analyze.py,sha256=O_HGifodg03VZbmTve-U6Cmo0T03AmuK86t1v1J9X-Q,6897
+mineru/backend/pipeline/pipeline_analyze.py,sha256=GkGOrWGnBSswUik3nt_m76bCwDISC9sxXZ6xRX3L154,6528
 mineru/backend/pipeline/pipeline_magic_model.py,sha256=w8jGx8f6yZN0Wf2yPP3L9rYKc9rogxreZCrUJzJvPO8,14974
 mineru/backend/pipeline/pipeline_middle_json_mkcontent.py,sha256=YlnEbbUnkniZXS13aLo5mjfFQvQM5SrIVvTAGBZsLmw,14478
 mineru/backend/vlm/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
 mineru/backend/vlm/model_output_to_middle_json.py,sha256=AqYX44gS9crUO_t7SuUatD71EVjow6pI6yA2Ik3gQ0s,5139
-mineru/backend/vlm/utils.py,sha256=taiPNKtsykImUYkkosk1CjxFIJEutygK8iZTLly-ZqU,3905
+mineru/backend/vlm/utils.py,sha256=JMgS3SMFcHJYH2jIx-Xhs-P2a1bmT8U6Kn60IL0OmQA,3570
 mineru/backend/vlm/vlm_analyze.py,sha256=wP3vuYGVec0hRsDAuzfSm2HD4Muu7wSWL767qxd_yqw,11690
-mineru/backend/vlm/vlm_magic_model.py,sha256=Pd0sOr7G1crAJIVeq6h_03gNSuxmV5U8dvGTGT_rrjs,23452
-mineru/backend/vlm/vlm_middle_json_mkcontent.py,sha256=5V-AU9KkxxMn0DDSQBrb15I4GVpEyiQy8uNI_tQhS6M,13498
+mineru/backend/vlm/vlm_magic_model.py,sha256=e8BWkfeRkZNJjFdm9oPmvIs9ATjdzCy_5OZw2qTziZA,23839
+mineru/backend/vlm/vlm_middle_json_mkcontent.py,sha256=9ig7kfbKDrG5QOwvYoXfcsK5EY9_VyAFmqz733Il-P0,24071
 mineru/cli/__init__.py,sha256=8CRrCQVuExa0BttRFh3Z40lFy2K5jN0sp67KWjOlj5c,50
-mineru/cli/client.py,sha256=__CQknekVeq6s72JzHQRPSpR7mfNaO-ob9wq6oiEj6s,7047
-mineru/cli/common.py,sha256=zhNOJCOnTSMbWdUWSZG-nf0odv5vBRtdZYZ1UbUPH3g,14369
-mineru/cli/fast_api.py,sha256=t5bda769VbM5iokAboiJfPIOnm-r5GTFReE-KQy8L3g,10941
+mineru/cli/client.py,sha256=XSEIr4klUuufMAWn5IioZdXpg1xAxqRZF0HkaVIhxh0,6815
+mineru/cli/common.py,sha256=Rld3P4FzbTYngy3BP-Irto9aMS7EPq2lIWgeIF1xPMc,14667
+mineru/cli/fast_api.py,sha256=lLxQKKHmD8ruoZGcE6LrXzr3pQIxvw8OdJrQq_FNLSM,14447
 mineru/cli/gradio_app.py,sha256=EUPuRHHCOECrE3E3VNEeuMDYeC3nicurOYfk8YJSOMw,15646
 mineru/cli/models_download.py,sha256=LNfoIpUlJM7m7qb2SiCxtjMDw4jILBQtZwNP2JoY81U,4815
 mineru/cli/vlm_server.py,sha256=27HaqO3wpMXSA_nA3CC6JOBTHK3q66SP00cD6m9HuQE,1974
@@ -159,14 +159,14 @@ mineru/utils/cli_parser.py,sha256=4seFAu1kulsYnw6WM2q_cxgEOt2tErZVkI-LNEF_kGw,14
 mineru/utils/config_reader.py,sha256=IRVWTpBnbnRpck6eXZUKw-fcLt7hon5S4uqWW-RBb1w,4075
 mineru/utils/cut_image.py,sha256=g3m4nfcJNWlxi-P0kpXTtlmspXkMcLCfGwmYuQ-Z2hE,751
 mineru/utils/draw_bbox.py,sha256=FkgppjUzRhN-uxvChdkhHXcDavJEaApMD6qC6qoRwfQ,20292
-mineru/utils/enum_class.py,sha256=-_Ey03vGNEQHkl6x7pZ43GgrakwhSCOa1RXdr1m-I3A,2503
+mineru/utils/enum_class.py,sha256=NO9FiM2Ni6-SI1rzMm5X98ssNDxgJTyykZddJSscgJs,3221
 mineru/utils/format_utils.py,sha256=2s89vHcSISjuolk8Hvg3K-5-rRbiT3Us7eFLzUKrNKs,10233
-mineru/utils/guess_suffix_or_lang.py,sha256=nznyQpUn1BSA8JNw9HuG3pVV-xtVAtrtcGuHZ-VXt9M,856
+mineru/utils/guess_suffix_or_lang.py,sha256=aUC2wAJwa5LH0SHxwTbOEJqVVgvpdUCWFF6of8eDVkc,1198
 mineru/utils/hash_utils.py,sha256=UPS_8NRBmVumdyOv16Lmv6Ly2xK8OVDJEe5gG6gKIFk,857
 mineru/utils/language.py,sha256=7RT3mxSa7jdpoC5ySd7ZddHA7TO7UsnmDOWiYZAxuyg,1433
 mineru/utils/llm_aided.py,sha256=9WUytvxenSAuaWR4sTQhVPQ5h8pY0wVOH1O2sj_6dLs,5149
 mineru/utils/magic_model_utils.py,sha256=2xOvi4oqg3MSw1FUrJTnYDtWeFrrm6qbmlEorLZSaYs,5650
-mineru/utils/model_utils.py,sha256=6OsgFLsABX5JuShSzCMSNHWV-yi-1cjwHweafyxIgRo,18448
+mineru/utils/model_utils.py,sha256=6moOQqE5ShHaJKkENXP8BXJA7RCWtOGlYHZ3nidwmZs,18977
 mineru/utils/models_download_utils.py,sha256=UfjvwhxO6BkJHa5JSpEVNZ71GoLMPMmJpym3THET2T4,2957
 mineru/utils/ocr_utils.py,sha256=lPIrwNUib5mrzUkponRYHuUCdjV2qvETNLSzOLyflrU,15990
 mineru/utils/os_env_config.py,sha256=ZNtkR4KrJW72CeIoTNzGDL6tMKv_hL8nzvWIssGWbqY,842
@@ -178,10 +178,10 @@ mineru/utils/pdf_text_tool.py,sha256=KEztjfdqsIHHuiTEAMAL7Lr1OS3R7Ur-uTqGiCRjReQ
 mineru/utils/run_async.py,sha256=rPeP4BCZerR8VByRDhiYzfZiahLVqoZEBVAS54dAjNg,1286
 mineru/utils/span_block_fix.py,sha256=0eVQjJCrT03woRt9hoh6Uu42Tp1dacfGTv2x3B9qq94,8797
 mineru/utils/span_pre_proc.py,sha256=h41q2uQajI0xQbc_30hqaju1dv3oVYxBAlKgURl8HIc,13692
-mineru/utils/table_merge.py,sha256=d98zNbM1ZQ8V1kUt6RugParNUNPv7DGL-XKIzR3iJVQ,15360
-mineru-2.6.5.dist-info/licenses/LICENSE.md,sha256=jVa0BUaKrRH4erV2P5AeJ24I2WRv9chIGxditreJ6e0,34524
-mineru-2.6.5.dist-info/METADATA,sha256=BUj9fYR_NiRpYGqXWd3J_fOTE8IN0bdl0PgY6FUGVcg,72362
-mineru-2.6.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mineru-2.6.5.dist-info/entry_points.txt,sha256=JbtrCPhx1T32s7TONUsteKg-24ZwRT1HSiFtW5jypVw,376
-mineru-2.6.5.dist-info/top_level.txt,sha256=zuGQfZcbsHv4I4oKI9gaKPqEWBFm6xJroKuug2LnKP8,7
-mineru-2.6.5.dist-info/RECORD,,
+mineru/utils/table_merge.py,sha256=pZHP0mIUcWPcdu0HecZDEFZ_ms_rT7C43m0Eg2iEEF4,15592
+mineru-2.6.7.dist-info/licenses/LICENSE.md,sha256=jVa0BUaKrRH4erV2P5AeJ24I2WRv9chIGxditreJ6e0,34524
+mineru-2.6.7.dist-info/METADATA,sha256=2SEMOR8-0ZzJL85pLQR4Gkm61YroeJ6BSDjB44YCUhs,73142
+mineru-2.6.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mineru-2.6.7.dist-info/entry_points.txt,sha256=JbtrCPhx1T32s7TONUsteKg-24ZwRT1HSiFtW5jypVw,376
+mineru-2.6.7.dist-info/top_level.txt,sha256=zuGQfZcbsHv4I4oKI9gaKPqEWBFm6xJroKuug2LnKP8,7
+mineru-2.6.7.dist-info/RECORD,,