vlmparse 0.1.0__py3-none-any.whl → 0.1.3__py3-none-any.whl

vlmparse/clients/mineru.py

@@ -0,0 +1,119 @@
+import asyncio
+import io
+import os
+
+import orjson
+from loguru import logger
+from pydantic import Field
+
+from vlmparse.clients.pipe_utils.html_to_md_conversion import html_to_md_keep_tables
+from vlmparse.clients.pipe_utils.utils import clean_response
+from vlmparse.converter import BaseConverter, ConverterConfig
+from vlmparse.data_model.document import BoundingBox, Item, Page
+from vlmparse.servers.docker_server import DockerServerConfig
+
+
+class MinerUDockerServerConfig(DockerServerConfig):
+    """Configuration for MinerU Docker server."""
+
+    model_name: str = "mineru25"
+    docker_image: str = "pulsia/mineru25apipulsia:latest"
+    docker_port: int = 4299
+    container_port: int = 8000
+
+    @property
+    def client_config(self):
+        return MinerUConverterConfig(api_url=f"http://localhost:{self.docker_port}")
+
+
+class MinerUConverterConfig(ConverterConfig):
+    """Configuration for MinerU API converter."""
+
+    base_url: str = Field(
+        default_factory=lambda: os.getenv("MINERU_API_URL", "http://localhost:4299")
+    )
+    timeout: int = 600
+
+    def get_client(self, **kwargs) -> "MinerUConverter":
+        return MinerUConverter(config=self, **kwargs)
+
+
+def to_bytes_io(image):
+    img_byte_arr = io.BytesIO()
+    image.save(img_byte_arr, format="PNG")
+    img_byte_arr.seek(0)
+    return img_byte_arr
+
+
+class MinerUConverter(BaseConverter):
+    """MinerU HTTP API converter."""
+
+    config: MinerUConverterConfig
+
+    def __init__(self, config: MinerUConverterConfig, **kwargs):
+        super().__init__(config=config, **kwargs)
+        from httpx import AsyncClient
+
+        self.client = AsyncClient(base_url=config.api_url, timeout=config.timeout)
+
+    async def _async_inference_with_api(self, image) -> list:
+        """Run async inference with MinerU API."""
+
+        img_byte_arr = await asyncio.to_thread(to_bytes_io, image)
+
+        response = await self.client.post(
+            "process-image",
+            files={"image": ("image.png", img_byte_arr, "image/png")},
+        )
+
+        response.raise_for_status()
+
+        res = orjson.loads(response.content)
+
+        return res
+
+    async def _parse_image_with_api(self, origin_image):
+        response = await self._async_inference_with_api(origin_image)
+
+        original_width, original_height = origin_image.size
+
+        for cell in response:
+            bbox = cell["bbox"]
+            bbox_resized = [
+                bbox[0] * original_width,
+                bbox[1] * original_height,
+                bbox[2] * original_width,
+                bbox[3] * original_height,
+            ]
+
+            cell["bbox"] = bbox_resized
+
+        return response
+
+    async def async_call_inside_page(self, page: Page) -> Page:
+        image = page.image
+
+        # Call MinerU API
+        response = await self._parse_image_with_api(image)
+        logger.info("Response: " + str(response))
+
+        contents = [item.get("content", "") for item in response]
+        text = "\n\n".join([content for content in contents if content is not None])
+        items = []
+        for item in response:
+            l, t, r, b = item["bbox"]
+            txt = item.get("content", "")
+
+            items.append(
+                Item(
+                    text=txt if txt is not None else "",
+                    box=BoundingBox(l=l, t=t, r=r, b=b),
+                    category=item["type"],
+                )
+            )
+            page.items = items
+
+        text = clean_response(text)
+        text = html_to_md_keep_tables(text)
+        page.text = text
+        return page

vlmparse/clients/nanonetocr.py

@@ -0,0 +1,29 @@
+from pydantic import Field
+
+from vlmparse.clients.openai_converter import OpenAIConverterConfig
+from vlmparse.servers.docker_server import VLLMDockerServerConfig
+
+
+class NanonetOCR2DockerServerConfig(VLLMDockerServerConfig):
+    """Configuration for NanonetOCR2 model."""
+
+    model_name: str = "nanonets/Nanonets-OCR2-3B"
+    aliases: list[str] = Field(default_factory=lambda: ["nanonetsocr2"])
+
+    @property
+    def client_config(self):
+        return NanonetOCR2ConverterConfig(llm_params=self.llm_params)
+
+
+class NanonetOCR2ConverterConfig(OpenAIConverterConfig):
+    """Configuration for NanonetOCR2 model."""
+
+    model_name: str = "nanonets/Nanonets-OCR2-3B"
+    preprompt: str | None = None
+    postprompt: str | None = (
+        "Extract the text from the above document as if you were reading it naturally. Return the tables in html format. Return the equations in LaTeX representation. If there is an image in the document and image caption is not present, add a small description of the image inside the <img></img> tag; otherwise, add the image caption inside <img></img>. Watermarks should be wrapped in brackets. Ex: <watermark>OFFICIAL COPY</watermark>. Page numbers should be wrapped in brackets. Ex: <page_number>14</page_number> or <page_number>9/22</page_number>. Prefer using ☐ and ☑ for check boxes."
+    )
+    completion_kwargs: dict | None = {"temperature": 0.0, "max_tokens": 15000}
+    max_image_size: int | None = None
+    dpi: int = 200
+    aliases: list[str] = Field(default_factory=lambda: ["nanonetsocr2"])

vlmparse/clients/olmocr.py

@@ -0,0 +1,46 @@
+from pydantic import Field
+
+from vlmparse.clients.openai_converter import OpenAIConverterConfig
+from vlmparse.servers.docker_server import VLLMDockerServerConfig
+
+
+class OlmOCRDockerServerConfig(VLLMDockerServerConfig):
+    """Configuration for OlmOCR model."""
+
+    model_name: str = "allenai/olmOCR-2-7B-1025-FP8"
+    command_args: list[str] = Field(
+        default_factory=lambda: [
+            "--limit-mm-per-prompt",
+            '{"image": 1, "video": 0}',
+            "--disable-log-requests",
+            "--uvicorn-log-level",
+            "warning",
+            "--max-model-len",
+            "16384",
+        ]
+    )
+    aliases: list[str] = Field(default_factory=lambda: ["olmocr-2-fp8"])
+
+    @property
+    def client_config(self):
+        return OlmOCRConverterConfig(llm_params=self.llm_params)
+
+
+class OlmOCRConverterConfig(OpenAIConverterConfig):
+    """OlmOCR converter"""
+
+    model_name: str = "allenai/olmOCR-2-7B-1025-FP8"
+    preprompt: str | None = (
+        "Attached is one page of a document that you must process. "
+        "Just return the plain text representation of this document as if you were reading it naturally. Convert equations to LateX and tables to HTML.\n"
+        "If there are any figures or charts, label them with the following markdown syntax ![Alt text describing the contents of the figure](page_startx_starty_width_height.png)\n"
+        "Return your output as markdown, with a front matter section on top specifying values for the primary_language, is_rotation_valid, rotation_correction, is_table, and is_diagram parameters."
+    )
+    postprompt: str | None = None
+    completion_kwargs: dict | None = {
+        "temperature": 0.1,
+        "max_tokens": 8000,
+    }
+    max_image_size: int | None = 1288
+    dpi: int = 200
+    aliases: list[str] = Field(default_factory=lambda: ["olmocr-2-fp8"])

vlmparse/clients/openai_converter.py

@@ -0,0 +1,173 @@
+import os
+from typing import Literal
+
+from loguru import logger
+from pydantic import Field
+
+from vlmparse.base_model import VLMParseBaseModel
+from vlmparse.clients.pipe_utils.html_to_md_conversion import html_to_md_keep_tables
+from vlmparse.clients.pipe_utils.utils import clean_response
+from vlmparse.converter import BaseConverter, ConverterConfig
+from vlmparse.data_model.document import Page
+from vlmparse.servers.docker_server import DEFAULT_MODEL_NAME
+from vlmparse.utils import to_base64
+
+from .prompts import PDF2MD_PROMPT
+
+GOOGLE_API_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"
+
+
+class LLMParams(VLMParseBaseModel):
+    api_key: str = ""
+    base_url: str | None = None
+    model_name: str = DEFAULT_MODEL_NAME
+    timeout: int | None = 500
+    max_retries: int = 1
+
+
+def get_llm_params(model_name: str, uri: str | None = None):
+    if uri is not None:
+        return LLMParams(base_url=uri, model_name="vllm-model", api_key="")
+    if model_name in [
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4.1",
+        "gpt-4.1-mini",
+        "gpt-4.1-nano",
+        "gpt-5",
+        "gpt-5-mini",
+        "gpt-5-nano",
+    ]:
+        base_url = None
+        api_key = os.getenv("OPENAI_API_KEY")
+    else:
+        if model_name in [
+            "gemini-2.5-flash-lite",
+            "gemini-2.5-flash",
+            "gemini-2.5-pro",
+        ]:
+            base_url = GOOGLE_API_BASE_URL
+            api_key = os.getenv("GOOGLE_API_KEY")
+        else:
+            return None
+    return LLMParams(base_url=base_url, model_name=model_name, api_key=api_key)
+
+
+class OpenAIConverterConfig(ConverterConfig):
+    llm_params: LLMParams
+    preprompt: str | None = None
+    postprompt: str | None = PDF2MD_PROMPT
+    completion_kwargs: dict = Field(default_factory=dict)
+    stream: bool = False
+
+    def get_client(self, **kwargs) -> "OpenAIConverterClient":
+        return OpenAIConverterClient(config=self, **kwargs)
+
+
+class OpenAIConverterClient(BaseConverter):
+    """Client for OpenAI-compatible API servers."""
+
+    def __init__(
+        self,
+        config: OpenAIConverterConfig,
+        num_concurrent_files: int = 10,
+        num_concurrent_pages: int = 10,
+        save_folder: str | None = None,
+        save_mode: Literal["document", "md", "md_page"] = "document",
+        debug: bool = False,
+        return_documents_in_batch_mode: bool = False,
+    ):
+        super().__init__(
+            config=config,
+            num_concurrent_files=num_concurrent_files,
+            num_concurrent_pages=num_concurrent_pages,
+            save_folder=save_folder,
+            save_mode=save_mode,
+            debug=debug,
+            return_documents_in_batch_mode=return_documents_in_batch_mode,
+        )
+        from openai import AsyncOpenAI
+
+        self.model = AsyncOpenAI(
+            base_url=self.config.llm_params.base_url,
+            api_key=self.config.llm_params.api_key,
+            timeout=self.config.llm_params.timeout,
+        )
+
+    async def _get_chat_completion(
+        self, messages: list[dict], completion_kwargs: dict | None = None
+    ) -> str:
+        """Helper to handle chat completion with optional streaming."""
+        if completion_kwargs is None:
+            completion_kwargs = self.config.completion_kwargs
+
+        if self.config.stream:
+            response_stream = await self.model.chat.completions.create(
+                model=self.config.llm_params.model_name,
+                messages=messages,
+                stream=True,
+                **completion_kwargs,
+            )
+            response_parts = []
+            async for chunk in response_stream:
+                if chunk.choices and chunk.choices[0].delta.content:
+                    response_parts.append(chunk.choices[0].delta.content)
+            return "".join(response_parts)
+        else:
+            response_obj = await self.model.chat.completions.create(
+                model=self.config.llm_params.model_name,
+                messages=messages,
+                **completion_kwargs,
+            )
+
+            if response_obj.choices[0].message.content is None:
+                raise ValueError(
+                    "Response is None, finish reason: "
+                    + response_obj.choices[0].finish_reason
+                )
+            return response_obj.choices[0].message.content
+
+    async def async_call_inside_page(self, page: Page) -> Page:
+        """Process a single page using OpenAI-compatible API."""
+        image = page.image
+        if self.config.preprompt:
+            preprompt = [
+                {
+                    "role": "system",
+                    "content": [{"type": "text", "text": self.config.preprompt}],
+                }
+            ]
+        else:
+            preprompt = []
+
+        postprompt = (
+            [{"type": "text", "text": self.config.postprompt}]
+            if self.config.postprompt
+            else []
+        )
+
+        messages = [
+            *preprompt,
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/png;base64,{to_base64(image)}"
+                        },
+                    },
+                    *postprompt,
+                ],
+            },
+        ]
+
+        response = await self._get_chat_completion(messages)
+        logger.info("Response: " + str(response))
+        page.raw_response = response
+        text = clean_response(response)
+
+        text = html_to_md_keep_tables(text)
+        page.text = text
+
+        return page

vlmparse/clients/paddleocrvl.py

@@ -0,0 +1,48 @@
+from pydantic import Field
+
+from vlmparse.clients.openai_converter import OpenAIConverterConfig
+from vlmparse.servers.docker_server import VLLMDockerServerConfig
+
+
+class PaddleOCRVLDockerServerConfig(VLLMDockerServerConfig):
+    """Configuration for PaddleOCRVL model."""
+
+    model_name: str = "PaddlePaddle/PaddleOCR-VL"
+    command_args: list[str] = Field(
+        default_factory=lambda: [
+            "--limit-mm-per-prompt",
+            '{"image": 1}',
+            "--async-scheduling",
+            "--trust-remote-code",
+            "--mm-processor-cache-gb",
+            "0",
+        ]
+    )
+    aliases: list[str] = Field(default_factory=lambda: ["paddleocrvl"])
+
+    @property
+    def client_config(self):
+        return PaddleOCRVLConverterConfig(llm_params=self.llm_params)
+
+
+# Task-specific base prompts
+TASKS = {
+    "ocr": "OCR:",
+    "table": "Table Recognition:",
+    "formula": "Formula Recognition:",
+    "chart": "Chart Recognition:",
+}
+
+
+class PaddleOCRVLConverterConfig(OpenAIConverterConfig):
+    """PaddleOCRVL converter"""
+
+    model_name: str = "PaddlePaddle/PaddleOCR-VL"
+    preprompt: str | None = None
+    postprompt: str | None = TASKS["ocr"]
+    completion_kwargs: dict | None = {
+        "temperature": 0.0,
+    }
+    max_image_size: int | None = 1540
+    dpi: int = 200
+    aliases: list[str] = Field(default_factory=lambda: ["paddleocrvl"])

vlmparse/clients/pipe_utils/cleaner.py

@@ -0,0 +1,74 @@
+import re
+import unicodedata
+
+# Code adapted from olmocr.bench.tests.normalize_text
+
+
+def normalize_text(
+    md_content: str,
+    additional_replacements: dict = {},
+    only_alphanum: bool = False,
+    remove_md_images: bool = True,
+) -> str:
+    """Normalise md text"""
+
+    if md_content is None:
+        return None
+
+    # Normalize <br> and <br/> to newlines
+    md_content = re.sub(r"<br/?>", " ", md_content)
+
+    # Normalize whitespace in the md_content
+    md_content = re.sub(r"[ \t]+", " ", md_content)
+
+    # remove_title_emphasis:
+    md_content = re.sub(r"[-=]{3,}", "", md_content)
+
+    # remove_space_with_newlines:
+    md_content = re.sub(r"\n *", "\n", md_content)
+
+    # remove_more_than_2_newlines:
+    md_content = re.sub(r"\n{2,}", "\n\n", md_content)
+
+    # Remove markdown bold formatting (** or __ for bold)
+    md_content = re.sub(r"\*\*(.*?)\*\*", r"\1", md_content)
+    # md_content = re.sub(r"__(.*?)__", r"\1", md_content)
+    md_content = re.sub(r"</?b>", "", md_content)  # Remove <b> tags if they exist
+    md_content = re.sub(r"</?i>", "", md_content)  # Remove <i> tags if they exist
+
+    # Remove markdown italics formatting (* or _ for italics)
+    md_content = re.sub(r"\*(.*?)\*", r"\1", md_content)
+    # md_content = re.sub(r"_(.*?)_", r"\1", md_content)
+
+    # remove_more_than_1_spaces:
+    md_content = re.sub(r" {2,}", " ", md_content)
+
+    # Convert down to a consistent unicode form, so é == e + accent, unicode forms
+    md_content = unicodedata.normalize("NFC", md_content)
+
+    # Dictionary of characters to replace: keys are fancy characters, values are ASCII equivalents, unicode micro with greek mu comes up often enough too
+    replacements = {
+        "‘": "'",
+        "’": "'",
+        "‚": "'",
+        "“": '"',
+        "”": '"',
+        "„": '"',
+        "＿": "_",
+        "–": "-",
+        "—": "-",
+        "‑": "-",
+        "‒": "-",
+        "−": "-",
+        "\u00b5": "\u03bc",
+    } | additional_replacements
+
+    # Apply all replacements from the dictionary
+    for fancy_char, ascii_char in replacements.items():
+        md_content = md_content.replace(fancy_char, ascii_char)
+
+    if only_alphanum:
+        md_content = re.sub(r"[^a-zA-Z0-9]", "", md_content)
+    if remove_md_images:
+        md_content = re.sub(r"!\[[\s\S]*?]\([\s\S]*?\)", "", md_content, flags=re.S)
+    return md_content.strip()

vlmparse/clients/pipe_utils/html_to_md_conversion.py

@@ -0,0 +1,136 @@
+import re
+
+from bs4 import BeautifulSoup
+from bs4.element import NavigableString
+from html_to_markdown import convert_to_markdown
+
+
+def html_to_md_keep_tables(html: str, remove_head: bool = False) -> str:
+    # remove the whole <img>…</img> block (tag + contents)
+    html = re.sub(r"<img>([\s\S]*?)</img>", "", html, flags=re.S)
+
+    soup = BeautifulSoup(html, "lxml")
+    if remove_head:
+        # Remove global <head> tags if present
+        if soup.head:
+            soup.head.decompose()
+
+    # --- recurse inside tables first ---------------------------------------
+    for table in soup.find_all("table"):
+        for cell in table.find_all(["td", "th"]):
+            inner = "".join(map(str, cell.contents))
+            cell_md = html_to_md_keep_tables(inner).strip()  # recursion
+            cell.clear()
+            cell.append(NavigableString(cell_md))
+
+    # --- protect current-level tables --------------------------------------
+    markers = {}
+    for i, t in enumerate(soup.find_all("table")):
+        key = f"%%%TAB{i}%%%"
+        markers[key] = str(t)
+        t.replace_with(key)
+
+    # --- html → markdown (tables excluded) ---------------------------------
+    if len(str(soup)) > 0:
+        md_txt = convert_to_markdown(
+            str(soup),
+            strip=["table", "b", "strong", "i", "em"],
+            heading_style="atx",
+            escape_misc=False,
+            bullets="-",
+        )
+    else:
+        md_txt = ""
+
+    # --- restore tables -----------------------------------------------------
+    for k, raw in markers.items():
+        md_txt = md_txt.replace(k, raw)
+        # print(md_txt)
+
+    return md_txt
+
+
+# %%
+def md_tables_to_html(md_text: str) -> str:
+    """
+    Convert all Markdown tables in the text to HTML tables.
+
+    Args:
+        md_text: Markdown text containing tables
+
+    Returns:
+        Text with Markdown tables replaced by HTML tables
+    """
+    # Pattern to match Markdown tables
+    table_pattern = r"(\|[^\n]*\|\n(?:\|[-\s:]*\|\n)?(?:\|[^\n]*\|\n?)*)"
+
+    def convert_md_table_to_html(match):
+        table_text = match.group(1).strip()
+        lines = table_text.split("\n")
+
+        # Remove empty lines
+        lines = [line for line in lines if line.strip()]
+
+        if len(lines) < 2:
+            return table_text  # Not a valid table
+
+        # Check if second line is a separator (contains only |, -, :, and spaces)
+        separator_pattern = r"^\|[-:\s|]+\|$"
+        has_separator = bool(re.match(separator_pattern, lines[1]))
+
+        if not has_separator:
+            return table_text  # Not a valid table
+
+        # Parse header
+        header_row = lines[0]
+        header_cells = [cell.strip() for cell in header_row.split("|")[1:-1]]
+
+        # Parse data rows
+        data_rows = []
+        for line in lines[2:]:
+            if line.strip():
+                cells = [cell.strip() for cell in line.split("|")[1:-1]]
+                data_rows.append(cells)
+
+        # Build HTML table
+        html_parts = ["<table>"]
+
+        # Add header
+        if header_cells:
+            html_parts.append("<thead>")
+            html_parts.append("<tr>")
+            for cell in header_cells:
+                html_parts.append(f"<th>{cell}</th>")
+            html_parts.append("</tr>")
+            html_parts.append("</thead>")
+
+        # Add body
+        if data_rows:
+            html_parts.append("<tbody>")
+            for row in data_rows:
+                html_parts.append("<tr>")
+                for cell in row:
+                    html_parts.append(f"<td>{cell}</td>")
+                html_parts.append("</tr>")
+            html_parts.append("</tbody>")
+
+        html_parts.append("</table>")
+
+        return "".join(html_parts)
+
+    # Replace all Markdown tables with HTML tables
+    result = re.sub(
+        table_pattern, convert_md_table_to_html, md_text, flags=re.MULTILINE
+    )
+
+    return result
+
+
+if __name__ == "__main__":
+    md_text = """| Name | Age | City |
+    |------|-----|------|
+    | John | 25  | NYC  |
+    | Jane | 30  | LA   |"""
+
+    html_result = md_tables_to_html(md_text)
+    print(html_result)

vlmparse/clients/pipe_utils/utils.py

@@ -0,0 +1,12 @@
+def clean_response(text):
+    """Clean markdown/html markers from response text."""
+    return (
+        text.strip()
+        .removeprefix("")
+        .removesuffix("")
+        .removeprefix("```")
+        .removesuffix("```")
+        .removeprefix("markdown")
+        .removeprefix("html")
+        .strip()
+    )