vlmparse 0.1.8__py3-none-any.whl → 0.1.9__py3-none-any.whl

vlmparse/clients/deepseekocr.py

@@ -13,6 +13,10 @@ from vlmparse.data_model.document import Item, Page
 from vlmparse.servers.docker_server import VLLMDockerServerConfig
 from vlmparse.utils import to_base64
 
+# ==============================================================================
+# DeepSeek-OCR (v1)
+# ==============================================================================
+
 
 class DeepSeekOCRDockerServerConfig(VLLMDockerServerConfig):
     """Configuration for DeepSeekOCR model."""
@@ -212,3 +216,169 @@ class DeepSeekOCRConverterClient(OpenAIConverterClient):
             page.completion_tokens = usage.completion_tokens
 
         return page
+
+
+# ==============================================================================
+# DeepSeek-OCR-2
+# ==============================================================================
+
+
+class DeepSeekOCR2DockerServerConfig(VLLMDockerServerConfig):
+    """Configuration for DeepSeek-OCR-2 model.
+
+    DeepSeek-OCR-2 uses a custom architecture that requires:
+    - Custom model registration via hf_overrides
+    - NoRepeatNGram logits processor with specific whitelist tokens
+    - Custom image processor (DeepseekOCR2Processor)
+    """
+
+    docker_image: str = "vllm/vllm-openai:nightly"
+    model_name: str = "deepseek-ai/DeepSeek-OCR-2"
+    command_args: list[str] = Field(
+        default_factory=lambda: [
+            "--limit-mm-per-prompt",
+            '{"image": 1}',
+            "--hf-overrides",
+            '{"architectures": ["DeepseekOCR2ForCausalLM"]}',
+            "--block-size",
+            "256",
+            "--trust-remote-code",
+            "--max-model-len",
+            "8192",
+            "--swap-space",
+            "0",
+            "--gpu-memory-utilization",
+            "0.9",
+            "--logits_processors",
+            "vllm.model_executor.models.deepseek_ocr:NGramPerReqLogitsProcessor",
+        ]
+    )
+    aliases: list[str] = Field(
+        default_factory=lambda: ["deepseekocr2", "DeepSeek-OCR-2"]
+    )
+
+    @property
+    def client_config(self):
+        return DeepSeekOCR2ConverterConfig(
+            **self._create_client_kwargs(
+                f"http://localhost:{self.docker_port}{self.get_base_url_suffix()}"
+            )
+        )
+
+
+class DeepSeekOCR2ConverterConfig(OpenAIConverterConfig):
+    """DeepSeek-OCR-2 converter configuration.
+
+    Key differences from DeepSeek-OCR v1:
+    - Uses DeepseekOCR2ForCausalLM architecture
+    - Different logits processor parameters (ngram_size=20, window_size=50)
+    - Supports cropping mode for image processing
+    """
+
+    model_name: str = "deepseek-ai/DeepSeek-OCR-2"
+    aliases: list[str] = Field(
+        default_factory=lambda: ["deepseekocr2", "DeepSeek-OCR-2"]
+    )
+    postprompt: str | None = None
+    prompts: dict[str, str] = {
+        "layout": "<|grounding|>Convert the document to markdown.",
+        "ocr": "Free OCR.",
+        "image_description": "Describe this image in detail.",
+    }
+    prompt_mode_map: dict[str, str] = {
+        "ocr_layout": "layout",
+        "table": "layout",
+    }
+
+    completion_kwargs: dict | None = {
+        "temperature": 0.0,
+        "max_tokens": 8180,
+        "extra_body": {
+            "skip_special_tokens": False,
+            # args used to control custom logits processor
+            "vllm_xargs": {
+                "ngram_size": 20,
+                "window_size": 50,
+                # whitelist: <td>, </td>
+                "whitelist_token_ids": [128821, 128822],
+            },
+        },
+    }
+    dpi: int = 144  # Default DPI used in reference implementation
+
+    def get_client(self, **kwargs) -> "DeepSeekOCR2ConverterClient":
+        return DeepSeekOCR2ConverterClient(config=self, **kwargs)
+
+
+class DeepSeekOCR2ConverterClient(DeepSeekOCRConverterClient):
+    """Client for DeepSeek-OCR-2 with specific post-processing.
+
+    Inherits from DeepSeekOCRConverterClient as the post-processing logic
+    for parsing grounding references and extracting items is the same.
+    The main differences are in the model configuration and logits processor.
+    """
+
+    async def async_call_inside_page(self, page: Page) -> Page:
+        # Prepare messages as in parent class
+        image = page.image
+
+        prompt_key = self.get_prompt_key() or "ocr"
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/png;base64,{to_base64(image)}"
+                        },
+                    },
+                    {"type": "text", "text": self.config.prompts[prompt_key]},
+                ],
+            },
+        ]
+
+        # Get raw response using parent's method
+        response, usage = await self._get_chat_completion(messages)
+        logger.info("Response length: " + str(len(response)))
+        page.raw_response = response
+
+        if prompt_key == "layout":
+            # Post-processing
+            matches, matches_image, matches_other = re_match(response)
+
+            # Extract items (bounding boxes)
+            page.items = self.extract_items(page.image, matches)
+
+            # Clean text
+            outputs = response
+
+            # Check for sentence end marker (indicates successful completion)
+            # If not present, it might be due to repetition detection
+            if "<｜end▁of▁sentence｜>" in outputs:
+                outputs = outputs.replace("<｜end▁of▁sentence｜>", "")
+
+            # Replace image references with a placeholder
+            for a_match_image in matches_image:
+                outputs = outputs.replace(a_match_image, "![image]")
+
+            # Replace other references (text grounding) and cleanup
+            for a_match_other in matches_other:
+                outputs = (
+                    outputs.replace(a_match_other, "")
+                    .replace("\\coloneqq", ":=")
+                    .replace("\\eqqcolon", "=:")
+                    .replace("\n\n\n\n", "\n\n")
+                    .replace("\n\n\n", "\n\n")
+                )
+        else:
+            outputs = response
+
+        page.text = outputs.strip()
+        logger.debug(page.text)
+        if usage is not None:
+            page.prompt_tokens = usage.prompt_tokens
+            page.completion_tokens = usage.completion_tokens
+
+        return page

vlmparse/clients/glmocr.py

@@ -0,0 +1,243 @@
+import asyncio
+import os
+from pathlib import Path
+from typing import Any
+
+import httpx
+import orjson
+from loguru import logger
+from pydantic import Field
+
+from vlmparse.clients.pipe_utils.html_to_md_conversion import html_to_md_keep_tables
+from vlmparse.clients.pipe_utils.utils import clean_response
+from vlmparse.converter import BaseConverter, ConverterConfig
+from vlmparse.data_model.document import BoundingBox, Item, Page
+from vlmparse.servers.docker_compose_server import DockerComposeServerConfig
+from vlmparse.utils import to_base64
+
+DOCKER_PIPELINE_DIR = (
+    Path(__file__).parent.parent.parent / "docker_pipelines" / "glmocr"
+)
+
+
+class GLMOCRDockerServerConfig(DockerComposeServerConfig):
+    """Docker Compose configuration for GLM-OCR server."""
+
+    model_name: str = "GLM-OCR"
+    aliases: list[str] = Field(default_factory=lambda: ["glmocr", "glm-ocr"])
+    compose_file: str = str(DOCKER_PIPELINE_DIR / "compose.yaml")
+    server_service: str = "glmocr-api"
+    compose_services: list[str] = Field(
+        default_factory=lambda: ["glmocr-api", "glmocr-vllm-server"]
+    )
+    gpu_service_names: list[str] = Field(default_factory=lambda: ["glmocr-vllm-server"])
+    docker_port: int = 5002
+    container_port: int = 5002
+    environment: dict[str, str] = Field(
+        default_factory=lambda: {
+            "VLM_BACKEND": "vllm",
+            "API_PORT": "8080",
+        }
+    )
+    environment_services: list[str] = Field(default_factory=lambda: ["glmocr-api"])
+    server_ready_indicators: list[str] = Field(
+        default_factory=lambda: ["Running on", "Application startup complete"]
+    )
+
+    def model_post_init(self, __context):
+        if not self.compose_env:
+            compose_env = {}
+            for key in [
+                "API_IMAGE_TAG_SUFFIX",
+                "VLM_IMAGE_TAG_SUFFIX",
+                "VLM_BACKEND",
+            ]:
+                value = os.getenv(key)
+                if value:
+                    compose_env[key] = value
+            if compose_env:
+                self.compose_env = compose_env
+
+    @property
+    def client_config(self):
+        return GLMOCRConverterConfig(
+            **self._create_client_kwargs(f"http://localhost:{self.docker_port}")
+        )
+
+
+class GLMOCRConverterConfig(ConverterConfig):
+    """Configuration for GLM-OCR API client."""
+
+    model_name: str = "GLM-OCR"
+    aliases: list[str] = Field(default_factory=lambda: ["glmocr", "glm-ocr"])
+    timeout: int = 600
+
+    endpoint_parse: str = "/glmocr/parse"
+
+    # GLM-OCR specific configuration
+
+    # Output format: "json", "markdown", or "both"
+    output_format: str = "both"
+
+    # Enable layout detection (PP-DocLayout)
+    enable_layout: bool = True
+
+    # GLM-OCR model parameters
+    max_tokens: int = 16384
+    temperature: float = 0.01
+    image_format: str = "JPEG"
+    min_pixels: int = 12544
+    max_pixels: int = 71372800
+
+    # Backward-compat escape hatch: if set, applied last to the payload.
+    request_overrides: dict[str, Any] = Field(default_factory=dict)
+
+    def get_client(self, **kwargs) -> "GLMOCRConverter":
+        return GLMOCRConverter(config=self, **kwargs)
+
+
+class GLMOCRConverter(BaseConverter):
+    """GLM-OCR HTTP API converter."""
+
+    config: GLMOCRConverterConfig
+
+    def _build_parse_payload(self, file_content_b64: str) -> dict:
+        """Build the request payload for the GLM-OCR parse endpoint.
+
+        Args:
+            file_content_b64: Base64 encoded image content
+
+        Returns:
+            Dictionary payload for the API request
+        """
+        # Wrap base64 in data URI format as expected by GLM-OCR
+        # Format: data:image/png;base64,<base64_data>
+        data_uri = f"data:image/png;base64,{file_content_b64}"
+
+        payload: dict[str, Any] = {
+            "images": [data_uri]  # GLM-OCR expects a list
+        }
+
+        # Apply any request overrides
+        if self.config.request_overrides:
+            payload.update(self.config.request_overrides)
+
+        return payload
+
+    async def _post_json(self, endpoint: str, payload: dict) -> dict:
+        """Make a POST request to the GLM-OCR API.
+
+        Args:
+            endpoint: API endpoint path
+            payload: Request payload
+
+        Returns:
+            Parsed JSON response
+
+        Raises:
+            RuntimeError: If the API returns an error
+        """
+        headers = {}
+
+        async with httpx.AsyncClient(
+            base_url=self.config.base_url, timeout=self.config.timeout, headers=headers
+        ) as client:
+            response = await client.post(endpoint, json=payload)
+
+        response.raise_for_status()
+        data = response.json()
+
+        # Check for error in response
+        if "error" in data:
+            raise RuntimeError(data.get("error", "Unknown error"))
+
+        return data
+
+    def _apply_markdown(self, page: Page, markdown_text: str | None):
+        """Apply markdown text to the page.
+
+        Args:
+            page: Page object to update
+            markdown_text: Markdown content from GLM-OCR
+        """
+        text = markdown_text or ""
+        text = clean_response(text)
+        text = html_to_md_keep_tables(text)
+        logger.debug(f"Converted markdown text: {text[:100]}...")
+        page.text = text
+
+    def _apply_items(self, page: Page, json_result: list[dict] | None):
+        """Apply structured items to the page from JSON result.
+
+        Args:
+            page: Page object to update
+            json_result: List of detected regions from GLM-OCR
+        """
+        if not json_result:
+            return
+
+        items: list[Item] = []
+
+        for block in json_result:
+            bbox = block.get("bbox_2d")
+            if not bbox or len(bbox) != 4:
+                # If no bbox, skip this item
+                continue
+
+            x1, y1, x2, y2 = bbox
+            text = block.get("content") or ""
+            label = block.get("label") or ""
+
+            items.append(
+                Item(
+                    text=text,
+                    box=BoundingBox(l=x1, t=y1, r=x2, b=y2),
+                    category=label,
+                )
+            )
+
+        page.items = items
+
+    async def async_call_inside_page(self, page: Page) -> Page:
+        """Process a single page through the GLM-OCR API.
+
+        Args:
+            page: Page object containing the image to process
+
+        Returns:
+            Updated Page object with OCR results
+        """
+        image = page.image
+
+        # Convert image to base64
+        file_content_b64 = await asyncio.to_thread(to_base64, image, "PNG")
+
+        # Build request payload
+        payload = self._build_parse_payload(file_content_b64)
+
+        # Call the GLM-OCR API
+        data = await self._post_json(self.config.endpoint_parse, payload)
+
+        # GLM-OCR returns results as a list (one per document)
+        # Since we send one image, we get one document result
+        result = data.get("markdown_result", None)
+
+        if result:
+            # Get markdown output if available
+            markdown_result = result
+            if markdown_result:
+                self._apply_markdown(page, markdown_result)
+
+            # Get JSON output if available and layout detection is enabled
+            json_result = data.get("json_result")
+            if json_result and isinstance(json_result, list) and len(json_result) > 0:
+                # json_result is a list of pages, take the first page
+                page_result = (
+                    json_result[0] if isinstance(json_result[0], list) else json_result
+                )
+                self._apply_items(page, page_result)
+
+            # Store raw response
+            page.raw_response = orjson.dumps(result).decode("utf-8")
+
+        return page

vlmparse/clients/paddleocrvl.py

@@ -1,56 +1,204 @@
+import asyncio
+import os
+from pathlib import Path
+from typing import Any
+
+import httpx
+import orjson
+from loguru import logger
 from pydantic import Field
 
-from vlmparse.clients.openai_converter import OpenAIConverterConfig
-from vlmparse.servers.docker_server import VLLMDockerServerConfig
+from vlmparse.clients.pipe_utils.html_to_md_conversion import html_to_md_keep_tables
+from vlmparse.clients.pipe_utils.utils import clean_response
+from vlmparse.converter import BaseConverter, ConverterConfig
+from vlmparse.data_model.document import BoundingBox, Item, Page
+from vlmparse.servers.docker_compose_server import DockerComposeServerConfig
+from vlmparse.utils import to_base64
+
+DOCKER_PIPELINE_DIR = (
+    Path(__file__).parent.parent.parent / "docker_pipelines" / "paddleocrvl"
+)
 
 
-class PaddleOCRVLDockerServerConfig(VLLMDockerServerConfig):
-    """Configuration for PaddleOCRVL model."""
+class PaddleOCRVLDockerServerConfig(DockerComposeServerConfig):
+    """Docker Compose configuration for PaddleOCR-VL server."""
 
-    model_name: str = "PaddlePaddle/PaddleOCR-VL"
-    command_args: list[str] = Field(
-        default_factory=lambda: [
-            "--limit-mm-per-prompt",
-            '{"image": 1}',
-            "--async-scheduling",
-            "--trust-remote-code",
-            "--mm-processor-cache-gb",
-            "0",
-        ]
+    model_name: str = "PaddleOCR-VL-1.5"
+    aliases: list[str] = Field(
+        default_factory=lambda: ["paddleocrvl1.5", "paddleocr-vl-1.5"]
+    )
+    compose_file: str = str(DOCKER_PIPELINE_DIR / "compose.yaml")
+    server_service: str = "paddleocr-vl-api"
+    compose_services: list[str] = Field(
+        default_factory=lambda: ["paddleocr-vl-api", "paddleocr-vlm-server"]
+    )
+    gpu_service_names: list[str] = Field(
+        default_factory=lambda: ["paddleocr-vl-api", "paddleocr-vlm-server"]
+    )
+    docker_port: int = 8080
+    container_port: int = 8080
+    environment: dict[str, str] = Field(
+        default_factory=lambda: {
+            "VLM_BACKEND": "vllm",
+        }
     )
-    aliases: list[str] = Field(default_factory=lambda: ["paddleocrvl"])
+    environment_services: list[str] = Field(
+        default_factory=lambda: ["paddleocr-vl-api"]
+    )
+    server_ready_indicators: list[str] = Field(
+        default_factory=lambda: ["Application startup complete", "Uvicorn running"]
+    )
+
+    def model_post_init(self, __context):
+        if not self.compose_env:
+            compose_env = {}
+            for key in [
+                "API_IMAGE_TAG_SUFFIX",
+                "VLM_IMAGE_TAG_SUFFIX",
+                "VLM_BACKEND",
+            ]:
+                value = os.getenv(key)
+                if value:
+                    compose_env[key] = value
+            if compose_env:
+                self.compose_env = compose_env
 
     @property
     def client_config(self):
         return PaddleOCRVLConverterConfig(
-            **self._create_client_kwargs(
-                f"http://localhost:{self.docker_port}{self.get_base_url_suffix()}"
-            )
+            **self._create_client_kwargs(f"http://localhost:{self.docker_port}")
         )
 
 
-# Task-specific base prompts
-TASKS = {
-    "ocr": "OCR:",
-    "table": "Table Recognition:",
-    "formula": "Formula Recognition:",
-    "chart": "Chart Recognition:",
-}
-
-
-class PaddleOCRVLConverterConfig(OpenAIConverterConfig):
-    """PaddleOCRVL converter"""
-
-    model_name: str = "PaddlePaddle/PaddleOCR-VL"
-    preprompt: str | None = None
-    postprompt: dict[str, str] = TASKS
-    prompt_mode_map: dict[str, str] = {
-        "ocr_layout": "ocr",
-    }
-    completion_kwargs: dict | None = {
-        "temperature": 0.0,
-        "max_completion_tokens": 16384,
-    }
-    dpi: int = 200
-    aliases: list[str] = Field(default_factory=lambda: ["paddleocrvl"])
-    stream: bool = True
+class PaddleOCRVLConverterConfig(ConverterConfig):
+    """Configuration for PaddleOCR-VL API client."""
+
+    model_name: str = "PaddleOCR-VL-1.5"
+    aliases: list[str] = Field(
+        default_factory=lambda: ["paddleocrvl1.5", "paddleocr-vl-1.5"]
+    )
+    timeout: int = 600
+
+    endpoint_layout_parsing: str = "/layout-parsing"
+    endpoint_restructure_pages: str = "/restructure-pages"
+
+    # Dict of PaddleOCR-VL API args.
+    # Keys should match the PaddleOCR-VL API JSON fields (camelCase), e.g.
+    # {"useLayoutDetection": true, "promptLabel": "..."}.
+    paddleocr_args: dict[str, Any] = Field(
+        default_factory=lambda: {
+            # Preserve previous default behavior (these were always sent before).
+            "prettifyMarkdown": True,
+            "showFormulaNumber": False,
+            "restructurePages": False,
+        }
+    )
+
+    # Optional args for the /restructure-pages endpoint (if/when used).
+    restructure_args: dict[str, Any] = Field(default_factory=dict)
+
+    # Backward-compat escape hatch: if set, applied last to the payload.
+    request_overrides: dict[str, Any] = Field(default_factory=dict)
+
+    def get_client(self, **kwargs) -> "PaddleOCRVLConverter":
+        return PaddleOCRVLConverter(config=self, **kwargs)
+
+
+class PaddleOCRVLConverter(BaseConverter):
+    """PaddleOCR-VL HTTP API converter."""
+
+    config: PaddleOCRVLConverterConfig
+
+    def _build_layout_payload(self, file_content_b64: str, file_type: int | None):
+        payload: dict[str, Any] = {"file": file_content_b64}
+
+        if self.config.paddleocr_args:
+            payload.update(self.config.paddleocr_args)
+
+        if file_type is not None:
+            payload["fileType"] = file_type
+
+        if self.config.request_overrides:
+            payload.update(self.config.request_overrides)
+
+        return payload
+
+    def _build_restructure_payload(self, layout_results: list[dict]) -> dict:
+        pages = []
+        for page_result in layout_results:
+            pruned = page_result.get("prunedResult")
+            markdown = page_result.get("markdown") or {}
+            if pruned is None:
+                continue
+            pages.append(
+                {
+                    "prunedResult": pruned,
+                    "markdownImages": markdown.get("images"),
+                }
+            )
+
+        payload: dict[str, Any] = {"pages": pages}
+
+        if self.config.restructure_args:
+            payload.update(self.config.restructure_args)
+
+        return payload
+
+    async def _post_json(self, endpoint: str, payload: dict) -> dict:
+        async with httpx.AsyncClient(
+            base_url=self.config.base_url, timeout=self.config.timeout
+        ) as client:
+            response = await client.post(endpoint, json=payload)
+
+        response.raise_for_status()
+        data = response.json()
+        if data.get("errorCode", 0) != 0:
+            raise RuntimeError(data.get("errorMsg", "Unknown error"))
+        return data
+
+    def _apply_markdown(self, page: Page, markdown_text: str | None):
+        text = markdown_text or ""
+        text = clean_response(text)
+        text = html_to_md_keep_tables(text)
+        logger.debug(f"Converted markdown text: {text}...")
+        page.text = text
+
+    def _apply_items(self, page: Page, pruned_result: dict | None):
+        if not pruned_result:
+            return
+        parsing_res_list = pruned_result.get("parsing_res_list") or []
+        items: list[Item] = []
+        for block in parsing_res_list:
+            bbox = block.get("block_bbox")
+            if not bbox or len(bbox) != 4:
+                logger.warning(f"Invalid bbox in block: {block}")
+                continue
+            l, t, r, b = bbox
+            text = block.get("block_content") or ""
+            items.append(
+                Item(
+                    text=text,
+                    box=BoundingBox(l=l, t=t, r=r, b=b),
+                    category=block.get("block_label") or "",
+                )
+            )
+
+        page.items = items
+
+    async def async_call_inside_page(self, page: Page) -> Page:
+        image = page.image
+        file_content_b64 = await asyncio.to_thread(to_base64, image, "PNG")
+        payload = self._build_layout_payload(file_content_b64, 1)
+
+        data = await self._post_json(self.config.endpoint_layout_parsing, payload)
+        result = data.get("result", {})
+        layout_results = result.get("layoutParsingResults", [])
+        if layout_results:
+            first = layout_results[0]
+
+            markdown = first.get("markdown") or {}
+            self._apply_markdown(page, markdown.get("text"))
+            self._apply_items(page, first.get("prunedResult"))
+            page.raw_response = orjson.dumps(first).decode("utf-8")
+
+        return page