langparse 0.1.0__py3-none-any.whl

langparse/engines/pdf/mineru_client.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import json
+import mimetypes
+import uuid
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any
+from urllib import request
+from urllib.error import HTTPError, URLError
+
+
+class _HTMLTableReader(HTMLParser):
+    """Collects rows/cells from the HTML fragment MinerU returns as ``table_body``."""
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.rows: list[list[str]] = []
+        self._row: list[str] | None = None
+        self._cell: list[str] | None = None
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "tr":
+            self._row = []
+        elif tag in ("td", "th"):
+            self._cell = []
+
+    def handle_endtag(self, tag):
+        if tag in ("td", "th") and self._cell is not None:
+            text = " ".join("".join(self._cell).split())
+            if self._row is None:
+                self._row = []
+            self._row.append(text)
+            self._cell = None
+        elif tag == "tr" and self._row is not None:
+            self.rows.append(self._row)
+            self._row = None
+
+    def handle_data(self, data):
+        if self._cell is not None:
+            self._cell.append(data)
+
+
+def parse_html_table(markup: str) -> list[list[str]]:
+    """Parse an HTML table fragment into rows of cell text. Returns [] if unparseable."""
+    if not markup:
+        return []
+    reader = _HTMLTableReader()
+    try:
+        reader.feed(markup)
+        reader.close()
+    except Exception:
+        return []
+    return [row for row in reader.rows if row]
+
+
+def rows_to_markdown(rows: list[list[str]]) -> str:
+    """Render rows as a Markdown table, padding short rows to the header width."""
+    if not rows:
+        return ""
+    width = max(len(row) for row in rows)
+    padded = [row + [""] * (width - len(row)) for row in rows]
+    lines = [f"| {' | '.join(padded[0])} |", f"| {' | '.join(['---'] * width)} |"]
+    lines.extend(f"| {' | '.join(row)} |" for row in padded[1:])
+    return "\n".join(lines)
+
+
+class MinerUClient:
+    def __init__(self, base_url: str, timeout: float = 300.0):
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+
+    def health(self) -> dict[str, Any]:
+        return self._request_json("GET", "/health")
+
+    def parse_file(self, file_path: Path, runtime_config: dict[str, Any]) -> list[dict[str, Any]]:
+        response = self._request_json(
+            "POST",
+            "/file_parse",
+            fields=self._build_form_fields(runtime_config),
+            file_path=file_path,
+        )
+        return self._normalize_parse_response(response)
+
+    def _build_form_fields(self, runtime_config: dict[str, Any]) -> dict[str, str]:
+        fields = {
+            "return_md": "true",
+            "return_content_list": "true",
+            "response_format_zip": "false",
+        }
+        extra_options = runtime_config.get("extra_options", {})
+        if runtime_config.get("enable_ocr") is False:
+            fields["parse_method"] = "txt"
+        if runtime_config.get("device"):
+            fields["device"] = str(runtime_config["device"])
+        if runtime_config.get("model_dir"):
+            fields["model_dir"] = str(runtime_config["model_dir"])
+        if runtime_config.get("download_dir"):
+            fields["download_dir"] = str(runtime_config["download_dir"])
+        for key, value in extra_options.items():
+            if value is None:
+                continue
+            fields[str(key)] = str(value)
+        return fields
+
+    def _request_json(
+        self,
+        method: str,
+        path: str,
+        fields: dict[str, str] | None = None,
+        file_path: Path | None = None,
+    ) -> dict[str, Any]:
+        headers = {"Accept": "application/json"}
+        data = None
+        if file_path is not None:
+            data, content_type = self._encode_multipart_form(fields or {}, file_path)
+            headers["Content-Type"] = content_type
+
+        req = request.Request(
+            f"{self.base_url}{path}",
+            data=data,
+            headers=headers,
+            method=method,
+        )
+        try:
+            with request.urlopen(req, timeout=self.timeout) as response:
+                payload = response.read().decode("utf-8")
+        except HTTPError as exc:
+            detail = exc.read().decode("utf-8", errors="replace")
+            raise RuntimeError(f"MinerU API request failed with HTTP {exc.code}: {detail}") from exc
+        except URLError as exc:
+            raise RuntimeError(f"MinerU API request failed: {exc.reason}") from exc
+
+        try:
+            return json.loads(payload)
+        except json.JSONDecodeError as exc:
+            raise RuntimeError("MinerU API returned a non-JSON response.") from exc
+
+    def _encode_multipart_form(self, fields: dict[str, str], file_path: Path) -> tuple[bytes, str]:
+        boundary = f"----langparse-mineru-{uuid.uuid4().hex}"
+        lines: list[bytes] = []
+        for name, value in fields.items():
+            lines.extend(
+                [
+                    f"--{boundary}".encode(),
+                    f'Content-Disposition: form-data; name="{name}"'.encode(),
+                    b"",
+                    str(value).encode("utf-8"),
+                ]
+            )
+
+        content_type = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream"
+        file_bytes = file_path.read_bytes()
+        lines.extend(
+            [
+                f"--{boundary}".encode(),
+                f'Content-Disposition: form-data; name="files"; filename="{file_path.name}"'.encode(),
+                f"Content-Type: {content_type}".encode(),
+                b"",
+                file_bytes,
+                f"--{boundary}--".encode(),
+                b"",
+            ]
+        )
+        return b"\r\n".join(lines), f"multipart/form-data; boundary={boundary}"
+
+    def _normalize_parse_response(self, response: dict[str, Any]) -> list[dict[str, Any]]:
+        markdown = self._extract_markdown(response)
+        content_list = self._extract_content_list(response)
+        if not content_list:
+            return [{"page_number": 1, "markdown": markdown}]
+
+        page_map: dict[int, list[dict[str, Any]]] = {}
+        for item in content_list:
+            page_idx = int(item.get("page_idx", 0))
+            page_map.setdefault(page_idx, []).append(item)
+
+        pages = []
+        for page_idx in sorted(page_map):
+            items = page_map[page_idx]
+            pages.append(self._build_page(page_idx, items, markdown))
+        return pages
+
+    def _build_page(
+        self, page_idx: int, items: list[dict[str, Any]], document_markdown: str
+    ) -> dict[str, Any]:
+        markdown_blocks: list[str] = []
+        text_lines: list[str] = []
+        tables: list[dict[str, Any]] = []
+        images: list[dict[str, Any]] = []
+        elements: list[dict[str, Any]] = []
+
+        for item in items:
+            kind = item.get("type", "text")
+            caption = self._join_caption(item.get(f"{kind}_caption"))
+
+            if kind == "table":
+                rows = parse_html_table(item.get("table_body", ""))
+                table_markdown = rows_to_markdown(rows)
+                tables.append(
+                    {
+                        "rows": rows,
+                        "caption": caption,
+                        "html": item.get("table_body", ""),
+                        "img_path": item.get("img_path"),
+                    }
+                )
+                block = "\n\n".join(part for part in (caption, table_markdown) if part)
+                if block:
+                    markdown_blocks.append(block)
+                element_text = table_markdown
+            elif kind == "image":
+                images.append(
+                    {
+                        "path": item.get("img_path"),
+                        "caption": caption,
+                        "footnote": self._join_caption(item.get("image_footnote")),
+                    }
+                )
+                block = f"![{caption}]({item.get('img_path') or ''})"
+                markdown_blocks.append(block)
+                element_text = caption
+            else:
+                text = item.get("text", "")
+                if text:
+                    markdown_blocks.append(text)
+                    text_lines.append(text)
+                element_text = text
+
+            elements.append(
+                {
+                    "kind": kind,
+                    "text": element_text,
+                    "bbox": item.get("bbox"),
+                    "metadata": {"page_idx": page_idx},
+                }
+            )
+
+        return {
+            "page_number": page_idx + 1,
+            "markdown": "\n\n".join(markdown_blocks) or document_markdown,
+            "plain_text": "\n".join(text_lines),
+            "elements": elements,
+            "tables": tables,
+            "images": images,
+            "engine_specific": {"content_list": items},
+        }
+
+    def _join_caption(self, caption: Any) -> str:
+        if isinstance(caption, str):
+            return caption.strip()
+        if isinstance(caption, list):
+            return " ".join(str(part).strip() for part in caption if str(part).strip())
+        return ""
+
+    def _extract_markdown(self, response: dict[str, Any]) -> str:
+        candidates = [
+            response.get("md_content"),
+            response.get("markdown"),
+            response.get("md"),
+            response.get("full_md"),
+        ]
+        result = response.get("result")
+        if isinstance(result, dict):
+            candidates.extend(
+                [
+                    result.get("md_content"),
+                    result.get("markdown"),
+                    result.get("md"),
+                    result.get("full_md"),
+                ]
+            )
+        results = response.get("results")
+        if isinstance(results, dict):
+            for file_result in results.values():
+                if isinstance(file_result, dict):
+                    candidates.extend(
+                        [
+                            file_result.get("md_content"),
+                            file_result.get("markdown"),
+                            file_result.get("md"),
+                            file_result.get("full_md"),
+                        ]
+                    )
+        for candidate in candidates:
+            if isinstance(candidate, str) and candidate:
+                return candidate
+        return ""
+
+    def _extract_content_list(self, response: dict[str, Any]) -> list[dict[str, Any]]:
+        for key in ("content_list", "content_list_v2"):
+            value = response.get(key)
+            if isinstance(value, str):
+                try:
+                    value = json.loads(value)
+                except json.JSONDecodeError:
+                    continue
+            if isinstance(value, list):
+                if value and isinstance(value[0], dict):
+                    return value
+                if value and isinstance(value[0], list):
+                    flattened = []
+                    for page_idx, page_items in enumerate(value):
+                        for item in page_items:
+                            if isinstance(item, dict):
+                                flattened.append({"page_idx": page_idx, **item})
+                    return flattened
+        result = response.get("result")
+        if isinstance(result, dict):
+            return self._extract_content_list(result)
+        results = response.get("results")
+        if isinstance(results, dict):
+            for file_result in results.values():
+                if isinstance(file_result, dict):
+                    content_list = self._extract_content_list(file_result)
+                    if content_list:
+                        return content_list
+        return []

langparse/engines/pdf/mineru_service.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+from langparse.engines.pdf.mineru_client import MinerUClient
+
+
+class MinerUServiceManager:
+    def __init__(
+        self,
+        api_url: str | None = None,
+        host: str = "127.0.0.1",
+        port: int = 8000,
+        command: str = "mineru-api",
+        start_timeout: float = 30.0,
+        request_timeout: float = 300.0,
+        model_dir: str | None = None,
+        download_dir: str | None = None,
+        model_policy: str = "download_if_missing",
+        model_source: str | None = None,
+        auto_install_runtime: bool = False,
+        runtime_package: str = "mineru>=3.4,<4",
+    ):
+        self.api_url = api_url.rstrip("/") if api_url else None
+        self.host = host
+        self.port = port
+        self.command = command
+        self.start_timeout = start_timeout
+        self.request_timeout = request_timeout
+        self.model_dir = model_dir
+        self.download_dir = download_dir
+        self.model_policy = model_policy
+        self.model_source = model_source
+        self.auto_install_runtime = auto_install_runtime
+        self.runtime_package = runtime_package
+
+    @contextmanager
+    def running_service(self):
+        if self.api_url:
+            yield self.api_url
+            return
+
+        base_url = f"http://{self.host}:{self.port}"
+        client = MinerUClient(base_url, timeout=self.request_timeout)
+        if self._is_healthy(client):
+            yield base_url
+            return
+
+        self._validate_model_policy()
+        home_override_cm = self._prepare_local_home()
+        with home_override_cm as home_override:
+            process = self._start_local_service(home_override=home_override)
+            try:
+                self._wait_until_ready(client)
+                yield base_url
+            finally:
+                self._stop_process(process)
+
+    def _is_healthy(self, client: MinerUClient) -> bool:
+        try:
+            client.health()
+        except Exception:
+            return False
+        return True
+
+    def _start_local_service(self, home_override: str | None = None) -> subprocess.Popen:
+        args = shlex.split(self.command) + ["--host", self.host, "--port", str(self.port)]
+        env = self._build_process_env(home_override=home_override)
+        if not self._command_available(self.command):
+            if self.auto_install_runtime:
+                self._install_runtime()
+            else:
+                raise RuntimeError(
+                    "Unable to start local mineru-api service using command: "
+                    f"{self.command}. MinerU runtime was not found. Install it with "
+                    '`pip install -U "mineru>=3.4,<4"` (plus the official backend extra '
+                    "needed for local inference) or retry with auto_install_runtime=True."
+                )
+        try:
+            return subprocess.Popen(
+                args,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                env=env,
+            )
+        except FileNotFoundError as exc:
+            raise RuntimeError(
+                f"Unable to start local mineru-api service using command: {self.command}"
+            ) from exc
+
+    def _command_available(self, command: str) -> bool:
+        executable = shlex.split(command)[0]
+        return shutil.which(executable) is not None
+
+    def _install_runtime(self) -> None:
+        try:
+            subprocess.run(
+                [sys.executable, "-m", "pip", "install", "-U", self.runtime_package],
+                check=True,
+            )
+        except subprocess.CalledProcessError as exc:
+            raise RuntimeError(
+                f"Failed to install MinerU runtime package: {self.runtime_package}"
+            ) from exc
+
+    def _wait_until_ready(self, client: MinerUClient) -> None:
+        deadline = time.time() + self.start_timeout
+        last_error: Exception | None = None
+        while time.time() < deadline:
+            try:
+                client.health()
+                return
+            except Exception as exc:  # pragma: no cover - exercised in timeout flow
+                last_error = exc
+                time.sleep(0.25)
+        raise RuntimeError(
+            "Timed out waiting for local mineru-api service to become ready."
+        ) from last_error
+
+    def _stop_process(self, process: subprocess.Popen) -> None:
+        if process.poll() is not None:
+            return
+        process.terminate()
+        try:
+            process.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            process.wait(timeout=5)
+
+    @staticmethod
+    def find_free_port() -> int:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.bind(("127.0.0.1", 0))
+            return int(sock.getsockname()[1])
+
+    def _validate_model_policy(self) -> None:
+        if self.model_policy not in {"download_if_missing", "require_existing"}:
+            raise ValueError(
+                "Unsupported MinerU model_policy: "
+                f"{self.model_policy}. Expected 'download_if_missing' or 'require_existing'."
+            )
+
+        if self.model_policy != "require_existing":
+            return
+
+        if self.model_dir:
+            model_path = Path(self.model_dir).expanduser()
+            if not model_path.exists() or not any(model_path.iterdir()):
+                raise RuntimeError(
+                    f"MinerU model_policy=require_existing but model_dir is missing or empty: {model_path}"
+                )
+            return
+
+        home_root = self._resolve_home_root()
+        mineru_config = home_root / "mineru.json"
+        mineru_cache = home_root / ".mineru"
+        if not mineru_config.exists() and not mineru_cache.exists():
+            raise RuntimeError(
+                "MinerU model_policy=require_existing requires either model_dir or an existing "
+                f"local MinerU home with mineru.json/.mineru under: {home_root}"
+            )
+
+    def _resolve_home_root(self) -> Path:
+        if self.download_dir:
+            return Path(self.download_dir).expanduser()
+        return Path.home()
+
+    def _build_process_env(self, home_override: str | None = None) -> dict[str, str]:
+        env = os.environ.copy()
+        if home_override:
+            env["HOME"] = home_override
+
+        effective_model_source = self.model_source
+        if effective_model_source is None and self.model_dir:
+            effective_model_source = "local"
+
+        if effective_model_source:
+            env["MINERU_MODEL_SOURCE"] = effective_model_source
+
+        return env
+
+    @contextmanager
+    def _prepare_local_home(self):
+        if self.model_dir:
+            configured_root = Path(self.download_dir).expanduser() if self.download_dir else None
+            if configured_root is not None:
+                configured_root.mkdir(parents=True, exist_ok=True)
+                self._write_mineru_config(configured_root)
+                yield str(configured_root)
+                return
+
+            with tempfile.TemporaryDirectory(prefix="langparse-mineru-home-") as temp_home:
+                temp_root = Path(temp_home)
+                self._write_mineru_config(temp_root)
+                yield temp_home
+                return
+
+        if self.download_dir:
+            configured_root = Path(self.download_dir).expanduser()
+            configured_root.mkdir(parents=True, exist_ok=True)
+            yield str(configured_root)
+            return
+
+        yield None
+
+    def _write_mineru_config(self, home_root: Path) -> None:
+        model_path = str(Path(self.model_dir).expanduser())
+        config_path = home_root / "mineru.json"
+        config = {
+            "models-dir": {
+                "pipeline": model_path,
+                "vlm": model_path,
+            }
+        }
+        config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8")

langparse/engines/pdf/ocr.py

@@ -0,0 +1,101 @@
+"""
+OCR fallback for PDF pages whose content is an image rather than text.
+
+The dangerous case is not an empty page -- it is a scanned page carrying a
+watermark, which leaves just enough extracted text that the parse reports
+success while the actual content is never read. Detection therefore looks for
+"almost no text next to an image", not "no text".
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, Protocol
+
+#: A full page of body text runs well over a thousand characters. This is the
+#: ceiling below which a page covered by an image is treated as a scan rather
+#: than as text -- data/domain/scan.pdf carries 145 characters of watermark.
+DEFAULT_MIN_CHARS = 500
+#: Fraction of the page an image must cover before the page counts as scanned.
+DEFAULT_MIN_IMAGE_COVERAGE = 0.5
+DEFAULT_RESOLUTION = 150
+
+
+class Recogniser(Protocol):
+    """Matches rapidocr_onnxruntime's RapidOCR call signature."""
+
+    def __call__(self, image: Any) -> tuple[Any, Any]: ...
+
+
+def image_coverage(page) -> float:
+    """Largest single image's area as a fraction of the page."""
+    images = getattr(page, "images", None)
+    if not images:
+        return 0.0
+
+    page_area = (getattr(page, "width", 0) or 0) * (getattr(page, "height", 0) or 0)
+    if page_area <= 0:
+        return 0.0
+
+    largest = 0.0
+    for image in images:
+        width = abs(float(image.get("x1", 0)) - float(image.get("x0", 0)))
+        height = abs(float(image.get("bottom", 0)) - float(image.get("top", 0)))
+        largest = max(largest, width * height)
+    return min(1.0, largest / page_area)
+
+
+def needs_ocr(
+    page,
+    min_chars: int = DEFAULT_MIN_CHARS,
+    min_image_coverage: float = DEFAULT_MIN_IMAGE_COVERAGE,
+) -> bool:
+    """
+    Whether a page's text layer is too thin to trust.
+
+    Text length alone is not enough: data/domain/scan.pdf carries 145 characters
+    of rotated watermark per page, which clears any threshold low enough to
+    avoid firing on genuinely sparse text pages. The page-covering image is the
+    signal that distinguishes them, so both conditions must hold.
+    """
+    if image_coverage(page) < min_image_coverage:
+        return False
+
+    text = page.extract_text() or ""
+    return len(text.strip()) < min_chars
+
+
+def ocr_page_text(
+    page,
+    recogniser: Recogniser,
+    resolution: int = DEFAULT_RESOLUTION,
+) -> str:
+    """Rasterise a page and return the recognised text, one line per detection."""
+    image = page.to_image(resolution=resolution).original
+    result, _elapsed = recogniser(image)
+    if not result:
+        return ""
+
+    lines = []
+    for detection in result:
+        # rapidocr yields [box, text, score]; be tolerant of shape changes.
+        if isinstance(detection, (list, tuple)) and len(detection) >= 2:
+            text = detection[1]
+        else:
+            text = detection
+        if isinstance(text, str) and text.strip():
+            lines.append(text.strip())
+    return "\n".join(lines)
+
+
+def load_recogniser() -> Callable[[Any], tuple[Any, Any]]:
+    """Build the default rapidocr recogniser, with an actionable error if absent."""
+    try:
+        from rapidocr_onnxruntime import RapidOCR
+    except ImportError as exc:
+        raise ImportError(
+            "OCR fallback needs rapidocr_onnxruntime. Install it with "
+            '`pip install "langparse[ocr]"`, or pass enable_ocr=False to skip it.'
+        ) from exc
+
+    return RapidOCR()

langparse/engines/pdf/other.py

@@ -0,0 +1,20 @@
+from collections.abc import Iterator
+from pathlib import Path
+
+from langparse.core.engine import PageResult
+from langparse.engines.pdf.simple import BasePDFEngine
+from langparse.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class PaddleOCRVLEngine(BasePDFEngine):
+    """
+    Adapter for PaddleOCR + Layout Analysis or PP-Structure.
+    Can be local or via API.
+    """
+
+    def process(self, file_path: Path, **kwargs) -> Iterator[PageResult]:
+        logger.debug("PaddleOCR processing %s", file_path)
+        # TODO: Integrate PaddleOCR / PP-Structure logic
+        raise NotImplementedError("PaddleOCR integration is pending.")