BabelDOC 0.5.24__tar.gz → 0.6.0__tar.gz

{babeldoc-0.5.24 → babeldoc-0.6.0}/.github/workflows/publish-to-pypi.yml

@@ -55,6 +55,11 @@ jobs:
           version-command: |
             cat pyproject.toml | grep "version = " | head -n 1 | awk -F'"' '{print $2}'
 
+      - name: Check manual release notes
+        if: steps.check-version.outputs.tag
+        run: |
+          test -f "docs/release-notes/${{ steps.check-version.outputs.tag }}.md"
+
       - name: Install Dependencies
         run: |
           uv sync
@@ -151,4 +156,22 @@ jobs:
           publish: ${{ needs.build.outputs.is_release != '' }}
           tag: ${{ needs.build.outputs.is_release }}
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Prepend manual release notes
+        if: needs.build.outputs.is_release != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ needs.build.outputs.is_release }}
+        run: |
+          set -euo pipefail
+          notes_path="docs/release-notes/${TAG}.md"
+          current_body="$(mktemp)"
+          combined_body="$(mktemp)"
+          gh release view "${TAG}" --json body --jq .body > "${current_body}"
+          {
+            cat "${notes_path}"
+            printf '\n\n---\n\n'
+            cat "${current_body}"
+          } > "${combined_body}"
+          gh release edit "${TAG}" --notes-file "${combined_body}"

{babeldoc-0.5.24 → babeldoc-0.6.0}/.gitignore

@@ -25,7 +25,9 @@ memray*
 .vscode
 __pycache__
 .ruff_cache
+.tmp
 yadt.toml
+yadt.toml*
 examples/
 /make_gif.py
 /dist
@@ -41,10 +43,9 @@ babeldoc/format/txt
 # uv
 uv.lock
 
-# Claude Code memory file
-CLAUDE.md
 /.claude
 babeldoc/format/playground
 temp.jpg
-AGENTS.md
-.serena
+.serena
+.codex
+

{babeldoc-0.5.24 → babeldoc-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.24
+Version: 0.6.0
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -34,7 +34,6 @@ Requires-Dist: psutil>=7.0.0
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: pymupdf>=1.25.1
 Requires-Dist: pyzstd>=0.17.0
-Requires-Dist: rapidocr-onnxruntime>=1.4.4
 Requires-Dist: rich>=13.9.4
 Requires-Dist: rtree>=1.4.0
 Requires-Dist: scikit-image>=0.25.2

babeldoc-0.6.0/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

{babeldoc-0.5.24 → babeldoc-0.6.0}/babeldoc/assets/assets.py

@@ -17,10 +17,6 @@ from babeldoc.assets.embedding_assets_metadata import (
 from babeldoc.assets.embedding_assets_metadata import EMBEDDING_FONT_METADATA
 from babeldoc.assets.embedding_assets_metadata import FONT_METADATA_URL
 from babeldoc.assets.embedding_assets_metadata import FONT_URL_BY_UPSTREAM
-from babeldoc.assets.embedding_assets_metadata import (
-    TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256,
-)
-from babeldoc.assets.embedding_assets_metadata import TABLE_DETECTION_RAPIDOCR_MODEL_URL
 from babeldoc.assets.embedding_assets_metadata import TIKTOKEN_CACHES
 from babeldoc.const import get_cache_file_path
 from tenacity import retry
@@ -38,20 +34,33 @@ _FASTEST_FONT_METADATA: dict | None = None
 class ResultContainer:
     def __init__(self):
         self.result = None
+        self.exception: BaseException | None = None
 
     def set_result(self, result):
         self.result = result
 
+    def set_exception(self, exc: BaseException):
+        self.exception = exc
+
 
 def run_in_another_thread(coro):
     result_container = ResultContainer()
 
     def _wrapper():
-        result_container.set_result(asyncio.run(coro))
+        try:
+            result_container.set_result(asyncio.run(coro))
+        except BaseException as exc:  # noqa: BLE001
+            result_container.set_exception(exc)
 
     thread = threading.Thread(target=_wrapper)
     thread.start()
     thread.join()
+    if result_container.exception is not None:
+        msg = (
+            "asset coroutine failed: "
+            f"{type(result_container.exception).__name__}: {result_container.exception}"
+        )
+        raise RuntimeError(msg) from result_container.exception
     return result_container.result
 
 
@@ -248,23 +257,10 @@ async def get_doclayout_onnx_model_path_async(client: httpx.AsyncClient | None =
 async def get_table_detection_rapidocr_model_path_async(
     client: httpx.AsyncClient | None = None,
 ):
-    onnx_path = get_cache_file_path("ch_PP-OCRv4_det_infer.onnx", "models")
-    if verify_file(onnx_path, TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256):
-        return onnx_path
-
-    logger.info("table detection rapidocr model not found or corrupted, downloading...")
-    fastest_upstream, _ = await get_fastest_upstream_for_model(client)
-    if fastest_upstream is None:
-        logger.error("Failed to get fastest upstream")
-        exit(1)
-
-    url = TABLE_DETECTION_RAPIDOCR_MODEL_URL[fastest_upstream]
-
-    await download_file(client, url, onnx_path, TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256)
-    logger.info(
-        f"Download table detection rapidocr model from {fastest_upstream} success"
-    )
-    return onnx_path
+    _ = client
+    dummy_path = get_cache_file_path("rapidocr-retired-dummy.txt", "models")
+    dummy_path.write_text("RapidOCR table detection is retired.\n", encoding="utf-8")
+    return dummy_path
 
 
 def get_doclayout_onnx_model_path():
@@ -457,12 +453,9 @@ async def async_warmup():
     _ = encoding_for_model("gpt-4o")
     async with httpx.AsyncClient() as client:
         onnx_task = asyncio.create_task(get_doclayout_onnx_model_path_async(client))
-        onnx_task2 = asyncio.create_task(
-            get_table_detection_rapidocr_model_path_async(client)
-        )
         font_tasks = asyncio.create_task(download_all_fonts_async(client))
         cmap_tasks = asyncio.create_task(download_all_cmaps_async(client))
-        await asyncio.gather(onnx_task, onnx_task2, font_tasks, cmap_tasks)
+        await asyncio.gather(onnx_task, font_tasks, cmap_tasks)
 
 
 def warmup():
@@ -502,12 +495,6 @@ def generate_all_assets_file_list():
             "sha3_256": DOCLAYOUT_YOLO_DOCSTRUCTBENCH_IMGSZ1024ONNX_SHA3_256,
         },
     )
-    result["models"].append(
-        {
-            "name": "ch_PP-OCRv4_det_infer.onnx",
-            "sha3_256": TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256,
-        },
-    )
     return result
 
 
@@ -620,6 +607,4 @@ if __name__ == "__main__":
     logging.getLogger("httpcore").setLevel(logging.WARNING)
     # warmup()
     # generate_offline_assets_package()
-    # restore_offline_assets_package(Path(
-    #     '/Users/aw/.cache/babeldoc/assets/offline_assets_33971e4940e90ba0c35baacda44bbe83b214f4703a7bdb8b837de97d0383508c.zip'))
     # warmup()

{babeldoc-0.5.24 → babeldoc-0.6.0}/babeldoc/assets/embedding_assets_metadata.py

@@ -4,10 +4,6 @@ DOCLAYOUT_YOLO_DOCSTRUCTBENCH_IMGSZ1024ONNX_SHA3_256 = (
     "60be061226930524958b5465c8c04af3d7c03bcb0beb66454f5da9f792e3cf2a"
 )
 
-TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256 = (
-    "062f4619afe91b33147c033acadecbb53f2a7b99ac703d157b96d5b10948da5e"
-)
-
 TIKTOKEN_CACHES = {
     "fb374d419588a4632f3f557e76b4b70aebbca790": "cb04bcda5782cfbbe77f2f991d92c0ea785d9496ef1137c91dfc3c8c324528d6"
 }
@@ -39,12 +35,6 @@ DOC_LAYOUT_ONNX_MODEL_URL = {
     "modelscope": "https://www.modelscope.cn/models/AI-ModelScope/DocLayout-YOLO-DocStructBench-onnx/resolve/master/doclayout_yolo_docstructbench_imgsz1024.onnx",
 }
 
-TABLE_DETECTION_RAPIDOCR_MODEL_URL = {
-    "huggingface": "https://huggingface.co/spaces/RapidAI/RapidOCR/resolve/main/models/text_det/ch_PP-OCRv4_det_infer.onnx",
-    "hf-mirror": "https://hf-mirror.com/spaces/RapidAI/RapidOCR/resolve/main/models/text_det/ch_PP-OCRv4_det_infer.onnx",
-    "modelscope": "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/master/onnx/PP-OCRv4/det/ch_PP-OCRv4_det_infer.onnx",
-}
-
 # from https://github.com/funstory-ai/BabelDOC-Assets/blob/main/font_metadata.json
 EMBEDDING_FONT_METADATA = {
     "GoNotoKurrent-Bold.ttf": {

{babeldoc-0.5.24 → babeldoc-0.6.0}/babeldoc/const.py

@@ -6,7 +6,7 @@ import subprocess
 import threading
 from pathlib import Path
 
-__version__ = "0.5.24"
+__version__ = "0.6.0"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc-0.6.0/babeldoc/docvision/rpc_doclayout8.py

@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+import base64
+import os
+import unicodedata
+from concurrent.futures import FIRST_COMPLETED
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import wait
+from dataclasses import dataclass
+from typing import Any
+
+import cv2
+import httpx
+import numpy as np
+import pymupdf
+
+from babeldoc.docvision.base_doclayout import DocLayoutModel
+from babeldoc.docvision.base_doclayout import YoloBox
+from babeldoc.docvision.base_doclayout import YoloResult
+from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.utils.extract_char import (
+    convert_page_to_char_boxes,
+)
+from babeldoc.format.pdf.document_il.utils.extract_char import (
+    process_page_chars_to_lines,
+)
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.layout_helper import SPACE_REGEX
+from babeldoc.format.pdf.document_il.utils.mupdf_helper import get_no_rotation_img
+
+DPI = 150
+LAYOUT_ERROR = "layout 解析失败"
+
+
+def _positive_int_env(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        value = int(raw)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be a positive integer") from exc
+    if value < 1:
+        raise ValueError(f"{name} must be a positive integer")
+    return value
+
+
+def _positive_float_env(name: str, default: float) -> float:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        value = float(raw)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be positive") from exc
+    if value <= 0:
+        raise ValueError(f"{name} must be positive")
+    return value
+
+
+def _encode_image(image: np.ndarray) -> bytes:
+    image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    ok, encoded = cv2.imencode(".jpg", image_bgr)
+    if not ok:
+        raise RuntimeError(LAYOUT_ERROR)
+    return encoded.tobytes()
+
+
+def _clip_num(num: float, min_value: float, max_value: float) -> float:
+    if num < min_value:
+        return min_value
+    if num > max_value:
+        return max_value
+    return num
+
+
+def _filter_text(text: str, font_mapper: FontMapper) -> str:
+    normalize = unicodedata.normalize("NFKC", text)
+    unicodes = [char for char in normalize if font_mapper.has_char(char)]
+    normalize = "".join(unicodes)
+    return SPACE_REGEX.sub(" ", normalize).strip()
+
+
+@dataclass(frozen=True)
+class _PageInput:
+    index: int
+    page: il_version_1.Page
+    image: np.ndarray
+    image_data: bytes
+    line_results: list[dict[str, Any]] | None
+
+
+class RpcDocLayoutModel(DocLayoutModel):
+    def __init__(self, host: str, requires_line_extraction: bool = False):
+        if not isinstance(host, str) or not host:
+            raise ValueError("rpc_doclayout8 host is required")
+        if ";" in host:
+            raise ValueError("rpc_doclayout8 host must be a single task-local URL")
+        self.host = host.rstrip("/")
+        self.requires_line_extraction = bool(requires_line_extraction)
+        self.layout_page_input_buffer_limit = _positive_int_env(
+            "BABELDOC_EXECUTOR_LAYOUT_PAGE_INPUT_BUFFER_LIMIT",
+            2,
+        )
+        self.line_extraction_max_workers = _positive_int_env(
+            "BABELDOC_EXECUTOR_LAYOUT_LINE_EXTRACTION_MAX_WORKERS",
+            1,
+        )
+        self.layout_request_max_workers = _positive_int_env(
+            "BABELDOC_EXECUTOR_LAYOUT_REQUEST_MAX_WORKERS",
+            8,
+        )
+        self.layout_request_timeout_seconds = _positive_float_env(
+            "BABELDOC_EXECUTOR_LAYOUT_REQUEST_TIMEOUT_SECONDS",
+            600.0,
+        )
+        self.font_mapper: FontMapper | None = None
+
+    @property
+    def stride(self) -> int:
+        return 32
+
+    def init_font_mapper(self, translation_config) -> None:
+        if self.requires_line_extraction:
+            self.font_mapper = FontMapper(translation_config)
+
+    def handle_document(
+        self,
+        pages: list[il_version_1.Page],
+        mupdf_doc: pymupdf.Document,
+        translate_config,
+        save_debug_image,
+    ):
+        with (
+            ThreadPoolExecutor(
+                max_workers=self.layout_request_max_workers,
+            ) as layout_executor,
+            ThreadPoolExecutor(
+                max_workers=self.line_extraction_max_workers,
+            ) as line_executor,
+        ):
+            futures = {}
+            completed = {}
+            next_submit = 0
+            next_yield = 0
+            max_in_flight = (
+                self.layout_request_max_workers + self.layout_page_input_buffer_limit
+            )
+
+            def fill_window() -> None:
+                nonlocal next_submit
+                while (
+                    next_submit < len(pages)
+                    and (len(futures) + len(completed)) < max_in_flight
+                ):
+                    page_input = self._prepare_page_input(
+                        next_submit,
+                        pages[next_submit],
+                        mupdf_doc,
+                        translate_config,
+                        line_executor,
+                    )
+                    future = layout_executor.submit(
+                        self._request_layout,
+                        page_input,
+                        translate_config,
+                    )
+                    futures[future] = next_submit
+                    next_submit += 1
+
+            fill_window()
+            while next_yield < len(pages):
+                if next_yield in completed:
+                    yield completed.pop(next_yield)
+                    next_yield += 1
+                    fill_window()
+                    continue
+
+                done, _pending = wait(futures.keys(), return_when=FIRST_COMPLETED)
+                for future in done:
+                    index = futures.pop(future)
+                    try:
+                        completed[index] = future.result()
+                    except Exception as exc:
+                        raise RuntimeError(LAYOUT_ERROR) from exc
+                fill_window()
+
+    def _prepare_page_input(
+        self,
+        index: int,
+        page: il_version_1.Page,
+        mupdf_doc: pymupdf.Document,
+        translate_config,
+        line_executor: ThreadPoolExecutor,
+    ) -> _PageInput:
+        translate_config.raise_if_cancelled()
+        line_future = (
+            line_executor.submit(self._extract_raw_lines, page)
+            if self.requires_line_extraction
+            else None
+        )
+        pix = get_no_rotation_img(mupdf_doc[page.page_number], dpi=DPI)
+        image = np.frombuffer(pix.samples, np.uint8).reshape(
+            pix.height,
+            pix.width,
+            3,
+        )
+        image_data = _encode_image(image)
+        line_results = (
+            self._convert_line_results(
+                line_future.result(),
+                image.shape[1],
+                image.shape[0],
+            )
+            if line_future is not None
+            else None
+        )
+        return _PageInput(index, page, image, image_data, line_results)
+
+    def _extract_raw_lines(self, page: il_version_1.Page):
+        if self.font_mapper is None:
+            raise RuntimeError(LAYOUT_ERROR)
+
+        char_boxes = convert_page_to_char_boxes(page)
+        return process_page_chars_to_lines(char_boxes)
+
+    def _convert_line_results(
+        self,
+        lines,
+        image_width: int,
+        image_height: int,
+    ) -> list[dict[str, Any]]:
+        results = []
+        for line in lines:
+            converted = self._convert_line(line, image_width, image_height)
+            if converted is not None:
+                results.append(converted)
+        return results
+
+    def _convert_line(
+        self,
+        line,
+        image_width: int,
+        image_height: int,
+    ) -> dict[str, Any] | None:
+        if not line.text or self.font_mapper is None:
+            return None
+        boxes = [char[0] for char in line.chars]
+        min_x = min(box.x for box in boxes)
+        max_x = max(box.x2 for box in boxes)
+        min_y = min(box.y for box in boxes)
+        max_y = max(box.y2 for box in boxes)
+
+        min_x = min_x / 72 * DPI
+        max_x = max_x / 72 * DPI
+        min_y = min_y / 72 * DPI
+        max_y = max_y / 72 * DPI
+        min_y, max_y = image_height - max_y, image_height - min_y
+
+        box_volume = (max_x - min_x) * (max_y - min_y)
+        if box_volume < 1:
+            return None
+
+        min_x = _clip_num(min_x, 0, image_width - 1)
+        max_x = _clip_num(max_x, 0, image_width - 1)
+        min_y = _clip_num(min_y, 0, image_height - 1)
+        max_y = _clip_num(max_y, 0, image_height - 1)
+
+        filtered_text = _filter_text(line.text, self.font_mapper)
+        if not filtered_text:
+            return None
+
+        return {
+            "box": [min_x, min_y, max_x, max_y],
+            "text": filtered_text,
+        }
+
+    def _request_layout(
+        self,
+        page_input: _PageInput,
+        translate_config,
+    ):
+        translate_config.raise_if_cancelled()
+        request_body: dict[str, Any] = {
+            "schema_version": 1,
+            "page_number": page_input.page.page_number,
+            "dpi": DPI,
+            "image": base64.b64encode(page_input.image_data).decode("utf-8"),
+            "image_size": [page_input.image.shape[1], page_input.image.shape[0]],
+        }
+        if self.requires_line_extraction:
+            request_body["line_results"] = page_input.line_results or []
+
+        try:
+            response = httpx.post(
+                f"{self.host}/inference",
+                json=request_body,
+                headers={
+                    "Accept": "application/json",
+                    "Content-Type": "application/json",
+                },
+                timeout=self.layout_request_timeout_seconds,
+                follow_redirects=True,
+            )
+            if response.status_code < 200 or response.status_code >= 300:
+                raise RuntimeError(LAYOUT_ERROR)
+            payload = response.json()
+            yolo_result = self._parse_response(
+                payload,
+                page_input.image.shape[1],
+                page_input.image.shape[0],
+            )
+            return page_input.page, yolo_result
+        except Exception as exc:
+            raise RuntimeError(LAYOUT_ERROR) from exc
+
+    def _parse_response(
+        self,
+        payload: Any,
+        _image_width: int,
+        _image_height: int,
+    ) -> YoloResult:
+        if not isinstance(payload, dict) or payload.get("schema_version") != 1:
+            raise RuntimeError(LAYOUT_ERROR)
+        boxes_payload = payload.get("boxes")
+        if not isinstance(boxes_payload, list):
+            raise RuntimeError(LAYOUT_ERROR)
+
+        names: dict[int, str] = {}
+        boxes: list[YoloBox] = []
+        scale = 72 / DPI
+        for item in boxes_payload:
+            if not isinstance(item, dict):
+                raise RuntimeError(LAYOUT_ERROR)
+            class_id = item.get("class_id")
+            label = item.get("label")
+            score = item.get("score")
+            box = item.get("box")
+            if (
+                not isinstance(class_id, int)
+                or not isinstance(label, str)
+                or not isinstance(score, int | float)
+                or not isinstance(box, list)
+                or len(box) != 4
+            ):
+                raise RuntimeError(LAYOUT_ERROR)
+            coords = [float(value) * scale for value in box]
+            names[class_id] = label
+            boxes.append(
+                YoloBox(
+                    None,
+                    np.array(coords),
+                    np.array(float(score)),
+                    class_id,
+                )
+            )
+        return YoloResult(boxes=boxes, names=names)
+
+    @staticmethod
+    def from_host(host: str) -> RpcDocLayoutModel:
+        return RpcDocLayoutModel(host=host, requires_line_extraction=False)

babeldoc-0.6.0/babeldoc/docvision/table_detection/rapidocr.py

@@ -0,0 +1,41 @@
+from collections.abc import Generator
+
+import babeldoc.format.pdf.document_il.il_version_1
+import numpy as np
+import pymupdf
+from babeldoc.docvision.base_doclayout import YoloResult
+
+
+class RapidOCRModel:
+    """Compatibility no-op for the retired RapidOCR table text detector."""
+
+    names = {0: "table_text"}
+
+    @property
+    def stride(self):
+        return 32
+
+    def predict(self, image, imgsz=800, batch_size=16, **kwargs):
+        _ = image, imgsz, batch_size, kwargs
+        return YoloResult(names=self.names, boxes=[])
+
+    def handle_document(
+        self,
+        pages: list[babeldoc.format.pdf.document_il.il_version_1.Page],
+        mupdf_doc: pymupdf.Document,
+        translate_config,
+        save_debug_image,
+    ) -> Generator[
+        tuple[babeldoc.format.pdf.document_il.il_version_1.Page, YoloResult], None, None
+    ]:
+        _ = mupdf_doc
+        for page in pages:
+            translate_config.raise_if_cancelled()
+            yolo_result = YoloResult(names=self.names, boxes=[])
+            if save_debug_image is not None:
+                save_debug_image(
+                    np.zeros((1, 1, 3), dtype=np.uint8),
+                    yolo_result,
+                    page.page_number + 1,
+                )
+            yield page, yolo_result