BabelDOC 0.6.2__tar.gz → 0.6.4__tar.gz

babeldoc-0.6.4/.github/workflows/checks.yml

@@ -0,0 +1,55 @@
+name: Checks
+
+on:
+  push:
+  pull_request:
+    branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  checks:
+    name: Package and lightweight PDF smoke
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Cache BabelDOC assets
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/babeldoc
+          key: babeldoc-assets-${{ hashFiles('babeldoc/assets/embedding_assets_metadata.py') }}
+
+      - name: Setup uv with Python 3.12
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
+        with:
+          python-version: "3.12"
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+          activate-environment: true
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Lint
+        run: uv run ruff check .
+
+      - name: Build package
+        run: uv build
+
+      - name: Import and CLI smoke
+        run: |
+          uv run python -m compileall babeldoc
+          uv run babeldoc --help
+
+      - name: Lightweight no-secret PDF E2E
+        run: |
+          uv run babeldoc \
+            --openai \
+            --openai-api-key ci-dummy-key \
+            --files examples/ci/test.pdf \
+            --only-parse-generate-pdf \
+            --skip-scanned-detection

{babeldoc-0.6.2 → babeldoc-0.6.4}/.github/workflows/labeler.yml

@@ -6,11 +6,11 @@ on:
       - 'main'
     paths:
       - '.github/labels.yml'
-      - '.github/workflows/labels.yml'
+      - '.github/workflows/labeler.yml'
   pull_request:
     paths:
       - '.github/labels.yml'
-      - '.github/workflows/labels.yml'
+      - '.github/workflows/labeler.yml'
 
 permissions:
   contents: read
@@ -33,4 +33,4 @@ jobs:
           yaml-file: .github/labels.yml
           exclude: |
             help*
-            *issue
+            *issue

{babeldoc-0.6.2 → babeldoc-0.6.4}/.github/workflows/lint.yml

@@ -14,4 +14,4 @@ jobs:
       - name: Ruff
         uses: astral-sh/ruff-action@v3
       - name: AutoCorrect
-        uses: huacnlee/autocorrect-action@main
+        uses: huacnlee/autocorrect-action@bf91ab3904c2908dd8e71312a8a83ed1eb632997 # v2.13.3

{babeldoc-0.6.2 → babeldoc-0.6.4}/.gitignore

@@ -48,4 +48,5 @@ babeldoc/format/playground
 temp.jpg
 .serena
 .codex
+.repomap.tags.cache.v1/
 

{babeldoc-0.6.2 → babeldoc-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.6.2
+Version: 0.6.4
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -16,7 +16,7 @@ Requires-Dist: bitstring>=4.3.0
 Requires-Dist: chardet>=5.2.0
 Requires-Dist: charset-normalizer>=2.0.0
 Requires-Dist: configargparse>=1.7
-Requires-Dist: cryptography>=36.0.0
+Requires-Dist: cryptography>=46.0.7
 Requires-Dist: freetype-py>=2.5.1
 Requires-Dist: httpx[socks]>=0.27.0
 Requires-Dist: huggingface-hub>=0.27.0
@@ -24,7 +24,7 @@ Requires-Dist: hyperscan>=0.7.13
 Requires-Dist: levenshtein>=0.27.1
 Requires-Dist: msgpack>=1.1.0
 Requires-Dist: numpy>=2.0.2
-Requires-Dist: onnx>=1.18.0
+Requires-Dist: onnx>=1.21.0
 Requires-Dist: onnxruntime<1.24,>=1.16.1; python_version < '3.11'
 Requires-Dist: onnxruntime>=1.16.1; python_version >= '3.11'
 Requires-Dist: openai>=1.59.3
@@ -33,7 +33,7 @@ Requires-Dist: orjson>=3.10.14
 Requires-Dist: peewee>=3.17.8
 Requires-Dist: psutil>=7.0.0
 Requires-Dist: pydantic>=2.10.6
-Requires-Dist: pymupdf>=1.25.1
+Requires-Dist: pymupdf>=1.26.7
 Requires-Dist: pyzstd>=0.17.0
 Requires-Dist: rich>=13.9.4
 Requires-Dist: rtree>=1.4.0

babeldoc-0.6.4/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.4"

{babeldoc-0.6.2 → babeldoc-0.6.4}/babeldoc/const.py

@@ -6,7 +6,7 @@ import subprocess
 import threading
 from pathlib import Path
 
-__version__ = "0.6.2"
+__version__ = "0.6.4"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.6.2 → babeldoc-0.6.4}/babeldoc/docvision/doclayout.py

@@ -10,7 +10,8 @@ import numpy as np
 
 from babeldoc.docvision.base_doclayout import DocLayoutModel
 from babeldoc.docvision.base_doclayout import YoloResult
-from babeldoc.format.pdf.document_il.utils.mupdf_helper import get_no_rotation_img
+from babeldoc.format.pdf.document_il.utils.raster_geometry import RasterGeometry
+from babeldoc.format.pdf.document_il.utils.raster_geometry import with_target_long_edge
 
 try:
     import onnx
@@ -178,7 +179,14 @@ class OnnxModel(DocLayoutModel):
         boxes[..., :4] = (boxes[..., :4] - [pad_x, pad_y, pad_x, pad_y]) / gain
         return boxes
 
-    def predict(self, image, imgsz=800, batch_size=16, **kwargs):
+    def predict(
+        self,
+        image,
+        imgsz=800,
+        batch_size=16,
+        geometry: RasterGeometry | None = None,
+        **kwargs,
+    ):
         """
         Predict the layout of document pages.
 
@@ -237,6 +245,13 @@ class OnnxModel(DocLayoutModel):
                         preds[..., :4],
                         orig_shapes[j],
                     )
+                    if geometry is not None:
+                        preds[..., [0, 2]] = geometry.px_len_to_pt(
+                            preds[..., [0, 2]], "x"
+                        )
+                        preds[..., [1, 3]] = geometry.px_len_to_pt(
+                            preds[..., [1, 3]], "y"
+                        )
                 results.append(YoloResult(boxes_data=preds, names=self._names))
 
         return results
@@ -253,14 +268,14 @@ class OnnxModel(DocLayoutModel):
         for page in pages:
             translate_config.raise_if_cancelled()
             with self.lock:
-                # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
-                pix = get_no_rotation_img(mupdf_doc[page.page_number])
-            image = np.frombuffer(pix.samples, np.uint8).reshape(
-                pix.height,
-                pix.width,
-                3,
-            )[:, :, ::-1]
-            predict_result = self.predict(image)[0]
+                geometry = with_target_long_edge(
+                    mupdf_doc[page.page_number],
+                    72,
+                    self._FIXED_IMGSZ,
+                    normalize_rotation=True,
+                )
+            image = geometry.image[:, :, ::-1]
+            predict_result = self.predict(image, geometry=geometry)[0]
             save_debug_image(
                 image,
                 predict_result,

{babeldoc-0.6.2 → babeldoc-0.6.4}/babeldoc/docvision/rpc_doclayout.py

@@ -17,7 +17,8 @@ import babeldoc
 from babeldoc.docvision.base_doclayout import DocLayoutModel
 from babeldoc.docvision.base_doclayout import YoloBox
 from babeldoc.docvision.base_doclayout import YoloResult
-from babeldoc.format.pdf.document_il.utils.mupdf_helper import get_no_rotation_img
+from babeldoc.format.pdf.document_il.utils.raster_geometry import RasterGeometry
+from babeldoc.format.pdf.document_il.utils.raster_geometry import with_target_long_edge
 
 logger = logging.getLogger(__name__)
 
@@ -209,6 +210,7 @@ class RpcDocLayoutModel(DocLayoutModel):
         host: str = None,
         result_container: ResultContainer | None = None,
         imgsz: int = 1024,
+        geometry: RasterGeometry | None = None,
     ) -> ResultContainer:
         """Predict the layout of document pages using RPC service."""
         if result_container is None:
@@ -232,6 +234,10 @@ class RpcDocLayoutModel(DocLayoutModel):
                     )
                     for x in pred["boxes"]
                 ]
+                if geometry is not None:
+                    for box in boxes:
+                        box.xyxy[[0, 2]] = geometry.px_len_to_pt(box.xyxy[[0, 2]], "x")
+                        box.xyxy[[1, 3]] = geometry.px_len_to_pt(box.xyxy[[1, 3]], "y")
                 result_container.result = YoloResult(
                     boxes=boxes,
                     names={int(k): v for k, v in pred["names"].items()},
@@ -244,11 +250,17 @@ class RpcDocLayoutModel(DocLayoutModel):
         if isinstance(image, np.ndarray) and len(image.shape) == 3:
             image = [image]
 
+        geometry = kwargs.get("geometry")
         result_containers = [ResultContainer() for _ in image]
         predict_thread = ThreadPoolExecutor(max_workers=len(image))
         for img, result_container in zip(image, result_containers, strict=True):
             predict_thread.submit(
-                self.predict_image, img, self.host, result_container, 800
+                self.predict_image,
+                img,
+                self.host,
+                result_container,
+                800,
+                geometry,
             )
         predict_thread.shutdown(wait=True)
         result = [result_container.result for result_container in result_containers]
@@ -259,14 +271,20 @@ class RpcDocLayoutModel(DocLayoutModel):
     ):
         translate_config.raise_if_cancelled()
         with self.lock:
-            # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
-            pix = get_no_rotation_img(mupdf_doc[page.page_number])
-        image = np.frombuffer(pix.samples, np.uint8).reshape(
-            pix.height,
-            pix.width,
-            3,
-        )[:, :, ::-1]
-        predict_result = self.predict_image(image, self.host, None, 800)
+            geometry = with_target_long_edge(
+                mupdf_doc[page.page_number],
+                72,
+                800,
+                normalize_rotation=True,
+            )
+        image = geometry.image[:, :, ::-1]
+        predict_result = self.predict_image(
+            image,
+            self.host,
+            None,
+            800,
+            geometry,
+        )
         save_debug_image(image, predict_result, page.page_number + 1)
         return page, predict_result
 

{babeldoc-0.6.2 → babeldoc-0.6.4}/babeldoc/docvision/rpc_doclayout6.py

@@ -4,7 +4,9 @@ import logging
 import threading
 import unicodedata
 from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Any
 
 import cv2
 import httpx
@@ -29,8 +31,10 @@ from babeldoc.format.pdf.document_il.utils.extract_char import (
 from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
 from babeldoc.format.pdf.document_il.utils.layout_helper import SPACE_REGEX
 from babeldoc.format.pdf.document_il.utils.mupdf_helper import (
-    get_no_rotation_img_multiprocess,
+    get_no_rotation_raster_geometry_multiprocess,
 )
+from babeldoc.format.pdf.document_il.utils.raster_geometry import DEFAULT_MAX_PIXELS
+from babeldoc.format.pdf.document_il.utils.raster_geometry import RasterGeometry
 
 logger = logging.getLogger(__name__)
 DPI = 150
@@ -84,12 +88,18 @@ def predict_layout(
     _imgsz: int = 1024,
     lines=None,
     font_mapper: FontMapper | None = None,
+    *,
+    geometry: RasterGeometry | None = None,
 ):
     """Predict document layout using OCR line information (RPC service)."""
 
     if lines is None:
         lines = []
 
+    if lines and geometry is None:
+        raise ValueError("geometry is required when service1 OCR lines are supplied")
+    if geometry is not None:
+        image = geometry.image
     image_data = encode_image(image)
 
     def convert_line(line):
@@ -101,13 +111,18 @@ def predict_layout(
         min_y = min(b.y for b in boxes)
         max_y = max(b.y2 for b in boxes)
 
-        image_height, image_width = image.shape[:2]
+        if geometry is None:
+            raise ValueError(
+                "geometry is required when service1 OCR lines are supplied"
+            )
+        image_width = geometry.pixel_width
+        image_height = geometry.pixel_height
 
         # Transform to image pixel coordinates
-        min_x = min_x / 72 * DPI
-        max_x = max_x / 72 * DPI
-        min_y = min_y / 72 * DPI
-        max_y = max_y / 72 * DPI
+        min_x = geometry.pt_len_to_px(min_x, "x")
+        max_x = geometry.pt_len_to_px(max_x, "x")
+        min_y = geometry.pt_len_to_px(min_y, "y")
+        max_y = geometry.pt_len_to_px(max_y, "y")
 
         min_y, max_y = image_height - max_y, image_height - min_y
 
@@ -136,7 +151,9 @@ def predict_layout(
     request_data = {
         "image": image_b64,
         "ocr_results": formatted_results,
-        "image_size": list(image.shape[:2])[::-1],  # (height, width)
+        "image_size": [geometry.pixel_width, geometry.pixel_height]
+        if geometry is not None
+        else [image.shape[1], image.shape[0]],
     }
 
     response = httpx.post(
@@ -200,6 +217,8 @@ def predict_layout2(
     image,
     host: str = "http://localhost:8000",
     _imgsz: int = 1024,
+    *,
+    geometry: RasterGeometry | None = None,
 ):
     """
     Predict document layout using the MOSEC service
@@ -214,7 +233,9 @@ def predict_layout2(
     """
     # Prepare request data
 
-    if not isinstance(image, list):
+    if geometry is not None:
+        image = [geometry.image]
+    elif not isinstance(image, list):
         image = [image]
     image_data = [encode_image(image) for image in image]
     data = {
@@ -286,6 +307,76 @@ class ResultContainer:
         self.result = YoloResult(boxes_data=np.array([]), names=[])
 
 
+@dataclass(frozen=True)
+class _PageInput:
+    page: Any
+    geometry: RasterGeometry
+    lines: list[Any]
+
+
+def _legacy_synthetic_geometry(image: np.ndarray) -> RasterGeometry:
+    """Build the legacy image-only coordinate mapping without PDF rendering."""
+
+    height, width = image.shape[:2]
+    return RasterGeometry(
+        image=image,
+        requested_dpi=DPI,
+        render_dpi=DPI,
+        pixel_width=width,
+        pixel_height=height,
+        page_width_pt=width * 72 / DPI,
+        page_height_pt=height * 72 / DPI,
+    )
+
+
+def _raster_box_to_page_points(
+    box: list[float] | np.ndarray,
+    geometry: RasterGeometry,
+) -> np.ndarray:
+    """Convert an uploaded-raster xyxy box to PDF point/top-left coordinates."""
+
+    return np.array(
+        [
+            geometry.px_len_to_pt(float(box[0]), "x"),
+            geometry.px_len_to_pt(float(box[1]), "y"),
+            geometry.px_len_to_pt(float(box[2]), "x"),
+            geometry.px_len_to_pt(float(box[3]), "y"),
+        ]
+    )
+
+
+def _project_result_to_raster(
+    result: YoloResult,
+    geometry: RasterGeometry,
+) -> YoloResult:
+    """Copy point-space boxes into pixel space for the debug callback only."""
+
+    boxes = [
+        YoloBox(
+            None,
+            _raster_box_from_page_points(box.xyxy, geometry),
+            box.conf,
+            box.cls,
+        )
+        for box in result.boxes
+    ]
+    return YoloResult(boxes=boxes, names=result.names)
+
+
+def _raster_box_from_page_points(
+    box: list[float] | np.ndarray,
+    geometry: RasterGeometry,
+) -> np.ndarray:
+    return np.array(
+        [
+            geometry.pt_len_to_px(float(box[0]), "x"),
+            geometry.pt_len_to_px(float(box[1]), "y"),
+            geometry.pt_len_to_px(float(box[2]), "x"),
+            geometry.pt_len_to_px(float(box[3]), "y"),
+        ]
+    )
+
+
 def filter_text(txt: str, font_mapper: FontMapper):
     normalize = unicodedata.normalize("NFKC", txt)
     unicodes = []
@@ -329,73 +420,6 @@ class RpcDocLayoutModel(DocLayoutModel):
         """Stride of the model input."""
         return self._stride
 
-    def resize_and_pad_image(self, image, new_shape):
-        """
-        Resize and pad the image to the specified size,
-        ensuring dimensions are multiples of stride.
-
-        Parameters:
-        - image: Input image
-        - new_shape: Target size (integer or (height, width) tuple)
-        - stride: Padding alignment stride, default 32
-
-        Returns:
-        - Processed image
-        """
-        if isinstance(new_shape, int):
-            new_shape = (new_shape, new_shape)
-
-        h, w = image.shape[:2]
-        new_h, new_w = new_shape
-
-        # Calculate scaling ratio
-        r = min(new_h / h, new_w / w)
-        resized_h, resized_w = int(round(h * r)), int(round(w * r))
-
-        # Resize image
-        image = cv2.resize(
-            image, (resized_w, resized_h), interpolation=cv2.INTER_LINEAR
-        )
-
-        # Calculate padding size
-        pad_h = new_h - resized_h
-        pad_w = new_w - resized_w
-        top, bottom = pad_h // 2, pad_h - pad_h // 2
-        left, right = pad_w // 2, pad_w - pad_w // 2
-
-        # Add padding
-        image = cv2.copyMakeBorder(
-            image, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(114, 114, 114)
-        )
-
-        return image
-
-    def scale_boxes(self, img1_shape, boxes, img0_shape):
-        """
-        Rescales bounding boxes (in the format of xyxy by default) from the shape of the image they were originally
-        specified in (img1_shape) to the shape of a different image (img0_shape).
-
-        Args:
-            img1_shape (tuple): The shape of the image that the bounding boxes are for,
-                in the format of (height, width).
-            boxes (torch.Tensor): the bounding boxes of the objects in the image, in the format of (x1, y1, x2, y2)
-            img0_shape (tuple): the shape of the target image, in the format of (height, width).
-
-        Returns:
-            boxes (torch.Tensor): The scaled bounding boxes, in the format of (x1, y1, x2, y2)
-        """
-
-        # Calculate scaling ratio
-        gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])
-
-        # Calculate padding size
-        pad_x = round((img1_shape[1] - img0_shape[1] * gain) / 2 - 0.1)
-        pad_y = round((img1_shape[0] - img0_shape[0] * gain) / 2 - 0.1)
-
-        # Remove padding and scale boxes
-        boxes = (boxes - [pad_x, pad_y, pad_x, pad_y]) / gain
-        return boxes
-
     def calculate_iou(self, box1, box2):
         """Calculate IoU between two boxes in xyxy format."""
         x1_1, y1_1, x2_1, y2_1 = box1
@@ -479,16 +503,14 @@ class RpcDocLayoutModel(DocLayoutModel):
         image,
         imgsz: int = 1024,
         lines=None,
+        *,
+        geometry: RasterGeometry | None = None,
     ) -> YoloResult:
         """Predict the layout of a single page and fuse results from two RPC services."""
 
-        # Resize/pad image if needed – use original size to avoid extra scaling artefacts
-        orig_h, orig_w = image.shape[:2]
-        target_imgsz = (orig_h, orig_w)
-        if image.shape[0] != target_imgsz[0] or image.shape[1] != target_imgsz[1]:
-            image_proc = self.resize_and_pad_image(image, new_shape=target_imgsz)
-        else:
-            image_proc = image
+        if geometry is None:
+            geometry = _legacy_synthetic_geometry(image)
+        image_proc = geometry.image
 
         # Parallel calls to both services; exceptions propagate if either fails
         with ThreadPoolExecutor(max_workers=2) as ex:
@@ -500,8 +522,15 @@ class RpcDocLayoutModel(DocLayoutModel):
                     imgsz,
                     lines,
                     self.font_mapper,
+                    geometry=geometry,
                 )
-            future2 = ex.submit(predict_layout2, image_proc, self.host2, imgsz)
+            future2 = ex.submit(
+                predict_layout2,
+                image_proc,
+                self.host2,
+                imgsz,
+                geometry=geometry,
+            )
 
             # .result() will re-raise any exception occurred in worker thread.
             if lines:
@@ -510,19 +539,13 @@ class RpcDocLayoutModel(DocLayoutModel):
                 preds1 = None
             preds2 = future2.result()
 
-        # Convert DPI to PDF points (72 dpi)
-        pdf_h, pdf_w = orig_h / DPI * 72, orig_w / DPI * 72
-
         merged_boxes: list[YoloBox] = []
         names: dict[int, str] = {}
 
         def _process_preds(preds, id_offset: int, label_suffix: str | None):
             for pred in preds or []:
                 for box in pred["boxes"]:
-                    # scale coords back to PDF space
-                    scaled_xyxy = self.scale_boxes(
-                        target_imgsz, np.array(box["xyxy"]), (pdf_h, pdf_w)
-                    )
+                    scaled_xyxy = _raster_box_to_page_points(box["xyxy"], geometry)
 
                     new_cls_id = box["cls"] + id_offset
 
@@ -558,7 +581,11 @@ class RpcDocLayoutModel(DocLayoutModel):
         return YoloResult(boxes=merged_boxes, names=names)
 
     def predict(self, image, imgsz=1024, **kwargs) -> list[YoloResult]:  # type: ignore[override]
-        """Predict the layout for one or multiple images."""
+        """Predict images using a legacy synthetic geometry.
+
+        This image-only path does not render a PDF page and is therefore not
+        protected by the PDF page pixel budget.
+        """
 
         # Normalize to list
         if isinstance(image, np.ndarray) and len(image.shape) == 3:
@@ -573,21 +600,26 @@ class RpcDocLayoutModel(DocLayoutModel):
 
     def predict_page(self, page, pdf_bytes: Path, translate_config, save_debug_image):
         translate_config.raise_if_cancelled()
-        # doc = pymupdf.open(io.BytesIO(pdf_bytes))
-        # with self.lock:
-        # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
-        image = get_no_rotation_img_multiprocess(
-            pdf_bytes.as_posix(), page.page_number, dpi=DPI
+        geometry = get_no_rotation_raster_geometry_multiprocess(
+            pdf_bytes.as_posix(),
+            page.page_number,
+            requested_dpi=DPI,
+            max_pixels=DEFAULT_MAX_PIXELS,
         )
-        # image = np.frombuffer(pix.samples, np.uint8).reshape(
-        #     pix.height,
-        #     pix.width,
-        #     3,
-        # )[:, :, ::-1]
         char_boxes = convert_page_to_char_boxes(page)
         lines = process_page_chars_to_lines(char_boxes)
-        predict_result = self.predict_image(image, 800, lines)
-        save_debug_image(image, predict_result, page.page_number + 1)
+        page_input = _PageInput(page, geometry, lines)
+        predict_result = self.predict_image(
+            page_input.geometry.image,
+            800,
+            page_input.lines,
+            geometry=page_input.geometry,
+        )
+        save_debug_image(
+            page_input.geometry.image,
+            _project_result_to_raster(predict_result, page_input.geometry),
+            page.page_number + 1,
+        )
         return page, predict_result
 
     def handle_document(  # type: ignore[override]