BabelDOC 0.2.14__tar.gz → 0.2.16__tar.gz

{babeldoc-0.2.14 → babeldoc-0.2.16}/.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@f94ec6bedd8674c4426838e6b50417d36b6ab231 # v5.3.1
+        uses: astral-sh/setup-uv@22695119d769bdb6f7032ad67b9bca0ef8c4a174 # v5.4.0
         with:
           python-version: "3.12"
           enable-cache: true

{babeldoc-0.2.14 → babeldoc-0.2.16}/.github/workflows/publish-to-pypi.yml

@@ -35,7 +35,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@f94ec6bedd8674c4426838e6b50417d36b6ab231 # v5.3.1
+        uses: astral-sh/setup-uv@22695119d769bdb6f7032ad67b9bca0ef8c4a174 # v5.4.0
         with:
           python-version: "3.12"
           enable-cache: true
@@ -68,7 +68,7 @@ jobs:
         run: "uv build"
 
       - name: Store the distribution packages
-        uses: actions/upload-artifact@v4.6.1
+        uses: actions/upload-artifact@v4.6.2
         with:
           name: python-package-distributions
           path: dist/
@@ -89,7 +89,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
+        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
         with:
           name: python-package-distributions
           path: dist/
@@ -113,7 +113,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
+        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
         with:
           name: python-package-distributions
           path: dist/

{babeldoc-0.2.14 → babeldoc-0.2.16}/.github/workflows/test.yml

@@ -28,7 +28,7 @@ jobs:
           path: ~/.cache/babeldoc
           key: babeldoc-assets-${{ hashFiles('babeldoc/assets/embedding_assets_metadata.py') }}
       - name: Setup uv with Python ${{ matrix.python-version }}
-        uses: astral-sh/setup-uv@f94ec6bedd8674c4426838e6b50417d36b6ab231 # v5.3.1
+        uses: astral-sh/setup-uv@22695119d769bdb6f7032ad67b9bca0ef8c4a174 # v5.4.0
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true

{babeldoc-0.2.14 → babeldoc-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.2.14
+Version: 0.2.16
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -30,6 +30,7 @@ Requires-Dist: psutil>=7.0.0
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: pymupdf>=1.25.1
 Requires-Dist: python-levenshtein>=0.27.1
+Requires-Dist: rapidocr-onnxruntime>=1.4.4
 Requires-Dist: rich>=13.9.4
 Requires-Dist: scikit-image>=0.25.2
 Requires-Dist: tenacity>=9.0.0
@@ -87,6 +88,10 @@ PDF scientific paper translation and bilingual comparison library.
 <img src="https://s.immersivetranslate.com/assets/r2-uploads/images/babeldoc-preview.png" width="80%"/>
 </div>
 
+## We are hiring
+
+See details: [EN](https://github.com/funstory-ai/jobs) | [ZH](https://github.com/funstory-ai/jobs/blob/main/README_ZH.md)
+
 ## Getting Started
 
 ### Install from PyPI
@@ -181,6 +186,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--watermark-output-mode`: Control watermark output mode: 'watermarked' (default) adds watermark to translated PDF, 'no_watermark' doesn't add watermark, 'both' outputs both versions.
 - `--max-pages-per-part`: Maximum number of pages per part for split translation. If not set, no splitting will be performed.
 - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
+- `--translate-table-text`: Translate table text (experimental, default: False)
 
 > [!TIP]
 > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers

{babeldoc-0.2.14 → babeldoc-0.2.16}/README.md

@@ -42,6 +42,10 @@ PDF scientific paper translation and bilingual comparison library.
 <img src="https://s.immersivetranslate.com/assets/r2-uploads/images/babeldoc-preview.png" width="80%"/>
 </div>
 
+## We are hiring
+
+See details: [EN](https://github.com/funstory-ai/jobs) | [ZH](https://github.com/funstory-ai/jobs/blob/main/README_ZH.md)
+
 ## Getting Started
 
 ### Install from PyPI
@@ -136,6 +140,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--watermark-output-mode`: Control watermark output mode: 'watermarked' (default) adds watermark to translated PDF, 'no_watermark' doesn't add watermark, 'both' outputs both versions.
 - `--max-pages-per-part`: Maximum number of pages per part for split translation. If not set, no splitting will be performed.
 - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
+- `--translate-table-text`: Translate table text (experimental, default: False)
 
 > [!TIP]
 > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers

babeldoc-0.2.16/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.16"

{babeldoc-0.2.14 → babeldoc-0.2.16}/babeldoc/assets/assets.py

@@ -14,6 +14,10 @@ from babeldoc.assets.embedding_assets_metadata import (
 from babeldoc.assets.embedding_assets_metadata import EMBEDDING_FONT_METADATA
 from babeldoc.assets.embedding_assets_metadata import FONT_METADATA_URL
 from babeldoc.assets.embedding_assets_metadata import FONT_URL_BY_UPSTREAM
+from babeldoc.assets.embedding_assets_metadata import (
+    TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256,
+)
+from babeldoc.assets.embedding_assets_metadata import TABLE_DETECTION_RAPIDOCR_MODEL_URL
 from babeldoc.assets.embedding_assets_metadata import TIKTOKEN_CACHES
 from babeldoc.const import get_cache_file_path
 from tenacity import retry
@@ -204,10 +208,36 @@ async def get_doclayout_onnx_model_path_async(client: httpx.AsyncClient | None =
     return onnx_path
 
 
+async def get_table_detection_rapidocr_model_path_async(
+    client: httpx.AsyncClient | None = None,
+):
+    onnx_path = get_cache_file_path("ch_PP-OCRv4_det_infer.onnx", "models")
+    if verify_file(onnx_path, TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256):
+        return onnx_path
+
+    logger.info("table detection rapidocr model not found or corrupted, downloading...")
+    fastest_upstream, _ = await get_fastest_upstream_for_model(client)
+    if fastest_upstream is None:
+        logger.error("Failed to get fastest upstream")
+        exit(1)
+
+    url = TABLE_DETECTION_RAPIDOCR_MODEL_URL[fastest_upstream]
+
+    await download_file(client, url, onnx_path, TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256)
+    logger.info(
+        f"Download table detection rapidocr model from {fastest_upstream} success"
+    )
+    return onnx_path
+
+
 def get_doclayout_onnx_model_path():
     return run_coro(get_doclayout_onnx_model_path_async())
 
 
+def get_table_detection_rapidocr_model_path():
+    return run_coro(get_table_detection_rapidocr_model_path_async())
+
+
 def get_font_url_by_name_and_upstream(font_file_name: str, upstream: str):
     if upstream not in FONT_URL_BY_UPSTREAM:
         logger.critical(f"Invalid upstream: {upstream}")
@@ -329,7 +359,11 @@ def generate_all_assets_file_list():
         {
             "name": "doclayout_yolo_docstructbench_imgsz1024.onnx",
             "sha3_256": DOCLAYOUT_YOLO_DOCSTRUCTBENCH_IMGSZ1024ONNX_SHA3_256,
-        }
+        },
+        {
+            "name": "ch_PP-OCRv4_det_infer.onnx",
+            "sha3_256": TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256,
+        },
     )
     return result
 

{babeldoc-0.2.14 → babeldoc-0.2.16}/babeldoc/assets/embedding_assets_metadata.py

@@ -2,6 +2,10 @@ DOCLAYOUT_YOLO_DOCSTRUCTBENCH_IMGSZ1024ONNX_SHA3_256 = (
     "60be061226930524958b5465c8c04af3d7c03bcb0beb66454f5da9f792e3cf2a"
 )
 
+TABLE_DETECTION_RAPIDOCR_MODEL_SHA3_256 = (
+    "062f4619afe91b33147c033acadecbb53f2a7b99ac703d157b96d5b10948da5e"
+)
+
 TIKTOKEN_CACHES = {
     "fb374d419588a4632f3f557e76b4b70aebbca790": "cb04bcda5782cfbbe77f2f991d92c0ea785d9496ef1137c91dfc3c8c324528d6"
 }
@@ -26,6 +30,12 @@ DOC_LAYOUT_ONNX_MODEL_URL = {
     "modelscope": "https://www.modelscope.cn/models/AI-ModelScope/DocLayout-YOLO-DocStructBench-onnx/resolve/master/doclayout_yolo_docstructbench_imgsz1024.onnx",
 }
 
+TABLE_DETECTION_RAPIDOCR_MODEL_URL = {
+    "huggingface": "https://huggingface.co/spaces/RapidAI/RapidOCR/resolve/main/models/text_det/ch_PP-OCRv4_det_infer.onnx",
+    "hf-mirror": "https://hf-mirror.com/spaces/RapidAI/RapidOCR/resolve/main/models/text_det/ch_PP-OCRv4_det_infer.onnx",
+    "modelscope": "https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/master/onnx/PP-OCRv4/det/ch_PP-OCRv4_det_infer.onnx",
+}
+
 # from https://github.com/funstory-ai/BabelDOC-Assets/blob/main/font_metadata.json
 EMBEDDING_FONT_METADATA = {
     "GoNotoKurrent-Bold.ttf": {

{babeldoc-0.2.14 → babeldoc-0.2.16}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.2.14"
+__version__ = "0.2.16"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.2.14 → babeldoc-0.2.16}/babeldoc/document_il/midend/paragraph_finder.py

@@ -287,6 +287,7 @@ class ParagraphFinder:
             "abandon",
             "figure_caption",
             "table_caption",
+            "table_text",
         ]
 
     def get_layout(
@@ -313,6 +314,7 @@ class ParagraphFinder:
             "table_footnote",
             "table_caption",
             "figure_caption",
+            "table_text",
             "table",
             "figure",
             "abandon",

{babeldoc-0.2.14 → babeldoc-0.2.16}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -428,6 +428,8 @@ class StylesAndFormulas:
                     formula.x_offset = 0  # 如果左边没有文字，x_offset 应该为 0
                 if abs(formula.x_offset) < 0.1:
                     formula.x_offset = 0
+                if formula.x_offset > 10:
+                    formula.x_offset = 0
                 # if formula.x_offset > 0:
                 #     formula.x_offset = 0
 
@@ -443,7 +445,7 @@ class StylesAndFormulas:
                 if abs(formula.y_offset) < 0.1:
                     formula.y_offset = 0
 
-                if max(abs(formula.y_offset), abs(formula.x_offset)) > 2:
+                if max(abs(formula.y_offset), abs(formula.x_offset)) > 10:
                     pass
                     # logging.debug(
                     #     f"公式 {formula.box} 的偏移量过大：{formula.x_offset}, {formula.y_offset}"

babeldoc-0.2.16/babeldoc/document_il/midend/table_parser.py

@@ -0,0 +1,164 @@
+import logging
+from pathlib import Path
+
+import cv2
+import numpy as np
+from pymupdf import Document
+
+from babeldoc.document_il import il_version_1
+from babeldoc.document_il.utils.style_helper import GREEN
+from babeldoc.translation_config import TranslationConfig
+
+logger = logging.getLogger(__name__)
+
+
+class TableParser:
+    stage_name = "Parse Table"
+
+    def __init__(self, translation_config: TranslationConfig):
+        self.translation_config = translation_config
+        self.model = translation_config.table_model
+
+    def _save_debug_image(self, image: np.ndarray, layouts, page_number: int):
+        """Save debug image with drawn boxes if debug mode is enabled."""
+        if not self.translation_config.debug:
+            return
+
+        if not isinstance(layouts, list):
+            layouts = [layouts]
+        debug_dir = Path(
+            self.translation_config.get_working_file_path("table-ocr-box-image")
+        )
+        debug_dir.mkdir(parents=True, exist_ok=True)
+
+        # Draw boxes on the image
+        debug_image = image.copy()
+        for layout in layouts:
+            for box in layout.boxes:
+                x0, y0, x1, y1 = box.xyxy
+                cv2.rectangle(
+                    debug_image,
+                    (int(x0), int(y0)),
+                    (int(x1), int(y1)),
+                    (0, 255, 0),
+                    2,
+                )
+                # Add text label
+                cv2.putText(
+                    debug_image,
+                    layout.names[box.cls],
+                    (int(x0), int(y0) - 5),
+                    cv2.FONT_HERSHEY_SIMPLEX,
+                    0.5,
+                    (0, 255, 0),
+                    1,
+                )
+
+        # Save the image
+        output_path = debug_dir / f"{page_number}.jpg"
+        cv2.imwrite(str(output_path), debug_image)
+
+    def _save_debug_box_to_page(self, page: il_version_1.Page):
+        """Save debug boxes and text labels to the PDF page."""
+        if not self.translation_config.debug:
+            return
+
+        color = GREEN
+
+        for layout in page.page_layout:
+            # Create a rectangle box
+            rect = il_version_1.PdfRectangle(
+                box=il_version_1.Box(
+                    x=layout.box.x,
+                    y=layout.box.y,
+                    x2=layout.box.x2,
+                    y2=layout.box.y2,
+                ),
+                graphic_state=color,
+                debug_info=True,
+            )
+            page.pdf_rectangle.append(rect)
+
+            # Create text label at top-left corner
+            # Note: PDF coordinates are from bottom-left,
+            # so we use y2 for top position
+            style = il_version_1.PdfStyle(
+                font_id="china-ss",
+                font_size=4,
+                graphic_state=color,
+            )
+            page.pdf_paragraph.append(
+                il_version_1.PdfParagraph(
+                    first_line_indent=False,
+                    box=il_version_1.Box(
+                        x=layout.box.x,
+                        y=layout.box.y2,
+                        x2=layout.box.x2,
+                        y2=layout.box.y2 + 5,
+                    ),
+                    vertical=False,
+                    pdf_style=style,
+                    unicode=layout.class_name,
+                    pdf_paragraph_composition=[
+                        il_version_1.PdfParagraphComposition(
+                            pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+                                unicode=layout.class_name,
+                                pdf_style=style,
+                                debug_info=True,
+                            ),
+                        ),
+                    ],
+                    xobj_id=-1,
+                ),
+            )
+
+    def process(self, docs: il_version_1.Document, mupdf_doc: Document):
+        """Generate layouts for all pages that need to be translated."""
+        # Get pages that need to be translated
+        have_table_pages = {}
+        for page in docs.page:
+            for layout in page.page_layout:
+                if layout.class_name == "table":
+                    have_table_pages[page.page_number] = page
+        with self.translation_config.progress_monitor.stage_start(
+            self.stage_name,
+            len(have_table_pages),
+        ) as progress:
+            # Process predictions for each page
+            for page, layouts in self.model.handle_document(
+                have_table_pages.values(),
+                mupdf_doc,
+                self.translation_config,
+                self._save_debug_image,
+            ):
+                page_layouts = []
+                for layout in layouts.boxes:
+                    # Convert coordinate system from picture to il
+                    # system to the il coordinate system
+                    x0, y0, x1, y1 = layout.xyxy
+                    pix = mupdf_doc[page.page_number].get_pixmap()
+                    h, w = pix.height, pix.width
+                    x0, y0, x1, y1 = (
+                        np.clip(int(x0 - 1), 0, w - 1),
+                        np.clip(int(h - y1 - 1), 0, h - 1),
+                        np.clip(int(x1 + 1), 0, w - 1),
+                        np.clip(int(h - y0 + 1), 0, h - 1),
+                    )
+                    page_layout = il_version_1.PageLayout(
+                        id=len(page_layouts) + 1,
+                        box=il_version_1.Box(
+                            x0.item(),
+                            y0.item(),
+                            x1.item(),
+                            y1.item(),
+                        ),
+                        conf=layout.conf.item(),
+                        class_name=layouts.names[layout.cls],
+                    )
+                    page_layouts.append(page_layout)
+
+                page.page_layout.extend(page_layouts)
+                self._save_debug_box_to_page(page)
+                progress.advance(1)
+
+        return docs

{babeldoc-0.2.14 → babeldoc-0.2.16}/babeldoc/document_il/midend/typesetting.py

@@ -1,3 +1,4 @@
+import copy
 import logging
 import re
 import statistics
@@ -603,17 +604,26 @@ class Typesetting:
             self.font_mapper.base_font.char_lengths("你", font_size * scale)[0] * 0.5
         )
 
-        # 计算平均行高
-        avg_height = (
-            sum(unit.height * scale for unit in typesetting_units)
-            / len(typesetting_units)
-            if typesetting_units
-            else 0
+        # 计算行高（使用众数）
+        unit_heights = (
+            [unit.height for unit in typesetting_units] if typesetting_units else []
         )
+        if not unit_heights:
+            avg_height = 0
+        elif len(unit_heights) == 1:
+            avg_height = unit_heights[0] * scale
+        else:
+            try:
+                avg_height = statistics.mode(unit_heights) * scale
+            except statistics.StatisticsError:
+                # 如果没有众数（所有值都出现相同次数），则使用平均值
+                avg_height = sum(unit_heights) / len(unit_heights) * scale
 
         # 初始化位置为右上角，并减去一个平均行高
         current_x = box.x
         current_y = box.y2 - avg_height
+        box = copy.deepcopy(box)
+        box.y -= avg_height * (line_spacing - 1)
         line_height = 0
         current_line_heights = []  # 存储当前行所有元素的高度