BabelDOC 0.4.21__tar.gz → 0.5.0__tar.gz

{babeldoc-0.4.21 → babeldoc-0.5.0}/.github/workflows/codeql.yml

@@ -55,7 +55,7 @@ jobs:
         # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

{babeldoc-0.4.21 → babeldoc-0.5.0}/.github/workflows/docs.yml

@@ -9,7 +9,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Configure Git Credentials

{babeldoc-0.4.21 → babeldoc-0.5.0}/.github/workflows/labeler.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Run Labeler
         uses: crazy-max/ghaction-github-labeler@24d110aa46a59976b8a7f35518cb7f14f434c916 # v5.3.0

{babeldoc-0.4.21 → babeldoc-0.5.0}/.github/workflows/lint.yml

@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Ruff
         uses: astral-sh/ruff-action@v3
       - name: AutoCorrect

{babeldoc-0.4.21 → babeldoc-0.5.0}/.github/workflows/pr-lint.yml

@@ -9,7 +9,7 @@ jobs:
     name: runner / ruff
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       
       - name: Install Python
         uses: actions/setup-python@v5
@@ -34,7 +34,7 @@ jobs:
     name: runner / autocorrect
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: AutoCorrect
         uses: huacnlee/autocorrect-action@bf91ab3904c2908dd8e71312a8a83ed1eb632997 # v2.13.3
       - name: Report ReviewDog

{babeldoc-0.4.21 → babeldoc-0.5.0}/.github/workflows/publish-to-pypi.yml

@@ -28,7 +28,7 @@ jobs:
     outputs:
       is_release: ${{ steps.check-version.outputs.tag }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           persist-credentials: true
           fetch-depth: 2
@@ -90,7 +90,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           name: python-package-distributions
           path: dist/
@@ -114,7 +114,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
         with:
           name: python-package-distributions
           path: dist/
@@ -139,7 +139,7 @@ jobs:
       contents: write
       pull-requests: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           persist-credentials: true
           fetch-depth: 2

{babeldoc-0.4.21 → babeldoc-0.5.0}/.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           persist-credentials: false
       - name: Cached Assets

{babeldoc-0.4.21 → babeldoc-0.5.0}/.gitignore

@@ -40,3 +40,7 @@ babeldoc/format/txt
 
 # uv
 uv.lock
+
+# Claude Code memory file
+CLAUDE.md
+/.claude

{babeldoc-0.4.21 → babeldoc-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.4.21
+Version: 0.5.0
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -38,6 +38,7 @@ Requires-Dist: rapidocr-onnxruntime>=1.4.4
 Requires-Dist: rich>=13.9.4
 Requires-Dist: rtree>=1.4.0
 Requires-Dist: scikit-image>=0.25.2
+Requires-Dist: scikit-learn>=1.7.1
 Requires-Dist: scipy>=1.15.3
 Requires-Dist: tenacity>=9.0.0
 Requires-Dist: tiktoken>=0.9.0
@@ -217,6 +218,13 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--auto-enable-ocr-workaround`: Enable automatic OCR workaround (default: False). If a document is detected as heavily scanned, this will attempt to enable OCR processing and skip further scan detection. See "Important Interaction Note" below for crucial details on how this interacts with `--ocr-workaround` and `--skip-scanned-detection`.
 - `--primary-font-family`: Override primary font family for translated text. Choices: 'serif' for serif fonts, 'sans-serif' for sans-serif fonts, 'script' for script/italic fonts. If not specified, uses automatic font selection based on original text properties.
 - `--only-include-translated-page`: Only include translated pages in the output PDF. This option is only effective when `--pages` is used. (default: False)
+- `--merge-alternating-line-numbers`: Enable post-processing to merge alternating line-number layouts (keep the number paragraph as an independent paragraph b; merge adjacent text paragraphs a and c across it when `layout_id` and `xobj_id` match, digits are ASCII and spaces only). Default: off.
+- `--skip-form-render`: Skip form rendering (default: False). When enabled, PDF forms will not be rendered in the output.
+- `--skip-curve-render`: Skip curve rendering (default: False). When enabled, PDF curves will not be rendered in the output.
+- `--only-parse-generate-pdf`: Only parse PDF and generate output PDF without translation (default: False). This skips all translation-related processing including layout analysis, paragraph finding, style processing, and translation itself. Useful for testing PDF parsing and reconstruction functionality.
+- `--remove-non-formula-lines`: Remove non-formula lines from paragraph areas (default: False). This removes decorative lines that are not part of formulas, while protecting lines in figure/table areas. Useful for cleaning up documents with decorative elements that interfere with text flow.
+- `--non-formula-line-iou-threshold`: IoU threshold for detecting paragraph overlap when removing non-formula lines (default: 0.9). Higher values are more conservative and will remove fewer lines.
+- `--figure-table-protection-threshold`: IoU threshold for protecting lines in figure/table areas when removing non-formula lines (default: 0.9). Higher values provide more protection for structural elements in figures and tables.
 
 - `--rpc-doclayout`: RPC service host address for document layout analysis (default: None)
 - `--working-dir`: Working directory for translation. If not set, use temp directory.
@@ -337,6 +345,12 @@ ocr_workaround = false # Use OCR workaround for scanned PDFs
 rpc_doclayout = "" # RPC service host for document layout analysis
 working_dir = "" # Working directory for translation
 auto_enable_ocr_workaround = false # Enable automatic OCR workaround for scanned PDFs. See docs for interaction with ocr_workaround and skip_scanned_detection.
+skip_form_render = false # Skip form rendering (default: False)
+skip_curve_render = false # Skip curve rendering (default: False)
+only_parse_generate_pdf = false # Only parse PDF and generate output PDF without translation (default: False)
+remove_non_formula_lines = false # Remove non-formula lines from paragraph areas (default: False)
+non_formula_line_iou_threshold = 0.2 # IoU threshold for paragraph overlap detection (default: 0.2)
+figure_table_protection_threshold = 0.3 # IoU threshold for figure/table protection (default: 0.3)
 
 # Translation service
 openai = true
@@ -362,48 +376,10 @@ report-interval = 0.5
 
 ## Python API
 
-> [!TIP]
->
-> 1. Before pdf2zh 2.0 is released, you can temporarily use BabelDOC's Python API. However, after pdf2zh 2.0 is released, please directly use pdf2zh's Python API.
->
-> 2. This project's Python API does not guarantee any compatibility. However, the Python API from pdf2zh will guarantee a certain level of compatibility.
->
-> 3. We do not provide any technical support for the BabelDOC API.
->
-> 4. When performing secondary development, please refer to [pdf2zh 2.0 high level](https://github.com/PDFMathTranslate/PDFMathTranslate-next/blob/main/pdf2zh_next/high_level.py) and ensure that BabelDOC runs in a subprocess.
-
-You can refer to the example in [main.py](https://github.com/funstory-ai/yadt/blob/main/babeldoc/main.py) to use BabelDOC's Python API.
-
-Please note:
-
-1. Make sure call `babeldoc.format.pdf.high_level.init()` before using the API
-
-2. The current `TranslationConfig` does not fully validate input parameters, so you need to ensure the validity of input parameters
-
-3. For offline assets management, you can use the following functions:
-   ```python
-   # Generate an offline assets package
-   from pathlib import Path
-   import babeldoc.assets.assets
-   
-   # Generate package to a specific directory
-   # path is optional, default is ~/.cache/babeldoc/assets/offline_assets_{hash}.zip
-   babeldoc.assets.assets.generate_offline_assets_package(Path("/path/to/output/dir"))
-   
-   # Restore from a package file
-   # path is optional, default is ~/.cache/babeldoc/assets/offline_assets_{hash}.zip
-   babeldoc.assets.assets.restore_offline_assets_package(Path("/path/to/offline_assets_package.zip"))
-   
-   # You can also restore from a directory containing the offline assets package
-   # The tool will automatically find the correct package file based on the hash
-   babeldoc.assets.assets.restore_offline_assets_package(Path("/path/to/directory"))
-   ```
+The current recommended way to call BabelDOC in Python is to call the `high_level.do_translate_async_stream` function of [pdf2zh next](https://github.com/PDFMathTranslate/PDFMathTranslate-next).
 
-> [!TIP]
-> 
-> 1. The offline assets package name cannot be modified because the file list hash is encoded in the name.
-> 2. When using in production environments, it's recommended to pre-generate the assets package and include it with your application distribution.
-> 3. The package verification ensures that all required assets are intact and match their expected checksums.
+> [!WARNING]
+> **All APIs of BabelDOC should be considered as internal APIs, and any direct use of BabelDOC is not supported.**
 
 ## Background
 
@@ -458,6 +434,11 @@ And meet the following requirements:
 
 This project uses a combination of [Semantic Versioning](https://semver.org/) and [Pride Versioning](https://pridever.org/). The version number format is: "0.MAJOR.MINOR".
 
+> [!NOTE]
+>
+> The API compatibility here mainly refers to the compatibility with [pdf2zh_next](https://github.com/PDFMathTranslate/PDFMathTranslate-next).
+
+
 - MAJOR: Incremented by 1 when API incompatible changes are made or when proud improvements are implemented.
 
 - MINOR: Incremented by 1 when any API compatible changes are made.

{babeldoc-0.4.21 → babeldoc-0.5.0}/README.md

@@ -162,6 +162,13 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--auto-enable-ocr-workaround`: Enable automatic OCR workaround (default: False). If a document is detected as heavily scanned, this will attempt to enable OCR processing and skip further scan detection. See "Important Interaction Note" below for crucial details on how this interacts with `--ocr-workaround` and `--skip-scanned-detection`.
 - `--primary-font-family`: Override primary font family for translated text. Choices: 'serif' for serif fonts, 'sans-serif' for sans-serif fonts, 'script' for script/italic fonts. If not specified, uses automatic font selection based on original text properties.
 - `--only-include-translated-page`: Only include translated pages in the output PDF. This option is only effective when `--pages` is used. (default: False)
+- `--merge-alternating-line-numbers`: Enable post-processing to merge alternating line-number layouts (keep the number paragraph as an independent paragraph b; merge adjacent text paragraphs a and c across it when `layout_id` and `xobj_id` match, digits are ASCII and spaces only). Default: off.
+- `--skip-form-render`: Skip form rendering (default: False). When enabled, PDF forms will not be rendered in the output.
+- `--skip-curve-render`: Skip curve rendering (default: False). When enabled, PDF curves will not be rendered in the output.
+- `--only-parse-generate-pdf`: Only parse PDF and generate output PDF without translation (default: False). This skips all translation-related processing including layout analysis, paragraph finding, style processing, and translation itself. Useful for testing PDF parsing and reconstruction functionality.
+- `--remove-non-formula-lines`: Remove non-formula lines from paragraph areas (default: False). This removes decorative lines that are not part of formulas, while protecting lines in figure/table areas. Useful for cleaning up documents with decorative elements that interfere with text flow.
+- `--non-formula-line-iou-threshold`: IoU threshold for detecting paragraph overlap when removing non-formula lines (default: 0.9). Higher values are more conservative and will remove fewer lines.
+- `--figure-table-protection-threshold`: IoU threshold for protecting lines in figure/table areas when removing non-formula lines (default: 0.9). Higher values provide more protection for structural elements in figures and tables.
 
 - `--rpc-doclayout`: RPC service host address for document layout analysis (default: None)
 - `--working-dir`: Working directory for translation. If not set, use temp directory.
@@ -282,6 +289,12 @@ ocr_workaround = false # Use OCR workaround for scanned PDFs
 rpc_doclayout = "" # RPC service host for document layout analysis
 working_dir = "" # Working directory for translation
 auto_enable_ocr_workaround = false # Enable automatic OCR workaround for scanned PDFs. See docs for interaction with ocr_workaround and skip_scanned_detection.
+skip_form_render = false # Skip form rendering (default: False)
+skip_curve_render = false # Skip curve rendering (default: False)
+only_parse_generate_pdf = false # Only parse PDF and generate output PDF without translation (default: False)
+remove_non_formula_lines = false # Remove non-formula lines from paragraph areas (default: False)
+non_formula_line_iou_threshold = 0.2 # IoU threshold for paragraph overlap detection (default: 0.2)
+figure_table_protection_threshold = 0.3 # IoU threshold for figure/table protection (default: 0.3)
 
 # Translation service
 openai = true
@@ -307,48 +320,10 @@ report-interval = 0.5
 
 ## Python API
 
-> [!TIP]
->
-> 1. Before pdf2zh 2.0 is released, you can temporarily use BabelDOC's Python API. However, after pdf2zh 2.0 is released, please directly use pdf2zh's Python API.
->
-> 2. This project's Python API does not guarantee any compatibility. However, the Python API from pdf2zh will guarantee a certain level of compatibility.
->
-> 3. We do not provide any technical support for the BabelDOC API.
->
-> 4. When performing secondary development, please refer to [pdf2zh 2.0 high level](https://github.com/PDFMathTranslate/PDFMathTranslate-next/blob/main/pdf2zh_next/high_level.py) and ensure that BabelDOC runs in a subprocess.
-
-You can refer to the example in [main.py](https://github.com/funstory-ai/yadt/blob/main/babeldoc/main.py) to use BabelDOC's Python API.
-
-Please note:
-
-1. Make sure call `babeldoc.format.pdf.high_level.init()` before using the API
-
-2. The current `TranslationConfig` does not fully validate input parameters, so you need to ensure the validity of input parameters
-
-3. For offline assets management, you can use the following functions:
-   ```python
-   # Generate an offline assets package
-   from pathlib import Path
-   import babeldoc.assets.assets
-   
-   # Generate package to a specific directory
-   # path is optional, default is ~/.cache/babeldoc/assets/offline_assets_{hash}.zip
-   babeldoc.assets.assets.generate_offline_assets_package(Path("/path/to/output/dir"))
-   
-   # Restore from a package file
-   # path is optional, default is ~/.cache/babeldoc/assets/offline_assets_{hash}.zip
-   babeldoc.assets.assets.restore_offline_assets_package(Path("/path/to/offline_assets_package.zip"))
-   
-   # You can also restore from a directory containing the offline assets package
-   # The tool will automatically find the correct package file based on the hash
-   babeldoc.assets.assets.restore_offline_assets_package(Path("/path/to/directory"))
-   ```
+The current recommended way to call BabelDOC in Python is to call the `high_level.do_translate_async_stream` function of [pdf2zh next](https://github.com/PDFMathTranslate/PDFMathTranslate-next).
 
-> [!TIP]
-> 
-> 1. The offline assets package name cannot be modified because the file list hash is encoded in the name.
-> 2. When using in production environments, it's recommended to pre-generate the assets package and include it with your application distribution.
-> 3. The package verification ensures that all required assets are intact and match their expected checksums.
+> [!WARNING]
+> **All APIs of BabelDOC should be considered as internal APIs, and any direct use of BabelDOC is not supported.**
 
 ## Background
 
@@ -403,6 +378,11 @@ And meet the following requirements:
 
 This project uses a combination of [Semantic Versioning](https://semver.org/) and [Pride Versioning](https://pridever.org/). The version number format is: "0.MAJOR.MINOR".
 
+> [!NOTE]
+>
+> The API compatibility here mainly refers to the compatibility with [pdf2zh_next](https://github.com/PDFMathTranslate/PDFMathTranslate-next).
+
+
 - MAJOR: Incremented by 1 when API incompatible changes are made or when proud improvements are implemented.
 
 - MINOR: Incremented by 1 when any API compatible changes are made.

babeldoc-0.5.0/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

{babeldoc-0.4.21 → babeldoc-0.5.0}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.4.21"
+__version__ = "0.5.0"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.4.21 → babeldoc-0.5.0}/babeldoc/docvision/base_doclayout.py

@@ -55,17 +55,6 @@ class DocLayoutModel(abc.ABC):
     def stride(self) -> int:
         """Stride of the model input."""
 
-    @abc.abstractmethod
-    def predict(self, image: bytes, imgsz: int = 1024, **kwargs) -> list[int]:
-        """
-        Predict the layout of a document page.
-
-        Args:
-            image: The image of the document page.
-            imgsz: Resize the image to this size. Must be a multiple of the stride.
-            **kwargs: Additional arguments.
-        """
-
     @abc.abstractmethod
     def handle_document(
         self,

{babeldoc-0.4.21 → babeldoc-0.5.0}/babeldoc/docvision/rpc_doclayout.py

@@ -38,6 +38,8 @@ def encode_image(image) -> bytes:
         img = image
 
     # logger.debug(f"Image shape: {img.shape}")
+    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
+
     encoded = cv2.imencode(".jpg", img)[1].tobytes()
     # logger.debug(f"Encoded image size: {len(encoded)} bytes")
     return encoded

{babeldoc-0.4.21 → babeldoc-0.5.0}/babeldoc/docvision/rpc_doclayout2.py

@@ -38,6 +38,7 @@ def encode_image(image) -> bytes:
     else:
         img = image
 
+    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
     # logger.debug(f"Image shape: {img.shape}")
     encoded = cv2.imencode(".jpg", img)[1].tobytes()
     # logger.debug(f"Encoded image size: {len(encoded)} bytes")

{babeldoc-0.4.21 → babeldoc-0.5.0}/babeldoc/docvision/rpc_doclayout3.py

@@ -38,6 +38,7 @@ def encode_image(image) -> bytes:
     else:
         img = image
 
+    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
     # logger.debug(f"Image shape: {img.shape}")
     encoded = cv2.imencode(".jpg", img)[1].tobytes()
     # logger.debug(f"Encoded image size: {len(encoded)} bytes")

{babeldoc-0.4.21 → babeldoc-0.5.0}/babeldoc/docvision/rpc_doclayout4.py

@@ -38,6 +38,7 @@ def encode_image(image) -> bytes:
     else:
         img = image
 
+    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
     # logger.debug(f"Image shape: {img.shape}")
     encoded = cv2.imencode(".jpg", img)[1].tobytes()
     # logger.debug(f"Encoded image size: {len(encoded)} bytes")