BabelDOC 0.5.20__tar.gz → 0.5.22__tar.gz

{babeldoc-0.5.20 → babeldoc-0.5.22}/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -28,7 +28,11 @@ body:
     attributes:
       value: |
         Thank you for using **BabelDOC** and helping us improve it! 🙏
+        Please confirm again that the above checklist items have been carefully executed! (If you have not carefully executed them, the issue will be closed directly without any response)
 
+        Please also note:
+        - If you are using a downstream project like pdf2zh-next, please submit an issue directly to the downstream application. Only when you confirm that this issue is a problem with the core library of BabelDOC, submit this issue.
+        - The CLI is only used for debugging purposes, we do not provide any technical support for CLI usage.
   - type: textarea
     id: environment
     attributes:

{babeldoc-0.5.20 → babeldoc-0.5.22}/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -16,6 +16,15 @@ body:
           required: true
         - label: I have fully read and understood the [README](https://github.com/funstory-ai/BabelDOC/blob/main/README.md)
           required: true
+        - label: This feature is not related to BabelDOC CLI. The CLI is only used for debugging purposes, we do not accept any feature requests related to the CLI.
+          required: true
+  
+  - type: markdown
+    attributes:
+      value: |
+        如果您想自部署 BabelDOC，请使用 [PDFMathTranslate-next](https://github.com/PDFMathTranslate-next/PDFMathTranslate-next) 代替。若其功能无法满足，请向 [PDFMathTranslate-next](https://github.com/PDFMathTranslate-next/PDFMathTranslate-next) 提交功能请求。
+        If you wish to self-host BabelDOC, please use [PDFMathTranslate-next](https://github.com/PDFMathTranslate-next/PDFMathTranslate-next) instead. If its features do not meet your needs, please submit a feature request to [PDFMathTranslate-next](https://github.com/PDFMathTranslate-next/PDFMathTranslate-next).
+
 
   - type: textarea
     id: describe

{babeldoc-0.5.20 → babeldoc-0.5.22}/.github/workflows/codeql.yml

@@ -59,7 +59,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
@@ -87,6 +87,6 @@ jobs:
         exit 1
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4
       with:
         category: "/language:${{matrix.language}}"

{babeldoc-0.5.20 → babeldoc-0.5.22}/.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v6.6.0
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
         with:
           python-version: "3.12"
           enable-cache: true

{babeldoc-0.5.20 → babeldoc-0.5.22}/.github/workflows/pr-lint.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v5
       
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.11'
           
@@ -20,7 +20,7 @@ jobs:
         run: pip install ruff
         
       - name: Install reviewdog
-        uses: reviewdog/action-setup@e04ffabe3898a0af8d0fb1af00c188831c4b5893 # v1.3.2
+        uses: reviewdog/action-setup@d8edfce3dd5e1ec6978745e801f9c50b5ef80252 # v1.4.0
         with:
           reviewdog_version: latest
           

{babeldoc-0.5.20 → babeldoc-0.5.22}/.github/workflows/publish-to-pypi.yml

@@ -35,7 +35,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v6.6.0
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
         with:
           python-version: "3.12"
           enable-cache: true
@@ -96,7 +96,7 @@ jobs:
           path: dist/
 
       - name: Publish distribution 📦 to PyPI
-        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
 
   publish-to-testpypi:
     name: Publish Python 🐍 distribution 📦 to TestPyPI
@@ -120,7 +120,7 @@ jobs:
           path: dist/
 
       - name: Publish distribution 📦 to TestPyPI
-        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
         with:
           repository-url: https://test.pypi.org/legacy/
 

{babeldoc-0.5.20 → babeldoc-0.5.22}/.github/workflows/test.yml

@@ -28,7 +28,7 @@ jobs:
           path: ~/.cache/babeldoc
           key: babeldoc-assets-${{ hashFiles('babeldoc/assets/embedding_assets_metadata.py') }}
       - name: Setup uv with Python ${{ matrix.python-version }}
-        uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v6.6.0
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true

{babeldoc-0.5.20 → babeldoc-0.5.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.20
+Version: 0.5.22
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -91,7 +91,7 @@ Description-Content-Type: text/markdown
 PDF scientific paper translation and bilingual comparison library.
 
 - **Online Service**: Beta version launched [Immersive Translate - BabelDOC](https://app.immersivetranslate.com/babel-doc/) Free usage quota is available; please refer to the FAQ section on the page for details.
-- **Self-deployment**: [PDFMathTranslate 2.0](https://github.com/PDFMathTranslate/PDFMathTranslate-next) support for BabelDOC, available for self-deployment + WebUI with more translation services.
+- **Self-deployment**: [PDFMathTranslate-next](https://github.com/PDFMathTranslate-next/PDFMathTranslate-next) support for BabelDOC, available for self-deployment + WebUI with more translation services.
 - Provides a simple [command line interface](#getting-started).
 - Provides a [Python API](#python-api).
 - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.

{babeldoc-0.5.20 → babeldoc-0.5.22}/README.md

@@ -35,7 +35,7 @@
 PDF scientific paper translation and bilingual comparison library.
 
 - **Online Service**: Beta version launched [Immersive Translate - BabelDOC](https://app.immersivetranslate.com/babel-doc/) Free usage quota is available; please refer to the FAQ section on the page for details.
-- **Self-deployment**: [PDFMathTranslate 2.0](https://github.com/PDFMathTranslate/PDFMathTranslate-next) support for BabelDOC, available for self-deployment + WebUI with more translation services.
+- **Self-deployment**: [PDFMathTranslate-next](https://github.com/PDFMathTranslate-next/PDFMathTranslate-next) support for BabelDOC, available for self-deployment + WebUI with more translation services.
 - Provides a simple [command line interface](#getting-started).
 - Provides a [Python API](#python-api).
 - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.

babeldoc-0.5.22/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.22"

{babeldoc-0.5.20 → babeldoc-0.5.22}/babeldoc/assets/assets.py

@@ -1,5 +1,6 @@
 import asyncio
 import hashlib
+import json
 import logging
 import threading
 import zipfile
@@ -7,6 +8,8 @@ from pathlib import Path
 
 import httpx
 from babeldoc.assets import embedding_assets_metadata
+from babeldoc.assets.embedding_assets_metadata import CMAP_METADATA
+from babeldoc.assets.embedding_assets_metadata import CMAP_URL_BY_UPSTREAM
 from babeldoc.assets.embedding_assets_metadata import DOC_LAYOUT_ONNX_MODEL_URL
 from babeldoc.assets.embedding_assets_metadata import (
     DOCLAYOUT_YOLO_DOCSTRUCTBENCH_IMGSZ1024ONNX_SHA3_256,
@@ -27,6 +30,11 @@ from tenacity import wait_exponential
 logger = logging.getLogger(__name__)
 
 
+_FASTEST_FONT_UPSTREAM_LOCK = asyncio.Lock()
+_FASTEST_FONT_UPSTREAM: str | None = None
+_FASTEST_FONT_METADATA: dict | None = None
+
+
 class ResultContainer:
     def __init__(self):
         self.result = None
@@ -140,9 +148,10 @@ async def get_font_metadata(
     return upstream, response.json()
 
 
-async def get_fastest_upstream_for_font(
-    client: httpx.AsyncClient | None = None, exclude_upstream: list[str] = None
-):
+async def _get_fastest_upstream_for_font_internal(
+    client: httpx.AsyncClient | None = None, exclude_upstream: list[str] | None = None
+) -> tuple[str | None, dict | None]:
+    """Find the fastest upstream for font metadata without using cached result."""
     tasks: list[asyncio.Task[tuple[str, dict]]] = []
     for upstream in FONT_METADATA_URL:
         if exclude_upstream and upstream in exclude_upstream:
@@ -161,6 +170,34 @@ async def get_fastest_upstream_for_font(
     return None, None
 
 
+async def get_fastest_upstream_for_font(
+    client: httpx.AsyncClient | None = None, exclude_upstream: list[str] | None = None
+) -> tuple[str | None, dict | None]:
+    """Get the fastest upstream for font metadata with cached result.
+
+    The cached upstream is only used when exclude_upstream is None.
+    """
+    global _FASTEST_FONT_UPSTREAM, _FASTEST_FONT_METADATA
+
+    if exclude_upstream is None and _FASTEST_FONT_UPSTREAM is not None:
+        return _FASTEST_FONT_UPSTREAM, _FASTEST_FONT_METADATA
+
+    if exclude_upstream is not None:
+        # Do not use or update cache when exclude_upstream is provided.
+        return await _get_fastest_upstream_for_font_internal(client, exclude_upstream)
+
+    async with _FASTEST_FONT_UPSTREAM_LOCK:
+        if _FASTEST_FONT_UPSTREAM is not None:
+            return _FASTEST_FONT_UPSTREAM, _FASTEST_FONT_METADATA
+
+        upstream, metadata = await _get_fastest_upstream_for_font_internal(client)
+        if upstream is not None:
+            _FASTEST_FONT_UPSTREAM = upstream
+            _FASTEST_FONT_METADATA = metadata
+            logger.info(f"Fastest font upstream determined: {upstream}")
+        return upstream, metadata
+
+
 async def get_fastest_upstream_for_model(client: httpx.AsyncClient | None = None):
     return await get_fastest_upstream_for_font(client, exclude_upstream=["github"])
 
@@ -289,6 +326,72 @@ def get_font_and_metadata(font_file_name: str):
     return run_coro(get_font_and_metadata_async(font_file_name))
 
 
+async def get_cmap_file_path_async(
+    name: str, client: httpx.AsyncClient | None = None
+) -> Path:
+    """Get cached cmap file path, downloading it if necessary."""
+    if name.endswith(".json"):
+        file_name = name
+    else:
+        file_name = f"{name}.json"
+
+    if file_name not in CMAP_METADATA:
+        logger.critical(f"CMap {file_name} not found in CMAP_METADATA")
+        exit(1)
+
+    meta = CMAP_METADATA[file_name]
+    cache_file_path = get_cache_file_path(file_name, "cmap")
+    if verify_file(cache_file_path, meta["sha3_256"]):
+        return cache_file_path
+
+    logger.info(f"CMap {cache_file_path} not found or corrupted, downloading...")
+    await download_cmap_file_async(file_name, client)
+    if not verify_file(cache_file_path, meta["sha3_256"]):
+        logger.critical(f"Failed to verify downloaded cmap file: {cache_file_path}")
+        exit(1)
+    return cache_file_path
+
+
+async def download_cmap_file_async(
+    file_name: str, client: httpx.AsyncClient | None = None
+) -> Path:
+    """Download a single cmap file to cache directory."""
+    if file_name not in CMAP_METADATA:
+        logger.critical(f"CMap {file_name} not found in CMAP_METADATA")
+        exit(1)
+
+    fastest_upstream, _ = await get_fastest_upstream_for_font(client)
+    if fastest_upstream is None:
+        logger.critical("Failed to get fastest upstream for cmap")
+        exit(1)
+
+    if fastest_upstream not in CMAP_URL_BY_UPSTREAM:
+        logger.critical(f"Invalid fastest upstream for cmap: {fastest_upstream}")
+        exit(1)
+
+    url = CMAP_URL_BY_UPSTREAM[fastest_upstream](file_name)
+    cache_file_path = get_cache_file_path(file_name, "cmap")
+    sha3_256 = CMAP_METADATA[file_name]["sha3_256"]
+    await download_file(client, url, cache_file_path, sha3_256)
+    return cache_file_path
+
+
+async def get_cmap_data_async(
+    name: str, client: httpx.AsyncClient | None = None
+) -> dict:
+    """Load cmap json data from cached file, downloading it if necessary."""
+    path = await get_cmap_file_path_async(name, client)
+    return json.loads(path.read_text())
+
+
+def get_cmap_file_path(name: str):
+    return run_coro(get_cmap_file_path_async(name))
+
+
+def get_cmap_data(name: str):
+    return run_coro(get_cmap_data_async(name))
+
+
 def get_font_family(lang_code: str):
     font_family = embedding_assets_metadata.get_font_family(lang_code)
     return font_family
@@ -322,6 +425,31 @@ async def download_all_fonts_async(client: httpx.AsyncClient | None = None):
     await asyncio.gather(*font_tasks)
 
 
+async def download_all_cmaps_async(client: httpx.AsyncClient | None = None):
+    """Download all cmap files defined in CMAP_METADATA."""
+    for cmap_file_name, meta in CMAP_METADATA.items():
+        if not verify_file(
+            get_cache_file_path(cmap_file_name, "cmap"),
+            meta["sha3_256"],
+        ):
+            break
+    else:
+        logger.debug("All cmaps are already downloaded")
+        return
+
+    fastest_upstream, _ = await get_fastest_upstream_for_font(client)
+    if fastest_upstream is None:
+        logger.error("Failed to get fastest upstream for cmap")
+        exit(1)
+    logger.info(f"Downloading cmaps from {fastest_upstream}")
+
+    cmap_tasks = [
+        asyncio.create_task(get_cmap_file_path_async(cmap_file_name, client))
+        for cmap_file_name in CMAP_METADATA
+    ]
+    await asyncio.gather(*cmap_tasks)
+
+
 async def async_warmup():
     logger.info("Downloading all assets...")
     from tiktoken import encoding_for_model
@@ -333,7 +461,8 @@ async def async_warmup():
             get_table_detection_rapidocr_model_path_async(client)
         )
         font_tasks = asyncio.create_task(download_all_fonts_async(client))
-        await asyncio.gather(onnx_task, onnx_task2, font_tasks)
+        cmap_tasks = asyncio.create_task(download_all_cmaps_async(client))
+        await asyncio.gather(onnx_task, onnx_task2, font_tasks, cmap_tasks)
 
 
 def warmup():
@@ -341,10 +470,11 @@ def warmup():
 
 
 def generate_all_assets_file_list():
-    result = {}
+    result: dict[str, list[dict[str, str]]] = {}
     result["fonts"] = []
     result["models"] = []
     result["tiktoken"] = []
+    result["cmap"] = []
     for font_file_name in EMBEDDING_FONT_METADATA:
         result["fonts"].append(
             {
@@ -352,6 +482,13 @@ def generate_all_assets_file_list():
                 "sha3_256": EMBEDDING_FONT_METADATA[font_file_name]["sha3_256"],
             }
         )
+    for cmap_file_name in CMAP_METADATA:
+        result["cmap"].append(
+            {
+                "name": cmap_file_name,
+                "sha3_256": CMAP_METADATA[cmap_file_name]["sha3_256"],
+            }
+        )
     for tiktoken_file, sha3_256 in TIKTOKEN_CACHES.items():
         result["tiktoken"].append(
             {