BabelDOC 0.3.62__tar.gz → 0.3.64__tar.gz

{babeldoc-0.3.62 → babeldoc-0.3.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.62
+Version: 0.3.64
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -210,7 +210,8 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--show-char-box`: Show character bounding boxes (debug only, default: False)
 - `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
 - `--ocr-workaround`: Use OCR workaround (default: False). Only suitable for documents with black text on white background. When enabled, white rectangular blocks will be added below the translation to cover the original text content, and all text will be forced to black color.
-
+- `--auto-enable-ocr-workaround`: Enable automatic OCR workaround (default: False). If a document is detected as heavily scanned, this will attempt to enable OCR processing and skip further scan detection. See "Important Interaction Note" below for crucial details on how this interacts with `--ocr-workaround` and `--skip-scanned-detection`.
+- `--primary-font-family`: Override primary font family for translated text. Choices: 'serif' for serif fonts, 'sans-serif' for sans-serif fonts, 'script' for script/italic fonts. If not specified, uses automatic font selection based on original text properties.
 
 - `--rpc-doclayout`: RPC service host address for document layout analysis (default: None)
 - `--working-dir`: Working directory for translation. If not set, use temp directory.
@@ -328,6 +329,7 @@ show_char_box = false # Show character bounding boxes (debug)
 ocr_workaround = false # Use OCR workaround for scanned PDFs
 rpc_doclayout = "" # RPC service host for document layout analysis
 working_dir = "" # Working directory for translation
+auto_enable_ocr_workaround = false # Enable automatic OCR workaround for scanned PDFs. See docs for interaction with ocr_workaround and skip_scanned_detection.
 
 # Translation service
 openai = true
@@ -477,4 +479,15 @@ Everyone interacting in YADT and its sub-projects' codebases, issue trackers, ch
    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=funstory-ai/babeldoc&type=Date" />
    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=funstory-ai/babeldoc&type=Date"/>
  </picture>
-</a>
+</a>
+
+> [!WARNING]
+> **Important Interaction Note for `--auto-enable-ocr-workaround`:**
+>
+> When `--auto-enable-ocr-workaround` is set to `true` (either via command line or config file):
+>
+> 1.  During the initial setup, the values for `ocr_workaround` and `skip_scanned_detection` will be forced to `false` by `TranslationConfig`, regardless of whether you also set `--ocr-workaround` or `--skip-scanned-detection` flags.
+> 2.  Then, during the scanned document detection phase (`DetectScannedFile` stage):
+>     *   If the document is identified as heavily scanned (e.g., >80% scanned pages) AND `auto_enable_ocr_workaround` is `true` (i.e., `translation_config.auto_enable_ocr_workaround` is true), the system will then attempt to set both `ocr_workaround` to `true` and `skip_scanned_detection` to `true`.
+>
+> This means that `--auto-enable-ocr-workaround` effectively gives the system control to enable OCR processing for scanned documents, potentially overriding manual settings for `--ocr-workaround` and `--skip_scanned_detection` based on its detection results. If the document is *not* detected as heavily scanned, then the initial `false` values for `ocr_workaround` and `skip_scanned_detection` (forced by `--auto-enable-ocr-workaround` at the `TranslationConfig` initialization stage) will remain in effect unless changed by other logic.

{babeldoc-0.3.62 → babeldoc-0.3.64}/README.md

@@ -159,7 +159,8 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--show-char-box`: Show character bounding boxes (debug only, default: False)
 - `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
 - `--ocr-workaround`: Use OCR workaround (default: False). Only suitable for documents with black text on white background. When enabled, white rectangular blocks will be added below the translation to cover the original text content, and all text will be forced to black color.
-
+- `--auto-enable-ocr-workaround`: Enable automatic OCR workaround (default: False). If a document is detected as heavily scanned, this will attempt to enable OCR processing and skip further scan detection. See "Important Interaction Note" below for crucial details on how this interacts with `--ocr-workaround` and `--skip-scanned-detection`.
+- `--primary-font-family`: Override primary font family for translated text. Choices: 'serif' for serif fonts, 'sans-serif' for sans-serif fonts, 'script' for script/italic fonts. If not specified, uses automatic font selection based on original text properties.
 
 - `--rpc-doclayout`: RPC service host address for document layout analysis (default: None)
 - `--working-dir`: Working directory for translation. If not set, use temp directory.
@@ -277,6 +278,7 @@ show_char_box = false # Show character bounding boxes (debug)
 ocr_workaround = false # Use OCR workaround for scanned PDFs
 rpc_doclayout = "" # RPC service host for document layout analysis
 working_dir = "" # Working directory for translation
+auto_enable_ocr_workaround = false # Enable automatic OCR workaround for scanned PDFs. See docs for interaction with ocr_workaround and skip_scanned_detection.
 
 # Translation service
 openai = true
@@ -426,4 +428,15 @@ Everyone interacting in YADT and its sub-projects' codebases, issue trackers, ch
    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=funstory-ai/babeldoc&type=Date" />
    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=funstory-ai/babeldoc&type=Date"/>
  </picture>
-</a>
+</a>
+
+> [!WARNING]
+> **Important Interaction Note for `--auto-enable-ocr-workaround`:**
+>
+> When `--auto-enable-ocr-workaround` is set to `true` (either via command line or config file):
+>
+> 1.  During the initial setup, the values for `ocr_workaround` and `skip_scanned_detection` will be forced to `false` by `TranslationConfig`, regardless of whether you also set `--ocr-workaround` or `--skip-scanned-detection` flags.
+> 2.  Then, during the scanned document detection phase (`DetectScannedFile` stage):
+>     *   If the document is identified as heavily scanned (e.g., >80% scanned pages) AND `auto_enable_ocr_workaround` is `true` (i.e., `translation_config.auto_enable_ocr_workaround` is true), the system will then attempt to set both `ocr_workaround` to `true` and `skip_scanned_detection` to `true`.
+>
+> This means that `--auto-enable-ocr-workaround` effectively gives the system control to enable OCR processing for scanned documents, potentially overriding manual settings for `--ocr-workaround` and `--skip_scanned_detection` based on its detection results. If the document is *not* detected as heavily scanned, then the initial `false` values for `ocr_workaround` and `skip_scanned_detection` (forced by `--auto-enable-ocr-workaround` at the `TranslationConfig` initialization stage) will remain in effect unless changed by other logic.

babeldoc-0.3.64/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.64"

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/assets/embedding_assets_metadata.py

@@ -49,7 +49,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Go Noto Kurrent-Bold Bold",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "000b37f592477945b27b7702dcad39f73e23e140e66ddff9847eb34f32389566",
         "size": 15303772,
     },
@@ -62,7 +62,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Go Noto Kurrent-Regular Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "4324a60d507c691e6efc97420647f4d2c2d86d9de35009d1c769861b76074ae6",
         "size": 15515760,
     },
@@ -75,7 +75,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Klee One Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "8585c29f89b322d937f83739f61ede5d84297873e1465cad9a120a208ac55ce0",
         "size": 8724704,
     },
@@ -88,7 +88,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "LXGW WenKai GB Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "b563a5e8d9db4cd15602a3a3700b01925e80a21f99fb88e1b763b1fb8685f8ee",
         "size": 19558756,
     },
@@ -101,7 +101,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "LXGW WenKai Mono TC Regular",
         "italic": 0,
         "monospace": 1,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "596b278d11418d374a1cfa3a50cbfb82b31db82d3650cfacae8f94311b27fdc5",
         "size": 13115416,
     },
@@ -114,7 +114,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "LXGW WenKai TC Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "66ccd0ffe8e56cd585dabde8d1292c3f551b390d8ed85f81d7a844825f9c2379",
         "size": 13100328,
     },
@@ -127,7 +127,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "MaruBuri Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "abb672dde7b89e06914ce27c59159b7a2933f26207bfcc47981c67c11c41e6d1",
         "size": 3268988,
     },
@@ -140,7 +140,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Noto Sans Bold",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "ecd38d472c1cad07d8a5dffd2b5a0f72edcd40fff2b4e68d770da8f2ef343a82",
         "size": 630964,
     },
@@ -153,7 +153,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Noto Sans Bold Italic",
         "italic": 1,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "0b6c690a4a6b7d605b2ecbde00c7ac1a23e60feb17fa30d8b972d61ec3ff732b",
         "size": 644340,
     },
@@ -166,7 +166,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Noto Sans Italic",
         "italic": 1,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "830652f61724c017e5a29a96225b484a2ccbd25f69a1b3f47e5f466a2dbed1ad",
         "size": 642344,
     },
@@ -179,7 +179,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Noto Sans Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "7dfe2bbf97dc04c852d1223b220b63430e6ad03b0dbb28ebe6328a20a2d45eb8",
         "size": 629024,
     },
@@ -244,7 +244,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans CN Bold",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "82314c11016a04ef03e7afd00abe0ccc8df54b922dee79abf6424f3002a31825",
         "size": 10174460,
     },
@@ -257,7 +257,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans CN Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "b45a80cf3650bfc62aa014e58243c6325e182c4b0c5819e41a583c699cce9a8f",
         "size": 10397552,
     },
@@ -270,7 +270,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans HK Bold",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "3eecd57457ba9a0fbad6c794f40e7ae704c4f825091aef2ac18902ffdde50608",
         "size": 6856692,
     },
@@ -283,7 +283,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans HK Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "5fe4141f9164c03616323400b2936ee4c8265314492e2b822c3a6fbfb63ffe08",
         "size": 6999792,
     },
@@ -296,7 +296,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans JP Bold",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "fb05bd84d62e8064117ee357ab6a4481e1cde931e8e984c0553c8c4b09dc3938",
         "size": 5603068,
     },
@@ -309,7 +309,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans JP Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "722cfbdcc0fd83fe07a3d1b10e9e64343c924a351d02cfe8dbb6ec4c6bc38230",
         "size": 5723960,
     },
@@ -322,7 +322,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans KR Bold",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "02959eb2c1eea0786a736aeb50b6e61f2ab873cd69c659389b7511f80f734838",
         "size": 5858892,
     },
@@ -335,7 +335,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans KR Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "aba70109eff718e8f796f0185f8dca38026c1661b43c195883c84577e501adf2",
         "size": 5961704,
     },
@@ -348,7 +348,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans TW Bold",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "4a92730e644a1348e87bba7c77e9b462f257f381bd6abbeac5860d8f8306aee6",
         "size": 6883224,
     },
@@ -361,7 +361,7 @@ EMBEDDING_FONT_METADATA = {
         "font_name": "Source Han Sans TW Regular",
         "italic": 0,
         "monospace": 0,
-        "serif": 1,
+        "serif": 0,
         "sha3_256": "6129b68ff4b0814624cac7edca61fbacf8f4d79db6f4c3cfc46b1c48ea2f81ac",
         "size": 7024812,
     },

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.62"
+__version__ = "0.3.64"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/document_il/backend/pdf_creater.py

@@ -99,6 +99,7 @@ def parse_truetype_data(data):
 TOUNICODE_HEAD = """\
 /CIDInit /ProcSet findresource begin
 12 dict begin
+begincmap
 /CIDSystemInfo <</Registry(Adobe)/Ordering(UCS)/Supplement 0>> def
 /CMapName /Adobe-Identity-UCS def
 /CMapType 2 def
@@ -124,7 +125,10 @@ def make_tounicode(cmap, used):
             if code < 0x10000:
                 line.append(f"<{glyph:04x}><{code:04x}>")
             else:
-                line.append(f"<{glyph:04x}><{code:08x}>")
+                code -= 0x10000
+                high = 0xD800 + (code >> 10)
+                low = 0xDC00 + (code & 0b1111111111)
+                line.append(f"<{glyph:04x}><{high:04x}{low:04x}>")
         line.append("endbfchar")
     line.append(TOUNICODE_TAIL)
     return "\n".join(line)

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/document_il/midend/detect_scanned_file.py

@@ -3,6 +3,7 @@ import logging
 import cv2
 import numpy as np
 import pymupdf
+import regex
 from skimage.metrics import structural_similarity
 
 from babeldoc.document_il import il_version_1
@@ -63,6 +64,20 @@ class DetectScannedFile:
             ),
         )
 
+    def fast_check(self, doc: pymupdf.Document) -> bool:
+        if doc:
+            hit_list = [0] * len(doc)
+            for page in doc:
+                contents_list = page.get_contents()
+                for index in contents_list:
+                    contents = doc.xref_stream(index)
+                    if regex.search(rb"/Artifact(\s*\<\<|\s+BDC)", contents):
+                        hit_list[page.number] += 1
+                    if regex.search(rb"\s3\s+Tr\s", contents):
+                        hit_list[page.number] += 1
+            return bool(sum(hit_list) > len(doc) * 0.8)
+        return False
+
     def process(self, docs: il_version_1.Document):
         """Generate layouts for all pages that need to be translated."""
         # Get pages that need to be translated
@@ -96,11 +111,20 @@ class DetectScannedFile:
                 progress.advance(1)
 
         if scanned > threshold:
-            logger.warning(
-                f"Detected {scanned} scanned pages, which is more than 80% of the total pages. "
-                "Please check the input PDF file.",
-            )
-            raise ScannedPDFError("Scanned PDF detected.")
+            if self.translation_config.auto_enable_ocr_workaround:
+                logger.warning(
+                    f"Detected {scanned} scanned pages, which is more than 80% of the total pages. "
+                    "Turning on OCR workaround.",
+                )
+                self.translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround = True
+                self.translation_config.ocr_workaround = True
+                self.translation_config.skip_scanned_detection = True
+            else:
+                logger.warning(
+                    f"Detected {scanned} scanned pages, which is more than 80% of the total pages. "
+                    "Please check the input PDF file.",
+                )
+                raise ScannedPDFError("Scanned PDF detected.")
 
     @staticmethod
     def detect_page_is_scanned(page: il_version_1.Page, pdf: pymupdf.Document) -> bool:
@@ -130,4 +154,4 @@ class DetectScannedFile:
         )[:, :, ::-1]
         before_page_image = cv2.cvtColor(before_page_image, cv2.COLOR_RGB2GRAY)
         after_page_image = cv2.cvtColor(after_page_image, cv2.COLOR_RGB2GRAY)
-        return structural_similarity(before_page_image, after_page_image) > 0.98
+        return structural_similarity(before_page_image, after_page_image) > 0.97

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/document_il/midend/il_translator.py

@@ -867,7 +867,7 @@ class ILTranslator:
             "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
         )
         llm_input.append(
-            "4. Only output the translation result without explanations and annotations."
+            f"4. Only output the translation result in {self.translation_config.lang_out} without explanations and annotations."
         )
         llm_input.append(f"5. Translate text into {self.translation_config.lang_out}.")
         prompt_template = f"""

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -284,7 +284,7 @@ class ILTranslatorLLMOnly:
                 llm_prompt_parts.append(self.translation_config.custom_system_prompt)
             else:
                 llm_prompt_parts.append(
-                    f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
+                    f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}.\n"
                     "When translating, strictly follow the instructions below to ensure translation quality and preserve all formatting, tags, and placeholders:\n"
                 )
 
@@ -378,7 +378,7 @@ class ILTranslatorLLMOnly:
                 '1. You will receive a JSON object with entries containing "id" and "input" fields.'
             )
             llm_prompt_parts.append(
-                '2. Your task is to translate the value of "input" into zh-CN, while applying the rules above.'
+                f'2. Your task is to translate the value of "input" into {self.translation_config.lang_out}, while applying the rules above.'
             )
             llm_prompt_parts.append(
                 '3. Return a new JSON object with the same "id" and the translated "output" field.'

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/document_il/utils/fontmap.py

@@ -1,3 +1,4 @@
+import enum
 import functools
 import logging
 import re
@@ -12,11 +13,39 @@ from babeldoc.translation_config import TranslationConfig
 logger = logging.getLogger(__name__)
 
 
+class PrimaryFontFamily(enum.IntEnum):
+    SERIF = 1
+    SANS_SERIF = 2
+    SCRIPT = 3
+    NONE = 4
+
+    @classmethod
+    def from_str(cls, value: str):
+        if value == "serif":
+            return cls.SERIF
+        elif value == "sans-serif":
+            return cls.SANS_SERIF
+        elif value == "script":
+            return cls.SCRIPT
+        else:
+            return cls.NONE
+
+
 class FontMapper:
     stage_name = "Add Fonts"
 
     def __init__(self, translation_config: TranslationConfig):
         self.translation_config = translation_config
+        assert translation_config.primary_font_family in [
+            None,
+            "serif",
+            "sans-serif",
+            "script",
+        ]
+        self.primary_font_family = PrimaryFontFamily.from_str(
+            translation_config.primary_font_family,
+        )
+
         font_family = assets.get_font_family(translation_config.lang_out)
         self.font_file_names = []
         for k in (
@@ -141,6 +170,14 @@ class FontMapper:
             )
             return None
 
+        if self.primary_font_family == PrimaryFontFamily.SERIF:
+            serif = True
+        elif self.primary_font_family == PrimaryFontFamily.SANS_SERIF:
+            serif = False
+        elif self.primary_font_family == PrimaryFontFamily.SCRIPT:
+            serif = False
+            italic = True
+
         script_font_map_result = self.map_in_type(
             bold, italic, monospaced, serif, char_unicode, "script"
         )

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/docvision/doclayout.py

@@ -217,7 +217,7 @@ class OnnxModel(DocLayoutModel):
             with self.lock:
                 # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
                 pix = get_no_rotation_img(mupdf_doc[page.page_number])
-            image = np.fromstring(pix.samples, np.uint8).reshape(
+            image = np.frombuffer(pix.samples, np.uint8).reshape(
                 pix.height,
                 pix.width,
                 3,

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/docvision/rpc_doclayout.py

@@ -259,7 +259,7 @@ class RpcDocLayoutModel(DocLayoutModel):
         with self.lock:
             # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
             pix = get_no_rotation_img(mupdf_doc[page.page_number])
-        image = np.fromstring(pix.samples, np.uint8).reshape(
+        image = np.frombuffer(pix.samples, np.uint8).reshape(
             pix.height,
             pix.width,
             3,

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/docvision/table_detection/rapidocr.py

@@ -239,7 +239,7 @@ class RapidOCRModel:
             with self.lock:
                 # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
                 pix = get_no_rotation_img(mupdf_doc[page.page_number])
-            image = np.fromstring(pix.samples, np.uint8).reshape(
+            image = np.frombuffer(pix.samples, np.uint8).reshape(
                 pix.height,
                 pix.width,
                 3,

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/high_level.py

@@ -4,6 +4,7 @@ import hashlib
 import io
 import logging
 import pathlib
+import re
 import shutil
 import threading
 import time
@@ -21,6 +22,8 @@ from babeldoc.assets.assets import warmup
 from babeldoc.const import CACHE_FOLDER
 from babeldoc.converter import TranslateConverter
 from babeldoc.document_il import il_version_1
+from babeldoc.document_il.babeldoc_exception.BabelDOCException import ExtractTextError
+from babeldoc.document_il.babeldoc_exception.BabelDOCException import ScannedPDFError
 from babeldoc.document_il.backend.pdf_creater import SAVE_PDF_STAGE_NAME
 from babeldoc.document_il.backend.pdf_creater import SUBSET_FONT_STAGE_NAME
 from babeldoc.document_il.backend.pdf_creater import PDFCreater
@@ -185,7 +188,7 @@ def start_parse_il(
         # but in order to facilitate the migration of pdf2zh,
         # the relevant code is temporarily retained.
         # pix = doc_zh[page.pageno].get_pixmap()
-        # image = np.fromstring(pix.samples, np.uint8).reshape(
+        # image = np.frombuffer(pix.samples, np.uint8).reshape(
         #     pix.height, pix.width, 3
         # )[:, :, ::-1]
         # page_layout = model.predict(
@@ -590,11 +593,7 @@ def do_translate(
         result.original_pdf_path = translation_config.input_file
         result.peak_memory_usage = peak_memory_usage
 
-        # should fix macOS preview compatibility
-        # Although the issue of Windows Edge
-        # not being able to copy translated text can be fixed,
-        # the macOS preview is broken
-        # fix_cmap(result, translation_config)
+        fix_cmap(result, translation_config)
         try:
             migrate_toc(translation_config, result)
         except Exception as e:
@@ -684,12 +683,30 @@ def fix_media_box(doc: Document) -> None:
     return mediabox_data
 
 
+def check_cid_char(il: il_version_1.Document):
+    chars = []
+    for page in il.page:
+        chars.extend(page.pdf_character)
+
+    cid_count = 0
+    for char in chars:
+        if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+            cid_count += 1
+
+    return cid_count > len(chars) * 0.8
+
+
 def _do_translate_single(
     pm: ProgressMonitor,
     translation_config: TranslationConfig,
 ) -> TranslateResult:
     """Original translation logic for a single document or part"""
     translation_config.progress_monitor = pm
+
+    if translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround:
+        translation_config.ocr_workaround = True
+        translation_config.skip_scanned_detection = True
+
     original_pdf_path = translation_config.input_file
     if translation_config.debug:
         doc_input = Document(original_pdf_path)
@@ -725,6 +742,23 @@ def _do_translate_single(
 
     resfont = None
     doc_pdf2zh.save(temp_pdf_path)
+
+    if not translation_config.skip_scanned_detection and DetectScannedFile(
+        translation_config
+    ).fast_check(doc_pdf2zh):
+        if translation_config.auto_enable_ocr_workaround:
+            logger.warning(
+                "Fast scanned check hit, Turning on OCR workaround.",
+            )
+            translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround = True
+            translation_config.ocr_workaround = True
+            translation_config.skip_scanned_detection = True
+        else:
+            logger.warning(
+                "Fast scanned check hit, Please check the input PDF file.",
+            )
+            raise ScannedPDFError("Scanned PDF detected.")
+
     il_creater = ILCreater(translation_config)
     il_creater.mupdf = doc_pdf2zh
     xml_converter = XMLConverter()
@@ -747,6 +781,9 @@ def _do_translate_single(
             translation_config.get_working_file_path("create_il.debug.json"),
         )
 
+    if check_cid_char(docs):
+        raise ExtractTextError("The document contains too many CID chars.")
+
     # Rest of the original translation logic...
     # [Previous implementation of do_translate continues here]
 

{babeldoc-0.3.62 → babeldoc-0.3.64}/babeldoc/main.py

@@ -21,7 +21,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.62"
+__version__ = "0.3.64"
 
 
 def create_parser():
@@ -248,6 +248,19 @@ def create_parser():
         default=True,
         help="Disable automatic term extraction. (Config file: set auto_extract_glossary = false)",
     )
+    translation_group.add_argument(
+        "--auto-enable-ocr-workaround",
+        action="store_true",
+        default=False,
+        help="Enable automatic OCR workaround. If a document is detected as heavily scanned, this will attempt to enable OCR processing and skip further scan detection. Note: This option interacts with `--ocr-workaround` and `--skip-scanned-detection`. See documentation for details. (default: False)",
+    )
+    translation_group.add_argument(
+        "--primary-font-family",
+        type=str,
+        choices=["serif", "sans-serif", "script"],
+        default=None,
+        help="Override primary font family for translated text. Choices: 'serif' for serif fonts, 'sans-serif' for sans-serif fonts, 'script' for script/italic fonts. If not specified, uses automatic font selection based on original text properties.",
+    )
     # service option argument group
     service_group = translation_group.add_mutually_exclusive_group()
     service_group.add_argument(
@@ -468,6 +481,8 @@ async def main():
             glossaries=loaded_glossaries,
             pool_max_workers=args.pool_max_workers,
             auto_extract_glossary=args.auto_extract_glossary,
+            auto_enable_ocr_workaround=args.auto_enable_ocr_workaround,
+            primary_font_family=args.primary_font_family,
         )
 
         # Create progress handler
@@ -602,6 +617,7 @@ def cli():
             or v.name.startswith("httpx")
             or "http11" in v.name
             or "openai" in v.name
+            or "pdfminer" in v.name
         ):
             v.disabled = True
             v.propagate = False