BabelDOC 0.2.32__tar.gz → 0.3.0__tar.gz

{babeldoc-0.2.32 → babeldoc-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.2.32
+Version: 0.3.0
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -187,6 +187,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--max-pages-per-part`: Maximum number of pages per part for split translation. If not set, no splitting will be performed.
 - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
 - `--translate-table-text`: Translate table text (experimental, default: False)
+- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
 
 > [!TIP]
 > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -194,6 +195,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 > - However, using `--skip-clean` will result in larger file sizes
 > - If you encounter any compatibility issues, try using `--enhance-compatibility` first
 > - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
+> - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
 
 ### Translation Service Options
 
@@ -268,6 +270,7 @@ use-alternating-pages-dual = false
 watermark-output-mode = "watermarked"  # Choices: "watermarked", "no_watermark", "both"
 max-pages-per-part = 50  # Automatically split the document for translation and merge it back.
 # no-watermark = false  # DEPRECATED: Use watermark-output-mode instead
+skip-scanned-detection = false  # Skip scanned document detection for faster processing
 
 # Translation service
 openai = true

{babeldoc-0.2.32 → babeldoc-0.3.0}/README.md

@@ -141,6 +141,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--max-pages-per-part`: Maximum number of pages per part for split translation. If not set, no splitting will be performed.
 - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
 - `--translate-table-text`: Translate table text (experimental, default: False)
+- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
 
 > [!TIP]
 > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -148,6 +149,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 > - However, using `--skip-clean` will result in larger file sizes
 > - If you encounter any compatibility issues, try using `--enhance-compatibility` first
 > - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
+> - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
 
 ### Translation Service Options
 
@@ -222,6 +224,7 @@ use-alternating-pages-dual = false
 watermark-output-mode = "watermarked"  # Choices: "watermarked", "no_watermark", "both"
 max-pages-per-part = 50  # Automatically split the document for translation and merge it back.
 # no-watermark = false  # DEPRECATED: Use watermark-output-mode instead
+skip-scanned-detection = false  # Skip scanned document detection for faster processing
 
 # Translation service
 openai = true

babeldoc-0.3.0/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.2.32"
+__version__ = "0.3.0"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/document_il/midend/il_translator.py

@@ -181,6 +181,7 @@ class ILTranslator:
                 for font in xobj.pdf_font:
                     page_xobj_font_map[xobj.xobj_id][font.font_id] = font
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
+            paragraph_token_count = self.calc_token_count(paragraph.unicode)
             executor.submit(
                 self.translate_paragraph,
                 paragraph,
@@ -188,7 +189,8 @@ class ILTranslator:
                 tracker.new_paragraph(),
                 page_font_map,
                 page_xobj_font_map,
-                priority=1048576 - self.calc_token_count(paragraph.unicode),
+                priority=1048576 - paragraph_token_count,
+                paragraph_token_count=paragraph_token_count,
             )
 
     class TranslateInput:
@@ -597,6 +599,7 @@ class ILTranslator:
         tracker: ParagraphTranslateTracker = None,
         page_font_map: dict[str, PdfFont] = None,
         xobj_font_map: dict[int, dict[str, PdfFont]] = None,
+        paragraph_token_count: int = 0,
     ):
         """Translate a paragraph using pre and post processing functions."""
         self.translation_config.raise_if_cancelled()
@@ -610,7 +613,10 @@ class ILTranslator:
                     return
 
                 # Perform translation
-                translated_text = self.translate_engine.translate(text)
+                translated_text = self.translate_engine.translate(
+                    text,
+                    rate_limit_params={"paragraph_token_count": paragraph_token_count},
+                )
                 translated_text = re.sub(r"[. 。…，]{20,}", ".", translated_text)
 
                 # Post-translation processing

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -179,6 +179,7 @@ class ILTranslatorLLMOnly:
                     self.translation_config.shared_context_cross_split_part.recent_title_paragraph,
                     executor2,
                     priority=1048576 - total_token_count,
+                    paragraph_token_count=total_token_count,
                 )
                 paragraphs = []
                 total_token_count = 0
@@ -194,6 +195,7 @@ class ILTranslatorLLMOnly:
                 self.translation_config.shared_context_cross_split_part.recent_title_paragraph,
                 executor2,
                 priority=1048576 - total_token_count,
+                paragraph_token_count=total_token_count,
             )
 
     def translate_paragraph(
@@ -205,6 +207,7 @@ class ILTranslatorLLMOnly:
         title_paragraph: PdfParagraph | None = None,
         local_title_paragraph: PdfParagraph | None = None,
         executor: PriorityThreadPoolExecutor | None = None,
+        paragraph_token_count: int = 0,
     ):
         """Translate a paragraph using pre and post processing functions."""
         self.translation_config.raise_if_cancelled()
@@ -293,7 +296,10 @@ class ILTranslatorLLMOnly:
             llm_input.append(prompt_template)
 
             final_input = "\n".join(llm_input).strip()
-            llm_output = self.translate_engine.llm_translate(final_input)
+            llm_output = self.translate_engine.llm_translate(
+                final_input,
+                rate_limit_params={"paragraph_token_count": paragraph_token_count},
+            )
             llm_output = llm_output.strip()
 
             llm_output = self._clean_json_output(llm_output)
@@ -364,6 +370,9 @@ class ILTranslatorLLMOnly:
                         logger.warning(
                             f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
                         )
+                        paragraph_token_count = self.calc_token_count(
+                            inputs[id_][2].unicode
+                        )
                         executor.submit(
                             self.il_translator.translate_paragraph,
                             inputs[id_][2],
@@ -371,7 +380,8 @@ class ILTranslatorLLMOnly:
                             inputs[id_][3],
                             page_font_map,
                             xobj_font_map,
-                            priority=1048576 - self.calc_token_count(paragraph.unicode),
+                            priority=1048576 - paragraph_token_count,
+                            paragraph_token_count=paragraph_token_count,
                         )
 
         except Exception as e:
@@ -385,6 +395,7 @@ class ILTranslatorLLMOnly:
                 tracker = batch_paragraph.trackers[i]
                 if paragraph.debug_id is None:
                     continue
+                paragraph_token_count = self.calc_token_count(paragraph.unicode)
                 executor.submit(
                     self.il_translator.translate_paragraph,
                     paragraph,
@@ -392,7 +403,8 @@ class ILTranslatorLLMOnly:
                     tracker,
                     page_font_map,
                     xobj_font_map,
-                    priority=1048576 - self.calc_token_count(paragraph.unicode),
+                    priority=1048576 - paragraph_token_count,
+                    paragraph_token_count=paragraph_token_count,
                 )
 
     def _clean_json_output(self, llm_output: str) -> str:

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -541,6 +541,8 @@ class StylesAndFormulas:
             r"|Times-Italic.*"
             r"|MeridienLTStd.*"
             r"|Calibri.*"
+            r"|STIXMathJax_Main-Regular.*"
+            r"|STIXMathJax_Main-Italic.*"
             r")$"
         )
         if self.translation_config.formular_font_pattern:

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/document_il/translator/translator.py

@@ -104,7 +104,7 @@ class BaseTranslator(ABC):
         """
         self.cache.add_params(k, v)
 
-    def translate(self, text, ignore_cache=False):
+    def translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
         """
         Translate the text, and the other part should call this method.
         :param text: text to translate
@@ -117,12 +117,12 @@ class BaseTranslator(ABC):
                 self.translate_cache_call_count += 1
                 return cache
         _translate_rate_limiter.wait()
-        translation = self.do_translate(text)
+        translation = self.do_translate(text, rate_limit_params)
         if not (self.ignore_cache or ignore_cache):
             self.cache.set(text, translation)
         return translation
 
-    def llm_translate(self, text, ignore_cache=False):
+    def llm_translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
         """
         Translate the text, and the other part should call this method.
         :param text: text to translate
@@ -135,13 +135,13 @@ class BaseTranslator(ABC):
                 self.translate_cache_call_count += 1
                 return cache
         _translate_rate_limiter.wait()
-        translation = self.do_llm_translate(text)
+        translation = self.do_llm_translate(text, rate_limit_params)
         if not (self.ignore_cache or ignore_cache):
             self.cache.set(text, translation)
         return translation
 
     @abstractmethod
-    def do_llm_translate(self, text):
+    def do_llm_translate(self, text, rate_limit_params: dict = None):
         """
         Actual translate text, override this method
         :param text: text to translate
@@ -150,7 +150,7 @@ class BaseTranslator(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def do_translate(self, text):
+    def do_translate(self, text, rate_limit_params: dict = None):
         """
         Actual translate text, override this method
         :param text: text to translate
@@ -209,7 +209,7 @@ class OpenAITranslator(BaseTranslator):
             f"(Attempt {retry_state.attempt_number}/100)"
         ),
     )
-    def do_translate(self, text) -> str:
+    def do_translate(self, text, rate_limit_params: dict = None) -> str:
         response = self.client.chat.completions.create(
             model=self.model,
             **self.options,
@@ -241,7 +241,7 @@ class OpenAITranslator(BaseTranslator):
             f"(Attempt {retry_state.attempt_number}/100)"
         ),
     )
-    def do_llm_translate(self, text):
+    def do_llm_translate(self, text, rate_limit_params: dict = None):
         if text is None:
             return None
 

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/high_level.py

@@ -431,6 +431,10 @@ def do_translate(
                                     (x, x) for x in should_translate_pages
                                 ]
 
+                                # Only first part should do scanned detection if enabled
+                                if i > 0:
+                                    part_config.skip_scanned_detection = True
+
                                 part_config.working_dir = (
                                     translation_config.get_part_working_dir(i)
                                 )
@@ -576,14 +580,17 @@ def _do_translate_single(
     # [Previous implementation of do_translate continues here]
 
     # 检测是否为扫描文件
-    logger.debug("start detect scanned file")
-    DetectScannedFile(translation_config).process(docs)
-    logger.debug("finish detect scanned file")
-    if translation_config.debug:
-        xml_converter.write_json(
-            docs,
-            translation_config.get_working_file_path("detect_scanned_file.json"),
-        )
+    if translation_config.skip_scanned_detection:
+        logger.debug("skipping scanned file detection")
+    else:
+        logger.debug("start detect scanned file")
+        DetectScannedFile(translation_config).process(docs)
+        logger.debug("finish detect scanned file")
+        if translation_config.debug:
+            xml_converter.write_json(
+                docs,
+                translation_config.get_working_file_path("detect_scanned_file.json"),
+            )
 
     # Generate layouts for all pages
     logger.debug("start generating layouts")

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.2.32"
+__version__ = "0.3.0"
 
 
 def create_parser():
@@ -204,6 +204,12 @@ def create_parser():
         default=False,
         help="Show character box (debug only)",
     )
+    translation_group.add_argument(
+        "--skip-scanned-detection",
+        action="store_true",
+        default=False,
+        help="Skip scanned document detection (speeds up processing for non-scanned documents)",
+    )
     # service option argument group
     service_group = translation_group.add_mutually_exclusive_group()
     service_group.add_argument(
@@ -370,6 +376,7 @@ async def main():
             split_strategy=split_strategy,
             table_model=table_model,
             show_char_box=args.show_char_box,
+            skip_scanned_detection=args.skip_scanned_detection,
         )
 
         # Create progress handler

{babeldoc-0.2.32 → babeldoc-0.3.0}/babeldoc/translation_config.py

@@ -67,6 +67,7 @@ class TranslationConfig:
         split_strategy: BaseSplitStrategy | None = None,
         table_model=None,
         show_char_box: bool = False,
+        skip_scanned_detection: bool = False,
     ):
         self.translator = translator
 
@@ -97,6 +98,7 @@ class TranslationConfig:
         self.doc_layout_model = doc_layout_model
 
         self.skip_clean = skip_clean or enhance_compatibility
+        self.skip_scanned_detection = skip_scanned_detection
 
         self.dual_translate_first = dual_translate_first or enhance_compatibility
         self.disable_rich_text_translate = (

{babeldoc-0.2.32 → babeldoc-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.2.32"
+version = "0.3.0"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -148,7 +148,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.2.32"
+current_version = "0.3.0"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.2.32 → babeldoc-0.3.0}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.2.32"
+version = "0.3.0"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.2.32/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.32"