BabelDOC 0.5.14__tar.gz → 0.5.16__tar.gz

{babeldoc-0.5.14 → babeldoc-0.5.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.14
+Version: 0.5.16
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.5.16/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.16"

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/const.py

@@ -6,7 +6,7 @@ import subprocess
 import threading
 from pathlib import Path
 
-__version__ = "0.5.14"
+__version__ = "0.5.16"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/docvision/rpc_doclayout6.py

@@ -58,17 +58,6 @@ def encode_image(image) -> bytes:
     return encoded
 
 
-@retry(
-    stop=stop_after_attempt(3),  # 最多重试 3 次
-    wait=wait_exponential(
-        multiplier=1, min=1, max=10
-    ),  # 指数退避策略，初始 1 秒，最大 10 秒
-    retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
-    before_sleep=lambda retry_state: logger.warning(
-        f"Request failed, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
-        f"(Attempt {retry_state.attempt_number}/3)"
-    ),
-)
 def clip_num(num: float, min_value: float, max_value: float) -> float:
     """Clip a number to a specified range."""
     if num < min_value:
@@ -78,6 +67,17 @@ def clip_num(num: float, min_value: float, max_value: float) -> float:
     return num
 
 
+@retry(
+    stop=stop_after_attempt(5),  # 最多重试 3 次
+    wait=wait_exponential(
+        multiplier=1, min=1, max=10
+    ),  # 指数退避策略，初始 1 秒，最大 10 秒
+    retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
+    before_sleep=lambda retry_state: logger.warning(
+        f"Request failed VLM, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
+        f"(Attempt {retry_state.attempt_number}/5)"
+    ),
+)
 def predict_layout(
     image,
     host: str = "http://localhost:8000",
@@ -186,14 +186,14 @@ def predict_layout(
 
 
 @retry(
-    stop=stop_after_attempt(3),  # 最多重试 3 次
+    stop=stop_after_attempt(5),  # 最多重试 3 次
     wait=wait_exponential(
         multiplier=1, min=1, max=10
     ),  # 指数退避策略，初始 1 秒，最大 10 秒
     retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
     before_sleep=lambda retry_state: logger.warning(
-        f"Request failed, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
-        f"(Attempt {retry_state.attempt_number}/3)"
+        f"Request failed PADDLE, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
+        f"(Attempt {retry_state.attempt_number}/5)"
     ),
 )
 def predict_layout2(
@@ -599,7 +599,7 @@ class RpcDocLayoutModel(DocLayoutModel):
     ):
         layout_temp_path = translate_config.get_working_file_path("layout.temp.pdf")
         mupdf_doc.save(layout_temp_path.as_posix())
-        with ThreadPoolExecutor(max_workers=8) as executor:
+        with ThreadPoolExecutor(max_workers=32) as executor:
             yield from executor.map(
                 self.predict_page,
                 pages,

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/format/pdf/document_il/midend/automatic_term_extractor.py

@@ -138,18 +138,24 @@ class AutomaticTermExtractor:
         except Exception:
             return 0
 
-    def _snapshot_token_usage(self) -> tuple[int, int, int]:
+    def _snapshot_token_usage(self) -> tuple[int, int, int, int]:
         if not self.translate_engine:
-            return 0, 0, 0
+            return 0, 0, 0, 0
         token_counter = getattr(self.translate_engine, "token_count", None)
         prompt_counter = getattr(self.translate_engine, "prompt_token_count", None)
         completion_counter = getattr(
             self.translate_engine, "completion_token_count", None
         )
+        cache_hit_prompt_counter = getattr(
+            self.translate_engine, "cache_hit_prompt_token_count", None
+        )
         total_tokens = token_counter.value if token_counter else 0
         prompt_tokens = prompt_counter.value if prompt_counter else 0
         completion_tokens = completion_counter.value if completion_counter else 0
-        return total_tokens, prompt_tokens, completion_tokens
+        cache_hit_prompt_tokens = (
+            cache_hit_prompt_counter.value if cache_hit_prompt_counter else 0
+        )
+        return total_tokens, prompt_tokens, completion_tokens, cache_hit_prompt_tokens
 
     def _clean_json_output(self, llm_output: str) -> str:
         llm_output = llm_output.strip()
@@ -327,7 +333,9 @@ class AutomaticTermExtractor:
 
     def procress(self, doc_il: ILDocument):
         logger.info(f"{self.stage_name}: Starting term extraction for document.")
-        start_total, start_prompt, start_completion = self._snapshot_token_usage()
+        start_total, start_prompt, start_completion, start_cache_hit_prompt = (
+            self._snapshot_token_usage()
+        )
         tracker = DocumentTermExtractTracker()
         total = sum(len(page.pdf_paragraph) for page in doc_il.page)
         with self.translation_config.progress_monitor.stage_start(
@@ -341,11 +349,14 @@ class AutomaticTermExtractor:
                     self.process_page(page, executor, pbar, tracker.new_page())
 
         self.shared_context.finalize_auto_extracted_glossary()
-        end_total, end_prompt, end_completion = self._snapshot_token_usage()
+        end_total, end_prompt, end_completion, end_cache_hit_prompt = (
+            self._snapshot_token_usage()
+        )
         self.translation_config.record_term_extraction_usage(
             end_total - start_total,
             end_prompt - start_prompt,
             end_completion - start_completion,
+            end_cache_hit_prompt - start_cache_hit_prompt,
         )
 
         if self.translation_config.debug:

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/format/pdf/document_il/midend/detect_scanned_file.py

@@ -131,6 +131,7 @@ class DetectScannedFile:
                 self.translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround = True
                 self.translation_config.ocr_workaround = True
                 self.translation_config.skip_scanned_detection = True
+                self.translation_config.disable_rich_text_translate = True
                 self.clean_render_order_for_chars(docs)
                 self.translation_config.remove_non_formula_lines = False
             else:

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/format/pdf/document_il/midend/il_translator.py

@@ -890,6 +890,38 @@ class ILTranslator:
                 f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
             ]
 
+        llm_input.append("When translating, please follow the following rules:")
+
+        rich_text_left_placeholder = (
+            self.translate_engine.get_rich_text_left_placeholder(1)
+        )
+        if isinstance(rich_text_left_placeholder, tuple):
+            rich_text_left_placeholder = rich_text_left_placeholder[0]
+        rich_text_right_placeholder = (
+            self.translate_engine.get_rich_text_right_placeholder(2)
+        )
+        if isinstance(rich_text_right_placeholder, tuple):
+            rich_text_right_placeholder = rich_text_right_placeholder[0]
+
+        # Create a structured prompt template for LLM translation
+        llm_input.append(
+            f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
+        )
+
+        formula_placeholder = self.translate_engine.get_formular_placeholder(3)
+        if isinstance(formula_placeholder, tuple):
+            formula_placeholder = formula_placeholder[0]
+
+        llm_input.append(
+            f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
+        )
+        llm_input.append(
+            "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
+        )
+        llm_input.append(
+            f"4. Only output the translation result in {self.translation_config.lang_out} without explanations and annotations."
+        )
+
         llm_context_hints = []
 
         if title_paragraph:
@@ -952,41 +984,8 @@ class ILTranslator:
                 for md_block in active_glossary_markdown_blocks:
                     llm_input.append(f"\n{md_block}\n")
 
-        llm_input.append("When translating, please follow the following rules:")
-
-        rich_text_left_placeholder = (
-            self.translate_engine.get_rich_text_left_placeholder(1)
-        )
-        if isinstance(rich_text_left_placeholder, tuple):
-            rich_text_left_placeholder = rich_text_left_placeholder[0]
-        rich_text_right_placeholder = (
-            self.translate_engine.get_rich_text_right_placeholder(2)
-        )
-        if isinstance(rich_text_right_placeholder, tuple):
-            rich_text_right_placeholder = rich_text_right_placeholder[0]
-
-        # Create a structured prompt template for LLM translation
-        llm_input.append(
-            f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
-        )
-
-        formula_placeholder = self.translate_engine.get_formular_placeholder(3)
-        if isinstance(formula_placeholder, tuple):
-            formula_placeholder = formula_placeholder[0]
-
-        llm_input.append(
-            f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
-        )
-        llm_input.append(
-            "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
-        )
-        llm_input.append(
-            f"4. Only output the translation result in {self.translation_config.lang_out} without explanations and annotations."
-        )
-        llm_input.append(f"5. Translate text into {self.translation_config.lang_out}.")
         prompt_template = f"""
 Now, please carefully read the following text to be translated and directly output your translation.\n\n{text}
-
 """
         llm_input.append(prompt_template)
 

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py

@@ -643,69 +643,6 @@ class ILTranslatorLLMOnly:
                     "When translating, strictly follow the instructions below to ensure translation quality and preserve all formatting, tags, and placeholders:\n"
                 )
 
-            # 2. ##Contextual Hints for Better Translation
-            contextual_hints_section: list[str] = []
-            hint_idx = 1
-            if title_paragraph:
-                contextual_hints_section.append(
-                    f"{hint_idx}. First title in full text: {title_paragraph.unicode}"
-                )
-                hint_idx += 1
-
-            if local_title_paragraph:
-                is_different_from_global = True
-                if title_paragraph:
-                    if local_title_paragraph.debug_id == title_paragraph.debug_id:
-                        is_different_from_global = False
-
-                if is_different_from_global:
-                    contextual_hints_section.append(
-                        f"{hint_idx}. Most similar section title: {local_title_paragraph.unicode}"
-                    )
-                    hint_idx += 1
-
-            # --- ADD GLOSSARY HINTS ---
-            batch_text_for_glossary_matching = "\n".join(
-                item.get("input", "") for item in json_format_input
-            )
-
-            active_glossary_markdown_blocks: list[str] = []
-            # Use cached glossaries
-            if self._cached_glossaries:
-                for glossary in self._cached_glossaries:
-                    # Get active entries for the current batch_text_for_glossary_matching
-                    active_entries = glossary.get_active_entries_for_text(
-                        batch_text_for_glossary_matching
-                    )
-
-                    if active_entries:
-                        current_glossary_md_entries: list[str] = []
-                        for original_source, target_text in sorted(active_entries):
-                            current_glossary_md_entries.append(
-                                f"| {original_source} | {target_text} |"
-                            )
-
-                        if current_glossary_md_entries:
-                            glossary_table_md = (
-                                f"### Glossary: {glossary.name}\n\n"
-                                "| Source Term | Target Term |\n"
-                                "|-------------|-------------|\n"
-                                + "\n".join(current_glossary_md_entries)
-                            )
-                            active_glossary_markdown_blocks.append(glossary_table_md)
-
-            if contextual_hints_section or active_glossary_markdown_blocks:
-                llm_prompt_parts.append("\n## Contextual Hints for Better Translation")
-                llm_prompt_parts.extend(contextual_hints_section)
-
-                if active_glossary_markdown_blocks:
-                    llm_prompt_parts.append(
-                        f"{hint_idx}. You MUST strictly adhere to the following glossaries. auto_extracted_glossary has a lower priority; please give preference to other glossaries. If a source term from a table appears in the text, use the corresponding target term in your translation:"
-                    )
-                    # hint_idx += 1 # No need to increment if tables are part of this point
-                    for md_block in active_glossary_markdown_blocks:
-                        llm_prompt_parts.append(f"\n{md_block}\n")
-
             # 3. ## Strict Rules:
             llm_prompt_parts.append("\n## Strict Rules:")
             llm_prompt_parts.append(
@@ -771,6 +708,69 @@ class ILTranslatorLLMOnly:
             llm_prompt_parts.append("```")
             llm_prompt_parts.append("</example>")
 
+            # 2. ##Contextual Hints for Better Translation
+            contextual_hints_section: list[str] = []
+            hint_idx = 1
+            if title_paragraph:
+                contextual_hints_section.append(
+                    f"{hint_idx}. First title in full text: {title_paragraph.unicode}"
+                )
+                hint_idx += 1
+
+            if local_title_paragraph:
+                is_different_from_global = True
+                if title_paragraph:
+                    if local_title_paragraph.debug_id == title_paragraph.debug_id:
+                        is_different_from_global = False
+
+                if is_different_from_global:
+                    contextual_hints_section.append(
+                        f"{hint_idx}. The most recent title is: {local_title_paragraph.unicode}"
+                    )
+                    hint_idx += 1
+
+            # --- ADD GLOSSARY HINTS ---
+            batch_text_for_glossary_matching = "\n".join(
+                item.get("input", "") for item in json_format_input
+            )
+
+            active_glossary_markdown_blocks: list[str] = []
+            # Use cached glossaries
+            if self._cached_glossaries:
+                for glossary in self._cached_glossaries:
+                    # Get active entries for the current batch_text_for_glossary_matching
+                    active_entries = glossary.get_active_entries_for_text(
+                        batch_text_for_glossary_matching
+                    )
+
+                    if active_entries:
+                        current_glossary_md_entries: list[str] = []
+                        for original_source, target_text in sorted(active_entries):
+                            current_glossary_md_entries.append(
+                                f"| {original_source} | {target_text} |"
+                            )
+
+                        if current_glossary_md_entries:
+                            glossary_table_md = (
+                                f"### Glossary: {glossary.name}\n\n"
+                                "| Source Term | Target Term |\n"
+                                "|-------------|-------------|\n"
+                                + "\n".join(current_glossary_md_entries)
+                            )
+                            active_glossary_markdown_blocks.append(glossary_table_md)
+
+            if contextual_hints_section or active_glossary_markdown_blocks:
+                llm_prompt_parts.append("\n## Contextual Hints for Better Translation")
+                llm_prompt_parts.extend(contextual_hints_section)
+
+                if active_glossary_markdown_blocks:
+                    llm_prompt_parts.append(
+                        f"{hint_idx}. You MUST strictly adhere to the following glossaries. please give preference to other glossaries. If a source term from a table appears in the text, use the corresponding target term in your translation:"
+                    )
+                    # hint_idx += 1 # No need to increment if tables are part of this point
+                    for md_block in active_glossary_markdown_blocks:
+                        llm_prompt_parts.append(f"\n{md_block}\n")
+
             # 6. ## Here is the input:
             llm_prompt_parts.append("\n## Here is the input:")
 

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/format/pdf/high_level.py

@@ -62,6 +62,7 @@ from babeldoc.pdfminer.pdfinterp import PDFResourceManager
 from babeldoc.pdfminer.pdfpage import PDFPage
 from babeldoc.pdfminer.pdfparser import PDFParser
 from babeldoc.progress_monitor import ProgressMonitor
+from babeldoc.utils import memory
 
 logger = logging.getLogger(__name__)
 
@@ -455,20 +456,10 @@ class MemoryMonitor:
         self.peak_memory_usage = 0
         self.monitor_thread = None
         self.stop_event = None
-
-        try:
-            import psutil
-
-            self.psutil = psutil
-        except ImportError:
-            logger.warning("psutil not installed, memory monitoring disabled")
-            self.psutil = None
+        self.last_pss_check_time = None
 
     def __enter__(self):
         """Start memory monitoring."""
-        if not self.psutil:
-            return self
-
         self.stop_event = threading.Event()
         self.monitor_thread = threading.Thread(
             target=self._monitor_memory_usage, daemon=True
@@ -479,7 +470,7 @@ class MemoryMonitor:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """Stop monitoring and log peak memory usage."""
-        if not self.psutil or not self.monitor_thread:
+        if not self.monitor_thread:
             return
 
         self.stop_event.set()
@@ -490,14 +481,15 @@ class MemoryMonitor:
         """Background thread that periodically checks memory usage."""
         while not self.stop_event.is_set():
             try:
-                current_process = self.psutil.Process()
-                # Get memory usage of current process and all children
-                total_memory = current_process.memory_info().rss
-                for child in current_process.children(recursive=True):
-                    try:
-                        total_memory += child.memory_info().rss
-                    except (self.psutil.NoSuchProcess, self.psutil.AccessDenied):
-                        pass
+                # Use throttled memory check with 2-second PSS throttle
+                total_memory, self.last_pss_check_time = (
+                    memory.get_memory_usage_with_throttle(
+                        include_children=True,
+                        prefer_pss=True,
+                        last_pss_check_time=self.last_pss_check_time,
+                        pss_throttle_seconds=2.0,
+                    )
+                )
 
                 # Convert to MB for better readability
                 total_memory_mb = total_memory / (1024 * 1024)
@@ -508,6 +500,10 @@ class MemoryMonitor:
 
             time.sleep(self.interval)
 
+    def get_peek_memory_psutil(self):
+        """Get peak memory usage using psutil (for backwards compatibility)."""
+        return memory.get_memory_usage_bytes(include_children=True, prefer_pss=True)
+
 
 def fix_null_page_content(doc: Document) -> list[int]:
     invalid_page = []

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/format/pdf/translation_config.py

@@ -227,6 +227,7 @@ class TranslationConfig:
 
         if self.ocr_workaround:
             self.skip_scanned_detection = True
+            self.disable_rich_text_translate = True
 
         # for backward compatibility
         if use_side_by_side_dual is False and use_alternating_pages_dual is False:
@@ -321,6 +322,7 @@ class TranslationConfig:
             "total_tokens": 0,
             "prompt_tokens": 0,
             "completion_tokens": 0,
+            "cache_hit_prompt_tokens": 0,
         }
 
         if self.ocr_workaround:
@@ -437,6 +439,7 @@ class TranslationConfig:
         total_tokens: int,
         prompt_tokens: int,
         completion_tokens: int,
+        cache_hit_prompt_tokens: int,
     ) -> None:
         """Accumulate token usage for automatic term extraction."""
         if total_tokens > 0:
@@ -445,6 +448,10 @@ class TranslationConfig:
             self.term_extraction_token_usage["prompt_tokens"] += prompt_tokens
         if completion_tokens > 0:
             self.term_extraction_token_usage["completion_tokens"] += completion_tokens
+        if cache_hit_prompt_tokens > 0:
+            self.term_extraction_token_usage["cache_hit_prompt_tokens"] += (
+                cache_hit_prompt_tokens
+            )
 
 
 class TranslateResult:

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/main.py

@@ -26,7 +26,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.5.14"
+__version__ = "0.5.16"
 
 
 def create_parser():
@@ -636,6 +636,7 @@ async def main():
     total_term_extraction_total_tokens = 0
     total_term_extraction_prompt_tokens = 0
     total_term_extraction_completion_tokens = 0
+    total_term_extraction_cache_hit_prompt_tokens = 0
 
     for file in pending_files:
         # 清理文件路径，去除两端的引号
@@ -721,21 +722,29 @@ async def main():
         total_term_extraction_total_tokens += usage["total_tokens"]
         total_term_extraction_prompt_tokens += usage["prompt_tokens"]
         total_term_extraction_completion_tokens += usage["completion_tokens"]
+        total_term_extraction_cache_hit_prompt_tokens += usage[
+            "cache_hit_prompt_tokens"
+        ]
     logger.info(f"Total tokens: {translator.token_count.value}")
     logger.info(f"Prompt tokens: {translator.prompt_token_count.value}")
     logger.info(f"Completion tokens: {translator.completion_token_count.value}")
     logger.info(
-        "Term extraction tokens: total=%s prompt=%s completion=%s",
+        f"Cache hit prompt tokens: {translator.cache_hit_prompt_token_count.value}"
+    )
+    logger.info(
+        "Term extraction tokens: total=%s prompt=%s completion=%s cache_hit_prompt=%s",
         total_term_extraction_total_tokens,
         total_term_extraction_prompt_tokens,
         total_term_extraction_completion_tokens,
+        total_term_extraction_cache_hit_prompt_tokens,
     )
     if term_extraction_translator is not translator:
         logger.info(
-            "Term extraction translator raw tokens: total=%s prompt=%s completion=%s",
+            "Term extraction translator raw tokens: total=%s prompt=%s completion=%s cache_hit_prompt=%s",
             term_extraction_translator.token_count.value,
             term_extraction_translator.prompt_token_count.value,
             term_extraction_translator.completion_token_count.value,
+            term_extraction_translator.cache_hit_prompt_token_count.value,
         )
 
 

{babeldoc-0.5.14 → babeldoc-0.5.16}/babeldoc/translator/translator.py

@@ -249,6 +249,7 @@ class OpenAITranslator(BaseTranslator):
         self.token_count = AtomicInteger()
         self.prompt_token_count = AtomicInteger()
         self.completion_token_count = AtomicInteger()
+        self.cache_hit_prompt_token_count = AtomicInteger()
 
     @retry(
         retry=retry_if_exception_type(openai.RateLimitError),
@@ -338,6 +339,10 @@ class OpenAITranslator(BaseTranslator):
                 self.prompt_token_count.inc(response.usage.prompt_tokens)
             if response.usage and response.usage.completion_tokens:
                 self.completion_token_count.inc(response.usage.completion_tokens)
+            if response.usage and (
+                hit_count := getattr(response.usage, "prompt_cache_hit_tokens", 0)
+            ):
+                self.cache_hit_prompt_token_count.inc(hit_count)
         except Exception as e:
             logger.exception("Error updating token count")