BabelDOC 0.5.15__tar.gz → 0.5.17__tar.gz

{babeldoc-0.5.15 → babeldoc-0.5.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.15
+Version: 0.5.17
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -90,7 +90,7 @@ Description-Content-Type: text/markdown
 
 PDF scientific paper translation and bilingual comparison library.
 
-- **Online Service**: Beta version launched [Immersive Translate - BabelDOC](https://app.immersivetranslate.com/babel-doc/) 1000 free pages per month.
+- **Online Service**: Beta version launched [Immersive Translate - BabelDOC](https://app.immersivetranslate.com/babel-doc/) Free usage quota is available; please refer to the FAQ section on the page for details.
 - **Self-deployment**: [PDFMathTranslate 2.0](https://github.com/PDFMathTranslate/PDFMathTranslate-next) support for BabelDOC, available for self-deployment + WebUI with more translation services.
 - Provides a simple [command line interface](#getting-started).
 - Provides a [Python API](#python-api).
@@ -489,4 +489,4 @@ Everyone interacting in YADT and its sub-projects' codebases, issue trackers, ch
 > 2.  Then, during the scanned document detection phase (`DetectScannedFile` stage):
 >     *   If the document is identified as heavily scanned (e.g., >80% scanned pages) AND `auto_enable_ocr_workaround` is `true` (i.e., `translation_config.auto_enable_ocr_workaround` is true), the system will then attempt to set both `ocr_workaround` to `true` and `skip_scanned_detection` to `true`.
 >
-> This means that `--auto-enable-ocr-workaround` effectively gives the system control to enable OCR processing for scanned documents, potentially overriding manual settings for `--ocr-workaround` and `--skip_scanned_detection` based on its detection results. If the document is *not* detected as heavily scanned, then the initial `false` values for `ocr_workaround` and `skip_scanned_detection` (forced by `--auto-enable-ocr-workaround` at the `TranslationConfig` initialization stage) will remain in effect unless changed by other logic.
+> This means that `--auto-enable-ocr-workaround` effectively gives the system control to enable OCR processing for scanned documents, potentially overriding manual settings for `--ocr-workaround` and `--skip_scanned_detection` based on its detection results. If the document is *not* detected as heavily scanned, then the initial `false` values for `ocr_workaround` and `skip_scanned_detection` (forced by `--auto-enable-ocr-workaround` at the `TranslationConfig` initialization stage) will remain in effect unless changed by other logic.

{babeldoc-0.5.15 → babeldoc-0.5.17}/README.md

@@ -34,7 +34,7 @@
 
 PDF scientific paper translation and bilingual comparison library.
 
-- **Online Service**: Beta version launched [Immersive Translate - BabelDOC](https://app.immersivetranslate.com/babel-doc/) 1000 free pages per month.
+- **Online Service**: Beta version launched [Immersive Translate - BabelDOC](https://app.immersivetranslate.com/babel-doc/) Free usage quota is available; please refer to the FAQ section on the page for details.
 - **Self-deployment**: [PDFMathTranslate 2.0](https://github.com/PDFMathTranslate/PDFMathTranslate-next) support for BabelDOC, available for self-deployment + WebUI with more translation services.
 - Provides a simple [command line interface](#getting-started).
 - Provides a [Python API](#python-api).
@@ -433,4 +433,4 @@ Everyone interacting in YADT and its sub-projects' codebases, issue trackers, ch
 > 2.  Then, during the scanned document detection phase (`DetectScannedFile` stage):
 >     *   If the document is identified as heavily scanned (e.g., >80% scanned pages) AND `auto_enable_ocr_workaround` is `true` (i.e., `translation_config.auto_enable_ocr_workaround` is true), the system will then attempt to set both `ocr_workaround` to `true` and `skip_scanned_detection` to `true`.
 >
-> This means that `--auto-enable-ocr-workaround` effectively gives the system control to enable OCR processing for scanned documents, potentially overriding manual settings for `--ocr-workaround` and `--skip_scanned_detection` based on its detection results. If the document is *not* detected as heavily scanned, then the initial `false` values for `ocr_workaround` and `skip_scanned_detection` (forced by `--auto-enable-ocr-workaround` at the `TranslationConfig` initialization stage) will remain in effect unless changed by other logic.
+> This means that `--auto-enable-ocr-workaround` effectively gives the system control to enable OCR processing for scanned documents, potentially overriding manual settings for `--ocr-workaround` and `--skip_scanned_detection` based on its detection results. If the document is *not* detected as heavily scanned, then the initial `false` values for `ocr_workaround` and `skip_scanned_detection` (forced by `--auto-enable-ocr-workaround` at the `TranslationConfig` initialization stage) will remain in effect unless changed by other logic.

babeldoc-0.5.17/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.17"

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/const.py

@@ -6,7 +6,7 @@ import subprocess
 import threading
 from pathlib import Path
 
-__version__ = "0.5.15"
+__version__ = "0.5.17"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/docvision/rpc_doclayout6.py

@@ -58,17 +58,6 @@ def encode_image(image) -> bytes:
     return encoded
 
 
-@retry(
-    stop=stop_after_attempt(3),  # 最多重试 3 次
-    wait=wait_exponential(
-        multiplier=1, min=1, max=10
-    ),  # 指数退避策略，初始 1 秒，最大 10 秒
-    retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
-    before_sleep=lambda retry_state: logger.warning(
-        f"Request failed, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
-        f"(Attempt {retry_state.attempt_number}/3)"
-    ),
-)
 def clip_num(num: float, min_value: float, max_value: float) -> float:
     """Clip a number to a specified range."""
     if num < min_value:
@@ -78,6 +67,17 @@ def clip_num(num: float, min_value: float, max_value: float) -> float:
     return num
 
 
+@retry(
+    stop=stop_after_attempt(5),  # 最多重试 3 次
+    wait=wait_exponential(
+        multiplier=1, min=1, max=10
+    ),  # 指数退避策略，初始 1 秒，最大 10 秒
+    retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
+    before_sleep=lambda retry_state: logger.warning(
+        f"Request failed VLM, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
+        f"(Attempt {retry_state.attempt_number}/5)"
+    ),
+)
 def predict_layout(
     image,
     host: str = "http://localhost:8000",
@@ -186,14 +186,14 @@ def predict_layout(
 
 
 @retry(
-    stop=stop_after_attempt(3),  # 最多重试 3 次
+    stop=stop_after_attempt(5),  # 最多重试 3 次
     wait=wait_exponential(
         multiplier=1, min=1, max=10
     ),  # 指数退避策略，初始 1 秒，最大 10 秒
     retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
     before_sleep=lambda retry_state: logger.warning(
-        f"Request failed, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
-        f"(Attempt {retry_state.attempt_number}/3)"
+        f"Request failed PADDLE, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
+        f"(Attempt {retry_state.attempt_number}/5)"
     ),
 )
 def predict_layout2(
@@ -599,7 +599,7 @@ class RpcDocLayoutModel(DocLayoutModel):
     ):
         layout_temp_path = translate_config.get_working_file_path("layout.temp.pdf")
         mupdf_doc.save(layout_temp_path.as_posix())
-        with ThreadPoolExecutor(max_workers=8) as executor:
+        with ThreadPoolExecutor(max_workers=32) as executor:
             yield from executor.map(
                 self.predict_page,
                 pages,

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/format/pdf/babelpdf/cidfont.py

@@ -41,9 +41,12 @@ def get_descendant_fonts(doc, xref):
 
 
 def get_glyph_bbox(face, g):
-    face.load_glyph(g, freetype.FT_LOAD_NO_SCALE)
-    cbox = face.glyph.outline.get_bbox()
-    return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
+    try:
+        face.load_glyph(g, freetype.FT_LOAD_NO_SCALE)
+        cbox = face.glyph.outline.get_bbox()
+        return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
+    except Exception:
+        return 0, 0, 0, 0
 
 
 def get_face_bbox(blob):

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/format/pdf/document_il/frontend/il_creater.py

@@ -3,12 +3,14 @@ import functools
 import logging
 import math
 import re
+import unicodedata
 from io import BytesIO
 from itertools import islice
 from typing import Literal
 
 import freetype
 import pymupdf
+import tiktoken
 
 import babeldoc.pdfminer.pdfinterp
 from babeldoc.format.pdf.babelpdf.base14 import get_base14_bbox
@@ -18,6 +20,7 @@ from babeldoc.format.pdf.babelpdf.encoding import get_type1_encoding
 from babeldoc.format.pdf.babelpdf.utils import guarded_bbox
 from babeldoc.format.pdf.document_il import il_version_1
 from babeldoc.format.pdf.document_il.utils import zstd_helper
+from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
 from babeldoc.format.pdf.document_il.utils.matrix_helper import decompose_ctm
 from babeldoc.format.pdf.document_il.utils.style_helper import BLACK
 from babeldoc.format.pdf.document_il.utils.style_helper import YELLOW
@@ -358,6 +361,10 @@ class ILCreater:
         self.render_order = 0
         self.current_clip_paths: list[tuple] = []
         self.clip_paths_stack: list[list[tuple]] = []
+        # For valid character collection
+        self.font_mapper = FontMapper(translation_config)
+        self.tokenizer = tiktoken.encoding_for_model("gpt-4o")
+        self._page_valid_chars_buffer: list[str] | None = None
 
     def transform_clip_path(
         self,
@@ -566,8 +573,32 @@ class ILCreater:
         self.current_clip_paths = []
         self.clip_paths_stack = []
         self.docs.page.append(self.current_page)
+        # Prepare per-page buffer for valid characters on translated pages
+        self._page_valid_chars_buffer = []
 
     def on_page_end(self):
+        # Accumulate this page's valid characters and tokens into shared context
+        try:
+            if (
+                self._page_valid_chars_buffer is not None
+                and len(self._page_valid_chars_buffer) > 0
+            ):
+                page_text = "".join(self._page_valid_chars_buffer)
+                char_count = len(page_text)
+                try:
+                    token_count = len(
+                        self.tokenizer.encode(page_text, disallowed_special=())
+                    )
+                except Exception as e:
+                    logger.warning("Failed to compute token count for page: %s", e)
+                    token_count = 0
+                self.translation_config.shared_context_cross_split_part.add_valid_counts(
+                    char_count, token_count
+                )
+        except Exception as e:
+            logger.warning("Failed to accumulate page valid stats: %s", e)
+        finally:
+            self._page_valid_chars_buffer = []
         self.progress.advance(1)
 
     def on_page_crop_box(
@@ -848,6 +879,11 @@ class ILCreater:
                 "Failed to get rotation angle for char %s",
                 char.get_text(),
             )
+        # Collect valid characters for statistics
+        try:
+            self._collect_valid_char(char.get_text())
+        except Exception as e:
+            logger.warning("Error collecting valid char: %s", e)
         gs = self.create_graphic_state(char.graphicstate)
         # Get font from current page or xobject
         font = None
@@ -983,6 +1019,43 @@ class ILCreater:
                 )
             )
 
+    def _collect_valid_char(self, ch: str):
+        """Append a valid character into the current page buffer according to rules.
+        Rules:
+        - Include whitespace matched by space_regex directly.
+        - Ignore categories that are never normal text: {Cc, Cs, Co, Cn}.
+        - Apply inverted criteria from formular_helper.py (21-28):
+          empty -> invalid, contains '(cid:' -> invalid,
+          not has_char(ch) -> invalid unless len(ch) > 1 and all(has_char(x)).
+        """
+        if self._page_valid_chars_buffer is None:
+            return
+        if space_regex.match(ch):
+            self._page_valid_chars_buffer.append(ch)
+            return
+        try:
+            cat = unicodedata.category(ch[0]) if ch else None
+        except Exception:
+            cat = None
+        if cat in {"Cc", "Cs", "Co", "Cn"}:
+            return
+        is_invalid = False
+        if not ch:
+            is_invalid = True
+        elif "(cid:" in ch:
+            is_invalid = True
+        else:
+            try:
+                if not self.font_mapper.has_char(ch):
+                    if len(ch) > 1 and all(self.font_mapper.has_char(x) for x in ch):
+                        is_invalid = False
+                    else:
+                        is_invalid = True
+            except Exception:
+                is_invalid = True
+        if not is_invalid:
+            self._page_valid_chars_buffer.append(ch)
+
     def on_lt_curve(self, curve: babeldoc.pdfminer.layout.LTCurve):
         if not self.enable_graphic_element_process:
             return

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/format/pdf/document_il/midend/detect_scanned_file.py

@@ -131,6 +131,7 @@ class DetectScannedFile:
                 self.translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround = True
                 self.translation_config.ocr_workaround = True
                 self.translation_config.skip_scanned_detection = True
+                self.translation_config.disable_rich_text_translate = True
                 self.clean_render_order_for_chars(docs)
                 self.translation_config.remove_non_formula_lines = False
             else:

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/format/pdf/high_level.py

@@ -62,6 +62,7 @@ from babeldoc.pdfminer.pdfinterp import PDFResourceManager
 from babeldoc.pdfminer.pdfpage import PDFPage
 from babeldoc.pdfminer.pdfparser import PDFParser
 from babeldoc.progress_monitor import ProgressMonitor
+from babeldoc.utils import memory
 
 logger = logging.getLogger(__name__)
 
@@ -455,20 +456,10 @@ class MemoryMonitor:
         self.peak_memory_usage = 0
         self.monitor_thread = None
         self.stop_event = None
-
-        try:
-            import psutil
-
-            self.psutil = psutil
-        except ImportError:
-            logger.warning("psutil not installed, memory monitoring disabled")
-            self.psutil = None
+        self.last_pss_check_time = None
 
     def __enter__(self):
         """Start memory monitoring."""
-        if not self.psutil:
-            return self
-
         self.stop_event = threading.Event()
         self.monitor_thread = threading.Thread(
             target=self._monitor_memory_usage, daemon=True
@@ -479,7 +470,7 @@ class MemoryMonitor:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """Stop monitoring and log peak memory usage."""
-        if not self.psutil or not self.monitor_thread:
+        if not self.monitor_thread:
             return
 
         self.stop_event.set()
@@ -490,14 +481,15 @@ class MemoryMonitor:
         """Background thread that periodically checks memory usage."""
         while not self.stop_event.is_set():
             try:
-                current_process = self.psutil.Process()
-                # Get memory usage of current process and all children
-                total_memory = current_process.memory_info().rss
-                for child in current_process.children(recursive=True):
-                    try:
-                        total_memory += child.memory_info().rss
-                    except (self.psutil.NoSuchProcess, self.psutil.AccessDenied):
-                        pass
+                # Use throttled memory check with 2-second PSS throttle
+                total_memory, self.last_pss_check_time = (
+                    memory.get_memory_usage_with_throttle(
+                        include_children=True,
+                        prefer_pss=True,
+                        last_pss_check_time=self.last_pss_check_time,
+                        pss_throttle_seconds=2.0,
+                    )
+                )
 
                 # Convert to MB for better readability
                 total_memory_mb = total_memory / (1024 * 1024)
@@ -508,6 +500,10 @@ class MemoryMonitor:
 
             time.sleep(self.interval)
 
+    def get_peek_memory_psutil(self):
+        """Get peak memory usage using psutil (for backwards compatibility)."""
+        return memory.get_memory_usage_bytes(include_children=True, prefer_pss=True)
+
 
 def fix_null_page_content(doc: Document) -> list[int]:
     invalid_page = []
@@ -759,6 +755,23 @@ def do_translate(
         logger.info(
             f"finish translate: {original_pdf_path}, cost: {finish_time - start_time} s",
         )
+        # Populate aggregate valid text statistics into result
+        try:
+            sc = translation_config.shared_context_cross_split_part
+            result.total_valid_character_count = getattr(
+                sc, "valid_char_count_total", 0
+            )
+            token_total = getattr(sc, "total_valid_text_token_count", None)
+            result.total_valid_text_token_count = (
+                token_total if isinstance(token_total, int) else 0
+            )
+        except Exception as e:
+            logger.warning("Failed to populate valid text statistics: %s", e)
+            try:
+                result.total_valid_character_count = 0
+                result.total_valid_text_token_count = 0
+            except Exception:
+                pass
         result.original_pdf_path = translation_config.input_file
         result.peak_memory_usage = peak_memory_usage
 

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/format/pdf/translation_config.py

@@ -32,6 +32,9 @@ class SharedContextCrossSplitPart:
         self.auto_extracted_glossary: Glossary | None = None
         self.raw_extracted_terms: list[tuple[str, str]] = []
         self.auto_enabled_ocr_workaround = False
+        # Statistics for valid characters/text across the whole file
+        self.valid_char_count_total: int = 0
+        self.total_valid_text_token_count: int = 0
 
     def initialize_glossaries(self, initial_glossaries: list[Glossary] | None):
         with self._lock:
@@ -45,6 +48,9 @@ class SharedContextCrossSplitPart:
             for g in self.user_glossaries:
                 for entity in g.normalized_lookup:
                     self.norm_terms.add(entity)
+            # reset statistics buffer when initializing
+            self.valid_char_count_total = 0
+            self.total_valid_text_token_count = 0
 
     def add_raw_extracted_term_pair(self, src: str, tgt: str):
         with self._lock:
@@ -67,7 +73,11 @@ class SharedContextCrossSplitPart:
         return current_name
 
     def contains_term(self, term: str) -> bool:
-        pass
+        with self._lock:
+            try:
+                return term in self.norm_terms
+            except Exception:
+                return False
 
     def finalize_auto_extracted_glossary(self):
         with self._lock:
@@ -112,6 +122,16 @@ class SharedContextCrossSplitPart:
                     all_glossaries.append(self.auto_extracted_glossary)
                 return all_glossaries
 
+    def add_valid_counts(self, char_count: int, token_count: int):
+        """Accumulate valid character and token counts in a threadsafe way."""
+        if char_count <= 0 and token_count <= 0:
+            return
+        with self._lock:
+            if char_count > 0:
+                self.valid_char_count_total += char_count
+            if token_count > 0:
+                self.total_valid_text_token_count += token_count
+
 
 class TranslationConfig:
     @staticmethod
@@ -227,6 +247,7 @@ class TranslationConfig:
 
         if self.ocr_workaround:
             self.skip_scanned_detection = True
+            self.disable_rich_text_translate = True
 
         # for backward compatibility
         if use_side_by_side_dual is False and use_alternating_pages_dual is False:
@@ -462,6 +483,8 @@ class TranslateResult:
     no_watermark_dual_pdf_path: Path | None
     peak_memory_usage: int | None
     auto_extracted_glossary_path: Path | None
+    total_valid_character_count: int | None
+    total_valid_text_token_count: int | None
 
     def __init__(
         self,
@@ -478,6 +501,8 @@ class TranslateResult:
         self.no_watermark_dual_pdf_path = dual_pdf_path
 
         self.auto_extracted_glossary_path = auto_extracted_glossary_path
+        self.total_valid_character_count = None
+        self.total_valid_text_token_count = None
 
     def __str__(self):
         """Return a human-readable string representation of the translation result."""
@@ -523,6 +548,20 @@ class TranslateResult:
         if hasattr(self, "peak_memory_usage") and self.peak_memory_usage:
             result.append(f"\tPeak memory usage: {self.peak_memory_usage} MB")
 
+        if hasattr(self, "total_valid_character_count") and isinstance(
+            self.total_valid_character_count, int
+        ):
+            result.append(
+                f"\tTotal valid character count: {self.total_valid_character_count}"
+            )
+
+        if hasattr(self, "total_valid_text_token_count") and isinstance(
+            self.total_valid_text_token_count, int
+        ):
+            result.append(
+                f"\tTotal valid text token count (gpt-4o): {self.total_valid_text_token_count}"
+            )
+
         if result:
             result.insert(0, "Translation results:")
 

{babeldoc-0.5.15 → babeldoc-0.5.17}/babeldoc/main.py

@@ -26,7 +26,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.5.15"
+__version__ = "0.5.17"
 
 
 def create_parser():