BabelDOC 0.5.15__tar.gz → 0.5.16__tar.gz

{babeldoc-0.5.15 → babeldoc-0.5.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.15
+Version: 0.5.16
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.5.16/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.16"

{babeldoc-0.5.15 → babeldoc-0.5.16}/babeldoc/const.py

@@ -6,7 +6,7 @@ import subprocess
 import threading
 from pathlib import Path
 
-__version__ = "0.5.15"
+__version__ = "0.5.16"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.5.15 → babeldoc-0.5.16}/babeldoc/docvision/rpc_doclayout6.py

@@ -58,17 +58,6 @@ def encode_image(image) -> bytes:
     return encoded
 
 
-@retry(
-    stop=stop_after_attempt(3),  # 最多重试 3 次
-    wait=wait_exponential(
-        multiplier=1, min=1, max=10
-    ),  # 指数退避策略，初始 1 秒，最大 10 秒
-    retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
-    before_sleep=lambda retry_state: logger.warning(
-        f"Request failed, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
-        f"(Attempt {retry_state.attempt_number}/3)"
-    ),
-)
 def clip_num(num: float, min_value: float, max_value: float) -> float:
     """Clip a number to a specified range."""
     if num < min_value:
@@ -78,6 +67,17 @@ def clip_num(num: float, min_value: float, max_value: float) -> float:
     return num
 
 
+@retry(
+    stop=stop_after_attempt(5),  # 最多重试 3 次
+    wait=wait_exponential(
+        multiplier=1, min=1, max=10
+    ),  # 指数退避策略，初始 1 秒，最大 10 秒
+    retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
+    before_sleep=lambda retry_state: logger.warning(
+        f"Request failed VLM, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
+        f"(Attempt {retry_state.attempt_number}/5)"
+    ),
+)
 def predict_layout(
     image,
     host: str = "http://localhost:8000",
@@ -186,14 +186,14 @@ def predict_layout(
 
 
 @retry(
-    stop=stop_after_attempt(3),  # 最多重试 3 次
+    stop=stop_after_attempt(5),  # 最多重试 3 次
     wait=wait_exponential(
         multiplier=1, min=1, max=10
     ),  # 指数退避策略，初始 1 秒，最大 10 秒
     retry=retry_if_exception_type((httpx.HTTPError, Exception)),  # 针对哪些异常重试
     before_sleep=lambda retry_state: logger.warning(
-        f"Request failed, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
-        f"(Attempt {retry_state.attempt_number}/3)"
+        f"Request failed PADDLE, retrying in {getattr(retry_state.next_action, 'sleep', 'unknown')} seconds... "
+        f"(Attempt {retry_state.attempt_number}/5)"
     ),
 )
 def predict_layout2(
@@ -599,7 +599,7 @@ class RpcDocLayoutModel(DocLayoutModel):
     ):
         layout_temp_path = translate_config.get_working_file_path("layout.temp.pdf")
         mupdf_doc.save(layout_temp_path.as_posix())
-        with ThreadPoolExecutor(max_workers=8) as executor:
+        with ThreadPoolExecutor(max_workers=32) as executor:
             yield from executor.map(
                 self.predict_page,
                 pages,

{babeldoc-0.5.15 → babeldoc-0.5.16}/babeldoc/format/pdf/document_il/midend/detect_scanned_file.py

@@ -131,6 +131,7 @@ class DetectScannedFile:
                 self.translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround = True
                 self.translation_config.ocr_workaround = True
                 self.translation_config.skip_scanned_detection = True
+                self.translation_config.disable_rich_text_translate = True
                 self.clean_render_order_for_chars(docs)
                 self.translation_config.remove_non_formula_lines = False
             else:

{babeldoc-0.5.15 → babeldoc-0.5.16}/babeldoc/format/pdf/high_level.py

@@ -62,6 +62,7 @@ from babeldoc.pdfminer.pdfinterp import PDFResourceManager
 from babeldoc.pdfminer.pdfpage import PDFPage
 from babeldoc.pdfminer.pdfparser import PDFParser
 from babeldoc.progress_monitor import ProgressMonitor
+from babeldoc.utils import memory
 
 logger = logging.getLogger(__name__)
 
@@ -455,20 +456,10 @@ class MemoryMonitor:
         self.peak_memory_usage = 0
         self.monitor_thread = None
         self.stop_event = None
-
-        try:
-            import psutil
-
-            self.psutil = psutil
-        except ImportError:
-            logger.warning("psutil not installed, memory monitoring disabled")
-            self.psutil = None
+        self.last_pss_check_time = None
 
     def __enter__(self):
         """Start memory monitoring."""
-        if not self.psutil:
-            return self
-
         self.stop_event = threading.Event()
         self.monitor_thread = threading.Thread(
             target=self._monitor_memory_usage, daemon=True
@@ -479,7 +470,7 @@ class MemoryMonitor:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """Stop monitoring and log peak memory usage."""
-        if not self.psutil or not self.monitor_thread:
+        if not self.monitor_thread:
             return
 
         self.stop_event.set()
@@ -490,14 +481,15 @@ class MemoryMonitor:
         """Background thread that periodically checks memory usage."""
         while not self.stop_event.is_set():
             try:
-                current_process = self.psutil.Process()
-                # Get memory usage of current process and all children
-                total_memory = current_process.memory_info().rss
-                for child in current_process.children(recursive=True):
-                    try:
-                        total_memory += child.memory_info().rss
-                    except (self.psutil.NoSuchProcess, self.psutil.AccessDenied):
-                        pass
+                # Use throttled memory check with 2-second PSS throttle
+                total_memory, self.last_pss_check_time = (
+                    memory.get_memory_usage_with_throttle(
+                        include_children=True,
+                        prefer_pss=True,
+                        last_pss_check_time=self.last_pss_check_time,
+                        pss_throttle_seconds=2.0,
+                    )
+                )
 
                 # Convert to MB for better readability
                 total_memory_mb = total_memory / (1024 * 1024)
@@ -508,6 +500,10 @@ class MemoryMonitor:
 
             time.sleep(self.interval)
 
+    def get_peek_memory_psutil(self):
+        """Get peak memory usage using psutil (for backwards compatibility)."""
+        return memory.get_memory_usage_bytes(include_children=True, prefer_pss=True)
+
 
 def fix_null_page_content(doc: Document) -> list[int]:
     invalid_page = []

{babeldoc-0.5.15 → babeldoc-0.5.16}/babeldoc/format/pdf/translation_config.py

@@ -227,6 +227,7 @@ class TranslationConfig:
 
         if self.ocr_workaround:
             self.skip_scanned_detection = True
+            self.disable_rich_text_translate = True
 
         # for backward compatibility
         if use_side_by_side_dual is False and use_alternating_pages_dual is False:

{babeldoc-0.5.15 → babeldoc-0.5.16}/babeldoc/main.py

@@ -26,7 +26,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.5.15"
+__version__ = "0.5.16"
 
 
 def create_parser():

babeldoc-0.5.16/babeldoc/utils/memory.py

@@ -0,0 +1,251 @@
+import os
+import sys
+import time
+from pathlib import Path
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+
+def _parse_pss_from_smaps_rollup(pid: int) -> int | None:
+    """
+    Try to read PSS from /proc/<pid>/smaps_rollup.
+    Returns PSS in bytes, or None if not available/readable.
+    """
+    try:
+        smaps_rollup_path = Path(f"/proc/{pid}/smaps_rollup")
+        with smaps_rollup_path.open() as f:
+            for line in f:
+                if line.startswith("Pss:"):
+                    # Format: "Pss:            1234 kB"
+                    parts = line.split()
+                    if len(parts) >= 2:
+                        pss_kb = int(parts[1])
+                        return pss_kb * 1024  # Convert to bytes
+        return None
+    except (FileNotFoundError, PermissionError, ValueError, OSError):
+        return None
+
+
+def _parse_pss_from_smaps(pid: int) -> int | None:
+    """
+    Try to read PSS from /proc/<pid>/smaps and sum all Pss entries.
+    Returns PSS in bytes, or None if not available/readable.
+    """
+    try:
+        smaps_path = Path(f"/proc/{pid}/smaps")
+        total_pss_kb = 0
+        with smaps_path.open() as f:
+            for line in f:
+                if line.startswith("Pss:"):
+                    # Format: "Pss:            1234 kB"
+                    parts = line.split()
+                    if len(parts) >= 2:
+                        total_pss_kb += int(parts[1])
+        if total_pss_kb > 0:
+            return total_pss_kb * 1024  # Convert to bytes
+        return None
+    except (FileNotFoundError, PermissionError, ValueError, OSError):
+        return None
+
+
+def _get_pss_linux(pid: int) -> int | None:
+    """
+    Try to get PSS on Linux.
+    Priority: smaps_rollup -> smaps -> None
+    Returns PSS in bytes, or None if not available.
+    """
+    # Try smaps_rollup first (lightweight)
+    pss = _parse_pss_from_smaps_rollup(pid)
+    if pss is not None:
+        return pss
+
+    # Fallback to smaps (heavier)
+    pss = _parse_pss_from_smaps(pid)
+    if pss is not None:
+        return pss
+
+    return None
+
+
+def _get_rss_psutil(pid: int) -> int | None:
+    """
+    Get RSS using psutil for a single process.
+    Returns RSS in bytes, or None if psutil unavailable or process not found.
+    """
+    if psutil is None:
+        return None
+
+    try:
+        process = psutil.Process(pid)
+        return process.memory_info().rss
+    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.TimeoutExpired):
+        return None
+
+
+def _get_single_process_memory(
+    pid: int, prefer_pss: bool = True, use_smaps_rollup_only: bool = False
+) -> int | None:
+    """
+    Get memory usage for a single process (no children).
+
+    Args:
+        pid: Process ID
+        prefer_pss: If True and on Linux, try PSS first; otherwise use RSS
+        use_smaps_rollup_only: If True, only try smaps_rollup (faster), fallback to RSS if not available
+
+    Returns:
+        Memory usage in bytes, or None if all methods fail
+    """
+    if sys.platform == "linux":
+        if prefer_pss:
+            if use_smaps_rollup_only:
+                # Only try smaps_rollup, then fallback to RSS
+                pss = _parse_pss_from_smaps_rollup(pid)
+                if pss is not None:
+                    return pss
+            else:
+                # Try full PSS (smaps_rollup -> smaps)
+                pss = _get_pss_linux(pid)
+                if pss is not None:
+                    return pss
+
+    # Fallback to RSS
+    return _get_rss_psutil(pid)
+
+
+def get_memory_usage_bytes(
+    pid: int | None = None,
+    include_children: bool = True,
+    prefer_pss: bool = True,
+) -> int:
+    """
+    Get memory usage of a process (and optionally its children).
+
+    On Linux with prefer_pss=True:
+      - Tries /proc/<pid>/smaps_rollup first (lightweight)
+      - Falls back to /proc/<pid>/smaps if smaps_rollup unavailable (heavier)
+      - Falls back to psutil RSS if smaps unavailable
+
+    On non-Linux systems or prefer_pss=False:
+      - Uses psutil RSS
+
+    Args:
+        pid: Process ID to monitor. If None, uses current process.
+        include_children: If True, also includes memory of child processes.
+        prefer_pss: If True on Linux, attempts to use PSS; otherwise uses RSS.
+
+    Returns:
+        Total memory usage in bytes (guaranteed non-negative).
+    """
+    if pid is None:
+        pid = os.getpid()
+
+    total_memory = 0
+
+    # Determine if we're using smaps (heavier) vs smaps_rollup (lighter)
+    use_smaps_rollup_only = False
+    if sys.platform == "linux" and prefer_pss:
+        # If we can read smaps_rollup, use rollup-only mode
+        test_rollup = _parse_pss_from_smaps_rollup(pid)
+        use_smaps_rollup_only = test_rollup is not None
+
+    # Get current process memory
+    memory = _get_single_process_memory(
+        pid, prefer_pss=prefer_pss, use_smaps_rollup_only=use_smaps_rollup_only
+    )
+    if memory is not None:
+        total_memory += memory
+
+    # Get children memory if requested
+    if include_children:
+        if psutil is None:
+            # Cannot get children without psutil
+            return total_memory
+
+        try:
+            parent_process = psutil.Process(pid)
+            children = parent_process.children(recursive=True)
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            # Parent process not found or no permission
+            return total_memory
+
+        for child in children:
+            try:
+                child_pid = child.pid
+                child_memory = _get_single_process_memory(
+                    child_pid,
+                    prefer_pss=prefer_pss,
+                    use_smaps_rollup_only=use_smaps_rollup_only,
+                )
+                if child_memory is not None:
+                    total_memory += child_memory
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                # Child process died or no permission; skip it
+                pass
+
+    return max(0, total_memory)
+
+
+def get_memory_usage_with_throttle(
+    pid: int | None = None,
+    include_children: bool = True,
+    prefer_pss: bool = True,
+    last_pss_check_time: float | None = None,
+    pss_throttle_seconds: float = 2.0,
+) -> tuple[int, float | None]:
+    """
+    Get memory usage with throttling for PSS checks on Linux.
+
+    When PSS is not available via smaps_rollup and must read smaps (expensive),
+    this throttles checks to at most once per pss_throttle_seconds.
+
+    Args:
+        pid: Process ID. If None, uses current process.
+        include_children: If True, includes child process memory.
+        prefer_pss: If True on Linux, attempts to use PSS.
+        last_pss_check_time: Timestamp of last PSS check. For throttling logic.
+        pss_throttle_seconds: Minimum interval (seconds) between smaps reads.
+
+    Returns:
+        Tuple of (memory_bytes, new_check_time).
+        If throttled, returns cached estimate (0) and original check time.
+    """
+    current_time = time.time()
+
+    # Check if we should throttle
+    if (
+        prefer_pss
+        and sys.platform == "linux"
+        and last_pss_check_time is not None
+        and (current_time - last_pss_check_time) < pss_throttle_seconds
+    ):
+        # Throttled: use RSS only as a fast estimate
+        memory = 0
+        pid_to_check = pid if pid is not None else os.getpid()
+        rss = _get_rss_psutil(pid_to_check)
+        if rss is not None:
+            memory += rss
+
+        if include_children and psutil is not None:
+            try:
+                parent_process = psutil.Process(pid_to_check)
+                for child in parent_process.children(recursive=True):
+                    try:
+                        child_rss = _get_rss_psutil(child.pid)
+                        if child_rss is not None:
+                            memory += child_rss
+                    except (psutil.NoSuchProcess, psutil.AccessDenied):
+                        pass
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+
+        return memory, last_pss_check_time
+
+    # Not throttled: do full check
+    memory = get_memory_usage_bytes(
+        pid=pid, include_children=include_children, prefer_pss=prefer_pss
+    )
+    return memory, current_time

{babeldoc-0.5.15 → babeldoc-0.5.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.5.15"
+version = "0.5.16"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -162,7 +162,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.5.15"
+current_version = "0.5.16"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

babeldoc-0.5.15/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.5.15"

{babeldoc-0.5.15 → babeldoc-0.5.16}/babeldoc/format/pdf/document_il/utils/layout_helper.py

@@ -659,7 +659,6 @@ def get_character_layout(
         layout_priority = [
             "number",
             "reference",
-            "reference_hybrid",
             "reference_content",
             "algorithm",
             "formula_caption",
@@ -703,6 +702,7 @@ def get_character_layout(
             "page_header",
             "page_footer",
             # --- hybrid labels ---
+            "reference_hybrid",
             "document_hybrid",
             "academic_paper_hybrid",
             "form_or_table_hybrid",