BabelDOC 0.3.55__tar.gz → 0.3.57__tar.gz

{babeldoc-0.3.55 → babeldoc-0.3.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.55
+Version: 0.3.57
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -17,6 +17,7 @@ Requires-Dist: configargparse>=1.7
 Requires-Dist: freetype-py>=2.5.1
 Requires-Dist: httpx[socks]>=0.27.0
 Requires-Dist: huggingface-hub>=0.27.0
+Requires-Dist: hyperscan>=0.7.13
 Requires-Dist: msgpack>=1.1.0
 Requires-Dist: numpy>=2.0.2
 Requires-Dist: onnx>=1.18.0

babeldoc-0.3.57/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.57"

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.55"
+__version__ = "0.3.57"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/document_il/backend/pdf_creater.py

@@ -781,10 +781,10 @@ class PDFCreater:
             return False
 
     def restore_media_box(self, doc: pymupdf.Document, mediabox_data: dict) -> None:
-        for pageno, page_box_data in mediabox_data.items():
+        for xref, page_box_data in mediabox_data.items():
             for name, box in page_box_data.items():
                 try:
-                    doc.xref_set_key(doc[pageno].xref, name, box)
+                    doc.xref_set_key(xref, name, box)
                 except Exception:
                     logger.debug(f"Error restoring media box {name} from PDF")
 

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/document_il/midend/automatic_term_extractor.py

@@ -185,6 +185,7 @@ class AutomaticTermExtractor:
         total_token_count = 0
         for paragraph in page.pdf_paragraph:
             if paragraph.debug_id is None or paragraph.unicode is None:
+                pbar.advance(1)
                 continue
             if is_cid_paragraph(paragraph):
                 pbar.advance(1)
@@ -227,7 +228,6 @@ class AutomaticTermExtractor:
             for u in inputs:
                 tracker.append_paragraph_unicode(u)
             if not inputs:
-                pbar.advance(len(paragraphs.paragraphs))
                 return
             prompt = LLM_PROMPT_TEMPLATE.format(
                 target_language=self.translation_config.lang_out,
@@ -254,8 +254,10 @@ class AutomaticTermExtractor:
                         )
 
         except Exception as e:
-            logger.error(f"Error preparing LLM prompt for automatic terms extract: {e}")
+            logger.error(f"Error during automatic terms extract: {e}")
             return
+        finally:
+            pbar.advance(len(paragraphs.paragraphs))
 
     def procress(self, doc_il: ILDocument):
         logger.info(f"{self.stage_name}: Starting term extraction for document.")

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/document_il/translator/translator.py

@@ -8,6 +8,7 @@ from abc import abstractmethod
 
 import httpx
 import openai
+from tenacity import before_sleep_log
 from tenacity import retry
 from tenacity import retry_if_exception_type
 from tenacity import stop_after_attempt
@@ -198,7 +199,8 @@ class OpenAITranslator(BaseTranslator):
             http_client=httpx.Client(
                 limits=httpx.Limits(
                     max_connections=None, max_keepalive_connections=None
-                )
+                ),
+                timeout=60,  # Set a reasonable timeout
             ),
         )
         self.add_cache_impact_parameters("temperature", self.options["temperature"])
@@ -213,10 +215,7 @@ class OpenAITranslator(BaseTranslator):
         retry=retry_if_exception_type(openai.RateLimitError),
         stop=stop_after_attempt(100),
         wait=wait_exponential(multiplier=1, min=1, max=15),
-        before_sleep=lambda retry_state: logger.warning(
-            f"RateLimitError, retrying in {retry_state.next_action.sleep} seconds... "
-            f"(Attempt {retry_state.attempt_number}/100)"
-        ),
+        before_sleep=before_sleep_log(logger, logging.WARNING),
     )
     def do_translate(self, text, rate_limit_params: dict = None) -> str:
         response = self.client.chat.completions.create(
@@ -243,10 +242,7 @@ class OpenAITranslator(BaseTranslator):
         retry=retry_if_exception_type(openai.RateLimitError),
         stop=stop_after_attempt(100),
         wait=wait_exponential(multiplier=1, min=1, max=15),
-        before_sleep=lambda retry_state: logger.warning(
-            f"RateLimitError, retrying in {retry_state.next_action.sleep} seconds... "
-            f"(Attempt {retry_state.attempt_number}/100)"
-        ),
+        before_sleep=before_sleep_log(logger, logging.WARNING),
     )
     def do_llm_translate(self, text, rate_limit_params: dict = None):
         if text is None:

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/glossary.py

@@ -1,8 +1,16 @@
 import csv
 import io
+import itertools
+import logging
 import re
+import time
 from pathlib import Path
 
+import hyperscan
+import regex
+
+logger = logging.getLogger(__name__)
+
 
 class GlossaryEntry:
     def __init__(self, source: str, target: str, target_language: str | None = None):
@@ -14,58 +22,92 @@ class GlossaryEntry:
         return f"GlossaryEntry(source='{self.source}', target='{self.target}', target_language='{self.target_language}')"
 
 
+def batched(iterable, n, *, strict=False):
+    # batched('ABCDEFG', 3) → ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    iterator = iter(iterable)
+    while batch := tuple(itertools.islice(iterator, n)):
+        if strict and len(batch) != n:
+            raise ValueError("batched(): incomplete batch")
+        yield batch
+
+
+TERM_NORM_PATTERN = re.compile(r"\s+", regex.UNICODE)
+
+
 class Glossary:
     def __init__(self, name: str, entries: list[GlossaryEntry]):
         self.name = name
         self.entries = entries
-        self.source_terms_regex: re.Pattern | None = None
         self.normalized_lookup: dict[str, tuple[str, str]] = {}
+        self.id_lookup: list[tuple[str, str]] = []
+        self.hs_dbs: list[hyperscan.Database] | None = None
         self._build_regex_and_lookup()
 
     @staticmethod
     def normalize_source(source_term: str) -> str:
         """Normalizes a source term by lowercasing and standardizing whitespace."""
         term = source_term.lower()
-        term = re.sub(
-            r"\s+", " ", term
+        term = TERM_NORM_PATTERN.sub(
+            " ", term
         )  # Replace multiple whitespace with single space
         return term.strip()
 
     def _build_regex_and_lookup(self):
+        logger.debug(
+            f"start build regex for glossary {self.name} with {len(self.entries)} entries"
+        )
         """
         Builds a combined regex for all source terms and a lookup dictionary
         from normalized source terms to (original_source, original_target).
         Regex patterns are sorted by length in descending order to prioritize longer matches.
         """
         self.normalized_lookup = {}
-        regex_patterns_for_or: list[str] = []
 
         if not self.entries:
             self.source_terms_regex = None
             return
 
-        for entry in self.entries:
+        self.hs_dbs = []
+        hs_pattern = []
+        start = time.time()
+        for idx, entry in enumerate(self.entries):
             normalized_key = self.normalize_source(entry.source)
             self.normalized_lookup[normalized_key] = (entry.source, entry.target)
+            self.id_lookup.append((entry.source, entry.target))
 
-            # Create a regex pattern for the source term, replacing spaces with \s+
-            # and escaping other special characters.
-            escaped_parts = [re.escape(part) for part in entry.source.split()]
-            if escaped_parts:  # Ensure there are parts to join
-                pattern_for_entry = r"\s+".join(escaped_parts)
-                regex_patterns_for_or.append(pattern_for_entry)
-
-        if regex_patterns_for_or:
-            # Sort patterns by length, longest first, to help with overlapping terms
-            regex_patterns_for_or.sort(key=len, reverse=True)
-            combined_regex_str = "|".join(
-                f"({pattern})" for pattern in regex_patterns_for_or
-            )  # Add capturing groups
-            self.source_terms_regex = re.compile(
-                combined_regex_str, flags=re.IGNORECASE
+            hs_pattern.append((re.escape(entry.source).encode("utf-8"), idx))
+
+        chunk_size = 20000
+        for i, pattern_chunk in enumerate(
+            batched(hs_pattern, chunk_size, strict=False)
+        ):
+            logger.debug(
+                f"building hs_db chunk {i + 1} / {len(self.entries) // chunk_size + 1}"
             )
-        else:
-            self.source_terms_regex = None
+            expressions, ids = zip(*pattern_chunk, strict=False)
+
+            hs_db = hyperscan.Database()
+            hs_db.compile(
+                expressions=expressions,
+                ids=ids,
+                elements=len(pattern_chunk),
+                flags=hyperscan.HS_FLAG_CASELESS | hyperscan.HS_FLAG_SINGLEMATCH,
+                # | hyperscan.HS_FLAG_UTF8
+                # | hyperscan.HS_FLAG_UCP,
+            )
+            self.hs_dbs.append(hs_db)
+
+        end = time.time()
+        logger.debug(
+            f"finished building regex for glossary {self.name} in {end - start:.2f} seconds"
+        )
+        logger.debug(
+            f"build hs database for glossary {self.name} with {len(self.entries)} entries, hs_info: {self.hs_dbs[0].info()}"
+        )
+        if not self.hs_dbs:
+            self.hs_dbs = None
 
     @classmethod
     def from_csv(cls, file_path: Path, target_lang_out: str) -> "Glossary":
@@ -136,24 +178,23 @@ class Glossary:
 
     def get_active_entries_for_text(self, text: str) -> list[tuple[str, str]]:
         """Returns a list of (original_source, target_text) tuples for terms found in the given text."""
-        if not self.source_terms_regex or not text:
+        if not self.hs_dbs or not text:
+            return []
+
+        text = TERM_NORM_PATTERN.sub(" ", text)  # Normalize whitespace in the text
+        if not text:
             return []
 
         active_entries = []
-        unique_added_original_sources = set()
-
-        # Find all non-overlapping matches for the combined regex
-        # The regex is constructed with capturing groups for each original pattern
-        # so findall will return tuples of all groups. We need to find which group matched.
-        # A simpler way with finditer:
-        for match in self.source_terms_regex.finditer(text):
-            # The matched text itself
-            matched_fragment = match.group(0)
-            normalized_frag = self.normalize_source(matched_fragment)
-
-            if normalized_frag in self.normalized_lookup:
-                original_source, target_text = self.normalized_lookup[normalized_frag]
-                if original_source not in unique_added_original_sources:
-                    active_entries.append((original_source, target_text))
-                    unique_added_original_sources.add(original_source)
+
+        def on_match(
+            idx: int, _from: int, _to: int, _flags: int, _context=None
+        ) -> bool | None:
+            active_entries.append(self.id_lookup[idx])
+            return False
+
+        for hs_db in self.hs_dbs:
+            # Scan the text with the hyperscan database
+            scratch = hyperscan.Scratch(hs_db)
+            hs_db.scan(text.encode("utf-8"), on_match, scratch=scratch)
         return active_entries

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/high_level.py

@@ -425,6 +425,8 @@ def fix_null_xref(doc: Document) -> None:
             elif obj and "/LZWDecode" in obj:
                 data = doc.xref_stream(i)
                 doc.update_stream(i, data)
+            elif obj and "/Annots" in obj:
+                doc.xref_set_key(i, "Annots", "null")
         except Exception:
             doc.update_object(i, "[]")
 
@@ -625,28 +627,22 @@ def do_translate(
 
 def fix_media_box(doc: Document) -> None:
     mediabox_data = {}
-    for page in doc:
-        page_box_data = {}
-        mediabox = doc.xref_get_key(page.xref, "MediaBox")
-        if mediabox[0] == "null":
-            mediabox = ("array", "[0 0 612 792]")
-            doc.xref_set_key(page.xref, "MediaBox", mediabox[1])
-        if page.mediabox.x0 != 0 or page.mediabox.y0 != 0:
-            x0 = page.mediabox.x0
-            y0 = page.mediabox.y0
-            x1 = page.mediabox.x1
-            y1 = page.mediabox.y1
-            page_box_data["MediaBox"] = doc.xref_get_key(page.xref, "MediaBox")[1]
-            doc.xref_set_key(page.xref, "MediaBox", f"[0 0 {x1 - x0} {y1 - y0}]")
-        if page.cropbox.x0 != 0 or page.cropbox.y0 != 0:
-            x0 = page.cropbox.x0
-            y0 = page.cropbox.y0
-            x1 = page.cropbox.x1
-            y1 = page.cropbox.y1
-            page_box_data["CropBox"] = doc.xref_get_key(page.xref, "CropBox")[1]
-            doc.xref_set_key(page.xref, "CropBox", f"[0 0 {x1 - x0} {y1 - x0}]")
-        if page_box_data:
-            mediabox_data[page.number] = page_box_data
+    for x in range(1, doc.xref_length()):
+        t = doc.xref_get_key(x, "Type")
+        box_set = {}
+        if t[1] in ["/Pages", "/Page"]:
+            mediabox = doc.xref_get_key(x, "MediaBox")
+            if mediabox[0] != "null":
+                _, _, x1, y1 = mediabox[1].replace("[", "").replace("]", "").split(" ")
+                doc.xref_set_key(x, "MediaBox", f"[0 0 {x1} {y1}]")
+                box_set["MediaBox"] = mediabox[1]
+            for k in ["CropBox", "BleedBox", "TrimBox", "ArtBox"]:
+                box = doc.xref_get_key(x, k)
+                if box[0] != "null":
+                    box_set[k] = box[1]
+                    doc.xref_set_key(x, k, "null")
+        if box_set:
+            mediabox_data[x] = box_set
     return mediabox_data
 
 

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/main.py

@@ -24,7 +24,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.55"
+__version__ = "0.3.57"
 
 
 def create_parser():

{babeldoc-0.3.55 → babeldoc-0.3.57}/babeldoc/result_merger.py

@@ -45,50 +45,85 @@ class ResultMerger:
         merged_dual_path = None
         merged_no_watermark_mono_path = None
         merged_no_watermark_dual_path = None
-
-        # Merge monolingual PDFs if they exist
-        if any(r.mono_pdf_path for r in results.values()):
-            merged_mono_path = self._merge_pdfs(
-                [r.mono_pdf_path for r in sorted_results.values() if r.mono_pdf_path],
-                mono_file_name,
-                tag="merged_mono",
-            )
-
-        # Merge dual-language PDFs if they exist
-        if any(r.dual_pdf_path for r in results.values()):
-            merged_dual_path = self._merge_pdfs(
-                [r.dual_pdf_path for r in sorted_results.values() if r.dual_pdf_path],
-                dual_file_name,
-                tag="merged_dual",
-            )
-
-        if any(
-            r.dual_pdf_path != r.no_watermark_dual_pdf_path
-            or r.mono_pdf_path != r.no_watermark_mono_pdf_path
-            for r in results.values()
-        ):
-            # Merge no-watermark PDFs if they exist
-            if any(r.no_watermark_mono_pdf_path for r in results.values()):
-                merged_no_watermark_mono_path = self._merge_pdfs(
+        try:
+            # Merge monolingual PDFs if they exist
+            if (
+                any(r.mono_pdf_path for r in results.values())
+                and not self.config.no_mono
+            ):
+                merged_mono_path = self._merge_pdfs(
                     [
-                        r.no_watermark_mono_pdf_path
+                        r.mono_pdf_path
                         for r in sorted_results.values()
-                        if r.no_watermark_mono_pdf_path
+                        if r.mono_pdf_path
                     ],
-                    mono_file_name_no_watermark,
-                    tag="merged_no_watermark_mono",
+                    mono_file_name,
+                    tag="merged_mono",
                 )
-
-            if any(r.no_watermark_dual_pdf_path for r in results.values()):
-                merged_no_watermark_dual_path = self._merge_pdfs(
+        except Exception as e:
+            logger.error(f"Error merging monolingual PDFs: {e}")
+            merged_mono_path = None
+
+        try:
+            # Merge dual-language PDFs if they exist
+            if (
+                any(r.dual_pdf_path for r in results.values())
+                and not self.config.no_dual
+            ):
+                merged_dual_path = self._merge_pdfs(
                     [
-                        r.no_watermark_dual_pdf_path
+                        r.dual_pdf_path
                         for r in sorted_results.values()
-                        if r.no_watermark_dual_pdf_path
+                        if r.dual_pdf_path
                     ],
-                    "merged_no_watermark_dual.pdf",
-                    tag="merged_no_watermark_dual",
+                    dual_file_name,
+                    tag="merged_dual",
                 )
+        except Exception as e:
+            logger.error(f"Error merging dual-language PDFs: {e}")
+            merged_dual_path = None
+
+        if any(
+            r.dual_pdf_path != r.no_watermark_dual_pdf_path
+            or r.mono_pdf_path != r.no_watermark_mono_pdf_path
+            for r in results.values()
+        ):
+            try:
+                # Merge no-watermark PDFs if they exist
+                if (
+                    any(r.no_watermark_mono_pdf_path for r in results.values())
+                    and not self.config.no_mono
+                ):
+                    merged_no_watermark_mono_path = self._merge_pdfs(
+                        [
+                            r.no_watermark_mono_pdf_path
+                            for r in sorted_results.values()
+                            if r.no_watermark_mono_pdf_path
+                        ],
+                        mono_file_name_no_watermark,
+                        tag="merged_no_watermark_mono",
+                    )
+            except Exception as e:
+                logger.error(f"Error merging no-watermark PDFs: {e}")
+                merged_no_watermark_mono_path = None
+
+            try:
+                if (
+                    any(r.no_watermark_dual_pdf_path for r in results.values())
+                    and not self.config.no_dual
+                ):
+                    merged_no_watermark_dual_path = self._merge_pdfs(
+                        [
+                            r.no_watermark_dual_pdf_path
+                            for r in sorted_results.values()
+                            if r.no_watermark_dual_pdf_path
+                        ],
+                        "merged_no_watermark_dual.pdf",
+                        tag="merged_no_watermark_dual",
+                    )
+            except Exception as e:
+                logger.error(f"Error merging no-watermark PDFs: {e}")
+                merged_no_watermark_dual_path = None
 
         # Create merged result
         merged_result = TranslateResult(

{babeldoc-0.3.55 → babeldoc-0.3.57}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.55"
+version = "0.3.57"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -44,6 +44,7 @@ dependencies = [
     "opencv-python-headless>=4.10.0.84",
     "rapidocr-onnxruntime>=1.4.4",
     "pyzstd>=0.17.0",
+    "hyperscan>=0.7.13",
 ]
 
 [project.optional-dependencies]
@@ -150,7 +151,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.55"
+current_version = "0.3.57"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]