BabelDOC 0.3.39__tar.gz → 0.3.41__tar.gz

{babeldoc-0.3.39 → babeldoc-0.3.41}/.github/workflows/publish-to-pypi.yml

@@ -61,8 +61,8 @@ jobs:
       - name: Bump version for developmental release
         if: "! steps.check-version.outputs.tag"
         run: |
-          version=$(bumpver update --patch --tag=final --dry 2>&1 | grep "New Version" | awk '{print $NF}') &&
-          bumpver update --set-version $version.dev$(date +%s)
+          version=$(uv run bumpver update --patch --tag=final --dry 2>&1 | grep "New Version" | awk '{print $NF}') &&
+          uv run bumpver update --set-version $version.dev$(date +%s)
 
       - name: Build package
         run: "uv build"

{babeldoc-0.3.39 → babeldoc-0.3.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.39
+Version: 0.3.41
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -216,6 +216,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--min-text-length`: Minimum text length to translate (default: 5)
 - `--openai`: Use OpenAI for translation (default: False)
 - `--custom-system-prompt`: Custom system prompt for translation.
+- `--add-formula-placehold-hint`: Add formula placeholder hint for translation. (Currently not recommended, it may affect translation quality, default: False)
 
 > [!TIP]
 >

{babeldoc-0.3.39 → babeldoc-0.3.41}/README.md

@@ -167,6 +167,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--min-text-length`: Minimum text length to translate (default: 5)
 - `--openai`: Use OpenAI for translation (default: False)
 - `--custom-system-prompt`: Custom system prompt for translation.
+- `--add-formula-placehold-hint`: Add formula placeholder hint for translation. (Currently not recommended, it may affect translation quality, default: False)
 
 > [!TIP]
 >

babeldoc-0.3.41/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.41"

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.39"
+__version__ = "0.3.41"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/midend/add_debug_information.py

@@ -34,7 +34,7 @@ class AddDebugInformation:
 
     def _create_text(self, text: str, color: GraphicState, box: il_version_1.Box):
         style = il_version_1.PdfStyle(
-            font_id="china-ss",
+            font_id="base",
             font_size=4,
             graphic_state=color,
         )

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/midend/detect_scanned_file.py

@@ -31,7 +31,7 @@ class DetectScannedFile:
         # Note: PDF coordinates are from bottom-left,
         # so we use y2 for top position
         style = il_version_1.PdfStyle(
-            font_id="china-ss",
+            font_id="base",
             font_size=4,
             graphic_state=color,
         )

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/midend/il_translator.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+import copy
 import json
 import logging
 import re
@@ -36,18 +39,29 @@ class RichTextPlaceholder:
         composition: PdfSameStyleCharacters,
         left_placeholder: str,
         right_placeholder: str,
+        left_regex_pattern: str = None,
+        right_regex_pattern: str = None,
     ):
         self.id = placeholder_id
         self.composition = composition
         self.left_placeholder = left_placeholder
         self.right_placeholder = right_placeholder
+        self.left_regex_pattern = left_regex_pattern
+        self.right_regex_pattern = right_regex_pattern
 
 
 class FormulaPlaceholder:
-    def __init__(self, placeholder_id: int, formula: PdfFormula, placeholder: str):
+    def __init__(
+        self,
+        placeholder_id: int,
+        formula: PdfFormula,
+        placeholder: str,
+        regex_pattern: str,
+    ):
         self.id = placeholder_id
         self.formula = formula
         self.placeholder = placeholder
+        self.regex_pattern = regex_pattern
 
 
 class PbarContext:
@@ -78,6 +92,10 @@ class DocumentTranslateTracker:
                 i_str = getattr(para, "input", None)
                 o_str = getattr(para, "output", None)
                 pdf_unicode = getattr(para, "pdf_unicode", None)
+                llm_translate_trackers = getattr(para, "llm_translate_trackers", None)
+                llm_translate_trackers_json = []
+                for tracker in llm_translate_trackers:
+                    llm_translate_trackers_json.append(tracker.to_dict())
                 if pdf_unicode is None or i_str is None:
                     continue
                 paragraphs.append(
@@ -85,6 +103,7 @@ class DocumentTranslateTracker:
                         "input": i_str,
                         "output": o_str,
                         "pdf_unicode": pdf_unicode,
+                        "llm_translate_trackers": llm_translate_trackers_json,
                     },
                 )
             pages.append({"paragraph": paragraphs})
@@ -103,7 +122,7 @@ class PageTranslateTracker:
 
 class ParagraphTranslateTracker:
     def __init__(self):
-        pass
+        self.llm_translate_trackers = []
 
     def set_pdf_unicode(self, unicode: str):
         self.pdf_unicode = unicode
@@ -114,6 +133,52 @@ class ParagraphTranslateTracker:
     def set_output(self, output: str):
         self.output = output
 
+    def new_llm_translate_tracker(self) -> LLMTranslateTracker:
+        tracker = LLMTranslateTracker()
+        self.llm_translate_trackers.append(tracker)
+        return tracker
+
+    def last_llm_translate_tracker(self) -> LLMTranslateTracker | None:
+        if self.llm_translate_trackers:
+            return self.llm_translate_trackers[-1]
+        return None
+
+
+class LLMTranslateTracker:
+    def __init__(self):
+        self.input = ""
+        self.output = ""
+        self.has_error = False
+        self.error_message = ""
+        self.placeholder_full_match = False
+        self.fallback_to_translate = False
+
+    def set_input(self, input_text: str):
+        self.input = input_text
+
+    def set_output(self, output_text: str):
+        self.output = output_text
+
+    def set_error_message(self, error_message: str):
+        self.has_error = True
+        self.error_message = error_message
+
+    def set_placeholder_full_match(self):
+        self.placeholder_full_match = True
+
+    def set_fallback_to_translate(self):
+        self.fallback_to_translate = True
+
+    def to_dict(self):
+        return {
+            "input": self.input,
+            "output": self.output,
+            "has_error": self.has_error,
+            "error_message": self.error_message,
+            "placeholder_full_match": self.placeholder_full_match,
+            "fallback_to_translate": self.fallback_to_translate,
+        }
+
 
 class ILTranslator:
     stage_name = "Translate Paragraphs"
@@ -156,9 +221,11 @@ class ILTranslator:
             # Try to find the first title paragraph
             title_paragraph = self.find_title_paragraph(docs)
             self.translation_config.shared_context_cross_split_part.first_paragraph = (
+                copy.deepcopy(title_paragraph)
+            )
+            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = copy.deepcopy(
                 title_paragraph
             )
-            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = title_paragraph
             if title_paragraph:
                 logger.info(f"Found first title paragraph: {title_paragraph.unicode}")
 
@@ -220,7 +287,9 @@ class ILTranslator:
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
             paragraph_token_count = self.calc_token_count(paragraph.unicode)
             if paragraph.layout_label == "title":
-                self.shared_context_cross_split_part.recent_title_paragraph = paragraph
+                self.shared_context_cross_split_part.recent_title_paragraph = (
+                    copy.deepcopy(paragraph)
+                )
             executor.submit(
                 self.translate_paragraph,
                 paragraph,
@@ -245,6 +314,24 @@ class ILTranslator:
             self.placeholders = placeholders
             self.base_style = base_style
 
+        def get_placeholders_hint(self) -> dict[str, str] | None:
+            hint = {}
+            for placeholder in self.placeholders:
+                if isinstance(placeholder, FormulaPlaceholder):
+                    cid_count = 0
+                    for char in placeholder.formula.pdf_character:
+                        if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+                            cid_count += 1
+                    if cid_count > len(placeholder.formula.pdf_character) * 0.8:
+                        continue
+
+                    hint[placeholder.placeholder] = get_char_unicode_string(
+                        placeholder.formula.pdf_character
+                    )
+            if hint:
+                return hint
+            return None
+
     def create_formula_placeholder(
         self,
         formula: PdfFormula,
@@ -252,10 +339,14 @@ class ILTranslator:
         paragraph: PdfParagraph,
     ):
         placeholder = self.translate_engine.get_formular_placeholder(formula_id)
+        if isinstance(placeholder, tuple):
+            placeholder, regex_pattern = placeholder
+        else:
+            regex_pattern = re.escape(placeholder)
         if placeholder in paragraph.unicode:
             return self.create_formula_placeholder(formula, formula_id + 1, paragraph)
 
-        return FormulaPlaceholder(formula_id, formula, placeholder)
+        return FormulaPlaceholder(formula_id, formula, placeholder, regex_pattern)
 
     def create_rich_text_placeholder(
         self,
@@ -269,6 +360,14 @@ class ILTranslator:
         right_placeholder = self.translate_engine.get_rich_text_right_placeholder(
             composition_id,
         )
+        if isinstance(left_placeholder, tuple):
+            left_placeholder, left_placeholder_regex_pattern = left_placeholder
+        else:
+            left_placeholder_regex_pattern = re.escape(left_placeholder)
+        if isinstance(right_placeholder, tuple):
+            right_placeholder, right_placeholder_regex_pattern = right_placeholder
+        else:
+            right_placeholder_regex_pattern = re.escape(right_placeholder)
         if (
             left_placeholder in paragraph.unicode
             or right_placeholder in paragraph.unicode
@@ -284,6 +383,8 @@ class ILTranslator:
             composition,
             left_placeholder,
             right_placeholder,
+            left_placeholder_regex_pattern,
+            right_placeholder_regex_pattern,
         )
 
     def get_translate_input(
@@ -452,9 +553,8 @@ class ILTranslator:
         self,
         input_text: TranslateInput,
         output: str,
+        llm_translate_tracker: LLMTranslateTracker | None = None,
     ) -> [PdfParagraphComposition]:
-        import re
-
         result = []
 
         # 如果没有占位符，直接返回整个文本
@@ -463,6 +563,8 @@ class ILTranslator:
             comp.pdf_same_style_unicode_characters = PdfSameStyleUnicodeCharacters()
             comp.pdf_same_style_unicode_characters.unicode = output
             comp.pdf_same_style_unicode_characters.pdf_style = input_text.base_style
+            if llm_translate_tracker:
+                llm_translate_tracker.set_placeholder_full_match()
             return [comp]
 
         # 构建正则表达式模式
@@ -473,18 +575,28 @@ class ILTranslator:
         for placeholder in input_text.placeholders:
             if isinstance(placeholder, FormulaPlaceholder):
                 # 转义特殊字符
-                pattern = re.escape(placeholder.placeholder)
+                # pattern = re.escape(placeholder.placeholder)
+                pattern = placeholder.regex_pattern
                 patterns.append(f"({pattern})")
                 placeholder_patterns.append(f"({pattern})")
                 placeholder_map[placeholder.placeholder] = placeholder
             else:
-                left = re.escape(placeholder.left_placeholder)
-                right = re.escape(placeholder.right_placeholder)
+                left = placeholder.left_regex_pattern
+                right = placeholder.right_regex_pattern
                 patterns.append(f"({left}.*?{right})")
                 placeholder_patterns.append(f"({left})")
                 placeholder_patterns.append(f"({right})")
                 placeholder_map[placeholder.left_placeholder] = placeholder
-
+        all_match = True
+        for pattern in patterns:
+            if not re.search(pattern, output, flags=re.IGNORECASE):
+                all_match = False
+                break
+        if all_match:
+            if llm_translate_tracker:
+                llm_translate_tracker.set_placeholder_full_match()
+        else:
+            logger.debug(f"Failed to match all placeholder for {input_text.unicode}")
         # 合并所有模式
         combined_pattern = "|".join(patterns)
         combined_placeholder_pattern = "|".join(placeholder_patterns)
@@ -534,13 +646,12 @@ class ILTranslator:
                     p
                     for p in input_text.placeholders
                     if not isinstance(p, FormulaPlaceholder)
-                    and matched_text.startswith(p.left_placeholder)
+                    and re.match(f"^{p.left_regex_pattern}", matched_text)
                 )
-                text = matched_text[
-                    len(placeholder.left_placeholder) : -len(
-                        placeholder.right_placeholder,
-                    )
-                ]
+                text = re.match(
+                    f"^{placeholder.left_regex_pattern}(.+){placeholder.right_regex_pattern}$",
+                    matched_text,
+                ).group(1)
 
                 if isinstance(
                     placeholder.composition,
@@ -626,10 +737,12 @@ class ILTranslator:
         """Post-translation processing: update paragraph with translated text."""
         tracker.set_output(translated_text)
         if translated_text == translate_input.unicode:
+            if llm_translate_tracker := tracker.last_llm_translate_tracker():
+                llm_translate_tracker.set_placeholder_full_match()
             return False
         paragraph.unicode = translated_text
         paragraph.pdf_paragraph_composition = self.parse_translate_output(
-            translate_input, translated_text
+            translate_input, translated_text, tracker.last_llm_translate_tracker()
         )
         for composition in paragraph.pdf_paragraph_composition:
             if (
@@ -646,35 +759,75 @@ class ILTranslator:
         text: str,
         title_paragraph: PdfParagraph | None = None,
         local_title_paragraph: PdfParagraph | None = None,
+        translate_input: TranslateInput | None = None,
     ):
         if self.translation_config.custom_system_prompt:
             llm_input = [self.translation_config.custom_system_prompt]
         else:
             llm_input = [
-                "You are a professional, authentic machine translation engine."
+                f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
             ]
+        llm_hint = []
 
         if title_paragraph:
-            llm_input.append(
+            llm_hint.append(
                 f"The first title in the full text: {title_paragraph.unicode}"
             )
         if (
             local_title_paragraph
             and local_title_paragraph.debug_id != title_paragraph.debug_id
         ):
-            llm_input.append(
+            llm_hint.append(
                 f"The most similar title in the full text: {local_title_paragraph.unicode}"
             )
+
+        if self.translation_config.add_formula_placehold_hint:
+            placeholders_hint = translate_input.get_placeholders_hint()
+            if placeholders_hint:
+                llm_hint.append(
+                    f"This is the formula placeholder hint: \n{placeholders_hint}"
+                )
+
+        if llm_hint:
+            llm_input.append(
+                "When translating, please refer to the following information to improve translation quality:"
+            )
+            for i, line in enumerate(llm_hint):
+                llm_input.append(f"{i}. {line}")
+        llm_input.append("When translating, please follow the following rules:")
+
+        rich_text_left_placeholder = (
+            self.translate_engine.get_rich_text_left_placeholder(1)
+        )
+        if isinstance(rich_text_left_placeholder, tuple):
+            rich_text_left_placeholder = rich_text_left_placeholder[0]
+        rich_text_right_placeholder = (
+            self.translate_engine.get_rich_text_right_placeholder(2)
+        )
+        if isinstance(rich_text_right_placeholder, tuple):
+            rich_text_right_placeholder = rich_text_right_placeholder[0]
+
         # Create a structured prompt template for LLM translation
         llm_input.append(
-            f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
+            f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
         )
 
+        formula_placeholder = self.translate_engine.get_formular_placeholder(3)
+        if isinstance(formula_placeholder, tuple):
+            formula_placeholder = formula_placeholder[0]
+
+        llm_input.append(
+            f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
+        )
         llm_input.append(
-            f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
+            "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
         )
+        llm_input.append(
+            "4. Only output the translation result without explanations and annotations."
+        )
+        llm_input.append(f"5. Translate text into {self.translation_config.lang_out}.")
         prompt_template = f"""
-;; Treat next line as plain text input and translate it into {self.translation_config.lang_out}, output translation ONLY. If translation is unnecessary (e.g. proper nouns, codes, {"{{1}}, etc. "}), return the original text. NO explanations. NO notes. Input:\n\n{text}
+Now, please carefully read the following text to be translated and directly output your translation.\n\n{text}
 
 """
         llm_input.append(prompt_template)
@@ -704,18 +857,20 @@ class ILTranslator:
                 )
                 if text is None:
                     return
-
+                llm_translate_tracker = tracker.new_llm_translate_tracker()
                 # Perform translation
                 if self.support_llm_translate:
                     llm_prompt = self.generate_prompt_for_llm(
-                        text, title_paragraph, local_title_paragraph
+                        text, title_paragraph, local_title_paragraph, translate_input
                     )
+                    llm_translate_tracker.set_input(llm_prompt)
                     translated_text = self.translate_engine.llm_translate(
                         llm_prompt,
                         rate_limit_params={
                             "paragraph_token_count": paragraph_token_count
                         },
                     )
+                    llm_translate_tracker.set_output(translated_text)
                 else:
                     translated_text = self.translate_engine.translate(
                         text,

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -1,3 +1,4 @@
+import copy
 import json
 import logging
 import re
@@ -11,6 +12,7 @@ from babeldoc.document_il import Document
 from babeldoc.document_il import Page
 from babeldoc.document_il import PdfFont
 from babeldoc.document_il import PdfParagraph
+from babeldoc.document_il.midend import il_translator
 from babeldoc.document_il.midend.il_translator import DocumentTranslateTracker
 from babeldoc.document_il.midend.il_translator import ILTranslator
 from babeldoc.document_il.midend.il_translator import PageTranslateTracker
@@ -94,9 +96,11 @@ class ILTranslatorLLMOnly:
             # Try to find the first title paragraph
             title_paragraph = self.find_title_paragraph(docs)
             self.translation_config.shared_context_cross_split_part.first_paragraph = (
+                copy.deepcopy(title_paragraph)
+            )
+            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = copy.deepcopy(
                 title_paragraph
             )
-            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = title_paragraph
             if title_paragraph:
                 logger.info(f"Found first title paragraph: {title_paragraph.unicode}")
 
@@ -169,9 +173,11 @@ class ILTranslatorLLMOnly:
             total_token_count += self.calc_token_count(paragraph.unicode)
             paragraphs.append(paragraph)
             if paragraph.layout_label == "title":
-                self.shared_context_cross_split_part.recent_title_paragraph = paragraph
+                self.shared_context_cross_split_part.recent_title_paragraph = (
+                    copy.deepcopy(paragraph)
+                )
 
-            if total_token_count > 400 or len(paragraphs) > 5:
+            if total_token_count > 200 or len(paragraphs) > 10:
                 executor.submit(
                     self.translate_paragraph,
                     BatchParagraph(paragraphs, tracker),
@@ -217,7 +223,7 @@ class ILTranslatorLLMOnly:
         should_translate_paragraph = []
         try:
             inputs = []
-
+            llm_translate_trackers = []
             for i in range(len(batch_paragraph.paragraphs)):
                 paragraph = batch_paragraph.paragraphs[i]
                 tracker = batch_paragraph.trackers[i]
@@ -227,20 +233,31 @@ class ILTranslatorLLMOnly:
                 if text is None:
                     pbar.advance(1)
                     continue
+                llm_translate_tracker = tracker.new_llm_translate_tracker()
                 should_translate_paragraph.append(i)
-                inputs.append((text, translate_input, paragraph, tracker))
+                llm_translate_trackers.append(llm_translate_tracker)
+                inputs.append(
+                    (text, translate_input, paragraph, tracker, llm_translate_tracker)
+                )
             if not inputs:
                 return
             json_format_input = []
 
             for id_, input_text in enumerate(inputs):
-                json_format_input.append(
-                    {
-                        "id": id_,
-                        "input": input_text[0],
-                        "layout_label": input_text[2].layout_label,
-                    }
-                )
+                ti: il_translator.ILTranslator.TranslateInput = input_text[1]
+                placeholders_hint = ti.get_placeholders_hint()
+                obj = {
+                    "id": id_,
+                    "input": input_text[0],
+                    "layout_label": input_text[2].layout_label,
+                }
+                if (
+                    placeholders_hint
+                    and self.translation_config.add_formula_placehold_hint
+                ):
+                    obj["formula_placeholders_hint"] = placeholders_hint
+                json_format_input.append(obj)
+
             json_format_input = json.dumps(
                 json_format_input, ensure_ascii=False, indent=2
             )
@@ -248,24 +265,57 @@ class ILTranslatorLLMOnly:
                 llm_input = [self.translation_config.custom_system_prompt]
             else:
                 llm_input = [
-                    "You are a professional, authentic machine translation engine."
+                    f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
                 ]
 
+            llm_hint = []
             if title_paragraph:
-                llm_input.append(
+                llm_hint.append(
                     f"The first title in the full text: {title_paragraph.unicode}"
                 )
             if (
                 local_title_paragraph
                 and local_title_paragraph.debug_id != title_paragraph.debug_id
             ):
-                llm_input.append(
+                llm_hint.append(
                     f"The most similar title in the full text: {local_title_paragraph.unicode}"
                 )
+
+            if llm_hint:
+                llm_input.append(
+                    "When translating, please refer to the following information to improve translation quality:"
+                )
+                for i, line in enumerate(llm_hint):
+                    llm_input.append(f"{i}. {line}")
+            llm_input.append("When translating, please follow the following rules:")
+            # Create a structured prompt template for LLM translation
+            rich_text_left_placeholder = (
+                self.translate_engine.get_rich_text_left_placeholder(1)
+            )
+            if isinstance(rich_text_left_placeholder, tuple):
+                rich_text_left_placeholder = rich_text_left_placeholder[0]
+            rich_text_right_placeholder = (
+                self.translate_engine.get_rich_text_right_placeholder(2)
+            )
+            if isinstance(rich_text_right_placeholder, tuple):
+                rich_text_right_placeholder = rich_text_right_placeholder[0]
+            llm_input.append(
+                f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
+            )
+
+            formula_placeholder = self.translate_engine.get_formular_placeholder(3)
+            if isinstance(formula_placeholder, tuple):
+                formula_placeholder = formula_placeholder[0]
+            llm_input.append(
+                f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
+            )
+            llm_input.append(
+                "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
+            )
             # Create a structured prompt template for LLM translation
             prompt_template = (
                 f"""
-    You will be given a JSON formatted input containing entries with "id" and "input" fields. 
+    4. You will be given a JSON formatted input containing entries with "id" and "input" fields. 
     
     For each entry in the JSON, translate the contents of the "input" field into {self.translation_config.lang_out}.
     Write the translation back into the "output" field for that entry.
@@ -280,7 +330,12 @@ class ILTranslatorLLMOnly:
     {
         "id": 1,
         "input": "Source",
-        "layout_label": "plain text"
+        "layout_label": "plain text",
+        // this is optional
+        "formula_placeholders_hint": {
+            "placeholder1": "hint1",
+            "placeholder2": "hint2"
+        }
     }
     ```
     Output:
@@ -296,13 +351,6 @@ class ILTranslatorLLMOnly:
     """
             )
             llm_input.append(prompt_template)
-            llm_input.append(
-                f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
-            )
-
-            llm_input.append(
-                f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
-            )
             llm_input.append(
                 f"""Here is the input:
     
@@ -311,17 +359,24 @@ class ILTranslatorLLMOnly:
     ```"""
             )
             final_input = "\n".join(llm_input).strip()
+            for llm_translate_tracker in llm_translate_trackers:
+                llm_translate_tracker.set_input(final_input)
             llm_output = self.translate_engine.llm_translate(
                 final_input,
                 rate_limit_params={"paragraph_token_count": paragraph_token_count},
             )
+            for llm_translate_tracker in llm_translate_trackers:
+                llm_translate_tracker.set_output(llm_output)
             llm_output = llm_output.strip()
 
             llm_output = self._clean_json_output(llm_output)
 
             parsed_output = json.loads(llm_output)
 
-            translation_results = {item["id"]: item["output"] for item in parsed_output}
+            translation_results = {
+                item["id"]: item.get("output", item.get("input"))
+                for item in parsed_output
+            }
 
             if len(translation_results) != len(inputs):
                 raise Exception(
@@ -347,6 +402,7 @@ class ILTranslatorLLMOnly:
 
                     # Get the original input for this translation
                     translate_input = inputs[id_][1]
+                    llm_translate_tracker = inputs[id_][4]
 
                     input_unicode = inputs[id_][2].unicode
                     output_unicode = translated_text
@@ -355,6 +411,9 @@ class ILTranslatorLLMOnly:
                     output_token_count = self.calc_token_count(output_unicode)
 
                     if not (0.3 < output_token_count / input_token_count < 3):
+                        llm_translate_tracker.set_error_message(
+                            f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
+                        )
                         logger.warning(
                             f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
                         )
@@ -362,6 +421,9 @@ class ILTranslatorLLMOnly:
 
                     edit_distance = Levenshtein.distance(input_unicode, output_unicode)
                     if edit_distance < 5 and input_token_count > 20:
+                        llm_translate_tracker.set_error_message(
+                            f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
+                        )
                         logger.warning(
                             f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
                         )
@@ -377,11 +439,15 @@ class ILTranslatorLLMOnly:
                     if pbar:
                         pbar.advance(1)
                 except Exception as e:
-                    logger.exception(f"Error translating paragraph. Error: {e}.")
+                    error_message = f"Error translating paragraph. Error: {e}."
+                    logger.exception(error_message)
                     # Ignore error and continue
+                    for llm_translate_tracker in llm_translate_trackers:
+                        llm_translate_tracker.set_error_message(error_message)
                     continue
                 finally:
                     if should_fallback:
+                        inputs[id_][4].set_fallback_to_translate()
                         logger.warning(
                             f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
                         )
@@ -402,7 +468,11 @@ class ILTranslatorLLMOnly:
                         )
 
         except Exception as e:
-            logger.warning(f"Error {e} during translation. try fallback")
+            error_message = f"Error {e} during translation. try fallback"
+            logger.warning(error_message)
+            for llm_translate_tracker in llm_translate_trackers:
+                llm_translate_tracker.set_error_message(error_message)
+                llm_translate_tracker.set_fallback_to_translate()
             if not should_translate_paragraph:
                 should_translate_paragraph = list(
                     range(len(batch_paragraph.paragraphs))

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/midend/layout_parser.py

@@ -78,7 +78,7 @@ class LayoutParser:
             # Note: PDF coordinates are from bottom-left,
             # so we use y2 for top position
             style = il_version_1.PdfStyle(
-                font_id="china-ss",
+                font_id="base",
                 font_size=4,
                 graphic_state=color,
             )

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -606,6 +606,7 @@ class StylesAndFormulas:
             r"|.*NimbusSan.*"
             r"|.*Mincho.*"
             r"|.*Amerigo.*"
+            r"|.*MSGloriolaIIStd.*"
             r")$"
         )
         precise_formula_font_pattern = (

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/midend/table_parser.py

@@ -83,7 +83,7 @@ class TableParser:
             # Note: PDF coordinates are from bottom-left,
             # so we use y2 for top position
             style = il_version_1.PdfStyle(
-                font_id="china-ss",
+                font_id="base",
                 font_size=4,
                 graphic_state=color,
             )

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/translator/translator.py

@@ -277,11 +277,14 @@ class OpenAITranslator(BaseTranslator):
             logger.exception("Error updating token count")
 
     def get_formular_placeholder(self, placeholder_id: int):
-        return "{{v" + str(placeholder_id) + "}}"
+        return "{v" + str(placeholder_id) + "}", f"{{\\s*v\\s*{placeholder_id}\\s*}}"
         return "{{" + str(placeholder_id) + "}}"
 
     def get_rich_text_left_placeholder(self, placeholder_id: int):
-        return f"<style id='{placeholder_id}'>"
+        return (
+            f"<style id='{placeholder_id}'>",
+            f"<\\s*style\\s*id\\s*=\\s*'\\s*{placeholder_id}\\s*'\\s*>",
+        )
 
     def get_rich_text_right_placeholder(self, placeholder_id: int):
-        return "</style>"
+        return "</style>", r"<\s*\/\s*style\s*>"

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/document_il/utils/style_helper.py

@@ -1,7 +1,7 @@
 from babeldoc.document_il import il_version_1
 
 
-def create_pdf_style(r, g, b, font_id="china-ss", font_size=6):
+def create_pdf_style(r, g, b, font_id="base", font_size=6):
     """
     Create a PdfStyle object from RGB values.
 

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/high_level.py

@@ -651,8 +651,8 @@ def _do_translate_single(
 
     mediabox_data = fix_media_box(doc_pdf2zh)
 
-    for page in doc_pdf2zh:
-        page.insert_font(resfont, None)
+    # for page in doc_pdf2zh:
+    #     page.insert_font(resfont, None)
 
     resfont = None
     doc_pdf2zh.save(temp_pdf_path)

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.39"
+__version__ = "0.3.41"
 
 
 def create_parser():

{babeldoc-0.3.39 → babeldoc-0.3.41}/babeldoc/translation_config.py

@@ -70,6 +70,7 @@ class TranslationConfig:
         skip_scanned_detection: bool = False,
         ocr_workaround: bool = False,
         custom_system_prompt: str | None = None,
+        add_formula_placehold_hint: bool = False,
     ):
         self.translator = translator
 
@@ -155,6 +156,7 @@ class TranslationConfig:
         self.table_model = table_model
         self.show_char_box = show_char_box
         self.custom_system_prompt = custom_system_prompt
+        self.add_formula_placehold_hint = add_formula_placehold_hint
 
     def parse_pages(self, pages_str: str | None) -> list[tuple[int, int]] | None:
         """解析页码字符串，返回页码范围列表

{babeldoc-0.3.39 → babeldoc-0.3.41}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.39"
+version = "0.3.41"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.39"
+current_version = "0.3.41"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.39 → babeldoc-0.3.41}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.39"
+version = "0.3.41"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.3.39/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.39"