BabelDOC 0.3.40__tar.gz → 0.3.42__tar.gz

{babeldoc-0.3.40 → babeldoc-0.3.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.40
+Version: 0.3.42
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.3.42/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.42"

{babeldoc-0.3.40 → babeldoc-0.3.42}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.40"
+__version__ = "0.3.42"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.40 → babeldoc-0.3.42}/babeldoc/document_il/midend/il_translator.py

@@ -39,18 +39,29 @@ class RichTextPlaceholder:
         composition: PdfSameStyleCharacters,
         left_placeholder: str,
         right_placeholder: str,
+        left_regex_pattern: str = None,
+        right_regex_pattern: str = None,
     ):
         self.id = placeholder_id
         self.composition = composition
         self.left_placeholder = left_placeholder
         self.right_placeholder = right_placeholder
+        self.left_regex_pattern = left_regex_pattern
+        self.right_regex_pattern = right_regex_pattern
 
 
 class FormulaPlaceholder:
-    def __init__(self, placeholder_id: int, formula: PdfFormula, placeholder: str):
+    def __init__(
+        self,
+        placeholder_id: int,
+        formula: PdfFormula,
+        placeholder: str,
+        regex_pattern: str,
+    ):
         self.id = placeholder_id
         self.formula = formula
         self.placeholder = placeholder
+        self.regex_pattern = regex_pattern
 
 
 class PbarContext:
@@ -328,10 +339,14 @@ class ILTranslator:
         paragraph: PdfParagraph,
     ):
         placeholder = self.translate_engine.get_formular_placeholder(formula_id)
+        if isinstance(placeholder, tuple):
+            placeholder, regex_pattern = placeholder
+        else:
+            regex_pattern = re.escape(placeholder)
         if placeholder in paragraph.unicode:
             return self.create_formula_placeholder(formula, formula_id + 1, paragraph)
 
-        return FormulaPlaceholder(formula_id, formula, placeholder)
+        return FormulaPlaceholder(formula_id, formula, placeholder, regex_pattern)
 
     def create_rich_text_placeholder(
         self,
@@ -345,6 +360,14 @@ class ILTranslator:
         right_placeholder = self.translate_engine.get_rich_text_right_placeholder(
             composition_id,
         )
+        if isinstance(left_placeholder, tuple):
+            left_placeholder, left_placeholder_regex_pattern = left_placeholder
+        else:
+            left_placeholder_regex_pattern = re.escape(left_placeholder)
+        if isinstance(right_placeholder, tuple):
+            right_placeholder, right_placeholder_regex_pattern = right_placeholder
+        else:
+            right_placeholder_regex_pattern = re.escape(right_placeholder)
         if (
             left_placeholder in paragraph.unicode
             or right_placeholder in paragraph.unicode
@@ -360,6 +383,8 @@ class ILTranslator:
             composition,
             left_placeholder,
             right_placeholder,
+            left_placeholder_regex_pattern,
+            right_placeholder_regex_pattern,
         )
 
     def get_translate_input(
@@ -550,13 +575,14 @@ class ILTranslator:
         for placeholder in input_text.placeholders:
             if isinstance(placeholder, FormulaPlaceholder):
                 # 转义特殊字符
-                pattern = re.escape(placeholder.placeholder)
+                # pattern = re.escape(placeholder.placeholder)
+                pattern = placeholder.regex_pattern
                 patterns.append(f"({pattern})")
                 placeholder_patterns.append(f"({pattern})")
                 placeholder_map[placeholder.placeholder] = placeholder
             else:
-                left = re.escape(placeholder.left_placeholder)
-                right = re.escape(placeholder.right_placeholder)
+                left = placeholder.left_regex_pattern
+                right = placeholder.right_regex_pattern
                 patterns.append(f"({left}.*?{right})")
                 placeholder_patterns.append(f"({left})")
                 placeholder_patterns.append(f"({right})")
@@ -569,6 +595,8 @@ class ILTranslator:
         if all_match:
             if llm_translate_tracker:
                 llm_translate_tracker.set_placeholder_full_match()
+        else:
+            logger.debug(f"Failed to match all placeholder for {input_text.unicode}")
         # 合并所有模式
         combined_pattern = "|".join(patterns)
         combined_placeholder_pattern = "|".join(placeholder_patterns)
@@ -618,13 +646,12 @@ class ILTranslator:
                     p
                     for p in input_text.placeholders
                     if not isinstance(p, FormulaPlaceholder)
-                    and matched_text.startswith(p.left_placeholder)
+                    and re.match(f"^{p.left_regex_pattern}", matched_text)
                 )
-                text = matched_text[
-                    len(placeholder.left_placeholder) : -len(
-                        placeholder.right_placeholder,
-                    )
-                ]
+                text = re.match(
+                    f"^{placeholder.left_regex_pattern}(.+){placeholder.right_regex_pattern}$",
+                    matched_text,
+                ).group(1)
 
                 if isinstance(
                     placeholder.composition,
@@ -738,39 +765,69 @@ class ILTranslator:
             llm_input = [self.translation_config.custom_system_prompt]
         else:
             llm_input = [
-                "You are a professional, authentic machine translation engine."
+                f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
             ]
+        llm_hint = []
 
         if title_paragraph:
-            llm_input.append(
+            llm_hint.append(
                 f"The first title in the full text: {title_paragraph.unicode}"
             )
         if (
             local_title_paragraph
             and local_title_paragraph.debug_id != title_paragraph.debug_id
         ):
-            llm_input.append(
+            llm_hint.append(
                 f"The most similar title in the full text: {local_title_paragraph.unicode}"
             )
-        # Create a structured prompt template for LLM translation
-        llm_input.append(
-            f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
-        )
 
-        llm_input.append(
-            f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
-        )
         if self.translation_config.add_formula_placehold_hint:
             placeholders_hint = translate_input.get_placeholders_hint()
             if placeholders_hint:
-                llm_input.append(
+                llm_hint.append(
                     f"This is the formula placeholder hint: \n{placeholders_hint}"
                 )
-                llm_input.append(
-                    "The system will automatically replace placeholders with corresponding formulas, please do not translate the placeholders!"
-                )
+
+        if llm_hint:
+            llm_input.append(
+                "When translating, please refer to the following information to improve translation quality:"
+            )
+            for i, line in enumerate(llm_hint):
+                llm_input.append(f"{i}. {line}")
+        llm_input.append("When translating, please follow the following rules:")
+
+        rich_text_left_placeholder = (
+            self.translate_engine.get_rich_text_left_placeholder(1)
+        )
+        if isinstance(rich_text_left_placeholder, tuple):
+            rich_text_left_placeholder = rich_text_left_placeholder[0]
+        rich_text_right_placeholder = (
+            self.translate_engine.get_rich_text_right_placeholder(2)
+        )
+        if isinstance(rich_text_right_placeholder, tuple):
+            rich_text_right_placeholder = rich_text_right_placeholder[0]
+
+        # Create a structured prompt template for LLM translation
+        llm_input.append(
+            f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
+        )
+
+        formula_placeholder = self.translate_engine.get_formular_placeholder(3)
+        if isinstance(formula_placeholder, tuple):
+            formula_placeholder = formula_placeholder[0]
+
+        llm_input.append(
+            f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
+        )
+        llm_input.append(
+            "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
+        )
+        llm_input.append(
+            "4. Only output the translation result without explanations and annotations."
+        )
+        llm_input.append(f"5. Translate text into {self.translation_config.lang_out}.")
         prompt_template = f"""
-;; Treat next line as plain text input and translate it into {self.translation_config.lang_out}, output translation ONLY. If translation is unnecessary (e.g. proper nouns, codes, {"{{1}}, etc. "}), return the original text. NO explanations. NO notes. Input:\n\n{text}
+Now, please carefully read the following text to be translated and directly output your translation.\n\n{text}
 
 """
         llm_input.append(prompt_template)

{babeldoc-0.3.40 → babeldoc-0.3.42}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -177,7 +177,7 @@ class ILTranslatorLLMOnly:
                     copy.deepcopy(paragraph)
                 )
 
-            if total_token_count > 400 or len(paragraphs) > 5:
+            if total_token_count > 200 or len(paragraphs) > 10:
                 executor.submit(
                     self.translate_paragraph,
                     BatchParagraph(paragraphs, tracker),
@@ -224,6 +224,7 @@ class ILTranslatorLLMOnly:
         try:
             inputs = []
             llm_translate_trackers = []
+            paragraph_unicodes = []
             for i in range(len(batch_paragraph.paragraphs)):
                 paragraph = batch_paragraph.paragraphs[i]
                 tracker = batch_paragraph.trackers[i]
@@ -237,8 +238,16 @@ class ILTranslatorLLMOnly:
                 should_translate_paragraph.append(i)
                 llm_translate_trackers.append(llm_translate_tracker)
                 inputs.append(
-                    (text, translate_input, paragraph, tracker, llm_translate_tracker)
+                    (
+                        text,
+                        translate_input,
+                        paragraph,
+                        tracker,
+                        llm_translate_tracker,
+                        paragraph_unicodes,
+                    )
                 )
+                paragraph_unicodes.append(paragraph.unicode)
             if not inputs:
                 return
             json_format_input = []
@@ -265,24 +274,57 @@ class ILTranslatorLLMOnly:
                 llm_input = [self.translation_config.custom_system_prompt]
             else:
                 llm_input = [
-                    "You are a professional, authentic machine translation engine."
+                    f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
                 ]
 
+            llm_hint = []
             if title_paragraph:
-                llm_input.append(
+                llm_hint.append(
                     f"The first title in the full text: {title_paragraph.unicode}"
                 )
             if (
                 local_title_paragraph
                 and local_title_paragraph.debug_id != title_paragraph.debug_id
             ):
-                llm_input.append(
+                llm_hint.append(
                     f"The most similar title in the full text: {local_title_paragraph.unicode}"
                 )
+
+            if llm_hint:
+                llm_input.append(
+                    "When translating, please refer to the following information to improve translation quality:"
+                )
+                for i, line in enumerate(llm_hint):
+                    llm_input.append(f"{i}. {line}")
+            llm_input.append("When translating, please follow the following rules:")
+            # Create a structured prompt template for LLM translation
+            rich_text_left_placeholder = (
+                self.translate_engine.get_rich_text_left_placeholder(1)
+            )
+            if isinstance(rich_text_left_placeholder, tuple):
+                rich_text_left_placeholder = rich_text_left_placeholder[0]
+            rich_text_right_placeholder = (
+                self.translate_engine.get_rich_text_right_placeholder(2)
+            )
+            if isinstance(rich_text_right_placeholder, tuple):
+                rich_text_right_placeholder = rich_text_right_placeholder[0]
+            llm_input.append(
+                f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
+            )
+
+            formula_placeholder = self.translate_engine.get_formular_placeholder(3)
+            if isinstance(formula_placeholder, tuple):
+                formula_placeholder = formula_placeholder[0]
+            llm_input.append(
+                f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
+            )
+            llm_input.append(
+                "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
+            )
             # Create a structured prompt template for LLM translation
             prompt_template = (
                 f"""
-    You will be given a JSON formatted input containing entries with "id" and "input" fields. 
+    4. You will be given a JSON formatted input containing entries with "id" and "input" fields. 
     
     For each entry in the JSON, translate the contents of the "input" field into {self.translation_config.lang_out}.
     Write the translation back into the "output" field for that entry.
@@ -318,16 +360,6 @@ class ILTranslatorLLMOnly:
     """
             )
             llm_input.append(prompt_template)
-            llm_input.append(
-                f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
-            )
-            llm_input.append(
-                f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
-            )
-            if self.translation_config.add_formula_placehold_hint:
-                llm_input.append(
-                    "The system will automatically replace placeholders with corresponding formulas, please do not translate the placeholders!"
-                )
             llm_input.append(
                 f"""Here is the input:
     
@@ -350,7 +382,10 @@ class ILTranslatorLLMOnly:
 
             parsed_output = json.loads(llm_output)
 
-            translation_results = {item["id"]: item["output"] for item in parsed_output}
+            translation_results = {
+                item["id"]: item.get("output", item.get("input"))
+                for item in parsed_output
+            }
 
             if len(translation_results) != len(inputs):
                 raise Exception(
@@ -428,6 +463,8 @@ class ILTranslatorLLMOnly:
                         paragraph_token_count = self.calc_token_count(
                             inputs[id_][2].unicode
                         )
+                        paragraph_unicodes = inputs[id_][5]
+                        inputs[id_][2].unicode = paragraph_unicodes[id_]
                         executor.submit(
                             self.il_translator.translate_paragraph,
                             inputs[id_][2],
@@ -446,6 +483,9 @@ class ILTranslatorLLMOnly:
             logger.warning(error_message)
             for llm_translate_tracker in llm_translate_trackers:
                 llm_translate_tracker.set_error_message(error_message)
+                llm_translate_tracker.set_fallback_to_translate()
+            for input_ in inputs:
+                input_[2].unicode = input_[5]
             if not should_translate_paragraph:
                 should_translate_paragraph = list(
                     range(len(batch_paragraph.paragraphs))

{babeldoc-0.3.40 → babeldoc-0.3.42}/babeldoc/document_il/translator/translator.py

@@ -277,11 +277,14 @@ class OpenAITranslator(BaseTranslator):
             logger.exception("Error updating token count")
 
     def get_formular_placeholder(self, placeholder_id: int):
-        return "{v" + str(placeholder_id) + "}"
+        return "{v" + str(placeholder_id) + "}", f"{{\\s*v\\s*{placeholder_id}\\s*}}"
         return "{{" + str(placeholder_id) + "}}"
 
     def get_rich_text_left_placeholder(self, placeholder_id: int):
-        return f"<style id='{placeholder_id}'>"
+        return (
+            f"<style id='{placeholder_id}'>",
+            f"<\\s*style\\s*id\\s*=\\s*'\\s*{placeholder_id}\\s*'\\s*>",
+        )
 
     def get_rich_text_right_placeholder(self, placeholder_id: int):
-        return "</style>"
+        return "</style>", r"<\s*\/\s*style\s*>"

{babeldoc-0.3.40 → babeldoc-0.3.42}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.40"
+__version__ = "0.3.42"
 
 
 def create_parser():

{babeldoc-0.3.40 → babeldoc-0.3.42}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.40"
+version = "0.3.42"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.40"
+current_version = "0.3.42"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.40 → babeldoc-0.3.42}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.40"
+version = "0.3.42"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.3.40/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.40"