BabelDOC 0.3.39__tar.gz → 0.3.40__tar.gz

{babeldoc-0.3.39 → babeldoc-0.3.40}/.github/workflows/publish-to-pypi.yml

@@ -61,8 +61,8 @@ jobs:
       - name: Bump version for developmental release
         if: "! steps.check-version.outputs.tag"
         run: |
-          version=$(bumpver update --patch --tag=final --dry 2>&1 | grep "New Version" | awk '{print $NF}') &&
-          bumpver update --set-version $version.dev$(date +%s)
+          version=$(uv run bumpver update --patch --tag=final --dry 2>&1 | grep "New Version" | awk '{print $NF}') &&
+          uv run bumpver update --set-version $version.dev$(date +%s)
 
       - name: Build package
         run: "uv build"

{babeldoc-0.3.39 → babeldoc-0.3.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.39
+Version: 0.3.40
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -216,6 +216,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--min-text-length`: Minimum text length to translate (default: 5)
 - `--openai`: Use OpenAI for translation (default: False)
 - `--custom-system-prompt`: Custom system prompt for translation.
+- `--add-formula-placehold-hint`: Add formula placeholder hint for translation. (Currently not recommended, it may affect translation quality, default: False)
 
 > [!TIP]
 >

{babeldoc-0.3.39 → babeldoc-0.3.40}/README.md

@@ -167,6 +167,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--min-text-length`: Minimum text length to translate (default: 5)
 - `--openai`: Use OpenAI for translation (default: False)
 - `--custom-system-prompt`: Custom system prompt for translation.
+- `--add-formula-placehold-hint`: Add formula placeholder hint for translation. (Currently not recommended, it may affect translation quality, default: False)
 
 > [!TIP]
 >

babeldoc-0.3.40/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.40"

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.39"
+__version__ = "0.3.40"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/midend/add_debug_information.py

@@ -34,7 +34,7 @@ class AddDebugInformation:
 
     def _create_text(self, text: str, color: GraphicState, box: il_version_1.Box):
         style = il_version_1.PdfStyle(
-            font_id="china-ss",
+            font_id="base",
             font_size=4,
             graphic_state=color,
         )

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/midend/detect_scanned_file.py

@@ -31,7 +31,7 @@ class DetectScannedFile:
         # Note: PDF coordinates are from bottom-left,
         # so we use y2 for top position
         style = il_version_1.PdfStyle(
-            font_id="china-ss",
+            font_id="base",
             font_size=4,
             graphic_state=color,
         )

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/midend/il_translator.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+import copy
 import json
 import logging
 import re
@@ -78,6 +81,10 @@ class DocumentTranslateTracker:
                 i_str = getattr(para, "input", None)
                 o_str = getattr(para, "output", None)
                 pdf_unicode = getattr(para, "pdf_unicode", None)
+                llm_translate_trackers = getattr(para, "llm_translate_trackers", None)
+                llm_translate_trackers_json = []
+                for tracker in llm_translate_trackers:
+                    llm_translate_trackers_json.append(tracker.to_dict())
                 if pdf_unicode is None or i_str is None:
                     continue
                 paragraphs.append(
@@ -85,6 +92,7 @@ class DocumentTranslateTracker:
                         "input": i_str,
                         "output": o_str,
                         "pdf_unicode": pdf_unicode,
+                        "llm_translate_trackers": llm_translate_trackers_json,
                     },
                 )
             pages.append({"paragraph": paragraphs})
@@ -103,7 +111,7 @@ class PageTranslateTracker:
 
 class ParagraphTranslateTracker:
     def __init__(self):
-        pass
+        self.llm_translate_trackers = []
 
     def set_pdf_unicode(self, unicode: str):
         self.pdf_unicode = unicode
@@ -114,6 +122,52 @@ class ParagraphTranslateTracker:
     def set_output(self, output: str):
         self.output = output
 
+    def new_llm_translate_tracker(self) -> LLMTranslateTracker:
+        tracker = LLMTranslateTracker()
+        self.llm_translate_trackers.append(tracker)
+        return tracker
+
+    def last_llm_translate_tracker(self) -> LLMTranslateTracker | None:
+        if self.llm_translate_trackers:
+            return self.llm_translate_trackers[-1]
+        return None
+
+
+class LLMTranslateTracker:
+    def __init__(self):
+        self.input = ""
+        self.output = ""
+        self.has_error = False
+        self.error_message = ""
+        self.placeholder_full_match = False
+        self.fallback_to_translate = False
+
+    def set_input(self, input_text: str):
+        self.input = input_text
+
+    def set_output(self, output_text: str):
+        self.output = output_text
+
+    def set_error_message(self, error_message: str):
+        self.has_error = True
+        self.error_message = error_message
+
+    def set_placeholder_full_match(self):
+        self.placeholder_full_match = True
+
+    def set_fallback_to_translate(self):
+        self.fallback_to_translate = True
+
+    def to_dict(self):
+        return {
+            "input": self.input,
+            "output": self.output,
+            "has_error": self.has_error,
+            "error_message": self.error_message,
+            "placeholder_full_match": self.placeholder_full_match,
+            "fallback_to_translate": self.fallback_to_translate,
+        }
+
 
 class ILTranslator:
     stage_name = "Translate Paragraphs"
@@ -156,9 +210,11 @@ class ILTranslator:
             # Try to find the first title paragraph
             title_paragraph = self.find_title_paragraph(docs)
             self.translation_config.shared_context_cross_split_part.first_paragraph = (
+                copy.deepcopy(title_paragraph)
+            )
+            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = copy.deepcopy(
                 title_paragraph
             )
-            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = title_paragraph
             if title_paragraph:
                 logger.info(f"Found first title paragraph: {title_paragraph.unicode}")
 
@@ -220,7 +276,9 @@ class ILTranslator:
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
             paragraph_token_count = self.calc_token_count(paragraph.unicode)
             if paragraph.layout_label == "title":
-                self.shared_context_cross_split_part.recent_title_paragraph = paragraph
+                self.shared_context_cross_split_part.recent_title_paragraph = (
+                    copy.deepcopy(paragraph)
+                )
             executor.submit(
                 self.translate_paragraph,
                 paragraph,
@@ -245,6 +303,24 @@ class ILTranslator:
             self.placeholders = placeholders
             self.base_style = base_style
 
+        def get_placeholders_hint(self) -> dict[str, str] | None:
+            hint = {}
+            for placeholder in self.placeholders:
+                if isinstance(placeholder, FormulaPlaceholder):
+                    cid_count = 0
+                    for char in placeholder.formula.pdf_character:
+                        if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+                            cid_count += 1
+                    if cid_count > len(placeholder.formula.pdf_character) * 0.8:
+                        continue
+
+                    hint[placeholder.placeholder] = get_char_unicode_string(
+                        placeholder.formula.pdf_character
+                    )
+            if hint:
+                return hint
+            return None
+
     def create_formula_placeholder(
         self,
         formula: PdfFormula,
@@ -452,9 +528,8 @@ class ILTranslator:
         self,
         input_text: TranslateInput,
         output: str,
+        llm_translate_tracker: LLMTranslateTracker | None = None,
     ) -> [PdfParagraphComposition]:
-        import re
-
         result = []
 
         # 如果没有占位符，直接返回整个文本
@@ -463,6 +538,8 @@ class ILTranslator:
             comp.pdf_same_style_unicode_characters = PdfSameStyleUnicodeCharacters()
             comp.pdf_same_style_unicode_characters.unicode = output
             comp.pdf_same_style_unicode_characters.pdf_style = input_text.base_style
+            if llm_translate_tracker:
+                llm_translate_tracker.set_placeholder_full_match()
             return [comp]
 
         # 构建正则表达式模式
@@ -484,7 +561,14 @@ class ILTranslator:
                 placeholder_patterns.append(f"({left})")
                 placeholder_patterns.append(f"({right})")
                 placeholder_map[placeholder.left_placeholder] = placeholder
-
+        all_match = True
+        for pattern in patterns:
+            if not re.search(pattern, output, flags=re.IGNORECASE):
+                all_match = False
+                break
+        if all_match:
+            if llm_translate_tracker:
+                llm_translate_tracker.set_placeholder_full_match()
         # 合并所有模式
         combined_pattern = "|".join(patterns)
         combined_placeholder_pattern = "|".join(placeholder_patterns)
@@ -626,10 +710,12 @@ class ILTranslator:
         """Post-translation processing: update paragraph with translated text."""
         tracker.set_output(translated_text)
         if translated_text == translate_input.unicode:
+            if llm_translate_tracker := tracker.last_llm_translate_tracker():
+                llm_translate_tracker.set_placeholder_full_match()
             return False
         paragraph.unicode = translated_text
         paragraph.pdf_paragraph_composition = self.parse_translate_output(
-            translate_input, translated_text
+            translate_input, translated_text, tracker.last_llm_translate_tracker()
         )
         for composition in paragraph.pdf_paragraph_composition:
             if (
@@ -646,6 +732,7 @@ class ILTranslator:
         text: str,
         title_paragraph: PdfParagraph | None = None,
         local_title_paragraph: PdfParagraph | None = None,
+        translate_input: TranslateInput | None = None,
     ):
         if self.translation_config.custom_system_prompt:
             llm_input = [self.translation_config.custom_system_prompt]
@@ -673,6 +760,15 @@ class ILTranslator:
         llm_input.append(
             f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
         )
+        if self.translation_config.add_formula_placehold_hint:
+            placeholders_hint = translate_input.get_placeholders_hint()
+            if placeholders_hint:
+                llm_input.append(
+                    f"This is the formula placeholder hint: \n{placeholders_hint}"
+                )
+                llm_input.append(
+                    "The system will automatically replace placeholders with corresponding formulas, please do not translate the placeholders!"
+                )
         prompt_template = f"""
 ;; Treat next line as plain text input and translate it into {self.translation_config.lang_out}, output translation ONLY. If translation is unnecessary (e.g. proper nouns, codes, {"{{1}}, etc. "}), return the original text. NO explanations. NO notes. Input:\n\n{text}
 
@@ -704,18 +800,20 @@ class ILTranslator:
                 )
                 if text is None:
                     return
-
+                llm_translate_tracker = tracker.new_llm_translate_tracker()
                 # Perform translation
                 if self.support_llm_translate:
                     llm_prompt = self.generate_prompt_for_llm(
-                        text, title_paragraph, local_title_paragraph
+                        text, title_paragraph, local_title_paragraph, translate_input
                     )
+                    llm_translate_tracker.set_input(llm_prompt)
                     translated_text = self.translate_engine.llm_translate(
                         llm_prompt,
                         rate_limit_params={
                             "paragraph_token_count": paragraph_token_count
                         },
                     )
+                    llm_translate_tracker.set_output(translated_text)
                 else:
                     translated_text = self.translate_engine.translate(
                         text,

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -1,3 +1,4 @@
+import copy
 import json
 import logging
 import re
@@ -11,6 +12,7 @@ from babeldoc.document_il import Document
 from babeldoc.document_il import Page
 from babeldoc.document_il import PdfFont
 from babeldoc.document_il import PdfParagraph
+from babeldoc.document_il.midend import il_translator
 from babeldoc.document_il.midend.il_translator import DocumentTranslateTracker
 from babeldoc.document_il.midend.il_translator import ILTranslator
 from babeldoc.document_il.midend.il_translator import PageTranslateTracker
@@ -94,9 +96,11 @@ class ILTranslatorLLMOnly:
             # Try to find the first title paragraph
             title_paragraph = self.find_title_paragraph(docs)
             self.translation_config.shared_context_cross_split_part.first_paragraph = (
+                copy.deepcopy(title_paragraph)
+            )
+            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = copy.deepcopy(
                 title_paragraph
             )
-            self.translation_config.shared_context_cross_split_part.recent_title_paragraph = title_paragraph
             if title_paragraph:
                 logger.info(f"Found first title paragraph: {title_paragraph.unicode}")
 
@@ -169,7 +173,9 @@ class ILTranslatorLLMOnly:
             total_token_count += self.calc_token_count(paragraph.unicode)
             paragraphs.append(paragraph)
             if paragraph.layout_label == "title":
-                self.shared_context_cross_split_part.recent_title_paragraph = paragraph
+                self.shared_context_cross_split_part.recent_title_paragraph = (
+                    copy.deepcopy(paragraph)
+                )
 
             if total_token_count > 400 or len(paragraphs) > 5:
                 executor.submit(
@@ -217,7 +223,7 @@ class ILTranslatorLLMOnly:
         should_translate_paragraph = []
         try:
             inputs = []
-
+            llm_translate_trackers = []
             for i in range(len(batch_paragraph.paragraphs)):
                 paragraph = batch_paragraph.paragraphs[i]
                 tracker = batch_paragraph.trackers[i]
@@ -227,20 +233,31 @@ class ILTranslatorLLMOnly:
                 if text is None:
                     pbar.advance(1)
                     continue
+                llm_translate_tracker = tracker.new_llm_translate_tracker()
                 should_translate_paragraph.append(i)
-                inputs.append((text, translate_input, paragraph, tracker))
+                llm_translate_trackers.append(llm_translate_tracker)
+                inputs.append(
+                    (text, translate_input, paragraph, tracker, llm_translate_tracker)
+                )
             if not inputs:
                 return
             json_format_input = []
 
             for id_, input_text in enumerate(inputs):
-                json_format_input.append(
-                    {
-                        "id": id_,
-                        "input": input_text[0],
-                        "layout_label": input_text[2].layout_label,
-                    }
-                )
+                ti: il_translator.ILTranslator.TranslateInput = input_text[1]
+                placeholders_hint = ti.get_placeholders_hint()
+                obj = {
+                    "id": id_,
+                    "input": input_text[0],
+                    "layout_label": input_text[2].layout_label,
+                }
+                if (
+                    placeholders_hint
+                    and self.translation_config.add_formula_placehold_hint
+                ):
+                    obj["formula_placeholders_hint"] = placeholders_hint
+                json_format_input.append(obj)
+
             json_format_input = json.dumps(
                 json_format_input, ensure_ascii=False, indent=2
             )
@@ -280,7 +297,12 @@ class ILTranslatorLLMOnly:
     {
         "id": 1,
         "input": "Source",
-        "layout_label": "plain text"
+        "layout_label": "plain text",
+        // this is optional
+        "formula_placeholders_hint": {
+            "placeholder1": "hint1",
+            "placeholder2": "hint2"
+        }
     }
     ```
     Output:
@@ -299,10 +321,13 @@ class ILTranslatorLLMOnly:
             llm_input.append(
                 f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
             )
-
             llm_input.append(
                 f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
             )
+            if self.translation_config.add_formula_placehold_hint:
+                llm_input.append(
+                    "The system will automatically replace placeholders with corresponding formulas, please do not translate the placeholders!"
+                )
             llm_input.append(
                 f"""Here is the input:
     
@@ -311,10 +336,14 @@ class ILTranslatorLLMOnly:
     ```"""
             )
             final_input = "\n".join(llm_input).strip()
+            for llm_translate_tracker in llm_translate_trackers:
+                llm_translate_tracker.set_input(final_input)
             llm_output = self.translate_engine.llm_translate(
                 final_input,
                 rate_limit_params={"paragraph_token_count": paragraph_token_count},
             )
+            for llm_translate_tracker in llm_translate_trackers:
+                llm_translate_tracker.set_output(llm_output)
             llm_output = llm_output.strip()
 
             llm_output = self._clean_json_output(llm_output)
@@ -347,6 +376,7 @@ class ILTranslatorLLMOnly:
 
                     # Get the original input for this translation
                     translate_input = inputs[id_][1]
+                    llm_translate_tracker = inputs[id_][4]
 
                     input_unicode = inputs[id_][2].unicode
                     output_unicode = translated_text
@@ -355,6 +385,9 @@ class ILTranslatorLLMOnly:
                     output_token_count = self.calc_token_count(output_unicode)
 
                     if not (0.3 < output_token_count / input_token_count < 3):
+                        llm_translate_tracker.set_error_message(
+                            f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
+                        )
                         logger.warning(
                             f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
                         )
@@ -362,6 +395,9 @@ class ILTranslatorLLMOnly:
 
                     edit_distance = Levenshtein.distance(input_unicode, output_unicode)
                     if edit_distance < 5 and input_token_count > 20:
+                        llm_translate_tracker.set_error_message(
+                            f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
+                        )
                         logger.warning(
                             f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
                         )
@@ -377,11 +413,15 @@ class ILTranslatorLLMOnly:
                     if pbar:
                         pbar.advance(1)
                 except Exception as e:
-                    logger.exception(f"Error translating paragraph. Error: {e}.")
+                    error_message = f"Error translating paragraph. Error: {e}."
+                    logger.exception(error_message)
                     # Ignore error and continue
+                    for llm_translate_tracker in llm_translate_trackers:
+                        llm_translate_tracker.set_error_message(error_message)
                     continue
                 finally:
                     if should_fallback:
+                        inputs[id_][4].set_fallback_to_translate()
                         logger.warning(
                             f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
                         )
@@ -402,7 +442,10 @@ class ILTranslatorLLMOnly:
                         )
 
         except Exception as e:
-            logger.warning(f"Error {e} during translation. try fallback")
+            error_message = f"Error {e} during translation. try fallback"
+            logger.warning(error_message)
+            for llm_translate_tracker in llm_translate_trackers:
+                llm_translate_tracker.set_error_message(error_message)
             if not should_translate_paragraph:
                 should_translate_paragraph = list(
                     range(len(batch_paragraph.paragraphs))

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/midend/layout_parser.py

@@ -78,7 +78,7 @@ class LayoutParser:
             # Note: PDF coordinates are from bottom-left,
             # so we use y2 for top position
             style = il_version_1.PdfStyle(
-                font_id="china-ss",
+                font_id="base",
                 font_size=4,
                 graphic_state=color,
             )

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -606,6 +606,7 @@ class StylesAndFormulas:
             r"|.*NimbusSan.*"
             r"|.*Mincho.*"
             r"|.*Amerigo.*"
+            r"|.*MSGloriolaIIStd.*"
             r")$"
         )
         precise_formula_font_pattern = (

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/midend/table_parser.py

@@ -83,7 +83,7 @@ class TableParser:
             # Note: PDF coordinates are from bottom-left,
             # so we use y2 for top position
             style = il_version_1.PdfStyle(
-                font_id="china-ss",
+                font_id="base",
                 font_size=4,
                 graphic_state=color,
             )

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/translator/translator.py

@@ -277,7 +277,7 @@ class OpenAITranslator(BaseTranslator):
             logger.exception("Error updating token count")
 
     def get_formular_placeholder(self, placeholder_id: int):
-        return "{{v" + str(placeholder_id) + "}}"
+        return "{v" + str(placeholder_id) + "}"
         return "{{" + str(placeholder_id) + "}}"
 
     def get_rich_text_left_placeholder(self, placeholder_id: int):

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/document_il/utils/style_helper.py

@@ -1,7 +1,7 @@
 from babeldoc.document_il import il_version_1
 
 
-def create_pdf_style(r, g, b, font_id="china-ss", font_size=6):
+def create_pdf_style(r, g, b, font_id="base", font_size=6):
     """
     Create a PdfStyle object from RGB values.
 

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/high_level.py

@@ -651,8 +651,8 @@ def _do_translate_single(
 
     mediabox_data = fix_media_box(doc_pdf2zh)
 
-    for page in doc_pdf2zh:
-        page.insert_font(resfont, None)
+    # for page in doc_pdf2zh:
+    #     page.insert_font(resfont, None)
 
     resfont = None
     doc_pdf2zh.save(temp_pdf_path)

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.39"
+__version__ = "0.3.40"
 
 
 def create_parser():

{babeldoc-0.3.39 → babeldoc-0.3.40}/babeldoc/translation_config.py

@@ -70,6 +70,7 @@ class TranslationConfig:
         skip_scanned_detection: bool = False,
         ocr_workaround: bool = False,
         custom_system_prompt: str | None = None,
+        add_formula_placehold_hint: bool = False,
     ):
         self.translator = translator
 
@@ -155,6 +156,7 @@ class TranslationConfig:
         self.table_model = table_model
         self.show_char_box = show_char_box
         self.custom_system_prompt = custom_system_prompt
+        self.add_formula_placehold_hint = add_formula_placehold_hint
 
     def parse_pages(self, pages_str: str | None) -> list[tuple[int, int]] | None:
         """解析页码字符串，返回页码范围列表

{babeldoc-0.3.39 → babeldoc-0.3.40}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.39"
+version = "0.3.40"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.39"
+current_version = "0.3.40"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.39 → babeldoc-0.3.40}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.39"
+version = "0.3.40"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.3.39/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.39"