BabelDOC 0.4.11__tar.gz → 0.4.12__tar.gz

{babeldoc-0.4.11 → babeldoc-0.4.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.4.11
+Version: 0.4.12
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.4.12/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.12"

{babeldoc-0.4.11 → babeldoc-0.4.12}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.4.11"
+__version__ = "0.4.12"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.4.11 → babeldoc-0.4.12}/babeldoc/format/pdf/document_il/midend/il_translator.py

@@ -746,7 +746,7 @@ class ILTranslator:
     ):
         """Post-translation processing: update paragraph with translated text."""
         tracker.set_output(translated_text)
-        if translated_text == translate_input.unicode:
+        if translated_text == translate_input:
             if llm_translate_tracker := tracker.last_llm_translate_tracker():
                 llm_translate_tracker.set_placeholder_full_match()
             return False

{babeldoc-0.4.11 → babeldoc-0.4.12}/babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py

@@ -70,6 +70,10 @@ class ILTranslatorLLMOnly:
         except NotImplementedError as e:
             raise ValueError("LLM translator not supported") from e
 
+        self.ok_count = 0
+        self.fallback_count = 0
+        self.total_count = 0
+
     def calc_token_count(self, text: str) -> int:
         try:
             return len(self.tokenizer.encode(text, disallowed_special=()))
@@ -145,6 +149,9 @@ class ILTranslatorLLMOnly:
             logger.debug(f"save translate tracking to {path}")
             with Path(path).open("w", encoding="utf-8") as f:
                 f.write(tracker.to_json())
+        logger.info(
+            f"Translation completed. Total: {self.total_count}, Successful: {self.ok_count}, Fallback: {self.fallback_count}"
+        )
 
     def process_page(
         self,
@@ -476,15 +483,15 @@ class ILTranslatorLLMOnly:
                     translate_input = inputs[id_][1]
                     llm_translate_tracker = inputs[id_][4]
 
-                    input_unicode = inputs[id_][2].unicode
+                    input_unicode = inputs[id_][0]
                     output_unicode = translated_text
 
                     trimed_input = re.sub(r"[. 。…，]{20,}", ".", input_unicode)
 
-                    input_token_count = self.calc_token_count(input_unicode)
+                    input_token_count = self.calc_token_count(trimed_input)
                     output_token_count = self.calc_token_count(output_unicode)
 
-                    if trimed_input == output_unicode and input_token_count > 20:
+                    if trimed_input == output_unicode and input_token_count > 10:
                         llm_translate_tracker.set_error_message(
                             "Translation result is the same as input, fallback."
                         )
@@ -529,7 +536,9 @@ class ILTranslatorLLMOnly:
                         llm_translate_tracker.set_error_message(error_message)
                     continue
                 finally:
+                    self.total_count += 1
                     if should_fallback:
+                        self.fallback_count += 1
                         inputs[id_][4].set_fallback_to_translate()
                         logger.warning(
                             f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
@@ -551,6 +560,8 @@ class ILTranslatorLLMOnly:
                             title_paragraph=title_paragraph,
                             local_title_paragraph=local_title_paragraph,
                         )
+                    else:
+                        self.ok_count += 1
 
         except Exception as e:
             error_message = f"Error {e} during translation. try fallback"

{babeldoc-0.4.11 → babeldoc-0.4.12}/babeldoc/format/pdf/document_il/utils/layout_helper.py

@@ -1,6 +1,7 @@
 import logging
 import math
 import re
+import unicodedata
 from typing import Literal
 
 from pymupdf import Font
@@ -255,7 +256,7 @@ def get_char_unicode_string(chars: list[PdfCharacter | str]) -> str:
         if not isinstance(chars[i], PdfCharacter):
             unicode_chars.append(chars[i])
             continue
-        unicode_chars.append(chars[i].char_unicode)
+        unicode_chars.append(unicodedata.normalize("NFKC", chars[i].char_unicode))
 
         # 如果是空格，跳过
         if chars[i].char_unicode == " ":

{babeldoc-0.4.11 → babeldoc-0.4.12}/babeldoc/main.py

@@ -21,7 +21,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.4.11"
+__version__ = "0.4.12"
 
 
 def create_parser():

{babeldoc-0.4.11 → babeldoc-0.4.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.4.11"
+version = "0.4.12"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -158,7 +158,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.4.11"
+current_version = "0.4.12"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.4.11 → babeldoc-0.4.12}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.4.11"
+version = "0.4.12"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.4.11/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.4.11"