BabelDOC 0.2.33__tar.gz → 0.3.0__tar.gz

{babeldoc-0.2.33 → babeldoc-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.2.33
+Version: 0.3.0
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.3.0/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{babeldoc-0.2.33 → babeldoc-0.3.0}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.2.33"
+__version__ = "0.3.0"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.2.33 → babeldoc-0.3.0}/babeldoc/document_il/midend/il_translator.py

@@ -181,6 +181,7 @@ class ILTranslator:
                 for font in xobj.pdf_font:
                     page_xobj_font_map[xobj.xobj_id][font.font_id] = font
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
+            paragraph_token_count = self.calc_token_count(paragraph.unicode)
             executor.submit(
                 self.translate_paragraph,
                 paragraph,
@@ -188,7 +189,8 @@ class ILTranslator:
                 tracker.new_paragraph(),
                 page_font_map,
                 page_xobj_font_map,
-                priority=1048576 - self.calc_token_count(paragraph.unicode),
+                priority=1048576 - paragraph_token_count,
+                paragraph_token_count=paragraph_token_count,
             )
 
     class TranslateInput:
@@ -597,6 +599,7 @@ class ILTranslator:
         tracker: ParagraphTranslateTracker = None,
         page_font_map: dict[str, PdfFont] = None,
         xobj_font_map: dict[int, dict[str, PdfFont]] = None,
+        paragraph_token_count: int = 0,
     ):
         """Translate a paragraph using pre and post processing functions."""
         self.translation_config.raise_if_cancelled()
@@ -610,7 +613,10 @@ class ILTranslator:
                     return
 
                 # Perform translation
-                translated_text = self.translate_engine.translate(text)
+                translated_text = self.translate_engine.translate(
+                    text,
+                    rate_limit_params={"paragraph_token_count": paragraph_token_count},
+                )
                 translated_text = re.sub(r"[. 。…，]{20,}", ".", translated_text)
 
                 # Post-translation processing

{babeldoc-0.2.33 → babeldoc-0.3.0}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -179,6 +179,7 @@ class ILTranslatorLLMOnly:
                     self.translation_config.shared_context_cross_split_part.recent_title_paragraph,
                     executor2,
                     priority=1048576 - total_token_count,
+                    paragraph_token_count=total_token_count,
                 )
                 paragraphs = []
                 total_token_count = 0
@@ -194,6 +195,7 @@ class ILTranslatorLLMOnly:
                 self.translation_config.shared_context_cross_split_part.recent_title_paragraph,
                 executor2,
                 priority=1048576 - total_token_count,
+                paragraph_token_count=total_token_count,
             )
 
     def translate_paragraph(
@@ -205,6 +207,7 @@ class ILTranslatorLLMOnly:
         title_paragraph: PdfParagraph | None = None,
         local_title_paragraph: PdfParagraph | None = None,
         executor: PriorityThreadPoolExecutor | None = None,
+        paragraph_token_count: int = 0,
     ):
         """Translate a paragraph using pre and post processing functions."""
         self.translation_config.raise_if_cancelled()
@@ -293,7 +296,10 @@ class ILTranslatorLLMOnly:
             llm_input.append(prompt_template)
 
             final_input = "\n".join(llm_input).strip()
-            llm_output = self.translate_engine.llm_translate(final_input)
+            llm_output = self.translate_engine.llm_translate(
+                final_input,
+                rate_limit_params={"paragraph_token_count": paragraph_token_count},
+            )
             llm_output = llm_output.strip()
 
             llm_output = self._clean_json_output(llm_output)
@@ -364,6 +370,9 @@ class ILTranslatorLLMOnly:
                         logger.warning(
                             f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
                         )
+                        paragraph_token_count = self.calc_token_count(
+                            inputs[id_][2].unicode
+                        )
                         executor.submit(
                             self.il_translator.translate_paragraph,
                             inputs[id_][2],
@@ -371,7 +380,8 @@ class ILTranslatorLLMOnly:
                             inputs[id_][3],
                             page_font_map,
                             xobj_font_map,
-                            priority=1048576 - self.calc_token_count(paragraph.unicode),
+                            priority=1048576 - paragraph_token_count,
+                            paragraph_token_count=paragraph_token_count,
                         )
 
         except Exception as e:
@@ -385,6 +395,7 @@ class ILTranslatorLLMOnly:
                 tracker = batch_paragraph.trackers[i]
                 if paragraph.debug_id is None:
                     continue
+                paragraph_token_count = self.calc_token_count(paragraph.unicode)
                 executor.submit(
                     self.il_translator.translate_paragraph,
                     paragraph,
@@ -392,7 +403,8 @@ class ILTranslatorLLMOnly:
                     tracker,
                     page_font_map,
                     xobj_font_map,
-                    priority=1048576 - self.calc_token_count(paragraph.unicode),
+                    priority=1048576 - paragraph_token_count,
+                    paragraph_token_count=paragraph_token_count,
                 )
 
     def _clean_json_output(self, llm_output: str) -> str:

{babeldoc-0.2.33 → babeldoc-0.3.0}/babeldoc/document_il/translator/translator.py

@@ -104,7 +104,7 @@ class BaseTranslator(ABC):
         """
         self.cache.add_params(k, v)
 
-    def translate(self, text, ignore_cache=False):
+    def translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
         """
         Translate the text, and the other part should call this method.
         :param text: text to translate
@@ -117,12 +117,12 @@ class BaseTranslator(ABC):
                 self.translate_cache_call_count += 1
                 return cache
         _translate_rate_limiter.wait()
-        translation = self.do_translate(text)
+        translation = self.do_translate(text, rate_limit_params)
         if not (self.ignore_cache or ignore_cache):
             self.cache.set(text, translation)
         return translation
 
-    def llm_translate(self, text, ignore_cache=False):
+    def llm_translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
         """
         Translate the text, and the other part should call this method.
         :param text: text to translate
@@ -135,13 +135,13 @@ class BaseTranslator(ABC):
                 self.translate_cache_call_count += 1
                 return cache
         _translate_rate_limiter.wait()
-        translation = self.do_llm_translate(text)
+        translation = self.do_llm_translate(text, rate_limit_params)
         if not (self.ignore_cache or ignore_cache):
             self.cache.set(text, translation)
         return translation
 
     @abstractmethod
-    def do_llm_translate(self, text):
+    def do_llm_translate(self, text, rate_limit_params: dict = None):
         """
         Actual translate text, override this method
         :param text: text to translate
@@ -150,7 +150,7 @@ class BaseTranslator(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def do_translate(self, text):
+    def do_translate(self, text, rate_limit_params: dict = None):
         """
         Actual translate text, override this method
         :param text: text to translate
@@ -209,7 +209,7 @@ class OpenAITranslator(BaseTranslator):
             f"(Attempt {retry_state.attempt_number}/100)"
         ),
     )
-    def do_translate(self, text) -> str:
+    def do_translate(self, text, rate_limit_params: dict = None) -> str:
         response = self.client.chat.completions.create(
             model=self.model,
             **self.options,
@@ -241,7 +241,7 @@ class OpenAITranslator(BaseTranslator):
             f"(Attempt {retry_state.attempt_number}/100)"
         ),
     )
-    def do_llm_translate(self, text):
+    def do_llm_translate(self, text, rate_limit_params: dict = None):
         if text is None:
             return None
 

{babeldoc-0.2.33 → babeldoc-0.3.0}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.2.33"
+__version__ = "0.3.0"
 
 
 def create_parser():

{babeldoc-0.2.33 → babeldoc-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.2.33"
+version = "0.3.0"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -148,7 +148,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.2.33"
+current_version = "0.3.0"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.2.33 → babeldoc-0.3.0}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.2.33"
+version = "0.3.0"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.2.33/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.33"