BabelDOC 0.2.33__tar.gz → 0.3.1__tar.gz

{babeldoc-0.2.33 → babeldoc-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.2.33
+Version: 0.3.1
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.3.1/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.1"

{babeldoc-0.2.33 → babeldoc-0.3.1}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.2.33"
+__version__ = "0.3.1"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.2.33 → babeldoc-0.3.1}/babeldoc/document_il/backend/pdf_creater.py

@@ -85,12 +85,14 @@ class PDFCreater:
         original_pdf_path: str,
         document: il_version_1.Document,
         translation_config: TranslationConfig,
+        mediabox_data: dict,
     ):
         self.original_pdf_path = original_pdf_path
         self.docs = document
         self.font_path = translation_config.font
         self.font_mapper = FontMapper(translation_config)
         self.translation_config = translation_config
+        self.mediabox_data = mediabox_data
 
     def render_graphic_state(
         self,
@@ -636,6 +638,10 @@ class PDFCreater:
 
             return False
 
+    def restore_media_box(self, doc: pymupdf.Document, mediabox_data: dict) -> None:
+        for pageno, mediabox in mediabox_data.items():
+            doc.xref_set_key(doc[pageno].xref, "MediaBox", mediabox[1])
+
     def write(
         self, translation_config: TranslationConfig, check_font_exists: bool = False
     ) -> TranslateResult:
@@ -776,6 +782,7 @@ class PDFCreater:
                     )
 
                 pbar.advance()
+            self.restore_media_box(pdf, self.mediabox_data)
             with self.translation_config.progress_monitor.stage_start(
                 SAVE_PDF_STAGE_NAME,
                 2,

{babeldoc-0.2.33 → babeldoc-0.3.1}/babeldoc/document_il/midend/il_translator.py

@@ -181,6 +181,7 @@ class ILTranslator:
                 for font in xobj.pdf_font:
                     page_xobj_font_map[xobj.xobj_id][font.font_id] = font
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
+            paragraph_token_count = self.calc_token_count(paragraph.unicode)
             executor.submit(
                 self.translate_paragraph,
                 paragraph,
@@ -188,7 +189,8 @@ class ILTranslator:
                 tracker.new_paragraph(),
                 page_font_map,
                 page_xobj_font_map,
-                priority=1048576 - self.calc_token_count(paragraph.unicode),
+                priority=1048576 - paragraph_token_count,
+                paragraph_token_count=paragraph_token_count,
             )
 
     class TranslateInput:
@@ -597,6 +599,7 @@ class ILTranslator:
         tracker: ParagraphTranslateTracker = None,
         page_font_map: dict[str, PdfFont] = None,
         xobj_font_map: dict[int, dict[str, PdfFont]] = None,
+        paragraph_token_count: int = 0,
     ):
         """Translate a paragraph using pre and post processing functions."""
         self.translation_config.raise_if_cancelled()
@@ -610,7 +613,10 @@ class ILTranslator:
                     return
 
                 # Perform translation
-                translated_text = self.translate_engine.translate(text)
+                translated_text = self.translate_engine.translate(
+                    text,
+                    rate_limit_params={"paragraph_token_count": paragraph_token_count},
+                )
                 translated_text = re.sub(r"[. 。…，]{20,}", ".", translated_text)
 
                 # Post-translation processing

{babeldoc-0.2.33 → babeldoc-0.3.1}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -179,6 +179,7 @@ class ILTranslatorLLMOnly:
                     self.translation_config.shared_context_cross_split_part.recent_title_paragraph,
                     executor2,
                     priority=1048576 - total_token_count,
+                    paragraph_token_count=total_token_count,
                 )
                 paragraphs = []
                 total_token_count = 0
@@ -194,6 +195,7 @@ class ILTranslatorLLMOnly:
                 self.translation_config.shared_context_cross_split_part.recent_title_paragraph,
                 executor2,
                 priority=1048576 - total_token_count,
+                paragraph_token_count=total_token_count,
             )
 
     def translate_paragraph(
@@ -205,6 +207,7 @@ class ILTranslatorLLMOnly:
         title_paragraph: PdfParagraph | None = None,
         local_title_paragraph: PdfParagraph | None = None,
         executor: PriorityThreadPoolExecutor | None = None,
+        paragraph_token_count: int = 0,
     ):
         """Translate a paragraph using pre and post processing functions."""
         self.translation_config.raise_if_cancelled()
@@ -293,7 +296,10 @@ class ILTranslatorLLMOnly:
             llm_input.append(prompt_template)
 
             final_input = "\n".join(llm_input).strip()
-            llm_output = self.translate_engine.llm_translate(final_input)
+            llm_output = self.translate_engine.llm_translate(
+                final_input,
+                rate_limit_params={"paragraph_token_count": paragraph_token_count},
+            )
             llm_output = llm_output.strip()
 
             llm_output = self._clean_json_output(llm_output)
@@ -364,6 +370,9 @@ class ILTranslatorLLMOnly:
                         logger.warning(
                             f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
                         )
+                        paragraph_token_count = self.calc_token_count(
+                            inputs[id_][2].unicode
+                        )
                         executor.submit(
                             self.il_translator.translate_paragraph,
                             inputs[id_][2],
@@ -371,7 +380,8 @@ class ILTranslatorLLMOnly:
                             inputs[id_][3],
                             page_font_map,
                             xobj_font_map,
-                            priority=1048576 - self.calc_token_count(paragraph.unicode),
+                            priority=1048576 - paragraph_token_count,
+                            paragraph_token_count=paragraph_token_count,
                         )
 
         except Exception as e:
@@ -385,6 +395,7 @@ class ILTranslatorLLMOnly:
                 tracker = batch_paragraph.trackers[i]
                 if paragraph.debug_id is None:
                     continue
+                paragraph_token_count = self.calc_token_count(paragraph.unicode)
                 executor.submit(
                     self.il_translator.translate_paragraph,
                     paragraph,
@@ -392,7 +403,8 @@ class ILTranslatorLLMOnly:
                     tracker,
                     page_font_map,
                     xobj_font_map,
-                    priority=1048576 - self.calc_token_count(paragraph.unicode),
+                    priority=1048576 - paragraph_token_count,
+                    paragraph_token_count=paragraph_token_count,
                 )
 
     def _clean_json_output(self, llm_output: str) -> str:

{babeldoc-0.2.33 → babeldoc-0.3.1}/babeldoc/document_il/translator/translator.py

@@ -104,7 +104,7 @@ class BaseTranslator(ABC):
         """
         self.cache.add_params(k, v)
 
-    def translate(self, text, ignore_cache=False):
+    def translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
         """
         Translate the text, and the other part should call this method.
         :param text: text to translate
@@ -117,12 +117,12 @@ class BaseTranslator(ABC):
                 self.translate_cache_call_count += 1
                 return cache
         _translate_rate_limiter.wait()
-        translation = self.do_translate(text)
+        translation = self.do_translate(text, rate_limit_params)
         if not (self.ignore_cache or ignore_cache):
             self.cache.set(text, translation)
         return translation
 
-    def llm_translate(self, text, ignore_cache=False):
+    def llm_translate(self, text, ignore_cache=False, rate_limit_params: dict = None):
         """
         Translate the text, and the other part should call this method.
         :param text: text to translate
@@ -135,13 +135,13 @@ class BaseTranslator(ABC):
                 self.translate_cache_call_count += 1
                 return cache
         _translate_rate_limiter.wait()
-        translation = self.do_llm_translate(text)
+        translation = self.do_llm_translate(text, rate_limit_params)
         if not (self.ignore_cache or ignore_cache):
             self.cache.set(text, translation)
         return translation
 
     @abstractmethod
-    def do_llm_translate(self, text):
+    def do_llm_translate(self, text, rate_limit_params: dict = None):
         """
         Actual translate text, override this method
         :param text: text to translate
@@ -150,7 +150,7 @@ class BaseTranslator(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def do_translate(self, text):
+    def do_translate(self, text, rate_limit_params: dict = None):
         """
         Actual translate text, override this method
         :param text: text to translate
@@ -209,7 +209,7 @@ class OpenAITranslator(BaseTranslator):
             f"(Attempt {retry_state.attempt_number}/100)"
         ),
     )
-    def do_translate(self, text) -> str:
+    def do_translate(self, text, rate_limit_params: dict = None) -> str:
         response = self.client.chat.completions.create(
             model=self.model,
             **self.options,
@@ -241,7 +241,7 @@ class OpenAITranslator(BaseTranslator):
             f"(Attempt {retry_state.attempt_number}/100)"
         ),
     )
-    def do_llm_translate(self, text):
+    def do_llm_translate(self, text, rate_limit_params: dict = None):
         if text is None:
             return None
 

{babeldoc-0.2.33 → babeldoc-0.3.1}/babeldoc/high_level.py

@@ -525,6 +525,17 @@ def do_translate(
         translation_config.cleanup_temp_files()
 
 
+def fix_media_box(doc: Document) -> None:
+    mediabox_data = {}
+    for page in doc:
+        if page.mediabox.x0 != 0 or page.mediabox.y0 != 0:
+            x1 = page.mediabox.x1
+            y1 = page.mediabox.y1
+            mediabox_data[page.number] = doc.xref_get_key(page.xref, "MediaBox")
+            doc.xref_set_key(page.xref, "MediaBox", f"[0 0 {x1} {y1}]")
+    return mediabox_data
+
+
 def _do_translate_single(
     pm: ProgressMonitor,
     translation_config: TranslationConfig,
@@ -541,6 +552,7 @@ def _do_translate_single(
         # Fix null xref in PDF file
         fix_null_xref(doc_input)
         doc_input.save(output_path, expand=True, pretty=True)
+        del doc_input
 
     # Continue with original processing
     temp_pdf_path = translation_config.get_working_file_path("input.pdf")
@@ -550,6 +562,8 @@ def _do_translate_single(
     # Fix null xref in PDF file
     fix_null_xref(doc_pdf2zh)
 
+    mediabox_data = fix_media_box(doc_pdf2zh)
+
     for page in doc_pdf2zh:
         page.insert_font(resfont, None)
 
@@ -677,7 +691,7 @@ def _do_translate_single(
             translation_config.get_working_file_path("typsetting.json"),
         )
 
-    pdf_creater = PDFCreater(temp_pdf_path, docs, translation_config)
+    pdf_creater = PDFCreater(temp_pdf_path, docs, translation_config, mediabox_data)
     result = pdf_creater.write(translation_config)
     try:
         if mono_watermark_first_page_doc_bytes:

{babeldoc-0.2.33 → babeldoc-0.3.1}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.2.33"
+__version__ = "0.3.1"
 
 
 def create_parser():

{babeldoc-0.2.33 → babeldoc-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.2.33"
+version = "0.3.1"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -148,7 +148,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.2.33"
+current_version = "0.3.1"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.2.33 → babeldoc-0.3.1}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.2.33"
+version = "0.3.1"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.2.33/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.33"