BabelDOC 0.5.12__tar.gz → 0.5.14__tar.gz

{babeldoc-0.5.12 → babeldoc-0.5.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.12
+Version: 0.5.14
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.5.14/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.14"

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/babeldoc_exception/BabelDOCException.py

@@ -11,3 +11,9 @@ class ExtractTextError(Exception):
 class InputFileGeneratedByBabelDOCError(Exception):
     def __init__(self, message):
         super().__init__(message)
+
+
+class ContentFilterError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+        self.message = message

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/const.py

@@ -6,7 +6,7 @@ import subprocess
 import threading
 from pathlib import Path
 
-__version__ = "0.5.12"
+__version__ = "0.5.14"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 
@@ -73,6 +73,17 @@ def get_process_pool():
         return _process_pool
 
 
+def close_process_pool():
+    if not _ENABLE_PROCESS_POOL:
+        return None
+    global _process_pool
+    with _process_pool_lock:
+        if _process_pool:
+            _process_pool.close()
+            _process_pool.join()
+            _process_pool = None
+
+
 def batched(iterable, n, *, strict=False):
     # batched('ABCDEFG', 3) → ABC DEF G
     if n < 1:

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/docvision/rpc_doclayout6.py

@@ -599,7 +599,7 @@ class RpcDocLayoutModel(DocLayoutModel):
     ):
         layout_temp_path = translate_config.get_working_file_path("layout.temp.pdf")
         mupdf_doc.save(layout_temp_path.as_posix())
-        with ThreadPoolExecutor(max_workers=32) as executor:
+        with ThreadPoolExecutor(max_workers=8) as executor:
             yield from executor.map(
                 self.predict_page,
                 pages,

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/format/pdf/document_il/frontend/il_creater.py

@@ -955,7 +955,7 @@ class ILCreater:
             )
             return
 
-        if char_bounding_box:
+        if char_bounding_box and len(char_bounding_box) == 4:
             x_min, y_min, x_max, y_max = char_bounding_box
             factor = 1 / 1000 * pdf_style.font_size
             x_min = x_min * factor

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/format/pdf/document_il/midend/il_translator.py

@@ -4,12 +4,16 @@ import copy
 import json
 import logging
 import re
+import threading
 from pathlib import Path
 
 import tiktoken
 from tqdm import tqdm
 
+import babeldoc.format.pdf.document_il.il_version_1 as il_version_1
+from babeldoc.babeldoc_exception.BabelDOCException import ContentFilterError
 from babeldoc.format.pdf.document_il import Document
+from babeldoc.format.pdf.document_il import GraphicState
 from babeldoc.format.pdf.document_il import Page
 from babeldoc.format.pdf.document_il import PdfFont
 from babeldoc.format.pdf.document_il import PdfFormula
@@ -34,6 +38,7 @@ from babeldoc.format.pdf.document_il.utils.paragraph_helper import (
 from babeldoc.format.pdf.document_il.utils.paragraph_helper import (
     is_pure_numeric_paragraph,
 )
+from babeldoc.format.pdf.document_il.utils.style_helper import GRAY80
 from babeldoc.format.pdf.translation_config import TranslationConfig
 from babeldoc.translator.translator import BaseTranslator
 from babeldoc.utils.priority_thread_pool_executor import PriorityThreadPoolExecutor
@@ -307,6 +312,8 @@ class ILTranslator:
             self.support_llm_translate = False
 
         self.use_as_fallback = False
+        self.add_content_filter_hint_lock = threading.Lock()
+        self.docs = None
 
     def calc_token_count(self, text: str) -> int:
         try:
@@ -315,6 +322,7 @@ class ILTranslator:
             return 0
 
     def translate(self, docs: Document):
+        self.docs = docs
         tracker = DocumentTranslateTracker()
 
         if not self.translation_config.shared_context_cross_split_part.first_paragraph:
@@ -390,6 +398,7 @@ class ILTranslator:
             executor.submit(
                 self.translate_paragraph,
                 paragraph,
+                page,
                 pbar,
                 tracker.new_paragraph(),
                 page_font_map,
@@ -985,9 +994,58 @@ Now, please carefully read the following text to be translated and directly outp
 
         return final_input
 
+    def add_content_filter_hint(self, page: Page, paragraph: PdfParagraph):
+        with self.add_content_filter_hint_lock:
+            new_box = il_version_1.Box(
+                x=paragraph.box.x,
+                y=paragraph.box.y2,
+                x2=paragraph.box.x2,
+                y2=paragraph.box.y2 + 1.1,
+            )
+            page.pdf_paragraph.append(
+                self._create_text(
+                    "翻译服务检测到内容可能包含不安全或敏感内容，请您避免翻译敏感内容，感谢您的配合。",
+                    GRAY80,
+                    new_box,
+                    1,
+                )
+            )
+            logger.info("success add content filter hint")
+
+    def _create_text(
+        self,
+        text: str,
+        color: GraphicState,
+        box: il_version_1.Box,
+        font_size: float = 4,
+    ):
+        style = il_version_1.PdfStyle(
+            font_id="base",
+            font_size=font_size,
+            graphic_state=color,
+        )
+        return il_version_1.PdfParagraph(
+            first_line_indent=False,
+            box=box,
+            vertical=False,
+            pdf_style=style,
+            unicode=text,
+            pdf_paragraph_composition=[
+                il_version_1.PdfParagraphComposition(
+                    pdf_same_style_unicode_characters=il_version_1.PdfSameStyleUnicodeCharacters(
+                        unicode=text,
+                        pdf_style=style,
+                        debug_info=True,
+                    ),
+                ),
+            ],
+            xobj_id=-1,
+        )
+
     def translate_paragraph(
         self,
         paragraph: PdfParagraph,
+        page: Page,
         pbar: tqdm | None = None,
         tracker: ParagraphTranslateTracker = None,
         page_font_map: dict[str, PdfFont] = None,
@@ -1039,6 +1097,10 @@ Now, please carefully read the following text to be translated and directly outp
                 self.post_translate_paragraph(
                     paragraph, tracker, translate_input, translated_text
                 )
+            except ContentFilterError as e:
+                logger.warning(f"ContentFilterError: {e.message}")
+                self.add_content_filter_hint(page, paragraph)
+                return
             except Exception as e:
                 logger.exception(
                     f"Error translating paragraph. Paragraph: {paragraph.debug_id} ({paragraph.unicode}). Error: {e}. ",

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py

@@ -40,9 +40,11 @@ class BatchParagraph:
     def __init__(
         self,
         paragraphs: list[PdfParagraph],
+        pages: list[Page],
         page_tracker: PageTranslateTracker,
     ):
         self.paragraphs = paragraphs
+        self.pages = pages
         self.trackers = [page_tracker.new_paragraph() for _ in paragraphs]
 
 
@@ -112,6 +114,7 @@ class ILTranslatorLLMOnly:
         return None
 
     def translate(self, docs: Document) -> None:
+        self.il_translator.docs = docs
         tracker = DocumentTranslateTracker()
         self.mid = 0
 
@@ -361,8 +364,9 @@ class ILTranslatorLLMOnly:
 
             # Create batch with both paragraphs
             cross_page_paragraphs = [last_curr_paragraph, first_next_paragraph]
+            cross_page_pages = [page_curr, page_next]
             batch_paragraph = BatchParagraph(
-                cross_page_paragraphs, tracker.new_cross_page()
+                cross_page_paragraphs, cross_page_pages, tracker.new_cross_page()
             )
 
             self.mid += 1
@@ -438,7 +442,7 @@ class ILTranslatorLLMOnly:
                 p1.unicode
             ) + self.calc_token_count(p2.unicode)
 
-            batch = BatchParagraph([p1, p2], tracker.new_cross_column())
+            batch = BatchParagraph([p1, p2], [page, page], tracker.new_cross_column())
             self.mid += 1
             executor.submit(
                 self.translate_paragraph,
@@ -523,7 +527,7 @@ class ILTranslatorLLMOnly:
                 self.mid += 1
                 executor.submit(
                     self.translate_paragraph,
-                    BatchParagraph(paragraphs, tracker),
+                    BatchParagraph(paragraphs, [page] * len(paragraphs), tracker),
                     pbar,
                     page_font_map,
                     page_xobj_font_map,
@@ -541,7 +545,7 @@ class ILTranslatorLLMOnly:
             self.mid += 1
             executor.submit(
                 self.translate_paragraph,
-                BatchParagraph(paragraphs, tracker),
+                BatchParagraph(paragraphs, [page] * len(paragraphs), tracker),
                 pbar,
                 page_font_map,
                 page_xobj_font_map,
@@ -630,6 +634,9 @@ class ILTranslatorLLMOnly:
             llm_prompt_parts.append("#role")
             if self.translation_config.custom_system_prompt:
                 llm_prompt_parts.append(self.translation_config.custom_system_prompt)
+                llm_prompt_parts.append(
+                    "When translating, strictly follow the instructions below to ensure translation quality and preserve all formatting, tags, and placeholders:\n"
+                )
             else:
                 llm_prompt_parts.append(
                     f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}.\n"
@@ -894,6 +901,7 @@ class ILTranslatorLLMOnly:
                         executor.submit(
                             self.il_translator.translate_paragraph,
                             inputs[id_][2],
+                            batch_paragraph.pages[id_],
                             pbar,
                             inputs[id_][3],
                             page_font_map,
@@ -929,6 +937,7 @@ class ILTranslatorLLMOnly:
                 executor.submit(
                     self.il_translator.translate_paragraph,
                     paragraph,
+                    batch_paragraph.pages[i],
                     pbar,
                     tracker,
                     page_font_map,

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/format/pdf/document_il/utils/style_helper.py

@@ -29,6 +29,10 @@ BLACK = il_version_1.GraphicState(passthrough_per_char_instruction="0 g 0 G")
 
 WHITE = il_version_1.GraphicState(passthrough_per_char_instruction="1 g 1 G")
 
+GRAY80 = il_version_1.GraphicState(passthrough_per_char_instruction="0.80 g 0.80 G")
+GRAY67 = il_version_1.GraphicState(passthrough_per_char_instruction="0.67 g 0.67 G")
+GRAY33 = il_version_1.GraphicState(passthrough_per_char_instruction="0.33 g 0.33 G")
+
 # Generate all color styles
 RED = il_version_1.GraphicState(
     passthrough_per_char_instruction="1.0000000000 0.2313725490 0.1882352941 rg "

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/format/pdf/high_level.py

@@ -25,6 +25,7 @@ from babeldoc.babeldoc_exception.BabelDOCException import (
 )
 from babeldoc.const import CACHE_FOLDER
 from babeldoc.const import WATERMARK_VERSION
+from babeldoc.const import close_process_pool
 from babeldoc.format.pdf.converter import TranslateConverter
 from babeldoc.format.pdf.document_il import il_version_1
 from babeldoc.format.pdf.document_il.backend.pdf_creater import SAVE_PDF_STAGE_NAME
@@ -149,6 +150,12 @@ def add_metadata(
             translated_by += f"_{translate_config.metadata_extra_data}"
         meta["producer"] = translated_by
         meta["creator"] = creator
+
+        for k, v in meta.items():
+            if v:
+                # 使用正则替换掉 surrogate 范围内的字符
+                meta[k] = re.sub(r"[\uD800-\uDFFF]", "", v)
+
         pdf.set_metadata(meta)
         safe_save(pdf, temp_path)
         shutil.move(temp_path, path)
@@ -1005,6 +1012,7 @@ def _do_translate_single(
     logger.debug("start generating layouts")
     docs = LayoutParser(translation_config).process(docs, doc_pdf2zh)
     logger.debug("finish generating layouts")
+    close_process_pool()
     if translation_config.debug:
         xml_converter.write_json(
             docs,

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/main.py

@@ -26,7 +26,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.5.12"
+__version__ = "0.5.14"
 
 
 def create_parser():

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/pdfminer/pdfinterp.py

@@ -119,6 +119,7 @@ class PDFTextState:
         obj.rise = self.rise
         obj.matrix = self.matrix
         obj.linematrix = self.linematrix
+        obj.font_id = getattr(self, "font_id", None)
         return obj
 
     def reset(self) -> None:

{babeldoc-0.5.12 → babeldoc-0.5.14}/babeldoc/translator/translator.py

@@ -14,6 +14,7 @@ from tenacity import retry_if_exception_type
 from tenacity import stop_after_attempt
 from tenacity import wait_exponential
 
+from babeldoc.babeldoc_exception.BabelDOCException import ContentFilterError
 from babeldoc.translator.cache import TranslationCache
 from babeldoc.utils.atomic_integer import AtomicInteger
 
@@ -155,7 +156,12 @@ class BaseTranslator(ABC):
         _translate_rate_limiter.wait()
         translation = self.do_llm_translate(text, rate_limit_params)
         if not (self.ignore_cache or ignore_cache):
-            self.cache.set(text, translation)
+            try:
+                self.cache.set(text, translation)
+            except Exception as e:
+                logger.debug(
+                    f"try set cache failed, ignore it: {e}, text: {text}, translation: {translation}"
+                )
         return translation
 
     @abstractmethod
@@ -299,22 +305,30 @@ class OpenAITranslator(BaseTranslator):
             extra_headers["X-DashScope-DataInspection"] = (
                 '{"input": "disable", "output": "disable"}'
             )
-
-        response = self.client.chat.completions.create(
-            model=self.model,
-            **options,
-            max_tokens=2048,
-            messages=[
-                {
-                    "role": "user",
-                    "content": text,
-                },
-            ],
-            extra_headers=extra_headers,
-            extra_body=self.extra_body,
-        )
-        self.update_token_count(response)
-        return response.choices[0].message.content.strip()
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                **options,
+                max_tokens=2048,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": text,
+                    },
+                ],
+                extra_headers=extra_headers,
+                extra_body=self.extra_body,
+            )
+            self.update_token_count(response)
+            return response.choices[0].message.content.strip()
+        except openai.BadRequestError as e:
+            if (
+                "系统检测到输入或生成内容可能包含不安全或敏感内容，请您避免输入易产生敏感内容的提示语，感谢您的配合。"
+                in e.message
+            ):
+                raise ContentFilterError(e.message) from e
+            else:
+                raise
 
     def update_token_count(self, response):
         try:

{babeldoc-0.5.12 → babeldoc-0.5.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.5.12"
+version = "0.5.14"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -162,7 +162,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.5.12"
+current_version = "0.5.14"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

babeldoc-0.5.12/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.5.12"