BabelDOC 0.3.63__tar.gz → 0.3.65__tar.gz

{babeldoc-0.3.63 → babeldoc-0.3.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.63
+Version: 0.3.65
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.3.65/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.65"

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.63"
+__version__ = "0.3.65"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/document_il/backend/pdf_creater.py

@@ -99,6 +99,7 @@ def parse_truetype_data(data):
 TOUNICODE_HEAD = """\
 /CIDInit /ProcSet findresource begin
 12 dict begin
+begincmap
 /CIDSystemInfo <</Registry(Adobe)/Ordering(UCS)/Supplement 0>> def
 /CMapName /Adobe-Identity-UCS def
 /CMapType 2 def
@@ -124,7 +125,10 @@ def make_tounicode(cmap, used):
             if code < 0x10000:
                 line.append(f"<{glyph:04x}><{code:04x}>")
             else:
-                line.append(f"<{glyph:04x}><{code:08x}>")
+                code -= 0x10000
+                high = 0xD800 + (code >> 10)
+                low = 0xDC00 + (code & 0b1111111111)
+                line.append(f"<{glyph:04x}><{high:04x}{low:04x}>")
         line.append("endbfchar")
     line.append(TOUNICODE_TAIL)
     return "\n".join(line)

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/document_il/midend/il_translator.py

@@ -867,7 +867,7 @@ class ILTranslator:
             "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
         )
         llm_input.append(
-            "4. Only output the translation result without explanations and annotations."
+            f"4. Only output the translation result in {self.translation_config.lang_out} without explanations and annotations."
         )
         llm_input.append(f"5. Translate text into {self.translation_config.lang_out}.")
         prompt_template = f"""

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -284,7 +284,7 @@ class ILTranslatorLLMOnly:
                 llm_prompt_parts.append(self.translation_config.custom_system_prompt)
             else:
                 llm_prompt_parts.append(
-                    f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
+                    f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}.\n"
                     "When translating, strictly follow the instructions below to ensure translation quality and preserve all formatting, tags, and placeholders:\n"
                 )
 
@@ -378,7 +378,7 @@ class ILTranslatorLLMOnly:
                 '1. You will receive a JSON object with entries containing "id" and "input" fields.'
             )
             llm_prompt_parts.append(
-                '2. Your task is to translate the value of "input" into zh-CN, while applying the rules above.'
+                f'2. Your task is to translate the value of "input" into {self.translation_config.lang_out}, while applying the rules above.'
             )
             llm_prompt_parts.append(
                 '3. Return a new JSON object with the same "id" and the translated "output" field.'

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/document_il/utils/layout_helper.py

@@ -39,7 +39,7 @@ LEFT_BRACKET = ("(cid:8)", "(", "(cid:16)", "{", "[", "(cid:104)", "(cid:2)")
 RIGHT_BRACKET = ("(cid:9)", ")", "(cid:17)", "}", "]", "(cid:105)", "(cid:3)")
 
 BULLET_POINT_PATTERN = re.compile(
-    r"[■•⚫⬤◆◇○●◦‣⁃▪▫∗†‡¹²³⁴⁵⁶⁷⁸⁹⁰₁₂₃₄₅₆₇₈₉₀ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖᵍʳˢᵗᵘᵛʷˣʸᶻ†‡§¶※⁑⁂⁕⁎⁜⁑❧☙⁋‖‽]"
+    r"[■•⚫⬤◆◇○●◦‣⁃▪▫∗†‡¹²³⁴⁵⁶⁷⁸⁹⁰₁₂₃₄₅₆₇₈₉₀ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖᵍʳˢᵗᵘᵛʷˣʸᶻ†‡§¶※⁑⁂⁕⁎⁜⁑❧☙⁋‖‽·]"
 )
 
 

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/docvision/doclayout.py

@@ -217,7 +217,7 @@ class OnnxModel(DocLayoutModel):
             with self.lock:
                 # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
                 pix = get_no_rotation_img(mupdf_doc[page.page_number])
-            image = np.fromstring(pix.samples, np.uint8).reshape(
+            image = np.frombuffer(pix.samples, np.uint8).reshape(
                 pix.height,
                 pix.width,
                 3,

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/docvision/rpc_doclayout.py

@@ -259,7 +259,7 @@ class RpcDocLayoutModel(DocLayoutModel):
         with self.lock:
             # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
             pix = get_no_rotation_img(mupdf_doc[page.page_number])
-        image = np.fromstring(pix.samples, np.uint8).reshape(
+        image = np.frombuffer(pix.samples, np.uint8).reshape(
             pix.height,
             pix.width,
             3,

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/docvision/table_detection/rapidocr.py

@@ -239,7 +239,7 @@ class RapidOCRModel:
             with self.lock:
                 # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
                 pix = get_no_rotation_img(mupdf_doc[page.page_number])
-            image = np.fromstring(pix.samples, np.uint8).reshape(
+            image = np.frombuffer(pix.samples, np.uint8).reshape(
                 pix.height,
                 pix.width,
                 3,

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/high_level.py

@@ -92,7 +92,7 @@ def fix_cmap(translate_result: TranslateResult, translate_config: TranslationCon
         "no_watermark_dual_pdf_path",
     ):
         path = getattr(translate_result, attr)
-        if path in processed:
+        if not path or path in processed:
             continue
         processed.append(path)
 
@@ -188,7 +188,7 @@ def start_parse_il(
         # but in order to facilitate the migration of pdf2zh,
         # the relevant code is temporarily retained.
         # pix = doc_zh[page.pageno].get_pixmap()
-        # image = np.fromstring(pix.samples, np.uint8).reshape(
+        # image = np.frombuffer(pix.samples, np.uint8).reshape(
         #     pix.height, pix.width, 3
         # )[:, :, ::-1]
         # page_layout = model.predict(
@@ -593,11 +593,7 @@ def do_translate(
         result.original_pdf_path = translation_config.input_file
         result.peak_memory_usage = peak_memory_usage
 
-        # should fix macOS preview compatibility
-        # Although the issue of Windows Edge
-        # not being able to copy translated text can be fixed,
-        # the macOS preview is broken
-        # fix_cmap(result, translation_config)
+        fix_cmap(result, translation_config)
         try:
             migrate_toc(translation_config, result)
         except Exception as e:

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/main.py

@@ -21,7 +21,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.63"
+__version__ = "0.3.65"
 
 
 def create_parser():

{babeldoc-0.3.63 → babeldoc-0.3.65}/babeldoc/tools/italic_assistance.py

@@ -5,6 +5,10 @@ from pathlib import Path
 
 import orjson
 from babeldoc.const import CACHE_FOLDER
+from babeldoc.document_il.midend.styles_and_formulas import StylesAndFormulas
+from babeldoc.translation_config import TranslationConfig
+from rich.console import Console
+from rich.table import Table
 
 WORKING_FOLDER = Path(CACHE_FOLDER) / "working"
 
@@ -210,6 +214,80 @@ def main():
             f"No fonts found for paragraphs matching debug_id pattern: {args.debug_id_regex}"
         )
 
+    fonts = []
+
+    # Read intermediate representation
+    with json_path.open(encoding="utf-8") as f:
+        pdf_data = json.load(f)
+
+    for page_index, page in enumerate(pdf_data["page"]):
+        for paragraph_index, paragraph_content in enumerate(page["pdf_paragraph"]):
+            font_debug_id = paragraph_content["debug_id"]
+            if font_debug_id:
+                # Create page font mapping
+                page_font_map = {}
+                for font in page["pdf_font"]:
+                    if "font_id" in font and "name" in font:
+                        page_font_map[font["font_id"]] = (font["font_id"], font["name"])
+
+                # Extract fonts from paragraph
+                name_list = []
+                paragraph_fonts = extract_fonts_from_paragraph(
+                    paragraph_content, page_font_map
+                )
+                for _font_id, font_name in paragraph_fonts:
+                    name_list.append(font_name)
+
+                font_list = []
+                for each in fonts:
+                    font_list.append(each[1])
+
+                for each_name in name_list:
+                    if each_name not in font_list:
+                        fonts.append(
+                            (page_index, each_name, paragraph_index, font_debug_id)
+                        )
+
+    # Initialize checker
+    translation_config = TranslationConfig(
+        *[None for _ in range(3)], lang_out="zh_cn", doc_layout_model=1
+    )
+    checker = StylesAndFormulas(translation_config)
+
+    # Create table
+    table = Table(title="Font Recognition Results")
+    table.add_column("Page #", justify="center", style="cyan")
+    table.add_column("Paragraph #", justify="center", style="cyan")
+    table.add_column("DEBUG_ID", justify="center", style="cyan")
+    table.add_column("Font Name", style="magenta")
+    table.add_column("Recognition Result", justify="center")
+
+    # Output results
+    for each_font in fonts:
+        page_index, font_name, paragraph_index, font_debug_id = each_font
+
+        if checker.is_formulas_font(font_name):
+            table.add_row(
+                str(page_index),
+                str(paragraph_index),
+                str(font_debug_id),
+                font_name,
+                "[bold red]Formula Font[/bold red]",
+            )
+        else:
+            table.add_row(
+                str(page_index),
+                str(paragraph_index),
+                str(font_debug_id),
+                font_name,
+                "[bold blue]Non-Formula Font[/bold blue]",
+            )
+
+    # Print table
+    console = Console()
+
+    console.print(table)
+
     return 0
 
 

{babeldoc-0.3.63 → babeldoc-0.3.65}/docs/CONTRIBUTOR_REWARD.md

@@ -16,7 +16,10 @@
 
 ### 二、申请流程
 #### **申请条件**
-   - PR 需被成功合并至[funstory-ai/BabelDOC](https://github.com/funstory-ai/BabelDOC/pulls) 仓库或 [Byaidu/PDFMathTranslate](https://github.com/Byaidu/PDFMathTranslate/pulls)的主分支。
+   - PR 需被成功合并至以下几个仓库：
+     1. [funstory-ai/BabelDOC](https://github.com/funstory-ai/BabelDOC/pulls) 仓库
+     2. [Byaidu/PDFMathTranslate](https://github.com/Byaidu/PDFMathTranslate/pulls) 的主分支 
+     3. [PDFMathTranslate/PDFMathTranslate-next](https://github.com/PDFMathTranslate/PDFMathTranslate-next/pulls) 的主分支。
    - 若目标为 [funstory-ai/BabelDOC](https://github.com/funstory-ai/BabelDOC/pulls) 的 PR 未被合并，但被维护者认定为有价值的概念验证，同样符合条件。
    - 审核 PR、撰写 wiki 等贡献也必须是以上两个仓库。
    - 同一贡献者每月仅可申请一次（无论提交 PR 数量）
@@ -33,6 +36,10 @@
    - 附件要求：
      - PR 页面完整截图（需包含合并状态、仓库名称及点击头像后弹出来的侧边栏，如下图所示）
 
+> [!IMPORTANT]
+>
+> 不满足上述格式要求的邮件会被直接忽略！
+
 ![附件示例](https://s.immersivetranslate.com/assets/r2-uploads/images/babeldoc-contributor_reward_example.png)
 
 #### **奖励说明**

{babeldoc-0.3.63 → babeldoc-0.3.65}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.63"
+version = "0.3.65"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -154,7 +154,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.63"
+current_version = "0.3.65"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]