BabelDOC 0.4.1__tar.gz → 0.4.6__tar.gz

{babeldoc-0.4.1 → babeldoc-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.4.1
+Version: 0.4.6
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.4.6/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.6"

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.4.1"
+__version__ = "0.4.6"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/docvision/rpc_doclayout2.py

@@ -286,7 +286,7 @@ class RpcDocLayoutModel(DocLayoutModel):
         translate_config,
         save_debug_image,
     ):
-        with ThreadPoolExecutor(max_workers=64) as executor:
+        with ThreadPoolExecutor(max_workers=128) as executor:
             yield from executor.map(
                 self.predict_page,
                 pages,

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/document_il/backend/pdf_creater.py

@@ -331,7 +331,7 @@ class PDFCreater:
         self,
         draw_op: BitStream,
         rectangle: il_version_1.PdfRectangle,
-        line_width: float = 0.5,
+        line_width: float = 0.4,
     ):
         """Draw a rectangle in PDF for visualization purposes.
 

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/document_il/frontend/il_creater.py

@@ -656,7 +656,7 @@ class ILCreater:
                 _, encoding = self.mupdf.xref_get_key(xref_id, "Encoding")
                 if encoding == "/Identity-H" or encoding == "/Identity-V":
                     encoding_length = 2
-                if encoding == "/WinAnsiEncoding":
+                elif encoding == "/WinAnsiEncoding":
                     encoding_length = 1
                 else:
                     _, to_unicode_id = self.mupdf.xref_get_key(xref_id, "ToUnicode")

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/document_il/midend/add_debug_information.py

@@ -3,7 +3,6 @@ import logging
 import babeldoc.format.pdf.document_il.il_version_1 as il_version_1
 from babeldoc.format.pdf.document_il import GraphicState
 from babeldoc.format.pdf.document_il.utils.style_helper import BLUE
-from babeldoc.format.pdf.document_il.utils.style_helper import CYAN
 from babeldoc.format.pdf.document_il.utils.style_helper import ORANGE
 from babeldoc.format.pdf.document_il.utils.style_helper import TEAL
 from babeldoc.format.pdf.document_il.utils.style_helper import YELLOW
@@ -133,9 +132,9 @@ class AddDebugInformation:
                                 char.visual_bbox.box, TEAL, line_width=0.2
                             ),
                         )
-                        page.pdf_rectangle.append(
-                            self._create_rectangle(char.box, CYAN, line_width=0.2),
-                        )
+                        # page.pdf_rectangle.append(
+                        #     self._create_rectangle(char.box, CYAN, line_width=0.2),
+                        # )
 
             for xobj in page.pdf_xobject:
                 new_paragraphs.append(

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/document_il/midend/styles_and_formulas.py

@@ -19,15 +19,14 @@ from babeldoc.format.pdf.document_il.utils.formular_helper import (
     is_formulas_middle_char,
 )
 from babeldoc.format.pdf.document_il.utils.formular_helper import is_formulas_start_char
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
 from babeldoc.format.pdf.document_il.utils.layout_helper import LEFT_BRACKET
 from babeldoc.format.pdf.document_il.utils.layout_helper import RIGHT_BRACKET
 from babeldoc.format.pdf.document_il.utils.layout_helper import build_layout_index
+from babeldoc.format.pdf.document_il.utils.layout_helper import calculate_iou_for_boxes
 from babeldoc.format.pdf.document_il.utils.layout_helper import (
     calculate_y_iou_for_boxes,
 )
-from babeldoc.format.pdf.document_il.utils.layout_helper import (
-    formular_height_ignore_char,
-)
 from babeldoc.format.pdf.document_il.utils.layout_helper import is_bullet_point
 from babeldoc.format.pdf.document_il.utils.layout_helper import is_same_style
 from babeldoc.format.pdf.translation_config import TranslationConfig
@@ -40,6 +39,9 @@ class StylesAndFormulas:
         self.translation_config = translation_config
         self.font_mapper = FontMapper(translation_config)
 
+    def update_formula_data(self, formula: PdfFormula):
+        update_formula_data(formula)
+
     def process(self, document: Document):
         with self.translation_config.progress_monitor.stage_start(
             self.stage_name,
@@ -50,6 +52,12 @@ class StylesAndFormulas:
                 self.process_page(page)
                 pbar.advance()
 
+    def update_all_formula_data(self, page: Page):
+        for para in page.pdf_paragraph:
+            for comp in para.pdf_paragraph_composition:
+                if comp.pdf_formula:
+                    self.update_formula_data(comp.pdf_formula)
+
     def process_page(self, page: Page):
         """处理页面，包括公式识别和偏移量计算"""
         build_layout_index(page)
@@ -59,7 +67,9 @@ class StylesAndFormulas:
         self.merge_overlapping_formulas(page)
         # self.process_page_offsets(page)
         self.process_translatable_formulas(page)
+        self.update_all_formula_data(page)
         self.process_page_offsets(page)
+        self.update_all_formula_data(page)
         self.process_page_styles(page)
 
         # clean up to save memory
@@ -468,6 +478,8 @@ class StylesAndFormulas:
             return
 
         for paragraph in page.pdf_paragraph:
+            if paragraph.debug_id is None:
+                continue
             if not paragraph.pdf_paragraph_composition:
                 continue
 
@@ -491,22 +503,28 @@ class StylesAndFormulas:
                     comp = paragraph.pdf_paragraph_composition[j]
                     if comp.pdf_line:
                         for char in reversed(comp.pdf_line.pdf_character):
+                            if not char.pdf_character_id:
+                                continue
                             # 检查 y 坐标是否接近，判断是否在同一行
                             left_iou = calculate_y_iou_for_boxes(char.box, formula.box)
                             if left_iou > 0.6:
                                 left_char = char
                                 break
+                    break
 
                 # 查找右边最近的同一行的文本
                 for j in range(i + 1, len(paragraph.pdf_paragraph_composition)):
                     comp = paragraph.pdf_paragraph_composition[j]
                     if comp.pdf_line:
                         for char in comp.pdf_line.pdf_character:
+                            if not char.pdf_character_id:
+                                continue
                             # 检查 y 坐标是否接近，判断是否在同一行
                             right_iou = calculate_y_iou_for_boxes(char.box, formula.box)
                             if right_iou > 0.6:
                                 right_char = char
                                 break
+                    break
 
                 # If both text segments exist, keep the one with higher IOU
                 if left_char and right_char:
@@ -590,33 +608,11 @@ class StylesAndFormulas:
             self.update_line_data(new_line)
             return PdfParagraphComposition(pdf_line=new_line)
 
-    def update_formula_data(self, formula: PdfFormula):
-        min_x = min(char.visual_bbox.box.x for char in formula.pdf_character)
-        max_x = max(char.visual_bbox.box.x2 for char in formula.pdf_character)
-        if not all(map(formular_height_ignore_char, formula.pdf_character)):
-            min_y = min(
-                char.visual_bbox.box.y
-                for char in formula.pdf_character
-                if not formular_height_ignore_char(char)
-            )
-            max_y = max(
-                char.visual_bbox.box.y2
-                for char in formula.pdf_character
-                if not formular_height_ignore_char(char)
-            )
-        else:
-            min_y = min(char.visual_bbox.box.y for char in formula.pdf_character)
-            max_y = max(char.visual_bbox.box.y2 for char in formula.pdf_character)
-        formula.box = Box(min_x, min_y, max_x, max_y)
-        if not formula.y_offset:
-            formula.y_offset = 0
-        if not formula.x_offset:
-            formula.x_offset = 0
-        if not formula.x_advance:
-            formula.x_advance = 0
-
     def is_translatable_formula(self, formula: PdfFormula) -> bool:
         """判断公式是否只包含需要正常翻译的字符（数字、空格和英文逗号）"""
+        if all(char.formula_layout_id for char in formula.pdf_character):
+            return False
+
         text = "".join(char.char_unicode for char in formula.pdf_character)
         if formula.y_offset > 0.1:
             return False
@@ -727,8 +723,11 @@ class StylesAndFormulas:
 
     def merge_overlapping_formulas(self, page: Page):
         """
-        合并 x 轴重叠且 y 轴有交集的相邻公式，或者 x 轴相邻且 y 轴 IOU > 0.5 的公式，
-        或者所有字符的 layout id 都相同的公式
+        合并符合以下条件的公式：
+        1. x 轴重叠且 y 轴有交集的相邻公式，或者
+        2. x 轴相邻且 y 轴 IOU > 0.5 的相邻公式，或者
+        3. 所有字符的 layout id 都相同的相邻公式，或者
+        4. 任意两个公式的 IOU > 0.8
         角标可能会被识别成单独的公式，需要合并
         """
         if not page.pdf_paragraph:
@@ -738,46 +737,76 @@ class StylesAndFormulas:
             if not paragraph.pdf_paragraph_composition:
                 continue
 
-            i = 0
-            while i < len(paragraph.pdf_paragraph_composition) - 1:
-                comp1 = paragraph.pdf_paragraph_composition[i]
-                comp2 = paragraph.pdf_paragraph_composition[i + 1]
-
-                # 检查是否都是公式
-                if comp1.pdf_formula is None or comp2.pdf_formula is None:
-                    i += 1
-                    continue
-
-                formula1 = comp1.pdf_formula
-                formula2 = comp2.pdf_formula
-
-                # 检查合并条件：
-                # 1. x 轴重叠且 y 轴有交集，或者
-                # 2. x 轴相邻且 y 轴 IOU > 0.5，或者
-                # 3. 所有字符的 layout id 都相同
-                should_merge = (
-                    (
-                        self.is_x_axis_contained(formula1.box, formula2.box)
-                        and self.has_y_intersection(formula1.box, formula2.box)
-                    )
-                    or (
-                        self.is_x_axis_adjacent(formula1.box, formula2.box)
-                        and self.calculate_y_iou(formula1.box, formula2.box) > 0.5
-                    )
-                    or (self._have_same_layout_ids(formula1, formula2, page))
-                )
+            # 重复执行合并过程，直到没有更多可以合并的公式
+            merged = True
+            while merged:
+                merged = False
+                for i in range(len(paragraph.pdf_paragraph_composition)):
+                    if merged:
+                        break
+                    comp1 = paragraph.pdf_paragraph_composition[i]
+                    if comp1.pdf_formula is None:
+                        continue
+
+                    for j in range(i + 1, len(paragraph.pdf_paragraph_composition)):
+                        comp2 = paragraph.pdf_paragraph_composition[j]
+                        if comp2.pdf_formula is None:
+                            continue
+
+                        formula1 = comp1.pdf_formula
+                        formula2 = comp2.pdf_formula
+
+                        # 检查合并条件：
+                        # 1. x 轴重叠且 y 轴有交集，或者
+                        # 2. x 轴相邻且 y 轴 IOU > 0.5，或者
+                        # 3. 所有字符的 layout id 都相同，或者
+                        # 4. 任意两个公式的 IOU > 0.8
+                        should_merge = (
+                            (
+                                j == i + 1
+                                and (
+                                    (
+                                        self.is_x_axis_contained(
+                                            formula1.box, formula2.box
+                                        )
+                                        and self.has_y_intersection(
+                                            formula1.box, formula2.box
+                                        )
+                                    )
+                                    or (
+                                        self.is_x_axis_adjacent(
+                                            formula1.box, formula2.box
+                                        )
+                                        and self.calculate_y_iou(
+                                            formula1.box, formula2.box
+                                        )
+                                        > 0.5
+                                    )
+                                )
+                            )
+                            or (self._have_same_layout_ids(formula1, formula2, page))
+                            or (
+                                calculate_iou_for_boxes(formula1.box, formula2.box)
+                                > 0.8
+                            )
+                            or (
+                                calculate_iou_for_boxes(formula2.box, formula1.box)
+                                > 0.8
+                            )
+                        )
 
-                if should_merge:
-                    # 合并公式
-                    merged_formula = self.merge_formulas(formula1, formula2)
-                    paragraph.pdf_paragraph_composition[i] = PdfParagraphComposition(
-                        pdf_formula=merged_formula,
-                    )
-                    # 删除第二个公式
-                    del paragraph.pdf_paragraph_composition[i + 1]
-                    # 不增加 i，因为合并后的公式可能还需要和下一个公式合并
-                else:
-                    i += 1
+                        if should_merge:
+                            # 合并公式
+                            merged_formula = self.merge_formulas(formula1, formula2)
+                            paragraph.pdf_paragraph_composition[i] = (
+                                PdfParagraphComposition(
+                                    pdf_formula=merged_formula,
+                                )
+                            )
+                            # 删除第二个公式
+                            del paragraph.pdf_paragraph_composition[j]
+                            merged = True
+                            break
 
     def _have_same_layout_ids(
         self, formula1: PdfFormula, formula2: PdfFormula, page: Page

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/document_il/midend/typesetting.py

@@ -15,6 +15,7 @@ from babeldoc.format.pdf.document_il import PdfParagraphComposition
 from babeldoc.format.pdf.document_il import PdfStyle
 from babeldoc.format.pdf.document_il import il_version_1
 from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.formular_helper import update_formula_data
 from babeldoc.format.pdf.translation_config import TranslationConfig
 from babeldoc.format.pdf.translation_config import WatermarkOutputMode
 
@@ -349,11 +350,12 @@ class TypesettingUnit:
 
             return box
         elif self.formular:
-            if self.formular.x_offset <= 0.5:
-                return self.formular.box
-            formular_box = copy.copy(self.formular.box)
-            formular_box.x2 += self.formular.x_advance
-            return formular_box
+            return self.formular.box
+            # if self.formular.x_offset <= 0.5:
+            #     return self.formular.box
+            # formular_box = copy.copy(self.formular.box)
+            # formular_box.x2 += self.formular.x_advance
+            # return formular_box
         elif self.unicode:
             char_width = self.font.char_lengths(self.unicode, self.font_size)[0]
             if self.x is None or self.y is None or self.scale is None:
@@ -406,8 +408,8 @@ class TypesettingUnit:
         elif self.formular:
             # 创建新的公式对象，保持内部字符的相对位置
             new_chars = []
-            min_x = min(char.visual_bbox.box.x for char in self.formular.pdf_character)
-            min_y = min(char.visual_bbox.box.y for char in self.formular.pdf_character)
+            min_x = self.formular.box.x
+            min_y = self.formular.box.y
 
             for char in self.formular.pdf_character:
                 # 计算相对位置
@@ -481,6 +483,7 @@ class TypesettingUnit:
                 y_offset=self.formular.y_offset * scale,
                 x_advance=self.formular.x_advance * scale,
             )
+            update_formula_data(new_formula)
             return TypesettingUnit(formular=new_formula)
 
         elif self.unicode:
@@ -819,7 +822,7 @@ class Typesetting:
             # 更新 x 坐标
             current_x = relocated_unit.box.x2
             if prev_x > current_x:
-                logger.warning(f"坐标回绕！！！TypesettingUnit: {unit}, ")
+                logger.warning(f"坐标回绕！！！TypesettingUnit: {unit.box}, ")
 
             last_unit = relocated_unit
 

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/document_il/utils/formular_helper.py

@@ -3,8 +3,13 @@ import functools
 import re
 import unicodedata
 
+from babeldoc.format.pdf.document_il.il_version_1 import Box
 from babeldoc.format.pdf.document_il.il_version_1 import Page
+from babeldoc.format.pdf.document_il.il_version_1 import PdfFormula
 from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
+from babeldoc.format.pdf.document_il.utils.layout_helper import (
+    formular_height_ignore_char,
+)
 from babeldoc.format.pdf.translation_config import TranslationConfig
 
 
@@ -301,3 +306,29 @@ def is_formulas_font(font_name: str, formular_font_pattern: str | None) -> bool:
         return True
 
     return False
+
+
+def update_formula_data(formula: PdfFormula):
+    min_x = min(char.visual_bbox.box.x for char in formula.pdf_character)
+    max_x = max(char.visual_bbox.box.x2 for char in formula.pdf_character)
+    if not all(map(formular_height_ignore_char, formula.pdf_character)):
+        min_y = min(
+            char.visual_bbox.box.y
+            for char in formula.pdf_character
+            if not formular_height_ignore_char(char)
+        )
+        max_y = max(
+            char.visual_bbox.box.y2
+            for char in formula.pdf_character
+            if not formular_height_ignore_char(char)
+        )
+    else:
+        min_y = min(char.visual_bbox.box.y for char in formula.pdf_character)
+        max_y = max(char.visual_bbox.box.y2 for char in formula.pdf_character)
+    formula.box = Box(min_x, min_y, max_x, max_y)
+    if not formula.y_offset:
+        formula.y_offset = 0
+    if not formula.x_offset:
+        formula.x_offset = 0
+    if not formula.x_advance:
+        formula.x_advance = 0

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/document_il/utils/layout_helper.py

@@ -634,6 +634,7 @@ def get_character_layout(
             "abstract",
             "content",
             "figure_title",
+            "chart_title",
             "table_title",
             "doc_title",
             "footnote",
@@ -722,6 +723,7 @@ def is_text_layout(layout: Layout):
         "footer",
         "seal",
         "text",
+        "chart_title",
     ]
 
 

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/high_level.py

@@ -491,7 +491,7 @@ def do_translate(
                         pm.total_parts = len(split_points)
 
                         # Process parts serially
-                        results: dict[int, TranslateResult] = {}
+                        results: dict[int, TranslateResult | None] = {}
                         original_watermark_mode = (
                             translation_config.watermark_output_mode
                         )
@@ -515,6 +515,12 @@ def do_translate(
                                 part_config.page_ranges = [
                                     (x, x) for x in should_translate_pages
                                 ]
+                                if (
+                                    translation_config.only_include_translated_page
+                                    and not should_translate_pages
+                                ):
+                                    results[i] = None
+                                    continue
 
                                 # Only first part should do scanned detection if enabled
                                 if i > 0:
@@ -640,7 +646,6 @@ def migrate_toc(
     old_doc = Document(translation_config.input_file)
     if not old_doc:
         return
-
     try:
         fix_filter(old_doc)
         fix_null_xref(old_doc)
@@ -653,6 +658,15 @@ def migrate_toc(
         logger.info("No TOC found in the original PDF, skipping migration.")
         return
 
+    if translation_config.only_include_translated_page:
+        total_page = set(range(0, len(old_doc)))
+
+        pages_to_translate = {
+            i for i in len(old_doc) if translation_config.should_translate_page(i + 1)
+        }
+
+        should_removed_page = list(total_page - pages_to_translate)
+
     files = {
         translate_result.dual_pdf_path,
         # translate_result.mono_pdf_path,
@@ -797,6 +811,9 @@ def _do_translate_single(
     docs = il_creater.create_il()
     logger.debug(f"finish create il from {temp_pdf_path}")
     del il_creater
+    if translation_config.only_include_translated_page and not docs.page:
+        return None
+
     if translation_config.debug:
         xml_converter.write_json(
             docs,

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/result_merger.py

@@ -16,7 +16,9 @@ class ResultMerger:
     def __init__(self, translation_config: TranslationConfig):
         self.config = translation_config
 
-    def merge_results(self, results: dict[int, TranslateResult]) -> TranslateResult:
+    def merge_results(
+        self, results: dict[int, TranslateResult | None]
+    ) -> TranslateResult:
         """Merge multiple translation results into one"""
         if not results:
             raise ValueError("No results to merge")
@@ -35,7 +37,7 @@ class ResultMerger:
         dual_file_name_no_watermark = (
             f"{basename}{debug_suffix}.{self.config.lang_out}.dual.pdf"
         )
-
+        results = {k: v for k, v in results.items() if v is not None}
         # Sort results by part index
         sorted_results = dict(sorted(results.items()))
         first_result = next(iter(sorted_results.values()))

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/format/pdf/translation_config.py

@@ -354,7 +354,7 @@ class TranslationConfig:
         if part_index in self._part_working_dirs:
             part_dir = self._part_working_dirs[part_index]
             if part_dir.exists():
-                shutil.rmtree(part_dir)
+                shutil.rmtree(part_dir, ignore_errors=True)
             del self._part_working_dirs[part_index]
 
     def cleanup_temp_files(self):
@@ -364,7 +364,7 @@ class TranslationConfig:
                 self.cleanup_part_working_dir(part_index)
             if self._is_temp_dir:
                 logger.info(f"cleanup temp files: {self.working_dir}")
-                shutil.rmtree(self.working_dir)
+                shutil.rmtree(self.working_dir, ignore_errors=True)
         except Exception:
             logger.exception("Error cleaning up temporary files")
 

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/glossary.py

@@ -39,7 +39,17 @@ TERM_NORM_PATTERN = re.compile(r"\s+", regex.UNICODE)
 class Glossary:
     def __init__(self, name: str, entries: list[GlossaryEntry]):
         self.name = name
-        self.entries = entries
+
+        # Deduplicate entries based on normalized source
+        unique_entries = []
+        seen_normalized_sources = set()
+        for entry in entries:
+            normalized_source = self.normalize_source(entry.source)
+            if normalized_source not in seen_normalized_sources:
+                unique_entries.append(entry)
+                seen_normalized_sources.add(normalized_source)
+        self.entries = unique_entries
+
         self.normalized_lookup: dict[str, tuple[str, str]] = {}
         self.id_lookup: list[tuple[str, str]] = []
         self.hs_dbs: list[hyperscan.Database] | None = None

{babeldoc-0.4.1 → babeldoc-0.4.6}/babeldoc/main.py

@@ -21,7 +21,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.4.1"
+__version__ = "0.4.6"
 
 
 def create_parser():