BabelDOC 0.4.18__tar.gz → 0.4.20__tar.gz

{babeldoc-0.4.18 → babeldoc-0.4.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.4.18
+Version: 0.4.20
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.4.20/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.20"

{babeldoc-0.4.18 → babeldoc-0.4.20}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.4.18"
+__version__ = "0.4.20"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.4.18 → babeldoc-0.4.20}/babeldoc/format/pdf/babelpdf/encoding.py

@@ -1045,3 +1045,263 @@ def get_type1_encoding(name):
             return win_ansi
         case "MacExpertEncoding":
             return mac_expert
+
+
+WinAnsiEncoding = [
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57,
+    58,
+    59,
+    60,
+    61,
+    62,
+    63,
+    64,
+    65,
+    66,
+    67,
+    68,
+    69,
+    70,
+    71,
+    72,
+    73,
+    74,
+    75,
+    76,
+    77,
+    78,
+    79,
+    80,
+    81,
+    82,
+    83,
+    84,
+    85,
+    86,
+    87,
+    88,
+    89,
+    90,
+    91,
+    92,
+    93,
+    94,
+    95,
+    96,
+    97,
+    98,
+    99,
+    100,
+    101,
+    102,
+    103,
+    104,
+    105,
+    106,
+    107,
+    108,
+    109,
+    110,
+    111,
+    112,
+    113,
+    114,
+    115,
+    116,
+    117,
+    118,
+    119,
+    120,
+    121,
+    122,
+    123,
+    124,
+    125,
+    126,
+    127,
+    8364,
+    0,
+    8218,
+    402,
+    8222,
+    8230,
+    8224,
+    8225,
+    710,
+    8240,
+    352,
+    8249,
+    338,
+    0,
+    381,
+    0,
+    0,
+    8216,
+    8217,
+    8220,
+    8221,
+    8226,
+    8211,
+    8212,
+    732,
+    8482,
+    353,
+    8250,
+    339,
+    0,
+    382,
+    376,
+    160,
+    161,
+    162,
+    163,
+    164,
+    165,
+    166,
+    167,
+    168,
+    169,
+    170,
+    171,
+    172,
+    173,
+    174,
+    175,
+    176,
+    177,
+    178,
+    179,
+    180,
+    181,
+    182,
+    183,
+    184,
+    185,
+    186,
+    187,
+    188,
+    189,
+    190,
+    191,
+    192,
+    193,
+    194,
+    195,
+    196,
+    197,
+    198,
+    199,
+    200,
+    201,
+    202,
+    203,
+    204,
+    205,
+    206,
+    207,
+    208,
+    209,
+    210,
+    211,
+    212,
+    213,
+    214,
+    215,
+    216,
+    217,
+    218,
+    219,
+    220,
+    221,
+    222,
+    223,
+    224,
+    225,
+    226,
+    227,
+    228,
+    229,
+    230,
+    231,
+    232,
+    233,
+    234,
+    235,
+    236,
+    237,
+    238,
+    239,
+    240,
+    241,
+    242,
+    243,
+    244,
+    245,
+    246,
+    247,
+    248,
+    249,
+    250,
+    251,
+    252,
+    253,
+    254,
+    255,
+]

{babeldoc-0.4.18 → babeldoc-0.4.20}/babeldoc/format/pdf/document_il/backend/pdf_creater.py

@@ -795,7 +795,9 @@ class PDFCreater:
                     logger.debug(f"Error restoring media box {name} from PDF")
 
     def write(
-        self, translation_config: TranslationConfig, check_font_exists: bool = False
+        self,
+        translation_config: TranslationConfig,
+        check_font_exists: bool = False,
     ) -> TranslateResult:
         try:
             basename = Path(translation_config.input_file).stem

{babeldoc-0.4.18 → babeldoc-0.4.20}/babeldoc/format/pdf/document_il/frontend/il_creater.py

@@ -10,6 +10,7 @@ import freetype
 import pymupdf
 
 import babeldoc.pdfminer.pdfinterp
+from babeldoc.format.pdf.babelpdf.encoding import WinAnsiEncoding
 from babeldoc.format.pdf.babelpdf.encoding import get_type1_encoding
 from babeldoc.format.pdf.document_il import il_version_1
 from babeldoc.format.pdf.document_il.utils import zstd_helper
@@ -75,15 +76,22 @@ def get_glyph_cbox(face, g):
     return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
 
 
+def get_char_cbox(face, idx):
+    g = face.get_char_index(idx)
+    return get_glyph_cbox(face, g)
+
+
 def get_name_cbox(face, name):
     if name:
+        if isinstance(name, str):
+            name = name.encode("utf-8")
         g = face.get_name_index(name)
         return get_glyph_cbox(face, g)
     return (0, 0, 0, 0)
 
 
 def font_encoding_lookup(doc, idx, key):
-    obj = doc.xref_get_key(doc, idx, key)
+    obj = doc.xref_get_key(idx, key)
     if obj[0] == "name":
         enc_name = obj[1][1:]
         if enc_vector := get_type1_encoding(enc_name):
@@ -98,7 +106,7 @@ def parse_font_encoding(doc, idx):
     return ("Custom", get_type1_encoding("StandardEncoding"))
 
 
-def parse_font_file(doc, idx, encoding, differences):
+def parse_font_file(doc, idx, encoding, differences, legacy_encoding):
     bbox_list = []
     data = doc.xref_stream(idx)
     face = freetype.Face(BytesIO(data))
@@ -111,6 +119,11 @@ def parse_font_file(doc, idx, encoding, differences):
                 face.select_charmap(charmap.encoding)
                 break
     bbox_list = [get_name_cbox(face, x) for x in enc_vector]
+    _, legacy_vector = legacy_encoding
+    legacy_bbox_list = [get_char_cbox(face, x) for x in legacy_vector]
+    for i, bbox in enumerate(bbox_list):
+        if sum(bbox) == 0:
+            bbox_list[i] = legacy_bbox_list[i]
     if differences:
         for code, name in differences:
             bbox_list[code] = get_name_cbox(face, name.encode("U8"))
@@ -525,14 +538,28 @@ class ILCreater:
                 self.current_page_font_char_bounding_box_map[font_id] = (
                     font_char_bounding_box_map
                 )
-        except Exception:
-            pass
+        except Exception as e:
+            logger.error("failed to parse font xobj id %d: %s", xref_id, e)
         self.current_page_font_name_id_map[xref_id] = font_id
         if self.xobj_id in self.xobj_map:
             self.xobj_map[self.xobj_id].pdf_font.append(il_font_metadata)
         else:
             self.current_page.pdf_font.append(il_font_metadata)
 
+    def get_legacy_encoding(self, xobj_id):
+        try:
+            encoding = ("Custom", list(range(256)))
+            font_encoding = self.mupdf.xref_get_key(xobj_id, "Encoding")
+            if font_encoding[1] == "/WinAnsiEncoding":
+                encoding = ("WinAnsi", WinAnsiEncoding)
+            base_encoding = self.mupdf.xref_get_key(xobj_id, "Encoding/BaseEncoding")
+            if base_encoding[1] == "/WinAnsiEncoding":
+                encoding = ("WinAnsi", WinAnsiEncoding)
+
+            return encoding
+        except Exception:
+            return ("Custom", list(range(256)))
+
     def parse_font_xobj_id(self, xobj_id: int):
         bbox_list = []
         encoding = parse_font_encoding(self.mupdf, xobj_id)
@@ -543,7 +570,13 @@ class ILCreater:
         for file_key in ["FontFile", "FontFile2", "FontFile3"]:
             font_file = self.mupdf.xref_get_key(xobj_id, f"FontDescriptor/{file_key}")
             if file_idx := indirect(font_file):
-                bbox_list = parse_font_file(self.mupdf, file_idx, encoding, differences)
+                bbox_list = parse_font_file(
+                    self.mupdf,
+                    file_idx,
+                    encoding,
+                    differences,
+                    self.get_legacy_encoding(xobj_id),
+                )
         cmap = {}
         to_unicode = self.mupdf.xref_get_key(xobj_id, "ToUnicode")
         if to_unicode_idx := indirect(to_unicode):

{babeldoc-0.4.18 → babeldoc-0.4.20}/babeldoc/format/pdf/document_il/il_version_1.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from dataclasses import field
 
 
-@dataclass
+@dataclass(slots=True)
 class BaseOperations:
     class Meta:
         name = "baseOperations"
@@ -15,7 +15,7 @@ class BaseOperations:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class Box:
     class Meta:
         name = "box"
@@ -50,7 +50,7 @@ class Box:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class GraphicState:
     class Meta:
         name = "graphicState"
@@ -138,7 +138,7 @@ class GraphicState:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfFontCharBoundingBox:
     class Meta:
         name = "pdfFontCharBoundingBox"
@@ -180,7 +180,7 @@ class PdfFontCharBoundingBox:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class Cropbox:
     class Meta:
         name = "cropbox"
@@ -194,7 +194,7 @@ class Cropbox:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class Mediabox:
     class Meta:
         name = "mediabox"
@@ -208,7 +208,7 @@ class Mediabox:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PageLayout:
     class Meta:
         name = "pageLayout"
@@ -243,7 +243,7 @@ class PageLayout:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfFigure:
     class Meta:
         name = "pdfFigure"
@@ -257,7 +257,7 @@ class PdfFigure:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfFont:
     class Meta:
         name = "pdfFont"
@@ -338,7 +338,7 @@ class PdfFont:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfRectangle:
     class Meta:
         name = "pdfRectangle"
@@ -386,7 +386,7 @@ class PdfRectangle:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfStyle:
     class Meta:
         name = "pdfStyle"
@@ -415,7 +415,7 @@ class PdfStyle:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class VisualBbox:
     class Meta:
         name = "visual_bbox"
@@ -429,7 +429,7 @@ class VisualBbox:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfCharacter:
     class Meta:
         name = "pdfCharacter"
@@ -508,7 +508,7 @@ class PdfCharacter:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfSameStyleUnicodeCharacters:
     class Meta:
         name = "pdfSameStyleUnicodeCharacters"
@@ -535,7 +535,7 @@ class PdfSameStyleUnicodeCharacters:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfXobject:
     class Meta:
         name = "pdfXobject"
@@ -580,7 +580,7 @@ class PdfXobject:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfFormula:
     class Meta:
         name = "pdfFormula"
@@ -622,7 +622,7 @@ class PdfFormula:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfLine:
     class Meta:
         name = "pdfLine"
@@ -644,7 +644,7 @@ class PdfLine:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfSameStyleCharacters:
     class Meta:
         name = "pdfSameStyleCharacters"
@@ -674,7 +674,7 @@ class PdfSameStyleCharacters:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfParagraphComposition:
     class Meta:
         name = "pdfParagraphComposition"
@@ -716,7 +716,7 @@ class PdfParagraphComposition:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class PdfParagraph:
     class Meta:
         name = "pdfParagraph"
@@ -802,7 +802,7 @@ class PdfParagraph:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class Page:
     class Meta:
         name = "page"
@@ -896,7 +896,7 @@ class Page:
     )
 
 
-@dataclass
+@dataclass(slots=True)
 class Document:
     class Meta:
         name = "document"

{babeldoc-0.4.18 → babeldoc-0.4.20}/babeldoc/format/pdf/document_il/midend/automatic_term_extractor.py

@@ -46,7 +46,9 @@ Result:
 
 class BatchParagraph:
     def __init__(
-        self, paragraphs: list[PdfParagraph], page_tracker: PageTermExtractTracker
+        self,
+        paragraphs: list[PdfParagraph],
+        page_tracker: PageTermExtractTracker,
     ):
         self.paragraphs = paragraphs
         self.tracker = page_tracker.new_paragraph()
@@ -105,7 +107,9 @@ class AutomaticTermExtractor:
     stage_name = "Automatic Term Extraction"
 
     def __init__(
-        self, translate_engine: BaseTranslator, translation_config: TranslationConfig
+        self,
+        translate_engine: BaseTranslator,
+        translation_config: TranslationConfig,
     ):
         self.translate_engine = translate_engine
         self.translation_config = translation_config

{babeldoc-0.4.18 → babeldoc-0.4.20}/babeldoc/format/pdf/document_il/midend/il_translator.py

@@ -81,36 +81,67 @@ class PbarContext:
 class DocumentTranslateTracker:
     def __init__(self):
         self.page = []
+        self.cross_page = []
+        # Track paragraphs that are combined due to cross-column detection within the same page
+        self.cross_column = []
 
     def new_page(self):
         page = PageTranslateTracker()
         self.page.append(page)
         return page
 
+    def new_cross_page(self):
+        page = PageTranslateTracker()
+        self.cross_page.append(page)
+        return page
+
+    def new_cross_column(self):
+        """Create and return a new PageTranslateTracker dedicated to cross-column merging."""
+        page = PageTranslateTracker()
+        self.cross_column.append(page)
+        return page
+
     def to_json(self):
         pages = []
         for page in self.page:
-            paragraphs = []
-            for para in page.paragraph:
-                i_str = getattr(para, "input", None)
-                o_str = getattr(para, "output", None)
-                pdf_unicode = getattr(para, "pdf_unicode", None)
-                llm_translate_trackers = getattr(para, "llm_translate_trackers", None)
-                llm_translate_trackers_json = []
-                for tracker in llm_translate_trackers:
-                    llm_translate_trackers_json.append(tracker.to_dict())
-                if pdf_unicode is None or i_str is None:
-                    continue
-                paragraphs.append(
-                    {
-                        "input": i_str,
-                        "output": o_str,
-                        "pdf_unicode": pdf_unicode,
-                        "llm_translate_trackers": llm_translate_trackers_json,
-                    },
-                )
+            paragraphs = self.convert_paragraph(page)
             pages.append({"paragraph": paragraphs})
-        return json.dumps({"page": pages}, ensure_ascii=False, indent=2)
+        cross_page = []
+        for page in self.cross_page:
+            paragraphs = self.convert_paragraph(page)
+            cross_page.append({"paragraph": paragraphs})
+        cross_column = []
+        for page in self.cross_column:
+            paragraphs = self.convert_paragraph(page)
+            cross_column.append({"paragraph": paragraphs})
+        return json.dumps(
+            {"cross_page": cross_page, "cross_column": cross_column, "page": pages},
+            ensure_ascii=False,
+            indent=2,
+        )
+
+    def convert_paragraph(self, page):
+        paragraphs = []
+        for para in page.paragraph:
+            i_str = getattr(para, "input", None)
+            o_str = getattr(para, "output", None)
+            pdf_unicode = getattr(para, "pdf_unicode", None)
+            llm_translate_trackers = getattr(para, "llm_translate_trackers", None)
+            llm_translate_trackers_json = []
+            for tracker in llm_translate_trackers:
+                llm_translate_trackers_json.append(tracker.to_dict())
+            if pdf_unicode is None or i_str is None:
+                continue
+            paragraph_json = {
+                "input": i_str,
+                "output": o_str,
+                "pdf_unicode": pdf_unicode,
+                "llm_translate_trackers": llm_translate_trackers_json,
+            }
+            paragraphs.append(
+                paragraph_json,
+            )
+        return paragraphs
 
 
 class PageTranslateTracker:
@@ -752,7 +783,9 @@ class ILTranslator:
             return False
         paragraph.unicode = translated_text
         paragraph.pdf_paragraph_composition = self.parse_translate_output(
-            translate_input, translated_text, tracker.last_llm_translate_tracker()
+            translate_input,
+            translated_text,
+            tracker.last_llm_translate_tracker(),
         )
         for composition in paragraph.pdf_paragraph_composition:
             if (
@@ -910,7 +943,10 @@ Now, please carefully read the following text to be translated and directly outp
                 # Perform translation
                 if self.support_llm_translate:
                     llm_prompt = self.generate_prompt_for_llm(
-                        text, title_paragraph, local_title_paragraph, translate_input
+                        text,
+                        title_paragraph,
+                        local_title_paragraph,
+                        translate_input,
                     )
                     llm_translate_tracker.set_input(llm_prompt)
                     translated_text = self.translate_engine.llm_translate(