BabelDOC 0.1.32__tar.gz → 0.2.0__tar.gz

{babeldoc-0.1.32 → babeldoc-0.2.0}/.github/workflows/test.yml

@@ -37,9 +37,13 @@ jobs:
         run: |
           uv run babeldoc --warmup
       - name: Run tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAIAPIKEY }}
+          OPENAI_BASE_URL: ${{ secrets.OPENAIAPIURL }}
+          OPENAI_MODEL: ${{ secrets.OPENAIMODEL }}
         run: |
           uv run babeldoc --help
-          uv run babeldoc --bing --files examples/ci/test.pdf
+          uv run babeldoc --openai --files examples/ci/test.pdf --openai-api-key ${{ env.OPENAI_API_KEY }} --openai-base-url ${{ env.OPENAI_BASE_URL }} --openai-model ${{ env.OPENAI_MODEL }}
       - name: Generate offline assets package
         run: |
           uv run babeldoc --generate-offline-assets /tmp/offline_assets

{babeldoc-0.1.32 → babeldoc-0.2.0}/LICENSE

@@ -629,7 +629,7 @@ to attach them to the start of each source file to most effectively
 state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
-    YADT is library for ultimated document translation solution.
+    BabelDOC is library for ultimated document translation solution.
     Copyright (C) 2024  <funstory.ai limited>
 
     This program is free software: you can redistribute it and/or modify

{babeldoc-0.1.32 → babeldoc-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.1.32
+Version: 0.2.0
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.13,>=3.10
 Requires-Dist: bitstring>=4.3.0
 Requires-Dist: configargparse>=1.7
+Requires-Dist: freetype-py>=2.5.1
 Requires-Dist: httpx[socks]>=0.27.0
 Requires-Dist: huggingface-hub>=0.27.0
 Requires-Dist: msgpack>=1.1.0
@@ -124,10 +125,10 @@ uv run babeldoc --help
 3. Use the `uv run babeldoc` command. For example:
 
 ```bash
-uv run babeldoc --bing --files example.pdf
+uv run babeldoc --files example.pdf --openai --openai-model "gpt-4o-mini" --openai-base-url "https://api.openai.com/v1" --openai-api-key "your-api-key-here"
 
 # multiple files
-uv run babeldoc --bing --files example.pdf --files example2.pdf
+uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model "gpt-4o-mini" --openai-base-url "https://api.openai.com/v1" --openai-api-key "your-api-key-here"
 ```
 
 > [!TIP]
@@ -175,12 +176,10 @@ uv run babeldoc --bing --files example.pdf --files example2.pdf
 - `--no-mono`: Do not output monolingual PDF files
 - `--min-text-length`: Minimum text length to translate (default: 5)
 - `--openai`: Use OpenAI for translation (default: False)
-- `--bing`: Use Bing for translation (default: False)
-- `--google`: Use Google Translate for translation (default: False)
 
 > [!TIP]
 >
-> 1. You must specify one translation service among `--openai`, `--bing`, `--google`.
+> 1. You must specify one translation service among `--openai`.
 > 2. It is recommended to use models with strong compatibility with OpenAI, such as: `glm-4-flash`, `deepseek-chat`, etc.
 > 3. Currently, it has not been optimized for traditional translation engines like Bing/Google, it is recommended to use LLMs.
 

{babeldoc-0.1.32 → babeldoc-0.2.0}/README.md

@@ -89,10 +89,10 @@ uv run babeldoc --help
 3. Use the `uv run babeldoc` command. For example:
 
 ```bash
-uv run babeldoc --bing --files example.pdf
+uv run babeldoc --files example.pdf --openai --openai-model "gpt-4o-mini" --openai-base-url "https://api.openai.com/v1" --openai-api-key "your-api-key-here"
 
 # multiple files
-uv run babeldoc --bing --files example.pdf --files example2.pdf
+uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model "gpt-4o-mini" --openai-base-url "https://api.openai.com/v1" --openai-api-key "your-api-key-here"
 ```
 
 > [!TIP]
@@ -140,12 +140,10 @@ uv run babeldoc --bing --files example.pdf --files example2.pdf
 - `--no-mono`: Do not output monolingual PDF files
 - `--min-text-length`: Minimum text length to translate (default: 5)
 - `--openai`: Use OpenAI for translation (default: False)
-- `--bing`: Use Bing for translation (default: False)
-- `--google`: Use Google Translate for translation (default: False)
 
 > [!TIP]
 >
-> 1. You must specify one translation service among `--openai`, `--bing`, `--google`.
+> 1. You must specify one translation service among `--openai`.
 > 2. It is recommended to use models with strong compatibility with OpenAI, such as: `glm-4-flash`, `deepseek-chat`, etc.
 > 3. Currently, it has not been optimized for traditional translation engines like Bing/Google, it is recommended to use LLMs.
 

babeldoc-0.2.0/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

{babeldoc-0.1.32 → babeldoc-0.2.0}/babeldoc/const.py

@@ -2,7 +2,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.1.32"
+__version__ = "0.2.0"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.1.32 → babeldoc-0.2.0}/babeldoc/document_il/__init__.py

@@ -9,6 +9,7 @@ from babeldoc.document_il.il_version_1 import PageLayout
 from babeldoc.document_il.il_version_1 import PdfCharacter
 from babeldoc.document_il.il_version_1 import PdfFigure
 from babeldoc.document_il.il_version_1 import PdfFont
+from babeldoc.document_il.il_version_1 import PdfFontCharBoundingBox
 from babeldoc.document_il.il_version_1 import PdfFormula
 from babeldoc.document_il.il_version_1 import PdfLine
 from babeldoc.document_il.il_version_1 import PdfParagraph
@@ -18,6 +19,7 @@ from babeldoc.document_il.il_version_1 import PdfSameStyleCharacters
 from babeldoc.document_il.il_version_1 import PdfSameStyleUnicodeCharacters
 from babeldoc.document_il.il_version_1 import PdfStyle
 from babeldoc.document_il.il_version_1 import PdfXobject
+from babeldoc.document_il.il_version_1 import VisualBbox
 
 __all__ = [
     "BaseOperations",
@@ -31,6 +33,7 @@ __all__ = [
     "PdfCharacter",
     "PdfFigure",
     "PdfFont",
+    "PdfFontCharBoundingBox",
     "PdfFormula",
     "PdfLine",
     "PdfParagraph",
@@ -40,4 +43,5 @@ __all__ = [
     "PdfSameStyleUnicodeCharacters",
     "PdfStyle",
     "PdfXobject",
+    "VisualBbox",
 ]

{babeldoc-0.1.32 → babeldoc-0.2.0}/babeldoc/document_il/frontend/il_creater.py

@@ -2,7 +2,10 @@ import base64
 import logging
 import re
 from functools import wraps
+from io import BytesIO
+from itertools import islice
 
+import freetype
 import pdfminer.pdfinterp
 import pymupdf
 from pdfminer.layout import LTChar
@@ -17,6 +20,18 @@ from pdfminer.psparser import PSLiteral
 from babeldoc.document_il import il_version_1
 from babeldoc.translation_config import TranslationConfig
 
+
+def batched(iterable, n, *, strict=False):
+    # batched('ABCDEFG', 3) → ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    iterator = iter(iterable)
+    while batch := tuple(islice(iterator, n)):
+        if strict and len(batch) != n:
+            raise ValueError("batched(): incomplete batch")
+        yield batch
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -40,6 +55,110 @@ PDFMinerPDFPage.__init__ = create_hook(
 )
 
 
+def indirect(obj):
+    if isinstance(obj, tuple) and obj[0] == "xref":
+        return int(obj[1].split(" ")[0])
+
+
+def get_glyph_cbox(face, g):
+    face.load_glyph(g, freetype.FT_LOAD_NO_SCALE)
+    cbox = face.glyph.outline.get_bbox()
+    return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
+
+
+def get_char_cbox(face, idx):
+    g = face.get_char_index(idx)
+    return get_glyph_cbox(face, g)
+
+
+def get_name_cbox(face, name):
+    g = face.get_name_index(name)
+    return get_glyph_cbox(face, g)
+
+
+def parse_font_file(doc, idx, encoding):
+    bbox_list = []
+    data = doc.xref_stream(idx)
+    face = freetype.Face(BytesIO(data))
+    for charmap in face.charmaps:
+        if charmap.encoding_name == "FT_ENCODING_ADOBE_CUSTOM":
+            face.select_charmap(freetype.FT_ENCODING_ADOBE_CUSTOM)
+            break
+    bbox_list = [get_char_cbox(face, x) for x in range(0, 256)]
+    if encoding:
+        for code, name in encoding:
+            bbox_list[code] = get_name_cbox(face, name.encode("U8"))
+    return bbox_list
+
+
+def parse_encoding(obj_str):
+    delta = []
+    current = 0
+    for x in re.finditer(
+        r"(?P<p>[\[\]])|(?P<c>\d+)|(?P<n>/[a-zA-Z0-9]+)|(?P<s>.)", obj_str
+    ):
+        key = x.lastgroup
+        val = x.group()
+        if key == "c":
+            current = int(val)
+        if key == "n":
+            delta.append((current, val[1:]))
+            current += 1
+    return delta
+
+
+def parse_mapping(text):
+    mapping = []
+    for x in re.finditer(r"<(?P<num>[a-fA-F0-9]+)>", text):
+        mapping.append(int(x.group("num"), 16))
+    return mapping
+
+
+def update_cmap_pair(cmap, data):
+    for start, stop, value in batched(data, 3):
+        for code in range(start, stop + 1):
+            cmap[code] = value
+
+
+def update_cmap_code(cmap, data):
+    for code, value in batched(data, 2):
+        cmap[code] = value
+
+
+def parse_cmap(cmap_str):
+    cmap = {}
+    for x in re.finditer(
+        r"\s+beginbfrange\s*(?P<r>(<[0-9a-fA-F]+>\s*)+)endbfrange\s+", cmap_str
+    ):
+        update_cmap_pair(cmap, parse_mapping(x.group("r")))
+    for x in re.finditer(
+        r"\s+beginbfchar\s*(?P<c>(<[0-9a-fA-F]+>\s*)+)endbfchar", cmap_str
+    ):
+        update_cmap_code(cmap, parse_mapping(x.group("c")))
+    return cmap
+
+
+def get_code(cmap, c):
+    for k, v in cmap.items():
+        if v == c:
+            return k
+    return -1
+
+
+def get_bbox(bbox, size, c, x, y):
+    x_min, y_min, x_max, y_max = bbox[c]
+    factor = 1 / 1000 * size
+    x_min = x_min * factor
+    y_min = -y_min * factor
+    x_max = x_max * factor
+    y_max = -y_max * factor
+    ll = (x + x_min, y + y_min)
+    lr = (x + x_max, y + y_min)
+    ul = (x + x_min, y + y_max)
+    ur = (x + x_max, y + y_max)
+    return pymupdf.Quad(ll, lr, ul, ur)
+
+
 class ILCreater:
     stage_name = "Parse PDF and Create Intermediate Representation"
 
@@ -58,6 +177,8 @@ class ILCreater:
         self.xobj_inc = 0
         self.xobj_map: dict[int, il_version_1.PdfXobject] = {}
         self.xobj_stack = []
+        self.current_page_font_name_id_map = {}
+        self.current_page_font_char_bounding_box_map = {}
 
     def on_finish(self):
         self.progress.__exit__(None, None, None)
@@ -121,12 +242,21 @@ class ILCreater:
 
     def push_xobj(self):
         self.xobj_stack.append(
-            (self.current_page_font_name_id_map.copy(), self.xobj_id),
+            (
+                self.current_page_font_name_id_map.copy(),
+                self.current_page_font_char_bounding_box_map.copy(),
+                self.xobj_id,
+            ),
         )
         self.current_page_font_name_id_map = {}
+        self.current_page_font_char_bounding_box_map = {}
 
     def pop_xobj(self):
-        self.current_page_font_name_id_map, self.xobj_id = self.xobj_stack.pop()
+        (
+            self.current_page_font_name_id_map,
+            self.current_page_font_char_bounding_box_map,
+            self.xobj_id,
+        ) = self.xobj_stack.pop()
 
     def on_xobj_begin(self, bbox, xref_id):
         self.push_passthrough_per_char_instruction()
@@ -164,6 +294,7 @@ class ILCreater:
             unit="point",
         )
         self.current_page_font_name_id_map = {}
+        self.current_page_font_char_bounding_box_map = {}
         self.passthrough_per_char_instruction_stack = []
         self.xobj_stack = []
         self.non_stroking_color_space_name = None
@@ -253,13 +384,73 @@ class ILCreater:
             serif=serif,
             ascent=font.ascent,
             descent=font.descent,
+            pdf_font_char_bounding_box=[],
         )
+        try:
+            bbox_list, cmap = self.parse_font_xobj_id(xref_id)
+            font_char_bounding_box_map = {}
+            if not cmap:
+                cmap = {x: x for x in range(257)}
+            for char_id in cmap:
+                if char_id < 0 or char_id >= len(bbox_list):
+                    continue
+                bbox = bbox_list[char_id]
+                x, y, x2, y2 = bbox
+                if (
+                    x == 0
+                    and y == 0
+                    and x2 == 500
+                    and y2 == 698
+                    or x == 0
+                    and y == 0
+                    and x2 == 0
+                    and y2 == 0
+                ):
+                    # ignore default bounding box
+                    continue
+                il_font_metadata.pdf_font_char_bounding_box.append(
+                    il_version_1.PdfFontCharBoundingBox(
+                        x=x,
+                        y=y,
+                        x2=x2,
+                        y2=y2,
+                        char_id=char_id,
+                    )
+                )
+                font_char_bounding_box_map[char_id] = bbox
+            if self.xobj_id in self.xobj_map:
+                if self.xobj_id not in self.current_page_font_char_bounding_box_map:
+                    self.current_page_font_char_bounding_box_map[self.xobj_id] = {}
+                self.current_page_font_char_bounding_box_map[self.xobj_id][font_id] = (
+                    font_char_bounding_box_map
+                )
+            else:
+                self.current_page_font_char_bounding_box_map[font_id] = (
+                    font_char_bounding_box_map
+                )
+        except Exception:
+            pass
         self.current_page_font_name_id_map[font_name] = font_id
         if self.xobj_id in self.xobj_map:
             self.xobj_map[self.xobj_id].pdf_font.append(il_font_metadata)
         else:
             self.current_page.pdf_font.append(il_font_metadata)
 
+    def parse_font_xobj_id(self, xobj_id: int):
+        encoding = []
+        font_encoding = self.mupdf.xref_get_key(xobj_id, "Encoding/Differences")
+        if font_encoding:
+            encoding = parse_encoding(font_encoding[1])
+        bbox_list = []
+        font_file = self.mupdf.xref_get_key(xobj_id, "FontDescriptor/FontFile")
+        if file_idx := indirect(font_file):
+            bbox_list = parse_font_file(self.mupdf, file_idx, encoding)
+        cmap = {}
+        to_unicode = self.mupdf.xref_get_key(xobj_id, "ToUnicode")
+        if to_unicode_idx := indirect(to_unicode):
+            cmap = parse_cmap(self.mupdf.xref_stream(to_unicode_idx).decode("U8"))
+        return bbox_list, cmap
+
     def create_graphic_state(self, gs: pdfminer.pdfinterp.PDFGraphicState):
         graphic_state = il_version_1.GraphicState()
         for k, v in gs.__dict__.items():
@@ -302,13 +493,27 @@ class ILCreater:
             descent = font.descent * char.size / 1000
 
         char_id = char.cid
+
+        if font_bounding_box_map := self.current_page_font_char_bounding_box_map.get(
+            self.xobj_id, self.current_page_font_char_bounding_box_map
+        ).get(font.font_id):
+            char_bounding_box = font_bounding_box_map.get(char_id, None)
+        else:
+            char_bounding_box = None
+
         char_unicode = char.get_text()
         if "(cid:" not in char_unicode and len(char_unicode) > 1:
             return
         advance = char.adv
+        bbox = il_version_1.Box(
+            x=char.bbox[0],
+            y=char.bbox[1],
+            x2=char.bbox[2],
+            y2=char.bbox[3],
+        )
         if char.matrix[0] == 0 and char.matrix[3] == 0:
             vertical = True
-            bbox = il_version_1.Box(
+            visual_bbox = il_version_1.Box(
                 x=char.bbox[0] - descent,
                 y=char.bbox[1],
                 x2=char.bbox[2] - descent,
@@ -317,12 +522,13 @@ class ILCreater:
         else:
             vertical = False
             # Add descent to y coordinates
-            bbox = il_version_1.Box(
+            visual_bbox = il_version_1.Box(
                 x=char.bbox[0],
                 y=char.bbox[1] + descent,
                 x2=char.bbox[2],
                 y2=char.bbox[3] + descent,
             )
+        visual_bbox = il_version_1.VisualBbox(box=visual_bbox)
         pdf_style = il_version_1.PdfStyle(
             font_id=char.aw_font_id,
             font_size=char.size,
@@ -336,6 +542,7 @@ class ILCreater:
             vertical=vertical,
             pdf_style=pdf_style,
             xobj_id=char.xobj_id,
+            visual_bbox=visual_bbox,
         )
         if pdf_style.font_size == 0.0:
             logger.warning(
@@ -343,6 +550,20 @@ class ILCreater:
                 char_unicode,
             )
             return
+
+        if char_bounding_box:
+            x_min, y_min, x_max, y_max = char_bounding_box
+            factor = 1 / 1000 * pdf_style.font_size
+            x_min = x_min * factor
+            y_min = y_min * factor
+            x_max = x_max * factor
+            y_max = y_max * factor
+            ll = (char.bbox[0] + x_min, char.bbox[1] + y_min)
+            ur = (char.bbox[0] + x_max, char.bbox[1] + y_max)
+            pdf_char.visual_bbox = il_version_1.VisualBbox(
+                il_version_1.Box(ll[0], ll[1], ur[0], ur[1])
+            )
+
         self.current_page.pdf_character.append(pdf_char)
 
     def create_il(self):