BabelDOC 0.6.1__tar.gz → 0.6.3__tar.gz

babeldoc-0.6.3/.github/workflows/checks.yml

@@ -0,0 +1,55 @@
+name: Checks
+
+on:
+  push:
+  pull_request:
+    branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  checks:
+    name: Package and lightweight PDF smoke
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Cache BabelDOC assets
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/babeldoc
+          key: babeldoc-assets-${{ hashFiles('babeldoc/assets/embedding_assets_metadata.py') }}
+
+      - name: Setup uv with Python 3.12
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
+        with:
+          python-version: "3.12"
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+          activate-environment: true
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Lint
+        run: uv run ruff check .
+
+      - name: Build package
+        run: uv build
+
+      - name: Import and CLI smoke
+        run: |
+          uv run python -m compileall babeldoc
+          uv run babeldoc --help
+
+      - name: Lightweight no-secret PDF E2E
+        run: |
+          uv run babeldoc \
+            --openai \
+            --openai-api-key ci-dummy-key \
+            --files examples/ci/test.pdf \
+            --only-parse-generate-pdf \
+            --skip-scanned-detection

{babeldoc-0.6.1 → babeldoc-0.6.3}/.github/workflows/labeler.yml

@@ -6,11 +6,11 @@ on:
       - 'main'
     paths:
       - '.github/labels.yml'
-      - '.github/workflows/labels.yml'
+      - '.github/workflows/labeler.yml'
   pull_request:
     paths:
       - '.github/labels.yml'
-      - '.github/workflows/labels.yml'
+      - '.github/workflows/labeler.yml'
 
 permissions:
   contents: read
@@ -33,4 +33,4 @@ jobs:
           yaml-file: .github/labels.yml
           exclude: |
             help*
-            *issue
+            *issue

{babeldoc-0.6.1 → babeldoc-0.6.3}/.github/workflows/lint.yml

@@ -14,4 +14,4 @@ jobs:
       - name: Ruff
         uses: astral-sh/ruff-action@v3
       - name: AutoCorrect
-        uses: huacnlee/autocorrect-action@main
+        uses: huacnlee/autocorrect-action@bf91ab3904c2908dd8e71312a8a83ed1eb632997 # v2.13.3

{babeldoc-0.6.1 → babeldoc-0.6.3}/.gitignore

@@ -48,4 +48,5 @@ babeldoc/format/playground
 temp.jpg
 .serena
 .codex
+.repomap.tags.cache.v1/
 

{babeldoc-0.6.1 → babeldoc-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.6.1
+Version: 0.6.3
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -16,7 +16,7 @@ Requires-Dist: bitstring>=4.3.0
 Requires-Dist: chardet>=5.2.0
 Requires-Dist: charset-normalizer>=2.0.0
 Requires-Dist: configargparse>=1.7
-Requires-Dist: cryptography>=36.0.0
+Requires-Dist: cryptography>=46.0.7
 Requires-Dist: freetype-py>=2.5.1
 Requires-Dist: httpx[socks]>=0.27.0
 Requires-Dist: huggingface-hub>=0.27.0
@@ -24,15 +24,16 @@ Requires-Dist: hyperscan>=0.7.13
 Requires-Dist: levenshtein>=0.27.1
 Requires-Dist: msgpack>=1.1.0
 Requires-Dist: numpy>=2.0.2
-Requires-Dist: onnx>=1.18.0
-Requires-Dist: onnxruntime>=1.16.1
+Requires-Dist: onnx>=1.21.0
+Requires-Dist: onnxruntime<1.24,>=1.16.1; python_version < '3.11'
+Requires-Dist: onnxruntime>=1.16.1; python_version >= '3.11'
 Requires-Dist: openai>=1.59.3
 Requires-Dist: opencv-python-headless>=4.10.0.84
 Requires-Dist: orjson>=3.10.14
 Requires-Dist: peewee>=3.17.8
 Requires-Dist: psutil>=7.0.0
 Requires-Dist: pydantic>=2.10.6
-Requires-Dist: pymupdf>=1.25.1
+Requires-Dist: pymupdf>=1.26.7
 Requires-Dist: pyzstd>=0.17.0
 Requires-Dist: rich>=13.9.4
 Requires-Dist: rtree>=1.4.0

babeldoc-0.6.3/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.3"

{babeldoc-0.6.1 → babeldoc-0.6.3}/babeldoc/const.py

@@ -6,7 +6,7 @@ import subprocess
 import threading
 from pathlib import Path
 
-__version__ = "0.6.1"
+__version__ = "0.6.3"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc-0.6.3/babeldoc/format/pdf/babelpdf/type3.py

@@ -0,0 +1,159 @@
+import io
+import math
+import re
+from dataclasses import dataclass
+
+import pymupdf
+
+_PDF_NUMBER_RE = re.compile(r"[+-]?(?:(?:\d+\.\d*)|(?:\.\d+)|(?:\d+))(?:[eE][+-]?\d+)?")
+
+
+@dataclass(frozen=True, slots=True)
+class Type3FontMetrics:
+    font_matrix: tuple[float, float, float, float, float, float]
+    font_bbox: tuple[float, float, float, float]
+    font_matrix_text: str
+    font_bbox_text: str
+    em_height: float
+    ascent: float
+    descent: float
+
+
+def _parse_number_array(text: str, *, expected_len: int) -> tuple[float, ...] | None:
+    values = [float(match.group(0)) for match in _PDF_NUMBER_RE.finditer(text)]
+    if len(values) < expected_len:
+        return None
+    values = values[:expected_len]
+    if not all(math.isfinite(value) for value in values):
+        return None
+    return tuple(values)
+
+
+def _format_number_array(values: tuple[float, ...]) -> str:
+    return " ".join(f"{value:.12g}" for value in values)
+
+
+def _transform_point(
+    matrix: tuple[float, float, float, float, float, float],
+    point: tuple[float, float],
+) -> tuple[float, float]:
+    a, b, c, d, e, f = matrix
+    x, y = point
+    return a * x + c * y + e, b * x + d * y + f
+
+
+def _transform_bbox(
+    matrix: tuple[float, float, float, float, float, float],
+    bbox: tuple[float, float, float, float],
+) -> tuple[float, float, float, float]:
+    x0, y0, x1, y1 = bbox
+    points = [
+        _transform_point(matrix, (x0, y0)),
+        _transform_point(matrix, (x0, y1)),
+        _transform_point(matrix, (x1, y0)),
+        _transform_point(matrix, (x1, y1)),
+    ]
+    xs = [point[0] for point in points]
+    ys = [point[1] for point in points]
+    return min(xs), min(ys), max(xs), max(ys)
+
+
+def build_type3_font_metrics(
+    font_matrix: tuple[float, ...],
+    font_bbox: tuple[float, ...],
+) -> Type3FontMetrics | None:
+    if len(font_matrix) < 6 or len(font_bbox) < 4:
+        return None
+    matrix = tuple(font_matrix[:6])
+    bbox = tuple(font_bbox[:4])
+    if not all(math.isfinite(value) for value in (*matrix, *bbox)):
+        return None
+    _, y0, _, y1 = _transform_bbox(matrix, bbox)
+    em_height = y1 - y0
+    if not math.isfinite(em_height) or em_height <= 0:
+        return None
+    scale = 1000.0 / em_height
+    return Type3FontMetrics(
+        font_matrix=matrix,
+        font_bbox=bbox,
+        font_matrix_text=_format_number_array(matrix),
+        font_bbox_text=_format_number_array(bbox),
+        em_height=em_height,
+        ascent=y1 * scale,
+        descent=y0 * scale,
+    )
+
+
+def get_type3_font_metrics(doc, obj: int) -> Type3FontMetrics | None:
+    try:
+        font_matrix_text = doc.xref_get_key(obj, "FontMatrix")[1]
+        font_bbox_text = doc.xref_get_key(obj, "FontBBox")[1]
+    except Exception:
+        return None
+    font_matrix = _parse_number_array(font_matrix_text, expected_len=6)
+    font_bbox = _parse_number_array(font_bbox_text, expected_len=4)
+    if font_matrix is None or font_bbox is None:
+        return None
+    return build_type3_font_metrics(font_matrix, font_bbox)
+
+
+def normalize_type3_bbox_to_1000_em(
+    bbox: tuple[float, float, float, float],
+    metrics: Type3FontMetrics,
+) -> tuple[float, float, float, float]:
+    scale = 1000.0 / metrics.em_height
+    return tuple(value * scale for value in bbox)
+
+
+def merge_bbox(bbox_list, factor=1):
+    if bbox_list:
+        base = bbox_list[0]
+        for bbox in bbox_list[1:]:
+            base.include_rect(bbox)
+        x0, y0, x1, y1 = [v / factor for v in tuple(base)]
+        return x0, -y1, x1, -y0
+
+
+def get_type3_bbox(
+    doc,
+    obj,
+    *,
+    normalize_to_1000_em: bool = False,
+    metrics: Type3FontMetrics | None = None,
+):
+    bbox_list = [(0, 0, 0, 0)] * 256
+    first = int(doc.xref_get_key(obj, "FirstChar")[1])
+    last = int(doc.xref_get_key(obj, "LastChar")[1])
+    normalize_metrics = None
+    if normalize_to_1000_em:
+        metrics = metrics or get_type3_font_metrics(doc, obj)
+        normalize_metrics = metrics
+        factor = 1
+    else:
+        factor_text = doc.xref_get_key(obj, "FontMatrix")[1]
+        factor = 1
+        if factor_m := re.search(_PDF_NUMBER_RE, factor_text):
+            factor = float(factor_m.group(0))
+    page = doc.new_page(width=10, height=10)
+    doc.xref_set_key(page.xref, "Resources", "<<>>")
+    doc.xref_set_key(page.xref, "Resources/Font", f"<</T0 {obj} 0 R>>")
+    text = doc.get_new_xref()
+    doc.update_object(text, "<<>>")
+    for x in range(first, last + 1):
+        doc.update_stream(text, b"1 0 0 1 0 10 cm BT /T0 1 Tf <%02X> Tj ET" % x)
+        doc.xref_set_key(page.xref, "Contents", f"{text} 0 R")
+        char_data = page.get_svg_image(text_as_path=True)
+        char_doc = pymupdf.Document(stream=io.BytesIO(char_data.encode("U8")))
+        char_bbox = []
+        for element in char_doc:
+            for item in element.get_drawings():
+                char_bbox.append(item["rect"])
+        if char_bbox_merged := merge_bbox(char_bbox, factor):
+            if normalize_metrics is not None:
+                char_bbox_merged = normalize_type3_bbox_to_1000_em(
+                    char_bbox_merged,
+                    normalize_metrics,
+                )
+            bbox_list[x] = char_bbox_merged
+    doc.delete_page(-1)
+    return bbox_list

{babeldoc-0.6.1 → babeldoc-0.6.3}/babeldoc/format/pdf/document_il/backend/pdf_creater.py

@@ -20,6 +20,9 @@ from babeldoc.format.pdf.document_il import PdfOriginalPath
 from babeldoc.format.pdf.document_il import il_version_1
 from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
 from babeldoc.format.pdf.document_il.utils.matrix_helper import matrix_to_bytes
+from babeldoc.format.pdf.document_il.utils.type3_font_metrics import (
+    inverse_type3_font_size_for_tf,
+)
 from babeldoc.format.pdf.document_il.utils.zstd_helper import zstd_decompress
 from babeldoc.format.pdf.new_parser.pdf_token_serializer import serialize_pdf_token
 from babeldoc.format.pdf.translation_config import TranslateResult
@@ -86,6 +89,7 @@ class CharacterRenderUnit(RenderUnit):
 
         char_size = char.pdf_style.font_size
         font_id = char.pdf_style.font_id
+        tf_font_size = self._font_size_for_pdf_tf(char_size, font_id, context)
 
         # Get encoding length map based on xobj_id
         if self.xobj_id in context.xobj_encoding_length_map:
@@ -106,11 +110,11 @@ class CharacterRenderUnit(RenderUnit):
 
         if char.vertical:
             draw_op.append(
-                f"BT /{font_id} {char_size:f} Tf 0 1 -1 0 {char.box.x2:f} {char.box.y:f} Tm ".encode(),
+                f"BT /{font_id} {tf_font_size:f} Tf 0 1 -1 0 {char.box.x2:f} {char.box.y:f} Tm ".encode(),
             )
         else:
             text_draw_op = (
-                f"BT /{font_id} {char_size:f} Tf "
+                f"BT /{font_id} {tf_font_size:f} Tf "
                 f"1 0 0 1 {char.box.x:f} {char.box.y:f} Tm "
             )
             draw_op.append(text_draw_op.encode())
@@ -130,6 +134,28 @@ class CharacterRenderUnit(RenderUnit):
         )
         draw_op.append(b" Tj ET Q \n")
 
+    def _font_size_for_pdf_tf(
+        self,
+        char_size: float,
+        font_id: str,
+        context: "RenderContext",
+    ) -> float:
+        get_scoped_font = getattr(context, "get_scoped_font", None)
+        font = (
+            get_scoped_font(font_id, self.xobj_id)
+            if callable(get_scoped_font)
+            else None
+        )
+        if font is None:
+            xobj_font_map = getattr(context, "xobj_font_map", {})
+            if self.xobj_id in xobj_font_map:
+                font = xobj_font_map[self.xobj_id].get(font_id)
+        if font is None:
+            font = getattr(context, "page_font_map", {}).get(font_id)
+        if font is None:
+            return char_size
+        return inverse_type3_font_size_for_tf(font, char_size)
+
 
 class FormRenderUnit(RenderUnit):
     """Render unit for PDF forms."""
@@ -358,6 +384,8 @@ class RenderContext:
         all_encoding_length_map: dict[str, int],
         xobj_available_fonts: dict[str, set[str]],
         xobj_encoding_length_map: dict[str, dict[str, int]],
+        page_font_map: dict[str, il_version_1.PdfFont],
+        xobj_font_map: dict[str, dict[str, il_version_1.PdfFont]],
         ctm_for_ops: bytes,
         check_font_exists: bool = False,
     ):
@@ -368,9 +396,22 @@ class RenderContext:
         self.all_encoding_length_map = all_encoding_length_map
         self.xobj_available_fonts = xobj_available_fonts
         self.xobj_encoding_length_map = xobj_encoding_length_map
+        self.page_font_map = page_font_map
+        self.xobj_font_map = xobj_font_map
         self.ctm_for_ops = ctm_for_ops
         self.check_font_exists = check_font_exists
 
+    def get_scoped_font(
+        self,
+        font_id: str,
+        xobj_id: str | None = None,
+    ) -> il_version_1.PdfFont | None:
+        if xobj_id in self.xobj_font_map:
+            scoped_font = self.xobj_font_map[xobj_id].get(font_id)
+            if scoped_font is not None:
+                return scoped_font
+        return self.page_font_map.get(font_id)
+
 
 def to_int(src):
     return int(re.search(r"\d+", src).group(0))
@@ -622,7 +663,15 @@ class PDFCreater:
             )
 
     @staticmethod
+    def _parse_xref_ref(value: str) -> int | None:
+        match = re.match(r"^\s*(\d+)\s+0\s+R\s*$", value)
+        if match is None:
+            return None
+        return int(match.group(1))
+
+    @classmethod
     def _find_resource_ref(
+        cls,
         pdf: pymupdf.Document,
         candidate_resource_xrefs: list[int],
         resource_kind: str,
@@ -640,6 +689,46 @@ class PDFCreater:
                 return value
         return None
 
+    @classmethod
+    def _set_resource_ref(
+        cls,
+        pdf: pymupdf.Document,
+        target_xref: int,
+        resource_kind: str,
+        name: str,
+        resource_ref: str,
+    ) -> None:
+        resource_key = f"Resources/{resource_kind}/{name}"
+        try:
+            pdf.xref_set_key(target_xref, resource_key, resource_ref)
+            return
+        except Exception as exc:
+            if "has indirects" not in str(exc):
+                raise
+            direct_set_error = exc
+
+        resources_type, resources_value = pdf.xref_get_key(
+            target_xref,
+            "Resources",
+        )
+        resources_xref = (
+            cls._parse_xref_ref(resources_value) if resources_type == "xref" else None
+        )
+        if resources_xref is None:
+            raise direct_set_error
+
+        kind_type, kind_value = pdf.xref_get_key(resources_xref, resource_kind)
+        kind_xref = cls._parse_xref_ref(kind_value) if kind_type == "xref" else None
+        if kind_xref is not None:
+            pdf.xref_set_key(kind_xref, name, resource_ref)
+            return
+
+        pdf.xref_set_key(
+            resources_xref,
+            f"{resource_kind}/{name}",
+            resource_ref,
+        )
+
     @classmethod
     def _ensure_stream_named_resources(
         cls,
@@ -676,9 +765,11 @@ class PDFCreater:
             )
             if resource_ref is None:
                 continue
-            pdf.xref_set_key(
+            cls._set_resource_ref(
+                pdf,
                 target_xref,
-                f"Resources/{resource_kind}/{name}",
+                resource_kind,
+                name,
                 resource_ref,
             )
 
@@ -1554,6 +1645,8 @@ class PDFCreater:
         xobj_draw_ops = {}
         xobj_encoding_length_map = {}
         available_font_list = self.get_available_font_list(pdf, page)
+        page_font_map = {f.font_id: f for f in page.pdf_font}
+        xobj_font_map = {}
         page_encoding_length_map: dict[str | None, int | None] = {
             f.font_id: f.encoding_length for f in page.pdf_font
         }
@@ -1566,6 +1659,8 @@ class PDFCreater:
                 )
             except Exception:
                 pass
+            xobj_font_map[xobj.xobj_id] = page_font_map.copy()
+            xobj_font_map[xobj.xobj_id].update({f.font_id: f for f in xobj.pdf_font})
             xobj_encoding_length_map[xobj.xobj_id] = {
                 f.font_id: f.encoding_length for f in xobj.pdf_font
             }
@@ -1594,6 +1689,8 @@ class PDFCreater:
             all_encoding_length_map=all_encoding_length_map,
             xobj_available_fonts=xobj_available_fonts,
             xobj_encoding_length_map=xobj_encoding_length_map,
+            page_font_map=page_font_map,
+            xobj_font_map=xobj_font_map,
             ctm_for_ops=ctm_for_ops,
             check_font_exists=check_font_exists,
         )
@@ -1639,13 +1736,13 @@ class PDFCreater:
         stream = draw_op.tobytes()
         self._ensure_stream_extgstate_resources(
             pdf,
-            op_container,
+            pdf[page.page_number].xref,
             stream,
             candidate_resource_xrefs,
         )
         self._ensure_stream_shading_resources(
             pdf,
-            op_container,
+            pdf[page.page_number].xref,
             stream,
             candidate_resource_xrefs,
         )