BabelDOC 0.5.10__tar.gz → 0.5.12__tar.gz

{babeldoc-0.5.10 → babeldoc-0.5.12}/.gitignore

@@ -45,4 +45,5 @@ uv.lock
 CLAUDE.md
 /.claude
 babeldoc/format/playground
-temp.jpg
+temp.jpg
+AGENTS.md

{babeldoc-0.5.10 → babeldoc-0.5.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.10
+Version: 0.5.12
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -81,6 +81,7 @@ Description-Content-Type: text/markdown
     <img src="https://img.shields.io/github/license/funstory-ai/BabelDOC"></a>
   <a href="https://t.me/+Z9_SgnxmsmA5NzBl">
     <img src="https://img.shields.io/badge/Telegram-2CA5E0?style=flat-squeare&logo=telegram&logoColor=white"></a>
+  <a href="https://deepwiki.com/funstory-ai/BabelDOC"><img src="https://deepwiki.com/badge.svg" alt="Ask DeepWiki"></a>
 </p>
 
 <a href="https://trendshift.io/repositories/13358" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13358" alt="funstory-ai%2FBabelDOC | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>

{babeldoc-0.5.10 → babeldoc-0.5.12}/README.md

@@ -25,6 +25,7 @@
     <img src="https://img.shields.io/github/license/funstory-ai/BabelDOC"></a>
   <a href="https://t.me/+Z9_SgnxmsmA5NzBl">
     <img src="https://img.shields.io/badge/Telegram-2CA5E0?style=flat-squeare&logo=telegram&logoColor=white"></a>
+  <a href="https://deepwiki.com/funstory-ai/BabelDOC"><img src="https://deepwiki.com/badge.svg" alt="Ask DeepWiki"></a>
 </p>
 
 <a href="https://trendshift.io/repositories/13358" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13358" alt="funstory-ai%2FBabelDOC | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>

babeldoc-0.5.12/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.12"

{babeldoc-0.5.10 → babeldoc-0.5.12}/babeldoc/babeldoc_exception/BabelDOCException.py

@@ -6,3 +6,8 @@ class ScannedPDFError(Exception):
 class ExtractTextError(Exception):
     def __init__(self, message):
         super().__init__(message)
+
+
+class InputFileGeneratedByBabelDOCError(Exception):
+    def __init__(self, message):
+        super().__init__(message)

{babeldoc-0.5.10 → babeldoc-0.5.12}/babeldoc/const.py

@@ -1,9 +1,12 @@
+import itertools
+import multiprocessing as mp
 import os
 import shutil
 import subprocess
+import threading
 from pathlib import Path
 
-__version__ = "0.5.10"
+__version__ = "0.5.12"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 
@@ -39,3 +42,43 @@ except (OSError, FileNotFoundError, subprocess.CalledProcessError):
 TIKTOKEN_CACHE_FOLDER = CACHE_FOLDER / "tiktoken"
 TIKTOKEN_CACHE_FOLDER.mkdir(parents=True, exist_ok=True)
 os.environ["TIKTOKEN_CACHE_DIR"] = str(TIKTOKEN_CACHE_FOLDER)
+
+
+_process_pool = None
+_process_pool_lock = threading.Lock()
+_ENABLE_PROCESS_POOL = False
+
+
+def enable_process_pool():
+    # Development and Testing ONLY API
+    global _ENABLE_PROCESS_POOL
+    _ENABLE_PROCESS_POOL = True
+
+
+# macos & windows use spawn mode
+# linux use forkserver mode
+
+
+def get_process_pool():
+    if not _ENABLE_PROCESS_POOL:
+        return None
+    global _process_pool
+    with _process_pool_lock:
+        if _process_pool is None:
+            # Create pool only in main process
+            if mp.current_process().name != "MainProcess":
+                return None
+
+            _process_pool = mp.Pool()
+        return _process_pool
+
+
+def batched(iterable, n, *, strict=False):
+    # batched('ABCDEFG', 3) → ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    iterator = iter(iterable)
+    while batch := tuple(itertools.islice(iterator, n)):
+        if strict and len(batch) != n:
+            raise ValueError("batched(): incomplete batch")
+        yield batch

{babeldoc-0.5.10 → babeldoc-0.5.12}/babeldoc/docvision/rpc_doclayout6.py

@@ -28,7 +28,9 @@ from babeldoc.format.pdf.document_il.utils.extract_char import (
 )
 from babeldoc.format.pdf.document_il.utils.fontmap import FontMapper
 from babeldoc.format.pdf.document_il.utils.layout_helper import SPACE_REGEX
-from babeldoc.format.pdf.document_il.utils.mupdf_helper import get_no_rotation_img
+from babeldoc.format.pdf.document_il.utils.mupdf_helper import (
+    get_no_rotation_img_multiprocess,
+)
 
 logger = logging.getLogger(__name__)
 DPI = 150
@@ -141,7 +143,7 @@ def predict_layout(
         f"{host}/inference",
         json=request_data,
         headers={"Accept": "application/json", "Content-Type": "application/json"},
-        timeout=1800,
+        timeout=30,
         follow_redirects=True,
     )
 
@@ -233,7 +235,7 @@ def predict_layout2(
             "Content-Type": "application/msgpack",
             "Accept": "application/msgpack",
         },
-        timeout=480,
+        timeout=30,
         follow_redirects=True,
     )
 
@@ -569,18 +571,19 @@ class RpcDocLayoutModel(DocLayoutModel):
 
         return results
 
-    def predict_page(
-        self, page, mupdf_doc: pymupdf.Document, translate_config, save_debug_image
-    ):
+    def predict_page(self, page, pdf_bytes: Path, translate_config, save_debug_image):
         translate_config.raise_if_cancelled()
-        with self.lock:
-            # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
-            pix = get_no_rotation_img(mupdf_doc[page.page_number], dpi=DPI)
-        image = np.frombuffer(pix.samples, np.uint8).reshape(
-            pix.height,
-            pix.width,
-            3,
-        )[:, :, ::-1]
+        # doc = pymupdf.open(io.BytesIO(pdf_bytes))
+        # with self.lock:
+        # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
+        image = get_no_rotation_img_multiprocess(
+            pdf_bytes.as_posix(), page.page_number, dpi=DPI
+        )
+        # image = np.frombuffer(pix.samples, np.uint8).reshape(
+        #     pix.height,
+        #     pix.width,
+        #     3,
+        # )[:, :, ::-1]
         char_boxes = convert_page_to_char_boxes(page)
         lines = process_page_chars_to_lines(char_boxes)
         predict_result = self.predict_image(image, 800, lines)
@@ -594,11 +597,13 @@ class RpcDocLayoutModel(DocLayoutModel):
         translate_config,
         save_debug_image,
     ):
+        layout_temp_path = translate_config.get_working_file_path("layout.temp.pdf")
+        mupdf_doc.save(layout_temp_path.as_posix())
         with ThreadPoolExecutor(max_workers=32) as executor:
             yield from executor.map(
                 self.predict_page,
                 pages,
-                (mupdf_doc for _ in range(len(pages))),
+                (layout_temp_path for _ in range(len(pages))),
                 (translate_config for _ in range(len(pages))),
                 (save_debug_image for _ in range(len(pages))),
             )

babeldoc-0.5.12/babeldoc/format/pdf/babelpdf/cidfont.py

@@ -0,0 +1,60 @@
+import re
+from io import BytesIO
+
+import freetype
+
+
+def indirect(obj):
+    if isinstance(obj, tuple) and obj[0] == "xref":
+        return int(obj[1].split(" ")[0])
+
+
+def get_xref(doc, xref, key):
+    obj = doc.xref_get_key(xref, key)
+    if obj[0] == "xref":
+        return indirect(obj)
+
+
+def get_font_file(doc, xref):
+    if idx := get_xref(doc, xref, "FontFile"):
+        return doc.xref_stream(idx)
+    if idx := get_xref(doc, xref, "FontFile2"):
+        return doc.xref_stream(idx)
+    if idx := get_xref(doc, xref, "FontFile3"):
+        return doc.xref_stream(idx)
+
+
+def get_font_descriptor(doc, xref):
+    if idx := get_xref(doc, xref, "FontDescriptor"):
+        return get_font_file(doc, idx)
+
+
+def get_descendant_fonts(doc, xref):
+    obj = doc.xref_get_key(xref, "DescendantFonts")
+    array_text = ""
+    if obj[0] == "xref":
+        array_text = doc.xref_object(indirect(obj))
+    elif obj[0] == "array":
+        array_text = obj[1]
+    if m := re.search(r"\d+", array_text):
+        return get_font_descriptor(doc, int(m.group(0)))
+
+
+def get_glyph_bbox(face, g):
+    face.load_glyph(g, freetype.FT_LOAD_NO_SCALE)
+    cbox = face.glyph.outline.get_bbox()
+    return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
+
+
+def get_face_bbox(blob):
+    face = freetype.Face(BytesIO(blob))
+    scale = 1000 / face.units_per_EM
+    bbox_list = [get_glyph_bbox(face, code) for code in range(face.num_glyphs)]
+    bbox_list = [[v * scale for v in bbox] for bbox in bbox_list]
+    return bbox_list
+
+
+def get_cidfont_bbox(doc, xref):
+    if doc.xref_get_key(xref, "Subtype")[1] == "/Type0":
+        if blob := get_descendant_fonts(doc, xref):
+            return get_face_bbox(blob)