BabelDOC 0.3.20__tar.gz → 0.3.22__tar.gz

{babeldoc-0.3.20 → babeldoc-0.3.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.20
+Version: 0.3.22
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -42,6 +42,8 @@ Provides-Extra: cuda
 Requires-Dist: onnxruntime-gpu>=1.16.1; extra == 'cuda'
 Provides-Extra: directml
 Requires-Dist: onnxruntime-directml>=1.16.1; extra == 'directml'
+Provides-Extra: memray
+Requires-Dist: memray>=1.17.1; extra == 'memray'
 Description-Content-Type: text/markdown
 
 <!-- # Yet Another Document Translator -->

babeldoc-0.3.22/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.22"

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/assets/embedding_assets_metadata.py

@@ -1,3 +1,5 @@
+import itertools
+
 DOCLAYOUT_YOLO_DOCSTRUCTBENCH_IMGSZ1024ONNX_SHA3_256 = (
     "60be061226930524958b5465c8c04af3d7c03bcb0beb66454f5da9f792e3cf2a"
 )
@@ -495,6 +497,7 @@ EMBEDDING_FONT_METADATA = {
     },
 }
 
+FONT_NAMES = {v["font_name"] for v in EMBEDDING_FONT_METADATA.values()}
 
 CN_FONT_FAMILY = {
     # 手写体
@@ -517,16 +520,12 @@ CN_FONT_FAMILY = {
 }
 
 HK_FONT_FAMILY = {
-    "script": ["LXGWWenKaiTC-Regular.ttf", "LXGWWenKaiGB-Regular.ttf"],
+    "script": ["LXGWWenKaiTC-Regular.ttf"],
     "normal": [
         "SourceHanSerifHK-Bold.ttf",
         "SourceHanSerifHK-Regular.ttf",
         "SourceHanSansHK-Bold.ttf",
         "SourceHanSansHK-Regular.ttf",
-        "SourceHanSerifCN-Bold.ttf",
-        "SourceHanSerifCN-Regular.ttf",
-        "SourceHanSansCN-Bold.ttf",
-        "SourceHanSansCN-Regular.ttf",
     ],
     "fallback": [
         "GoNotoKurrent-Regular.ttf",
@@ -536,16 +535,12 @@ HK_FONT_FAMILY = {
 }
 
 TW_FONT_FAMILY = {
-    "script": ["LXGWWenKaiTC-Regular.ttf", "LXGWWenKaiGB-Regular.ttf"],
+    "script": ["LXGWWenKaiTC-Regular.ttf"],
     "normal": [
         "SourceHanSerifTW-Bold.ttf",
         "SourceHanSerifTW-Regular.ttf",
         "SourceHanSansTW-Bold.ttf",
         "SourceHanSansTW-Regular.ttf",
-        "SourceHanSerifCN-Bold.ttf",
-        "SourceHanSerifCN-Regular.ttf",
-        "SourceHanSansCN-Bold.ttf",
-        "SourceHanSansCN-Regular.ttf",
     ],
     "fallback": [
         "GoNotoKurrent-Regular.ttf",
@@ -561,10 +556,6 @@ KR_FONT_FAMILY = {
         "SourceHanSerifKR-Regular.ttf",
         "SourceHanSansKR-Bold.ttf",
         "SourceHanSansKR-Regular.ttf",
-        "SourceHanSerifCN-Bold.ttf",
-        "SourceHanSerifCN-Regular.ttf",
-        "SourceHanSansCN-Bold.ttf",
-        "SourceHanSansCN-Regular.ttf",
     ],
     "fallback": [
         "GoNotoKurrent-Regular.ttf",
@@ -580,10 +571,6 @@ JP_FONT_FAMILY = {
         "SourceHanSerifJP-Regular.ttf",
         "SourceHanSansJP-Bold.ttf",
         "SourceHanSansJP-Regular.ttf",
-        "SourceHanSerifCN-Bold.ttf",
-        "SourceHanSerifCN-Regular.ttf",
-        "SourceHanSansCN-Bold.ttf",
-        "SourceHanSansCN-Regular.ttf",
     ],
     "fallback": [
         "GoNotoKurrent-Regular.ttf",
@@ -604,10 +591,6 @@ EN_FONT_FAMILY = {
         "NotoSerif-Bold.ttf",
         "NotoSans-Regular.ttf",
         "NotoSans-Bold.ttf",
-        "SourceHanSerifCN-Bold.ttf",
-        "SourceHanSerifCN-Regular.ttf",
-        "SourceHanSansCN-Bold.ttf",
-        "SourceHanSansCN-Regular.ttf",
     ],
     "fallback": [
         "GoNotoKurrent-Regular.ttf",
@@ -620,14 +603,32 @@ EN_FONT_FAMILY = {
 
 ALL_FONT_FAMILY = {
     "CN": CN_FONT_FAMILY,
-    "HK": HK_FONT_FAMILY,
     "TW": TW_FONT_FAMILY,
+    "HK": HK_FONT_FAMILY,
     "KR": KR_FONT_FAMILY,
     "JP": JP_FONT_FAMILY,
     "EN": EN_FONT_FAMILY,
 }
 
 
+def __add_fallback_to_font_family():
+    for lang1, family1 in ALL_FONT_FAMILY.items():
+        added_font = set()
+        for font in itertools.chain.from_iterable(family1.values()):
+            added_font.add(font)
+
+        for lang2, family2 in ALL_FONT_FAMILY.items():
+            if lang1 != lang2:
+                for type_ in family1:
+                    for font in family2[type_]:
+                        if font not in added_font:
+                            family1[type_].append(font)
+                            added_font.add(font)
+
+
+__add_fallback_to_font_family()
+
+
 def get_font_family(lang_code: str):
     lang_code = lang_code.upper()
     if "KR" in lang_code:

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/asynchronize/__init__.py

@@ -23,7 +23,7 @@ class AsyncCallback:
         self.loop.call_soon_threadsafe(self.queue.put_nowait, args)
 
         # Add a small delay to release the GIL, ensuring the event loop has time to process messages
-        time.sleep(0.01)
+        time.sleep(0.1)
 
     def finished_callback(self, *args, **kwargs):
         # Whenever a finished is called, add to the queue as with step, but also set finished to True, so __anext__

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.20"
+__version__ = "0.3.22"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/document_il/backend/pdf_creater.py

@@ -1,13 +1,18 @@
+import io
+import itertools
 import logging
 import os
 import re
 import time
+import unicodedata
 from multiprocessing import Process
 from pathlib import Path
 
+import freetype
 import pymupdf
 from bitstring import BitStream
 
+from babeldoc.assets.embedding_assets_metadata import FONT_NAMES
 from babeldoc.document_il import il_version_1
 from babeldoc.document_il.utils.fontmap import FontMapper
 from babeldoc.translation_config import TranslateResult
@@ -20,6 +25,138 @@ SUBSET_FONT_STAGE_NAME = "Subset font"
 SAVE_PDF_STAGE_NAME = "Save PDF"
 
 
+def to_int(src):
+    return int(re.search(r"\d+", src).group(0))
+
+
+def parse_mapping(text):
+    mapping = []
+    for x in re.finditer(rb"<(?P<num>[a-fA-F0-9]+)>", text):
+        mapping.append(int(x.group("num"), 16))
+    return mapping
+
+
+def apply_normalization(cmap, gid, code):
+    need = False
+    if 0x2F00 <= code <= 0x2FD5:  # Kangxi Radicals
+        need = True
+    if 0xF900 <= code <= 0xFAFF:  # CJK Compatibility Ideographs
+        need = True
+    if need:
+        norm = unicodedata.normalize("NFD", chr(code))
+        cmap[gid] = ord(norm)
+    else:
+        cmap[gid] = code
+
+
+def batched(iterable, n, *, strict=False):
+    # batched('ABCDEFG', 3) → ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    iterator = iter(iterable)
+    while batch := tuple(itertools.islice(iterator, n)):
+        if strict and len(batch) != n:
+            raise ValueError("batched(): incomplete batch")
+        yield batch
+
+
+def update_tounicode_cmap_pair(cmap, data):
+    for start, stop, value in batched(data, 3):
+        for gid in range(start, stop + 1):
+            code = value + gid - start
+            apply_normalization(cmap, gid, code)
+
+
+def update_tounicode_cmap_code(cmap, data):
+    for gid, code in batched(data, 2):
+        apply_normalization(cmap, gid, code)
+
+
+def parse_tounicode_cmap(data):
+    cmap = {}
+    for x in re.finditer(
+        rb"\s+beginbfrange\s*(?P<r>(<[0-9a-fA-F]+>\s*)+)endbfrange\s+", data
+    ):
+        update_tounicode_cmap_pair(cmap, parse_mapping(x.group("r")))
+    for x in re.finditer(
+        rb"\s+beginbfchar\s*(?P<c>(<[0-9a-fA-F]+>\s*)+)endbfchar", data
+    ):
+        update_tounicode_cmap_code(cmap, parse_mapping(x.group("c")))
+    return cmap
+
+
+def parse_truetype_data(data):
+    glyph_in_use = []
+    face = freetype.Face(io.BytesIO(data))
+    for i in range(face.num_glyphs):
+        face.load_glyph(i)
+        if face.glyph.outline.contours:
+            glyph_in_use.append(i)
+    return glyph_in_use
+
+
+TOUNICODE_HEAD = """\
+/CIDInit /ProcSet findresource begin
+12 dict begin
+/CIDSystemInfo <</Registry(Adobe)/Ordering(UCS)/Supplement 0>> def
+/CMapName /Adobe-Identity-UCS def
+/CMapType 2 def
+1 begincodespacerange
+<0000> <FFFF>
+endcodespacerange"""
+TOUNICODE_TAIL = """\
+endcmap
+CMapName currentdict /CMap defineresource pop
+end
+end"""
+
+
+def make_tounicode(cmap, used):
+    short = []
+    for x in used:
+        if x in cmap:
+            short.append((x, cmap[x]))
+    line = [TOUNICODE_HEAD]
+    for block in batched(short, 100):
+        line.append(f"{len(block)} beginbfchar")
+        for glyph, code in block:
+            if code < 0x10000:
+                line.append(f"<{glyph:04x}><{code:04x}>")
+            else:
+                line.append(f"<{glyph:04x}><{code:08x}>")
+        line.append("endbfchar")
+    line.append(TOUNICODE_TAIL)
+    return "\n".join(line)
+
+
+def reproduce_one_font(doc, index):
+    m = doc.xref_get_key(index, "ToUnicode")
+    f = doc.xref_get_key(index, "DescendantFonts")
+    if m[0] == "xref" and f[0] == "array":
+        mi = to_int(m[1])
+        fi = to_int(f[1])
+        ff = doc.xref_get_key(fi, "FontDescriptor/FontFile2")
+        ms = doc.xref_stream(mi)
+        fs = doc.xref_stream(to_int(ff[1]))
+        cmap = parse_tounicode_cmap(ms)
+        used = parse_truetype_data(fs)
+        text = make_tounicode(cmap, used)
+        doc.update_stream(mi, bytes(text, "U8"))
+
+
+def reproduce_cmap(doc):
+    assert doc
+    font_set = set()
+    for page in doc:
+        font_list = page.get_fonts()
+        for font in font_list:
+            if font[1] == "ttf" and font[3] in FONT_NAMES and ".ttf" in font[4]:
+                font_set.add(font)
+    for font in font_set:
+        reproduce_one_font(doc, font[0])
+    return doc
+
+
 def _subset_fonts_process(pdf_path, output_path):
     """Function to run in subprocess for font subsetting.
 

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/document_il/frontend/il_creater.py

@@ -49,8 +49,11 @@ def create_hook(func, hook):
 
 def hook_pdfminer_pdf_page_init(*args):
     attrs = args[3]
-    while isinstance(attrs["MediaBox"], PDFMinerPDFObjRef):
-        attrs["MediaBox"] = pdftypes_resolve1(attrs["MediaBox"])
+    try:
+        while isinstance(attrs["MediaBox"], PDFMinerPDFObjRef):
+            attrs["MediaBox"] = pdftypes_resolve1(attrs["MediaBox"])
+    except Exception:
+        logger.exception(f"try to fix mediabox failed: {attrs}")
 
 
 PDFMinerPDFPage.__init__ = create_hook(
@@ -343,6 +346,7 @@ def parse_font_file(doc, idx, encoding, differences):
     bbox_list = []
     data = doc.xref_stream(idx)
     face = freetype.Face(BytesIO(data))
+    scale = 1000 / face.units_per_EM
     for charmap in face.charmaps:
         if charmap.encoding_name == "FT_ENCODING_ADOBE_CUSTOM":
             face.select_charmap(freetype.FT_ENCODING_ADOBE_CUSTOM)
@@ -351,7 +355,8 @@ def parse_font_file(doc, idx, encoding, differences):
     if differences:
         for code, name in differences:
             bbox_list[code] = get_name_cbox(face, name.encode("U8"))
-    return bbox_list
+    norm_bbox_list = [[v * scale for v in box] for box in bbox_list]
+    return norm_bbox_list
 
 
 def parse_encoding(obj_str):

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/document_il/midend/paragraph_finder.py

@@ -12,6 +12,7 @@ from babeldoc.document_il import PdfRectangle
 from babeldoc.document_il.utils.layout_helper import Layout
 from babeldoc.document_il.utils.layout_helper import add_space_dummy_chars
 from babeldoc.document_il.utils.layout_helper import get_char_unicode_string
+from babeldoc.document_il.utils.layout_helper import is_bullet_point
 from babeldoc.document_il.utils.style_helper import WHITE
 from babeldoc.translation_config import TranslationConfig
 
@@ -30,9 +31,6 @@ class ParagraphFinder:
     stage_name = "Parse Paragraphs"
 
     # 定义项目符号的正则表达式模式
-    BULLET_POINT_PATTERN = re.compile(
-        r"[■•⚫⬤◆◇○●◦‣⁃▪▫∗†‡¹²³⁴⁵⁶⁷⁸⁹⁰₁₂₃₄₅₆₇₈₉₀ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖᵍʳˢᵗᵘᵛʷˣʸᶻ†‡§¶※⁑⁂⁕⁎⁜⁑❧☙⁋‖‽]"
-    )
 
     def __init__(self, translation_config: TranslationConfig):
         self.translation_config = translation_config
@@ -246,7 +244,7 @@ class ParagraphFinder:
                     and current_line_chars[-1].xobj_id != char.xobj_id
                 )
                 or (
-                    self.is_bullet_point(char)  # 如果是项目符号，开启新段落
+                    is_bullet_point(char)  # 如果是项目符号，开启新段落
                     and not current_line_chars
                 )
             ):
@@ -512,15 +510,3 @@ class ParagraphFinder:
                     break
                 j += 1
             i += 1
-
-    def is_bullet_point(self, char: PdfCharacter) -> bool:
-        """Check if the character is a bullet point.
-
-        Args:
-            char: The character to check
-
-        Returns:
-            bool: True if the character is a bullet point
-        """
-        is_bullet = bool(self.BULLET_POINT_PATTERN.match(char.char_unicode))
-        return is_bullet

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -17,6 +17,7 @@ from babeldoc.document_il.utils.fontmap import FontMapper
 from babeldoc.document_il.utils.layout_helper import LEFT_BRACKET
 from babeldoc.document_il.utils.layout_helper import RIGHT_BRACKET
 from babeldoc.document_il.utils.layout_helper import formular_height_ignore_char
+from babeldoc.document_il.utils.layout_helper import is_bullet_point
 from babeldoc.document_il.utils.layout_helper import is_same_style
 from babeldoc.translation_config import TranslationConfig
 
@@ -70,6 +71,7 @@ class StylesAndFormulas:
                 continue
 
             new_compositions = []
+            first_is_bullet = False
 
             for composition in paragraph.pdf_paragraph_composition:
                 current_chars = []
@@ -81,6 +83,8 @@ class StylesAndFormulas:
                     new_compositions.append(composition)
                     continue
                 for char in line.pdf_character:
+                    if not current_chars and is_bullet_point(char):
+                        first_is_bullet = True
                     is_formula = (
                         (  # 区分公式开头的字符&公式中间的字符。主要是逗号不能在公式开头，但是可以在中间。
                             (
@@ -112,6 +116,7 @@ class StylesAndFormulas:
                     is_corner_mark = (
                         len(current_chars) > 0
                         and not isspace
+                        and not first_is_bullet
                         # 角标字体，有 0.76 的角标和 0.799 的大写，这里用 0.79 取中，同时考虑首字母放大的情况
                         and char.pdf_style.font_size
                         < current_chars[-1].pdf_style.font_size * 0.79
@@ -119,6 +124,7 @@ class StylesAndFormulas:
                     ) or (
                         len(current_chars) > 0
                         and not isspace
+                        and not first_is_bullet
                         # 角标字体，有 0.76 的角标和 0.799 的大写，这里用 0.79 取中，同时考虑首字母放大的情况
                         and char.pdf_style.font_size
                         < current_chars[-1].pdf_style.font_size * 1.1
@@ -520,7 +526,7 @@ class StylesAndFormulas:
         return bool(re.match(r"^[0-9, ]+$", text))
 
     def is_formulas_font(self, font_name: str) -> bool:
-        pattern2 = (
+        pattern_text = (
             r"^("
             r"|Cambria.*"
             r"|EUAlbertina.+"
@@ -570,12 +576,90 @@ class StylesAndFormulas:
             r"|.*CronosPro.*"
             r"|.*ACaslon.*"
             r"|.*Frutiger.*"
+            r"|.*BrandonGrotesque.*"
+            r"|.*FairfieldLH.*"
+            r"|.*CaeciliaLTStd.*"
+            r"|.*Whitney.*"
+            r"|.*Mercury.*"
+            r"|.*SabonLTStd.*"
+            r"|.*AnonymousPro.*"
+            r"|.*SabonLTPro.*"
+            r"|.*ArnoPro.*"
+            r"|.*CharisSIL.*"
+            r")$"
+        )
+        precise_formula_font_pattern = (
+            r"^("
+            r"|.*CambriaMath.*"
+            r"|.*Cambria Math.*"
+            r"|.*Asana.*"
+            r"|.*MiriamMonoCLM-BookOblique.*"
+            r"|.*Miriam Mono CLM.*"
+            r"|.*Logix.*"
+            r"|.*AeBonum.*"
+            r"|.*AeMRoman.*"
+            r"|.*AePagella.*"
+            r"|.*AeSchola.*"
+            r"|.*Concrete.*"
+            r"|.*LatinModernMathCompanion.*"
+            r"|.*Latin Modern Math Companion.*"
+            r"|.*RalphSmithsFormalScriptCompanion.*"
+            r"|.*Ralph Smiths Formal Script Companion.*"
+            r"|.*TeXGyreBonumMathCompanion.*"
+            r"|.*TeX Gyre Bonum Companion.*"
+            r"|.*TeXGyrePagellaMathCompanion.*"
+            r"|.*TeX Gyre Pagella Math Companion.*"
+            r"|.*TeXGyreTermesMathCompanion.*"
+            r"|.*TeX Gyre Termes Math Companion.*"
+            r"|.*XITSMathCompanion.*"
+            r"|.*XITS Math Companion.*"
+            r"|.*Erewhon.*"
+            r"|.*Euler-Math.*"
+            r"|.*Euler Math.*"
+            r"|.*FiraMath-Regular.*"
+            r"|.*Fira Math.*"
+            r"|.*Garamond-Math.*"
+            r"|.*GFSNeohellenicMath.*"
+            r"|.*KpMath.*"
+            r"|.*Lete Sans Math.*"
+            r"|.*LeteSansMath.*"
+            r"|.*LinLibertineO.*"
+            r"|.*Linux Libertine O.*"
+            r"|.*LibertinusMath-Regular.*"
+            r"|.*Libertinus Math.*"
+            r"|.*LatinModernMath-Regular.*"
+            r"|.*Latin Modern Math.*"
+            r"|.*Luciole.*"
+            r"|.*NewCM.*"
+            r"|.*NewComputerModern.*"
+            r"|.*OldStandard-Math.*"
+            r"|.*STIXMath-Regular.*"
+            r"|.*STIX Math.*"
+            r"|.*STIXTwoMath-Regular.*"
+            r"|.*STIX Two Math.*"
+            r"|.*TeXGyreBonumMath.*"
+            r"|.*TeX Gyre Bonum Math.*"
+            r"|.*TeXGyreDejaVuMath.*"
+            r"|.*TeX Gyre DejaVu Math.*"
+            r"|.*TeXGyrePagellaMath.*"
+            r"|.*TeX Gyre Pagella Math.*"
+            r"|.*TeXGyreScholaMath.*"
+            r"|.*TeX Gyre Schola Math.*"
+            r"|.*TeXGyreTermesMath.*"
+            r"|.*TeX Gyre Termes Math.*"
+            r"|.*XCharter-Math.*"
+            r"|.*XCharter Math.*"
+            r"|.*XITSMath-Bold.*"
+            r"|.*XITS Math.*"
+            r"|.*XITSMath.*"
+            r"|.*IBMPlexMath.*"
+            r"|.*IBM Plex Math.*"
             r")$"
         )
         if self.translation_config.formular_font_pattern:
-            pattern = self.translation_config.formular_font_pattern
+            broad_formula_font_pattern = self.translation_config.formular_font_pattern
         else:
-            pattern = (
+            broad_formula_font_pattern = (
                 r"(CM[^RB]"
                 r"|(MS|XY|MT|BL|RM|EU|LA|RS)[A-Z]"
                 r"|LINE"
@@ -599,14 +683,16 @@ class StylesAndFormulas:
         if font_name.startswith("BASE64:"):
             font_name_bytes = base64.b64decode(font_name[7:])
             font = font_name_bytes.split(b"+")[-1]
-            pattern2 = pattern2.encode()
-            pattern = pattern.encode()
+            pattern_text = pattern_text.encode()
+            broad_formula_font_pattern = broad_formula_font_pattern.encode()
         else:
             font = font_name.split("+")[-1]
 
-        if re.match(pattern2, font):
+        if re.match(precise_formula_font_pattern, font):
+            return True
+        elif re.match(pattern_text, font):
             return False
-        if re.match(pattern, font):
+        elif re.match(broad_formula_font_pattern, font):
             return True
 
         return False
@@ -626,7 +712,7 @@ class StylesAndFormulas:
             and (
                 unicodedata.category(char[0])
                 in [
-                    "Lm",
+                    # "Lm",
                     "Mn",
                     "Sk",
                     "Sm",

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/document_il/translator/translator.py

@@ -215,9 +215,7 @@ class OpenAITranslator(BaseTranslator):
             **self.options,
             messages=self.prompt(text),
         )
-        self.token_count.inc(response.usage.total_tokens)
-        self.prompt_token_count.inc(response.usage.prompt_tokens)
-        self.completion_token_count.inc(response.usage.completion_tokens)
+        self.update_token_count(response)
         return response.choices[0].message.content.strip()
 
     def prompt(self, text):
@@ -255,11 +253,20 @@ class OpenAITranslator(BaseTranslator):
                 },
             ],
         )
-        self.token_count.inc(response.usage.total_tokens)
-        self.prompt_token_count.inc(response.usage.prompt_tokens)
-        self.completion_token_count.inc(response.usage.completion_tokens)
+        self.update_token_count(response)
         return response.choices[0].message.content.strip()
 
+    def update_token_count(self, response):
+        try:
+            if response.usage and response.usage.total_tokens:
+                self.token_count.inc(response.usage.total_tokens)
+            if response.usage and response.usage.prompt_tokens:
+                self.prompt_token_count.inc(response.usage.prompt_tokens)
+            if response.usage and response.usage.completion_tokens:
+                self.completion_token_count.inc(response.usage.completion_tokens)
+        except Exception as e:
+            logger.exception("Error updating token count")
+
     def get_formular_placeholder(self, placeholder_id: int):
         return "{{v" + str(placeholder_id) + "}}"
         return "{{" + str(placeholder_id) + "}}"

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/document_il/utils/fontmap.py

@@ -174,8 +174,24 @@ class FontMapper:
         )
         return None
 
+    def get_used_font_ids(self, il: il_version_1.Document) -> set[str]:
+        result = set()
+        for page in il.page:
+            for char in page.pdf_character:
+                if char.pdf_style and char.pdf_style.font_id:
+                    result.add(char.pdf_style.font_id)
+            for para in page.pdf_paragraph:
+                for comp in para.pdf_paragraph_composition:
+                    if char := comp.pdf_character:
+                        if char.pdf_style and char.pdf_style.font_id:
+                            result.add(char.pdf_style.font_id)
+        return result
+
     def add_font(self, doc_zh: pymupdf.Document, il: il_version_1.Document):
-        font_list = [(k, v) for k, v in self.fontid2fontpath.items()]
+        used_font_ids = self.get_used_font_ids(il)
+        font_list = [
+            (k, v) for k, v in self.fontid2fontpath.items() if k in used_font_ids
+        ]
 
         font_id = {}
         xreflen = doc_zh.xref_length()

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/document_il/utils/layout_helper.py

@@ -1,5 +1,6 @@
 import logging
 import math
+import re
 
 from babeldoc.document_il import GraphicState
 from babeldoc.document_il import il_version_1
@@ -37,6 +38,23 @@ HEIGHT_NOT_USFUL_CHAR_IN_CHAR = (
 LEFT_BRACKET = ("(cid:8)", "(", "(cid:16)", "{", "[", "(cid:104)", "(cid:2)")
 RIGHT_BRACKET = ("(cid:9)", ")", "(cid:17)", "}", "]", "(cid:105)", "(cid:3)")
 
+BULLET_POINT_PATTERN = re.compile(
+    r"[■•⚫⬤◆◇○●◦‣⁃▪▫∗†‡¹²³⁴⁵⁶⁷⁸⁹⁰₁₂₃₄₅₆₇₈₉₀ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖᵍʳˢᵗᵘᵛʷˣʸᶻ†‡§¶※⁑⁂⁕⁎⁜⁑❧☙⁋‖‽]"
+)
+
+
+def is_bullet_point(char: PdfCharacter) -> bool:
+    """Check if the character is a bullet point.
+
+    Args:
+        char: The character to check
+
+    Returns:
+        bool: True if the character is a bullet point
+    """
+    is_bullet = bool(BULLET_POINT_PATTERN.match(char.char_unicode))
+    return is_bullet
+
 
 def formular_height_ignore_char(char: PdfCharacter):
     return (

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/high_level.py

@@ -4,6 +4,7 @@ import hashlib
 import io
 import logging
 import pathlib
+import shutil
 import threading
 import time
 from asyncio import CancelledError
@@ -11,6 +12,7 @@ from pathlib import Path
 from typing import Any
 from typing import BinaryIO
 
+import pymupdf
 from pdfminer.pdfdocument import PDFDocument
 from pdfminer.pdfinterp import PDFResourceManager
 from pdfminer.pdfpage import PDFPage
@@ -26,6 +28,7 @@ from babeldoc.document_il import il_version_1
 from babeldoc.document_il.backend.pdf_creater import SAVE_PDF_STAGE_NAME
 from babeldoc.document_il.backend.pdf_creater import SUBSET_FONT_STAGE_NAME
 from babeldoc.document_il.backend.pdf_creater import PDFCreater
+from babeldoc.document_il.backend.pdf_creater import reproduce_cmap
 from babeldoc.document_il.frontend.il_creater import ILCreater
 from babeldoc.document_il.midend.add_debug_information import AddDebugInformation
 from babeldoc.document_il.midend.detect_scanned_file import DetectScannedFile
@@ -75,6 +78,26 @@ resfont_map = {
 }
 
 
+def fix_cmap(translate_result: TranslateResult, translate_config: TranslationConfig):
+    processed = []
+    for attr in (
+        "mono_pdf_path",
+        "dual_pdf_path",
+        "no_watermark_mono_pdf_path",
+        "no_watermark_dual_pdf_path",
+    ):
+        path = getattr(translate_result, attr)
+        if path in processed:
+            continue
+        processed.append(path)
+
+        temp_path = translate_config.get_working_file_path(f"{path.stem}.cmap.pdf")
+        pdf = pymupdf.open(path)
+        reproduce_cmap(pdf)
+        pdf.save(temp_path)
+        shutil.move(temp_path, path)
+
+
 def verify_file_hash(file_path: str, expected_hash: str) -> bool:
     """Verify the SHA256 hash of a file."""
     sha256_hash = hashlib.sha256()
@@ -514,11 +537,18 @@ def do_translate(
         )
         result.original_pdf_path = translation_config.input_file
         result.peak_memory_usage = peak_memory_usage
+
+        # should fix macOS preview compatibility
+        # Although the issue of Windows Edge
+        # not being able to copy translated text can be fixed,
+        # the macOS preview is broken
+        # fix_cmap(result, translation_config)
         pm.translate_done(result)
         return result
 
     except Exception as e:
         logger.exception(f"translate error: {e}")
+        pm.disable = False
         pm.translate_error(e)
         raise
     finally:
@@ -530,10 +560,16 @@ def do_translate(
 def fix_media_box(doc: Document) -> None:
     mediabox_data = {}
     for page in doc:
+        mediabox = doc.xref_get_key(page.xref, "MediaBox")
+
+        # Some PDF pages do not have a mediabox
+        if mediabox[0] == "null":
+            mediabox = ("array", "[0 0 612 792]")
+            doc.xref_set_key(page.xref, "MediaBox", mediabox[1])
         if page.mediabox.x0 != 0 or page.mediabox.y0 != 0:
             x1 = page.mediabox.x1
             y1 = page.mediabox.y1
-            mediabox_data[page.number] = doc.xref_get_key(page.xref, "MediaBox")
+            mediabox_data[page.number] = mediabox
             doc.xref_set_key(page.xref, "MediaBox", f"[0 0 {x1} {y1}]")
     return mediabox_data
 

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.20"
+__version__ = "0.3.22"
 
 
 def create_parser():

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/tools/italic_assistance.py

@@ -104,6 +104,7 @@ def extract_fonts_from_paragraph(
                 same_style_unicode = comp["pdf_same_style_unicode_characters"]
                 if (
                     "pdf_style" in same_style_unicode
+                    and same_style_unicode["pdf_style"] is not None
                     and "font_id" in same_style_unicode["pdf_style"]
                 ):
                     font_id = same_style_unicode["pdf_style"]["font_id"]

{babeldoc-0.3.20 → babeldoc-0.3.22}/babeldoc/tools/italic_recognize_tool.py

@@ -72,8 +72,14 @@ for each_font in fonts:
             font_name,
             "[bold red]Formula Font[/bold red]",
         )
-    # else:
-    #     table.add_row(str(page_index), font_name, "[bold blue]Non-Formula Font[/bold blue]")
+    else:
+        table.add_row(
+            str(page_index),
+            str(paragraph_index),
+            str(font_debug_id),
+            font_name,
+            "[bold blue]Non-Formula Font[/bold blue]",
+        )
 
 # Print table
 console.print(table)

{babeldoc-0.3.20 → babeldoc-0.3.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.20"
+version = "0.3.22"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -48,6 +48,7 @@ dependencies = [
 [project.optional-dependencies]
 directml = ["onnxruntime-directml>=1.16.1"]
 cuda = ["onnxruntime-gpu>=1.16.1"]
+memray = ["memray>=1.17.1"]
 
 [project.urls]
 Homepage = "https://github.com/funstory-ai/BabelDOC"
@@ -141,7 +142,6 @@ dev = [
     "pygments>=2.19.1",
     "ruff>=0.9.2",
     "pytest>=8.3.4",
-    "memray>=1.17.1",
 ]
 
 [tool.pytest.ini_options]
@@ -149,7 +149,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.20"
+current_version = "0.3.22"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.20 → babeldoc-0.3.22}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.20"
+version = "0.3.22"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },
@@ -86,13 +86,15 @@ cuda = [
 directml = [
     { name = "onnxruntime-directml" },
 ]
+memray = [
+    { name = "memray" },
+]
 
 [package.dev-dependencies]
 dev = [
     { name = "bumpver" },
     { name = "markdown-callouts" },
     { name = "markdown-include" },
-    { name = "memray" },
     { name = "mkdocs-git-authors-plugin" },
     { name = "mkdocs-git-committers-plugin-2" },
     { name = "mkdocs-git-revision-date-localized-plugin" },
@@ -110,6 +112,7 @@ requires-dist = [
     { name = "freetype-py", specifier = ">=2.5.1" },
     { name = "httpx", extras = ["socks"], specifier = ">=0.27.0" },
     { name = "huggingface-hub", specifier = ">=0.27.0" },
+    { name = "memray", marker = "extra == 'memray'", specifier = ">=1.17.1" },
     { name = "msgpack", specifier = ">=1.1.0" },
     { name = "numpy", specifier = ">=2.0.2" },
     { name = "onnx", specifier = ">=1.17.0" },
@@ -134,14 +137,13 @@ requires-dist = [
     { name = "tqdm", specifier = ">=4.67.1" },
     { name = "xsdata", extras = ["cli", "lxml", "soap"], specifier = ">=24.12" },
 ]
-provides-extras = ["directml", "cuda"]
+provides-extras = ["directml", "cuda", "memray"]
 
 [package.metadata.requires-dev]
 dev = [
     { name = "bumpver", specifier = ">=2024.1130" },
     { name = "markdown-callouts", specifier = ">=0.4.0" },
     { name = "markdown-include", specifier = ">=0.8.1" },
-    { name = "memray", specifier = ">=1.17.1" },
     { name = "mkdocs-git-authors-plugin", specifier = ">=0.9.2" },
     { name = "mkdocs-git-committers-plugin-2", specifier = ">=2.5.0" },
     { name = "mkdocs-git-revision-date-localized-plugin", specifier = ">=1.3.0" },

babeldoc-0.3.20/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.20"