BabelDOC 0.4.6__tar.gz → 0.4.8__tar.gz

{babeldoc-0.4.6 → babeldoc-0.4.8}/.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
+        uses: astral-sh/setup-uv@445689ea25e0de0a23313031f5fe577c74ae45a1 # v6.3.0
         with:
           python-version: "3.12"
           enable-cache: true

{babeldoc-0.4.6 → babeldoc-0.4.8}/.github/workflows/publish-to-pypi.yml

@@ -35,7 +35,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
+        uses: astral-sh/setup-uv@445689ea25e0de0a23313031f5fe577c74ae45a1 # v6.3.0
         with:
           python-version: "3.12"
           enable-cache: true

{babeldoc-0.4.6 → babeldoc-0.4.8}/.github/workflows/test.yml

@@ -28,7 +28,7 @@ jobs:
           path: ~/.cache/babeldoc
           key: babeldoc-assets-${{ hashFiles('babeldoc/assets/embedding_assets_metadata.py') }}
       - name: Setup uv with Python ${{ matrix.python-version }}
-        uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
+        uses: astral-sh/setup-uv@445689ea25e0de0a23313031f5fe577c74ae45a1 # v6.3.0
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true

{babeldoc-0.4.6 → babeldoc-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.4.6
+Version: 0.4.8
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -13,6 +13,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.14,>=3.10
 Requires-Dist: bitstring>=4.3.0
+Requires-Dist: chardet>=5.2.0
 Requires-Dist: charset-normalizer>=2.0.0
 Requires-Dist: configargparse>=1.7
 Requires-Dist: cryptography>=36.0.0

babeldoc-0.4.8/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.8"

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.4.6"
+__version__ = "0.4.8"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/format/pdf/document_il/backend/pdf_creater.py

@@ -289,7 +289,7 @@ class PDFCreater:
                 )
                 continue
             chars.append(composition.pdf_character)
-        if not chars and paragraph.unicode:
+        if not chars and paragraph.unicode and paragraph.debug_id:
             logger.error(
                 f"Unable to export paragraphs that have "
                 f"not yet been formatted: {paragraph}",

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/format/pdf/document_il/frontend/il_creater.py

@@ -348,11 +348,14 @@ def parse_font_file(doc, idx, encoding, differences):
     data = doc.xref_stream(idx)
     face = freetype.Face(BytesIO(data))
     scale = 1000 / face.units_per_EM
-    for charmap in face.charmaps:
-        if charmap.encoding_name == "FT_ENCODING_ADOBE_CUSTOM":
-            face.select_charmap(freetype.FT_ENCODING_ADOBE_CUSTOM)
-            break
-    bbox_list = [get_char_cbox(face, x) for x in encoding]
+    enc_name, enc_vector = encoding
+    if enc_name == "Custom":
+        for charmap in face.charmaps:
+            face.select_charmap(charmap.encoding)
+            if charmap.encoding_name == "FT_ENCODING_ADOBE_CUSTOM":
+                face.select_charmap(charmap.encoding)
+                break
+    bbox_list = [get_char_cbox(face, x) for x in enc_vector]
     if differences:
         for code, name in differences:
             bbox_list[code] = get_name_cbox(face, name.encode("U8"))
@@ -765,10 +768,13 @@ class ILCreater:
 
     def parse_font_xobj_id(self, xobj_id: int):
         bbox_list = []
-        encoding = list(range(256))
+        encoding = ("Custom", list(range(256)))
         font_encoding = self.mupdf.xref_get_key(xobj_id, "Encoding")
         if font_encoding[1] == "/WinAnsiEncoding":
-            encoding = WinAnsiEncoding
+            encoding = ("WinAnsi", WinAnsiEncoding)
+        base_encoding = self.mupdf.xref_get_key(xobj_id, "Encoding/BaseEncoding")
+        if base_encoding[1] == "/WinAnsiEncoding":
+            encoding = ("WinAnsi", WinAnsiEncoding)
         differences = []
         font_differences = self.mupdf.xref_get_key(xobj_id, "Encoding/Differences")
         if font_differences:

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/format/pdf/document_il/midend/il_translator_llm_only.py

@@ -479,9 +479,20 @@ class ILTranslatorLLMOnly:
                     input_unicode = inputs[id_][2].unicode
                     output_unicode = translated_text
 
+                    trimed_input = re.sub(r"[. 。…，]{20,}", ".", output)
+
                     input_token_count = self.calc_token_count(input_unicode)
                     output_token_count = self.calc_token_count(output_unicode)
 
+                    if trimed_input == output_unicode and input_token_count > 20:
+                        llm_translate_tracker.set_error_message(
+                            "Translation result is the same as input, fallback."
+                        )
+                        logger.warning(
+                            "Translation result is the same as input, fallback."
+                        )
+                        continue
+
                     if not (0.3 < output_token_count / input_token_count < 3):
                         llm_translate_tracker.set_error_message(
                             f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/format/pdf/document_il/midend/typesetting.py

@@ -871,7 +871,8 @@ class Typesetting:
                             PdfParagraphComposition(pdf_character=char),
                         )
                 return
-
+            if not paragraph.debug_id:
+                return
             # 如果当前行距大于最小行距，先减小行距
             if line_spacing > min_line_spacing:
                 line_spacing -= 0.1

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/format/pdf/document_il/utils/formular_helper.py

@@ -197,8 +197,8 @@ def is_formulas_font(font_name: str, formular_font_pattern: str | None) -> bool:
     )
     precise_formula_font_pattern = (
         r"^("
-        r"|.*CambriaMath.*"
-        r"|.*Cambria Math.*"
+        # r"|.*CambriaMath.*"
+        # r"|.*Cambria Math.*"
         r"|.*Asana.*"
         r"|.*MiriamMonoCLM-BookOblique.*"
         r"|.*Miriam Mono CLM.*"

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/format/pdf/document_il/utils/layout_helper.py

@@ -710,7 +710,7 @@ def is_text_layout(layout: Layout):
         "figure_caption",
         "table_caption",
         "table_text",
-        "reference",
+        # "reference",
         "title",
         "paragraph_title",
         "abstract",

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/glossary.py

@@ -6,6 +6,7 @@ import re
 import time
 from pathlib import Path
 
+import chardet
 import hyperscan
 import regex
 
@@ -134,8 +135,11 @@ class Glossary:
         normalized_target_lang_out = target_lang_out.lower().replace("-", "_")
 
         try:
-            with file_path.open("r", encoding="utf-8") as f:
-                reader = csv.DictReader(f, doublequote=True)
+            with file_path.open("rb") as f:
+                content = f.read()
+                encoding = chardet.detect(content)["encoding"]
+                buffer = io.StringIO(content.decode(encoding))
+                reader = csv.DictReader(buffer, doublequote=True)
                 if not all(col in reader.fieldnames for col in ["source", "target"]):
                     raise ValueError(
                         f"CSV file {file_path} must contain 'source' and 'target' columns."

{babeldoc-0.4.6 → babeldoc-0.4.8}/babeldoc/main.py

@@ -21,7 +21,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.4.6"
+__version__ = "0.4.8"
 
 
 def create_parser():

{babeldoc-0.4.6 → babeldoc-0.4.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.4.6"
+version = "0.4.8"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -47,7 +47,8 @@ dependencies = [
     "rapidocr-onnxruntime>=1.4.4",
     "pyzstd>=0.17.0",
     "hyperscan>=0.7.13",
-    "rtree>=1.4.0"
+    "rtree>=1.4.0",
+    "chardet>=5.2.0",
 ]
 
 [project.optional-dependencies]
@@ -157,7 +158,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.4.6"
+current_version = "0.4.8"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.4.6 → babeldoc-0.4.8}/uv.lock

@@ -48,10 +48,11 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.4.6"
+version = "0.4.8"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },
+    { name = "chardet" },
     { name = "charset-normalizer" },
     { name = "configargparse" },
     { name = "cryptography" },
@@ -114,6 +115,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "bitstring", specifier = ">=4.3.0" },
+    { name = "chardet", specifier = ">=5.2.0" },
     { name = "charset-normalizer", specifier = ">=2.0.0" },
     { name = "configargparse", specifier = ">=1.7" },
     { name = "cryptography", specifier = ">=36.0.0" },
@@ -368,6 +370,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" },
 ]
 
+[[package]]
+name = "chardet"
+version = "5.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" },
+]
+
 [[package]]
 name = "charset-normalizer"
 version = "3.4.2"

babeldoc-0.4.6/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.4.6"