BabelDOC 0.3.29__tar.gz → 0.3.31__tar.gz

{babeldoc-0.3.29 → babeldoc-0.3.31}/.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
+        uses: astral-sh/setup-uv@c7f87aa956e4c323abf06d5dec078e358f6b4d04 # v6.0.0
         with:
           python-version: "3.12"
           enable-cache: true

{babeldoc-0.3.29 → babeldoc-0.3.31}/.github/workflows/publish-to-pypi.yml

@@ -35,7 +35,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
+        uses: astral-sh/setup-uv@c7f87aa956e4c323abf06d5dec078e358f6b4d04 # v6.0.0
         with:
           python-version: "3.12"
           enable-cache: true
@@ -89,7 +89,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: python-package-distributions
           path: dist/
@@ -113,7 +113,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
           name: python-package-distributions
           path: dist/

{babeldoc-0.3.29 → babeldoc-0.3.31}/.github/workflows/test.yml

@@ -28,7 +28,7 @@ jobs:
           path: ~/.cache/babeldoc
           key: babeldoc-assets-${{ hashFiles('babeldoc/assets/embedding_assets_metadata.py') }}
       - name: Setup uv with Python ${{ matrix.python-version }}
-        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2
+        uses: astral-sh/setup-uv@c7f87aa956e4c323abf06d5dec078e358f6b4d04 # v6.0.0
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true

{babeldoc-0.3.29 → babeldoc-0.3.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.29
+Version: 0.3.31
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.3.31/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.31"

{babeldoc-0.3.29 → babeldoc-0.3.31}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.29"
+__version__ = "0.3.31"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc-0.3.31/babeldoc/document_il/babeldoc_exception/BabelDOCException.py

@@ -0,0 +1,8 @@
+class ScannedPDFError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class ExtractTextError(Exception):
+    def __init__(self, message):
+        super().__init__(message)

{babeldoc-0.3.29 → babeldoc-0.3.31}/babeldoc/document_il/midend/detect_scanned_file.py

@@ -130,4 +130,4 @@ class DetectScannedFile:
         )[:, :, ::-1]
         before_page_image = cv2.cvtColor(before_page_image, cv2.COLOR_RGB2GRAY)
         after_page_image = cv2.cvtColor(after_page_image, cv2.COLOR_RGB2GRAY)
-        return structural_similarity(before_page_image, after_page_image) > 0.9
+        return structural_similarity(before_page_image, after_page_image) > 0.98

{babeldoc-0.3.29 → babeldoc-0.3.31}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -16,6 +16,7 @@ from babeldoc.document_il.midend.il_translator import ILTranslator
 from babeldoc.document_il.midend.il_translator import PageTranslateTracker
 from babeldoc.document_il.translator.translator import BaseTranslator
 from babeldoc.document_il.utils.fontmap import FontMapper
+from babeldoc.document_il.utils.paragraph_helper import is_cid_paragraph
 from babeldoc.document_il.utils.priority_thread_pool_executor import (
     PriorityThreadPoolExecutor,
 )
@@ -162,6 +163,8 @@ class ILTranslatorLLMOnly:
         for paragraph in page.pdf_paragraph:
             if paragraph.debug_id is None or paragraph.unicode is None:
                 continue
+            if is_cid_paragraph(paragraph):
+                continue
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
             total_token_count += self.calc_token_count(paragraph.unicode)
             paragraphs.append(paragraph)

{babeldoc-0.3.29 → babeldoc-0.3.31}/babeldoc/document_il/midend/paragraph_finder.py

@@ -3,16 +3,19 @@ import random
 import re
 
 from babeldoc.document_il import Box
+from babeldoc.document_il import Document
 from babeldoc.document_il import Page
 from babeldoc.document_il import PdfCharacter
 from babeldoc.document_il import PdfLine
 from babeldoc.document_il import PdfParagraph
 from babeldoc.document_il import PdfParagraphComposition
 from babeldoc.document_il import PdfRectangle
+from babeldoc.document_il.babeldoc_exception.BabelDOCException import ExtractTextError
 from babeldoc.document_il.utils.layout_helper import Layout
 from babeldoc.document_il.utils.layout_helper import add_space_dummy_chars
 from babeldoc.document_il.utils.layout_helper import get_char_unicode_string
 from babeldoc.document_il.utils.layout_helper import is_bullet_point
+from babeldoc.document_il.utils.paragraph_helper import is_cid_paragraph
 from babeldoc.document_il.utils.style_helper import WHITE
 from babeldoc.translation_config import TranslationConfig
 
@@ -132,6 +135,25 @@ class ParagraphFinder:
                 self.process_page(page)
                 pbar.advance()
 
+            if self.check_cid_paragraph(document):
+                raise ExtractTextError("The document contains too many CID paragraphs.")
+
+            total_paragraph_count = 0
+            for page in document.page:
+                total_paragraph_count += len(page.pdf_paragraph)
+            if total_paragraph_count == 0:
+                raise ExtractTextError("The document contains no paragraphs.")
+
+    def check_cid_paragraph(self, doc: Document):
+        cid_para_count = 0
+        para_total = 0
+        for page in doc.page:
+            para_total += len(page.pdf_paragraph)
+            for para in page.pdf_paragraph:
+                if is_cid_paragraph(para):
+                    cid_para_count += 1
+        return cid_para_count / para_total > 0.8
+
     def bbox_overlap(self, bbox1: Box, bbox2: Box) -> bool:
         return (
             bbox1.x < bbox2.x2

{babeldoc-0.3.29 → babeldoc-0.3.31}/babeldoc/document_il/midend/typesetting.py

@@ -138,9 +138,18 @@ class TypesettingUnit:
             r"\u2D00-\u2D2F"  # Georgian Supplement
             r"\u1780-\u17FF"  # Khmer
             r"\u19E0-\u19FF"  # Khmer Symbols
+            r"\U00010B00-\U00010B3F"  # Avestan
+            r"\u1D00-\u1D7F"  # Phonetic Extensions
+            r"\u1400-\u167F"  # Unified Canadian Aboriginal Syllabics
+            r"\u0B00-\u0B7F"  # Oriya
+            r"\u0780-\u07BF"  # Thaana
+            r"\U0001E900-\U0001E95F"  # Adlam
+            r"\u1C80-\u1C8F"  # Cyrillic Extended-C
+            r"\U0001E030-\U0001E08F"  # Cyrillic Extended-D
             r"'"
             r"-"  # Hyphen
             r"·"  # Middle Dot (U+00B7) For Català
+            r"ʻ"  # Spacing Modifier Letters U+02BB
             r"]+$",
             unicode,
         ):

babeldoc-0.3.31/babeldoc/document_il/utils/paragraph_helper.py

@@ -0,0 +1,35 @@
+import logging
+import re
+
+from babeldoc.document_il import il_version_1
+
+logger = logging.getLogger(__name__)
+
+
+def is_cid_paragraph(paragraph: il_version_1.PdfParagraph):
+    chars: list[il_version_1.PdfCharacter] = []
+    for composition in paragraph.pdf_paragraph_composition:
+        if composition.pdf_line:
+            chars.extend(composition.pdf_line.pdf_character)
+        elif composition.pdf_same_style_characters:
+            chars.extend(composition.pdf_same_style_characters.pdf_character)
+        elif composition.pdf_same_style_unicode_characters:
+            chars.extend(composition.pdf_same_style_unicode_characters.unicode)
+        elif composition.pdf_formula:
+            chars.extend(composition.pdf_formula.pdf_character)
+        elif composition.pdf_character:
+            chars.append(composition.pdf_character)
+        else:
+            logger.error(
+                f"Unknown composition type. "
+                f"Composition: {composition}. "
+                f"Paragraph: {paragraph}. ",
+            )
+            continue
+
+    cid_count = 0
+    for char in chars:
+        if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+            cid_count += 1
+
+    return cid_count > len(chars) * 0.8

{babeldoc-0.3.29 → babeldoc-0.3.31}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.29"
+__version__ = "0.3.31"
 
 
 def create_parser():

{babeldoc-0.3.29 → babeldoc-0.3.31}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.29"
+version = "0.3.31"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.29"
+current_version = "0.3.31"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.29 → babeldoc-0.3.31}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.29"
+version = "0.3.31"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.3.29/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.29"

babeldoc-0.3.29/babeldoc/document_il/babeldoc_exception/BabelDOCException.py

@@ -1,3 +0,0 @@
-class ScannedPDFError(Exception):
-    def __init__(self, message):
-        super().__init__(message)