BabelDOC 0.3.30__tar.gz → 0.3.31__tar.gz

{babeldoc-0.3.30 → babeldoc-0.3.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.30
+Version: 0.3.31
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.3.31/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.31"

{babeldoc-0.3.30 → babeldoc-0.3.31}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.30"
+__version__ = "0.3.31"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc-0.3.31/babeldoc/document_il/babeldoc_exception/BabelDOCException.py

@@ -0,0 +1,8 @@
+class ScannedPDFError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class ExtractTextError(Exception):
+    def __init__(self, message):
+        super().__init__(message)

{babeldoc-0.3.30 → babeldoc-0.3.31}/babeldoc/document_il/midend/detect_scanned_file.py

@@ -130,4 +130,4 @@ class DetectScannedFile:
         )[:, :, ::-1]
         before_page_image = cv2.cvtColor(before_page_image, cv2.COLOR_RGB2GRAY)
         after_page_image = cv2.cvtColor(after_page_image, cv2.COLOR_RGB2GRAY)
-        return structural_similarity(before_page_image, after_page_image) > 0.9
+        return structural_similarity(before_page_image, after_page_image) > 0.98

{babeldoc-0.3.30 → babeldoc-0.3.31}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -16,6 +16,7 @@ from babeldoc.document_il.midend.il_translator import ILTranslator
 from babeldoc.document_il.midend.il_translator import PageTranslateTracker
 from babeldoc.document_il.translator.translator import BaseTranslator
 from babeldoc.document_il.utils.fontmap import FontMapper
+from babeldoc.document_il.utils.paragraph_helper import is_cid_paragraph
 from babeldoc.document_il.utils.priority_thread_pool_executor import (
     PriorityThreadPoolExecutor,
 )
@@ -162,6 +163,8 @@ class ILTranslatorLLMOnly:
         for paragraph in page.pdf_paragraph:
             if paragraph.debug_id is None or paragraph.unicode is None:
                 continue
+            if is_cid_paragraph(paragraph):
+                continue
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
             total_token_count += self.calc_token_count(paragraph.unicode)
             paragraphs.append(paragraph)

{babeldoc-0.3.30 → babeldoc-0.3.31}/babeldoc/document_il/midend/paragraph_finder.py

@@ -3,16 +3,19 @@ import random
 import re
 
 from babeldoc.document_il import Box
+from babeldoc.document_il import Document
 from babeldoc.document_il import Page
 from babeldoc.document_il import PdfCharacter
 from babeldoc.document_il import PdfLine
 from babeldoc.document_il import PdfParagraph
 from babeldoc.document_il import PdfParagraphComposition
 from babeldoc.document_il import PdfRectangle
+from babeldoc.document_il.babeldoc_exception.BabelDOCException import ExtractTextError
 from babeldoc.document_il.utils.layout_helper import Layout
 from babeldoc.document_il.utils.layout_helper import add_space_dummy_chars
 from babeldoc.document_il.utils.layout_helper import get_char_unicode_string
 from babeldoc.document_il.utils.layout_helper import is_bullet_point
+from babeldoc.document_il.utils.paragraph_helper import is_cid_paragraph
 from babeldoc.document_il.utils.style_helper import WHITE
 from babeldoc.translation_config import TranslationConfig
 
@@ -132,6 +135,25 @@ class ParagraphFinder:
                 self.process_page(page)
                 pbar.advance()
 
+            if self.check_cid_paragraph(document):
+                raise ExtractTextError("The document contains too many CID paragraphs.")
+
+            total_paragraph_count = 0
+            for page in document.page:
+                total_paragraph_count += len(page.pdf_paragraph)
+            if total_paragraph_count == 0:
+                raise ExtractTextError("The document contains no paragraphs.")
+
+    def check_cid_paragraph(self, doc: Document):
+        cid_para_count = 0
+        para_total = 0
+        for page in doc.page:
+            para_total += len(page.pdf_paragraph)
+            for para in page.pdf_paragraph:
+                if is_cid_paragraph(para):
+                    cid_para_count += 1
+        return cid_para_count / para_total > 0.8
+
     def bbox_overlap(self, bbox1: Box, bbox2: Box) -> bool:
         return (
             bbox1.x < bbox2.x2

babeldoc-0.3.31/babeldoc/document_il/utils/paragraph_helper.py

@@ -0,0 +1,35 @@
+import logging
+import re
+
+from babeldoc.document_il import il_version_1
+
+logger = logging.getLogger(__name__)
+
+
+def is_cid_paragraph(paragraph: il_version_1.PdfParagraph):
+    chars: list[il_version_1.PdfCharacter] = []
+    for composition in paragraph.pdf_paragraph_composition:
+        if composition.pdf_line:
+            chars.extend(composition.pdf_line.pdf_character)
+        elif composition.pdf_same_style_characters:
+            chars.extend(composition.pdf_same_style_characters.pdf_character)
+        elif composition.pdf_same_style_unicode_characters:
+            chars.extend(composition.pdf_same_style_unicode_characters.unicode)
+        elif composition.pdf_formula:
+            chars.extend(composition.pdf_formula.pdf_character)
+        elif composition.pdf_character:
+            chars.append(composition.pdf_character)
+        else:
+            logger.error(
+                f"Unknown composition type. "
+                f"Composition: {composition}. "
+                f"Paragraph: {paragraph}. ",
+            )
+            continue
+
+    cid_count = 0
+    for char in chars:
+        if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+            cid_count += 1
+
+    return cid_count > len(chars) * 0.8

{babeldoc-0.3.30 → babeldoc-0.3.31}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.30"
+__version__ = "0.3.31"
 
 
 def create_parser():

{babeldoc-0.3.30 → babeldoc-0.3.31}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.30"
+version = "0.3.31"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.30"
+current_version = "0.3.31"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.30 → babeldoc-0.3.31}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.30"
+version = "0.3.31"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.3.30/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.30"

babeldoc-0.3.30/babeldoc/document_il/babeldoc_exception/BabelDOCException.py

@@ -1,3 +0,0 @@
-class ScannedPDFError(Exception):
-    def __init__(self, message):
-        super().__init__(message)