BabelDOC 0.3.30__tar.gz → 0.3.32__tar.gz

{babeldoc-0.3.30 → babeldoc-0.3.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.30
+Version: 0.3.32
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -88,6 +88,8 @@ PDF scientific paper translation and bilingual comparison library.
 - Provides a [Python API](#python-api).
 - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.
 
+[Supported Language](https://funstory-ai.github.io/BabelDOC/supported_languages/)
+
 ## Preview
 
 <div align="center">

{babeldoc-0.3.30 → babeldoc-0.3.32}/README.md

@@ -39,6 +39,8 @@ PDF scientific paper translation and bilingual comparison library.
 - Provides a [Python API](#python-api).
 - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.
 
+[Supported Language](https://funstory-ai.github.io/BabelDOC/supported_languages/)
+
 ## Preview
 
 <div align="center">

babeldoc-0.3.32/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.32"

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.30"
+__version__ = "0.3.32"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc-0.3.32/babeldoc/document_il/babeldoc_exception/BabelDOCException.py

@@ -0,0 +1,8 @@
+class ScannedPDFError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class ExtractTextError(Exception):
+    def __init__(self, message):
+        super().__init__(message)

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/document_il/backend/pdf_creater.py

@@ -782,7 +782,7 @@ class PDFCreater:
                 try:
                     doc.xref_set_key(doc[pageno].xref, name, box)
                 except Exception:
-                    logger.error(f"Error restoring media box {name} from PDF")
+                    logger.debug(f"Error restoring media box {name} from PDF")
 
     def write(
         self, translation_config: TranslationConfig, check_font_exists: bool = False

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/document_il/midend/detect_scanned_file.py

@@ -130,4 +130,4 @@ class DetectScannedFile:
         )[:, :, ::-1]
         before_page_image = cv2.cvtColor(before_page_image, cv2.COLOR_RGB2GRAY)
         after_page_image = cv2.cvtColor(after_page_image, cv2.COLOR_RGB2GRAY)
-        return structural_similarity(before_page_image, after_page_image) > 0.9
+        return structural_similarity(before_page_image, after_page_image) > 0.98

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -16,6 +16,7 @@ from babeldoc.document_il.midend.il_translator import ILTranslator
 from babeldoc.document_il.midend.il_translator import PageTranslateTracker
 from babeldoc.document_il.translator.translator import BaseTranslator
 from babeldoc.document_il.utils.fontmap import FontMapper
+from babeldoc.document_il.utils.paragraph_helper import is_cid_paragraph
 from babeldoc.document_il.utils.priority_thread_pool_executor import (
     PriorityThreadPoolExecutor,
 )
@@ -162,6 +163,8 @@ class ILTranslatorLLMOnly:
         for paragraph in page.pdf_paragraph:
             if paragraph.debug_id is None or paragraph.unicode is None:
                 continue
+            if is_cid_paragraph(paragraph):
+                continue
             # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
             total_token_count += self.calc_token_count(paragraph.unicode)
             paragraphs.append(paragraph)

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/document_il/midend/paragraph_finder.py

@@ -3,16 +3,19 @@ import random
 import re
 
 from babeldoc.document_il import Box
+from babeldoc.document_il import Document
 from babeldoc.document_il import Page
 from babeldoc.document_il import PdfCharacter
 from babeldoc.document_il import PdfLine
 from babeldoc.document_il import PdfParagraph
 from babeldoc.document_il import PdfParagraphComposition
 from babeldoc.document_il import PdfRectangle
+from babeldoc.document_il.babeldoc_exception.BabelDOCException import ExtractTextError
 from babeldoc.document_il.utils.layout_helper import Layout
 from babeldoc.document_il.utils.layout_helper import add_space_dummy_chars
 from babeldoc.document_il.utils.layout_helper import get_char_unicode_string
 from babeldoc.document_il.utils.layout_helper import is_bullet_point
+from babeldoc.document_il.utils.paragraph_helper import is_cid_paragraph
 from babeldoc.document_il.utils.style_helper import WHITE
 from babeldoc.translation_config import TranslationConfig
 
@@ -132,6 +135,25 @@ class ParagraphFinder:
                 self.process_page(page)
                 pbar.advance()
 
+            total_paragraph_count = 0
+            for page in document.page:
+                total_paragraph_count += len(page.pdf_paragraph)
+            if total_paragraph_count == 0:
+                raise ExtractTextError("The document contains no paragraphs.")
+
+            if self.check_cid_paragraph(document):
+                raise ExtractTextError("The document contains too many CID paragraphs.")
+
+    def check_cid_paragraph(self, doc: Document):
+        cid_para_count = 0
+        para_total = 0
+        for page in doc.page:
+            para_total += len(page.pdf_paragraph)
+            for para in page.pdf_paragraph:
+                if is_cid_paragraph(para):
+                    cid_para_count += 1
+        return cid_para_count / para_total > 0.8
+
     def bbox_overlap(self, bbox1: Box, bbox2: Box) -> bool:
         return (
             bbox1.x < bbox2.x2

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -529,10 +529,10 @@ class StylesAndFormulas:
         pattern_text = (
             r"^("
             r"|Cambria.*"
-            r"|EUAlbertina.+"
-            r"|NimbusRomNo9L.+"
-            r"|GlosaMath.+"
-            r"|URWPalladioL.+"
+            r"|EUAlbertina.*"
+            r"|NimbusRomNo9L.*"
+            r"|GlosaMath.*"
+            r"|URWPalladioL.*"
             r"|CMSS.+"
             r"|Arial.*"
             r"|TimesNewRoman.*"

babeldoc-0.3.32/babeldoc/document_il/utils/paragraph_helper.py

@@ -0,0 +1,36 @@
+import logging
+import re
+
+from babeldoc.document_il import il_version_1
+
+logger = logging.getLogger(__name__)
+
+
+def is_cid_paragraph(paragraph: il_version_1.PdfParagraph):
+    chars: list[il_version_1.PdfCharacter] = []
+    for composition in paragraph.pdf_paragraph_composition:
+        if composition.pdf_line:
+            chars.extend(composition.pdf_line.pdf_character)
+        elif composition.pdf_same_style_characters:
+            chars.extend(composition.pdf_same_style_characters.pdf_character)
+        elif composition.pdf_same_style_unicode_characters:
+            continue
+        #     chars.extend(composition.pdf_same_style_unicode_characters.unicode)
+        elif composition.pdf_formula:
+            chars.extend(composition.pdf_formula.pdf_character)
+        elif composition.pdf_character:
+            chars.append(composition.pdf_character)
+        else:
+            logger.error(
+                f"Unknown composition type. "
+                f"Composition: {composition}. "
+                f"Paragraph: {paragraph}. ",
+            )
+            continue
+
+    cid_count = 0
+    for char in chars:
+        if re.match(r"^\(cid:\d+\)$", char.char_unicode):
+            cid_count += 1
+
+    return cid_count > len(chars) * 0.8

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/high_level.py

@@ -5,6 +5,7 @@ import io
 import logging
 import pathlib
 import shutil
+import struct
 import threading
 import time
 from asyncio import CancelledError
@@ -13,6 +14,7 @@ from typing import Any
 from typing import BinaryIO
 
 import pymupdf
+from pdfminer.cmapdb import IdentityCMap
 from pdfminer.pdfdocument import PDFDocument
 from pdfminer.pdfinterp import PDFResourceManager
 from pdfminer.pdfpage import PDFPage
@@ -49,6 +51,17 @@ from babeldoc.translation_config import TranslateResult
 from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
+
+def decode(_, code: bytes) -> tuple[int, ...]:
+    n = len(code) // 2
+    if n:
+        return struct.unpack_from(f">{n}H", code)
+    else:
+        return ()
+
+
+IdentityCMap.decode = decode
+
 logger = logging.getLogger(__name__)
 
 TRANSLATE_STAGES = [

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.30"
+__version__ = "0.3.32"
 
 
 def create_parser():

{babeldoc-0.3.30 → babeldoc-0.3.32}/babeldoc/pdfinterp.py

@@ -19,12 +19,15 @@ from pdfminer.pdfinterp import PDFPageInterpreter
 from pdfminer.pdfinterp import PDFResourceManager
 from pdfminer.pdfinterp import PDFStackT
 from pdfminer.pdfpage import PDFPage
+from pdfminer.pdftypes import LITERALS_ASCII85_DECODE
 from pdfminer.pdftypes import PDFObjRef
+from pdfminer.pdftypes import PDFStream
 from pdfminer.pdftypes import dict_value
 from pdfminer.pdftypes import list_value
 from pdfminer.pdftypes import resolve1
 from pdfminer.pdftypes import stream_value
 from pdfminer.psexceptions import PSEOF
+from pdfminer.psexceptions import PSTypeError
 from pdfminer.psparser import PSKeyword
 from pdfminer.psparser import PSLiteral
 from pdfminer.psparser import keyword_name
@@ -33,6 +36,7 @@ from pdfminer.utils import MATRIX_IDENTITY
 from pdfminer.utils import Matrix
 from pdfminer.utils import Rect
 from pdfminer.utils import apply_matrix_pt
+from pdfminer.utils import choplist
 from pdfminer.utils import mult_matrix
 
 from babeldoc.document_il.frontend.il_creater import ILCreater
@@ -47,6 +51,42 @@ def safe_float(o: Any) -> float | None:
         return None
 
 
+class PDFContentParserEx(PDFContentParser):
+    def __init__(self, streams: Sequence[object]) -> None:
+        super().__init__(streams)
+
+    def do_keyword(self, pos: int, token: PSKeyword) -> None:
+        if token is self.KEYWORD_BI:
+            # inline image within a content stream
+            self.start_type(pos, "inline")
+        elif token is self.KEYWORD_ID:
+            try:
+                (_, objs) = self.end_type("inline")
+                if len(objs) % 2 != 0:
+                    error_msg = f"Invalid dictionary construct: {objs!r}"
+                    raise PSTypeError(error_msg)
+                d = {literal_name(k): resolve1(v) for (k, v) in choplist(2, objs)}
+                eos = b"EI"
+                filter_ = d.get("F", None)
+                if filter_:
+                    if isinstance(filter_, PSLiteral):
+                        filter_ = [filter_]
+                    if filter_[0] in LITERALS_ASCII85_DECODE:
+                        eos = b"~>"
+                (pos, data) = self.get_inline_data(pos + len(b"ID "), target=eos)
+                if eos != b"EI":  # it may be necessary for decoding
+                    data += eos
+                obj = PDFStream(d, data)
+                self.push((pos, obj))
+                if eos == b"EI":  # otherwise it is still in the stream
+                    self.push((pos, self.KEYWORD_EI))
+            except PSTypeError:
+                if settings.STRICT:
+                    raise
+        else:
+            self.push((pos, token))
+
+
 class PDFPageInterpreterEx(PDFPageInterpreter):
     """Processor for the content of a PDF page
 
@@ -409,7 +449,7 @@ class PDFPageInterpreterEx(PDFPageInterpreter):
             self.il_creater.on_new_stream()
             # 重载返回指令流
             try:
-                parser = PDFContentParser([stream])
+                parser = PDFContentParserEx([stream])
             except PSEOF:
                 # empty page
                 return

babeldoc-0.3.32/docs/supported_languages.md

@@ -0,0 +1,140 @@
+# Supported Languages
+
+For languages in the table below that do not rely on ligature support, BabelDOC provides good support. For languages that partially rely on ligatures, BabelDOC's translation results can generally meet self-reading needs. For languages that completely rely on ligatures (such as some Indian languages), BabelDOC does not currently support them.
+
+We are working hard to develop support for ligatures as soon as possible.
+
+| Language             | Language Code | Ligature Dependency |
+| :------------------- | :------------ | :------------------ |
+| Simplified Chinese   | zh-CN         | None                |
+| Traditional Chinese  | zh-TW         | None                |
+| Japanese             | JP            | None                |
+| Korean               | KR            | None                |
+| Polish               | PL            | Partial             |
+| Russian              | RU            | None                |
+| Spanish              | es            | None                |
+| Portuguese           | pt            | None                |
+| French               | fr            | Partial             |
+| Malay                | ms            | None                |
+| Indonesian           | id            | None                |
+| Turkmen              | tk            | None                |
+| Filipino (Tagalog)   | tl            | None                |
+| Vietnamese           | vi            | None                |
+| Azerbaijani          | az            | None                |
+| Kazakh (Latin)       | kk            | None                |
+| Kazakh (Cyrillic)    | kk            | None                |
+| German               | de            | None                |
+| Dutch                | nl            | None                |
+| Irish                | ga            | None                |
+| Italian              | it            | None                |
+| Greek                | el            | None                |
+| Swedish              | sv            | None                |
+| Danish               | da            | None                |
+| Norwegian            | no            | None                |
+| Icelandic            | is            | None                |
+| Finnish              | fi            | None                |
+| Ukrainian            | uk            | None                |
+| Czech                | cs            | None                |
+| Romanian             | ro            | None                |
+| Hungarian            | hu            | None                |
+| Slovak               | sk            | None                |
+| Croatian             | hr            | None                |
+| Estonian             | et            | None                |
+| Latvian              | lv            | None                |
+| Lithuanian           | lt            | None                |
+| Belarusian           | be            | None                |
+| Macedonian           | mk            | None                |
+| Albanian             | sq            | None                |
+| Serbian (Cyrillic)   | sr            | Partial             |
+| Serbian (Latin)      | sr            | Partial             |
+| Slovenian            | sl            | None                |
+| Catalan              | ca            | None                |
+| Bulgarian            | bg            | None                |
+| Maltese              | mt            | None                |
+| Swahili              | sw            | None                |
+| Amharic              | am            | None                |
+| Oromo                | om            | None                |
+| Tigrinya             | ti            | None                |
+| Haitian Creole       | ht            | None                |
+| Latin                | la            | None                |
+| Lao                  | lo            | None                |
+| Malayalam            | ml            | None                |
+| Gujarati             | gu            | None                |
+| Thai                 | th            | None                |
+| Burmese              | my            | Partial             |
+| Tamil                | ta            | None                |
+| Telugu               | te            | None                |
+| Oriya                | or            | Partial             |
+| Armenian             | hy            | None                |
+| Mongolian (Cyrillic) | mn            | None                |
+| Georgian             | ka            | None                |
+| Khmer                | km            | None                |
+| Bosnian              | bs            | None                |
+| Luxembourgish        | lb            | None                |
+| Moldovan             | ro            | None                |
+| Moldovan (Cyrillic)  | ro            | None                |
+| Romansh              | rm            | None                |
+| Turkish              | tr            | None                |
+| Sinhala              | si            | None                |
+| Uzbek                | uz            | None                |
+| Kyrgyz               | ky            | None                |
+| Tajik                | tg            | None                |
+| Abkhazian            | ab            | None                |
+| Afar                 | aa            | None                |
+| Afrikaans            | af            | None                |
+| Akan                 | ak            | None                |
+| Aragonese            | an            | None                |
+| Avaric               | av            | None                |
+| Avestan              | ae            | None                |
+| Ewe                  | ee            | None                |
+| Aymara               | ay            | None                |
+| Ojibwa               | oj            | None                |
+| Occitan              | oc            | None                |
+| Oriya                | or            | None                |
+| Ossetian             | os            | None                |
+| Pali                 | pi            | None                |
+| Bashkir              | ba            | None                |
+| Basque               | eu            | None                |
+| Bambara              | bm            | None                |
+| Breton               | br            | None                |
+| Chamorro             | ch            | None                |
+| Chechen              | ce            | None                |
+| Chuvash              | cv            | None                |
+| Tswana               | tn            | None                |
+| Divehi               | dv            | None                |
+| Ndebele, South       | nr            | None                |
+| Ndonga               | ng            | None                |
+| Faroese              | fo            | None                |
+| Fijian               | fj            | None                |
+| Frisian, Western     | fy            | None                |
+| Fulah                | ff            | None                |
+| Ganda                | lg            | None                |
+| Kongo                | kg            | None                |
+| Kalaallisut          | kl            | None                |
+| Church Slavic        | cu            | None                |
+| Guarani              | gn            | None                |
+| Interlingua          | ia            | None                |
+| Hausa                | ha            | None                |
+| Herero               | hz            | None                |
+| Kikuyu               | ki            | None                |
+| Rundi                | rn            | None                |
+| Kinyarwanda          | rw            | None                |
+| Kirghiz              | ky            | None                |
+| Galician             | gl            | None                |
+| Kanuri               | kr            | None                |
+| Cornish              | kw            | None                |
+| Komi                 | kv            | None                |
+| Xhosa                | xh            | None                |
+| Corsican             | co            | None                |
+| Cree                 | cr            | None                |
+| Croatian             | hr            | None                |
+| Quechua              | qu            | None                |
+| Kurdish (Latin)      | ku            | None                |
+| Kuanyama             | kj            | None                |
+| Limburgan            | li            | None                |
+| Lingala              | ln            | None                |
+| Manx                 | gv            | None                |
+| Malagasy             | mg            | None                |
+| Marshallese          | mh            | None                |
+| Maori                | mi            | None                |
+

{babeldoc-0.3.30 → babeldoc-0.3.32}/mkdocs.yml

@@ -132,6 +132,7 @@ not_in_nav: |
 # Page tree
 nav:
   - Home: index.md
+  - Supported languages: supported_languages.md
   - API:
     - Async Translation API: ImplementationDetails/AsyncTranslate/AsyncTranslate.md
   - Implementation Details:

{babeldoc-0.3.30 → babeldoc-0.3.32}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.30"
+version = "0.3.32"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.30"
+current_version = "0.3.32"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.30 → babeldoc-0.3.32}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.30"
+version = "0.3.32"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.3.30/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.30"

babeldoc-0.3.30/babeldoc/document_il/babeldoc_exception/BabelDOCException.py

@@ -1,3 +0,0 @@
-class ScannedPDFError(Exception):
-    def __init__(self, message):
-        super().__init__(message)