BabelDOC 0.3.31__tar.gz → 0.3.32__tar.gz

{babeldoc-0.3.31 → babeldoc-0.3.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.3.31
+Version: 0.3.32
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -88,6 +88,8 @@ PDF scientific paper translation and bilingual comparison library.
 - Provides a [Python API](#python-api).
 - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.
 
+[Supported Language](https://funstory-ai.github.io/BabelDOC/supported_languages/)
+
 ## Preview
 
 <div align="center">

{babeldoc-0.3.31 → babeldoc-0.3.32}/README.md

@@ -39,6 +39,8 @@ PDF scientific paper translation and bilingual comparison library.
 - Provides a [Python API](#python-api).
 - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.
 
+[Supported Language](https://funstory-ai.github.io/BabelDOC/supported_languages/)
+
 ## Preview
 
 <div align="center">

babeldoc-0.3.32/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.32"

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.31"
+__version__ = "0.3.32"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/document_il/backend/pdf_creater.py

@@ -782,7 +782,7 @@ class PDFCreater:
                 try:
                     doc.xref_set_key(doc[pageno].xref, name, box)
                 except Exception:
-                    logger.error(f"Error restoring media box {name} from PDF")
+                    logger.debug(f"Error restoring media box {name} from PDF")
 
     def write(
         self, translation_config: TranslationConfig, check_font_exists: bool = False

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -529,10 +529,10 @@ class StylesAndFormulas:
         pattern_text = (
             r"^("
             r"|Cambria.*"
-            r"|EUAlbertina.+"
-            r"|NimbusRomNo9L.+"
-            r"|GlosaMath.+"
-            r"|URWPalladioL.+"
+            r"|EUAlbertina.*"
+            r"|NimbusRomNo9L.*"
+            r"|GlosaMath.*"
+            r"|URWPalladioL.*"
             r"|CMSS.+"
             r"|Arial.*"
             r"|TimesNewRoman.*"

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/document_il/utils/paragraph_helper.py

@@ -14,7 +14,8 @@ def is_cid_paragraph(paragraph: il_version_1.PdfParagraph):
         elif composition.pdf_same_style_characters:
             chars.extend(composition.pdf_same_style_characters.pdf_character)
         elif composition.pdf_same_style_unicode_characters:
-            chars.extend(composition.pdf_same_style_unicode_characters.unicode)
+            continue
+        #     chars.extend(composition.pdf_same_style_unicode_characters.unicode)
         elif composition.pdf_formula:
             chars.extend(composition.pdf_formula.pdf_character)
         elif composition.pdf_character:

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/high_level.py

@@ -5,6 +5,7 @@ import io
 import logging
 import pathlib
 import shutil
+import struct
 import threading
 import time
 from asyncio import CancelledError
@@ -13,6 +14,7 @@ from typing import Any
 from typing import BinaryIO
 
 import pymupdf
+from pdfminer.cmapdb import IdentityCMap
 from pdfminer.pdfdocument import PDFDocument
 from pdfminer.pdfinterp import PDFResourceManager
 from pdfminer.pdfpage import PDFPage
@@ -49,6 +51,17 @@ from babeldoc.translation_config import TranslateResult
 from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
+
+def decode(_, code: bytes) -> tuple[int, ...]:
+    n = len(code) // 2
+    if n:
+        return struct.unpack_from(f">{n}H", code)
+    else:
+        return ()
+
+
+IdentityCMap.decode = decode
+
 logger = logging.getLogger(__name__)
 
 TRANSLATE_STAGES = [

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.31"
+__version__ = "0.3.32"
 
 
 def create_parser():

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/pdfinterp.py

@@ -19,12 +19,15 @@ from pdfminer.pdfinterp import PDFPageInterpreter
 from pdfminer.pdfinterp import PDFResourceManager
 from pdfminer.pdfinterp import PDFStackT
 from pdfminer.pdfpage import PDFPage
+from pdfminer.pdftypes import LITERALS_ASCII85_DECODE
 from pdfminer.pdftypes import PDFObjRef
+from pdfminer.pdftypes import PDFStream
 from pdfminer.pdftypes import dict_value
 from pdfminer.pdftypes import list_value
 from pdfminer.pdftypes import resolve1
 from pdfminer.pdftypes import stream_value
 from pdfminer.psexceptions import PSEOF
+from pdfminer.psexceptions import PSTypeError
 from pdfminer.psparser import PSKeyword
 from pdfminer.psparser import PSLiteral
 from pdfminer.psparser import keyword_name
@@ -33,6 +36,7 @@ from pdfminer.utils import MATRIX_IDENTITY
 from pdfminer.utils import Matrix
 from pdfminer.utils import Rect
 from pdfminer.utils import apply_matrix_pt
+from pdfminer.utils import choplist
 from pdfminer.utils import mult_matrix
 
 from babeldoc.document_il.frontend.il_creater import ILCreater
@@ -47,6 +51,42 @@ def safe_float(o: Any) -> float | None:
         return None
 
 
+class PDFContentParserEx(PDFContentParser):
+    def __init__(self, streams: Sequence[object]) -> None:
+        super().__init__(streams)
+
+    def do_keyword(self, pos: int, token: PSKeyword) -> None:
+        if token is self.KEYWORD_BI:
+            # inline image within a content stream
+            self.start_type(pos, "inline")
+        elif token is self.KEYWORD_ID:
+            try:
+                (_, objs) = self.end_type("inline")
+                if len(objs) % 2 != 0:
+                    error_msg = f"Invalid dictionary construct: {objs!r}"
+                    raise PSTypeError(error_msg)
+                d = {literal_name(k): resolve1(v) for (k, v) in choplist(2, objs)}
+                eos = b"EI"
+                filter_ = d.get("F", None)
+                if filter_:
+                    if isinstance(filter_, PSLiteral):
+                        filter_ = [filter_]
+                    if filter_[0] in LITERALS_ASCII85_DECODE:
+                        eos = b"~>"
+                (pos, data) = self.get_inline_data(pos + len(b"ID "), target=eos)
+                if eos != b"EI":  # it may be necessary for decoding
+                    data += eos
+                obj = PDFStream(d, data)
+                self.push((pos, obj))
+                if eos == b"EI":  # otherwise it is still in the stream
+                    self.push((pos, self.KEYWORD_EI))
+            except PSTypeError:
+                if settings.STRICT:
+                    raise
+        else:
+            self.push((pos, token))
+
+
 class PDFPageInterpreterEx(PDFPageInterpreter):
     """Processor for the content of a PDF page
 
@@ -409,7 +449,7 @@ class PDFPageInterpreterEx(PDFPageInterpreter):
             self.il_creater.on_new_stream()
             # 重载返回指令流
             try:
-                parser = PDFContentParser([stream])
+                parser = PDFContentParserEx([stream])
             except PSEOF:
                 # empty page
                 return

babeldoc-0.3.32/docs/supported_languages.md

@@ -0,0 +1,140 @@
+# Supported Languages
+
+For languages in the table below that do not rely on ligature support, BabelDOC provides good support. For languages that partially rely on ligatures, BabelDOC's translation results can generally meet self-reading needs. For languages that completely rely on ligatures (such as some Indian languages), BabelDOC does not currently support them.
+
+We are working hard to develop support for ligatures as soon as possible.
+
+| Language             | Language Code | Ligature Dependency |
+| :------------------- | :------------ | :------------------ |
+| Simplified Chinese   | zh-CN         | None                |
+| Traditional Chinese  | zh-TW         | None                |
+| Japanese             | JP            | None                |
+| Korean               | KR            | None                |
+| Polish               | PL            | Partial             |
+| Russian              | RU            | None                |
+| Spanish              | es            | None                |
+| Portuguese           | pt            | None                |
+| French               | fr            | Partial             |
+| Malay                | ms            | None                |
+| Indonesian           | id            | None                |
+| Turkmen              | tk            | None                |
+| Filipino (Tagalog)   | tl            | None                |
+| Vietnamese           | vi            | None                |
+| Azerbaijani          | az            | None                |
+| Kazakh (Latin)       | kk            | None                |
+| Kazakh (Cyrillic)    | kk            | None                |
+| German               | de            | None                |
+| Dutch                | nl            | None                |
+| Irish                | ga            | None                |
+| Italian              | it            | None                |
+| Greek                | el            | None                |
+| Swedish              | sv            | None                |
+| Danish               | da            | None                |
+| Norwegian            | no            | None                |
+| Icelandic            | is            | None                |
+| Finnish              | fi            | None                |
+| Ukrainian            | uk            | None                |
+| Czech                | cs            | None                |
+| Romanian             | ro            | None                |
+| Hungarian            | hu            | None                |
+| Slovak               | sk            | None                |
+| Croatian             | hr            | None                |
+| Estonian             | et            | None                |
+| Latvian              | lv            | None                |
+| Lithuanian           | lt            | None                |
+| Belarusian           | be            | None                |
+| Macedonian           | mk            | None                |
+| Albanian             | sq            | None                |
+| Serbian (Cyrillic)   | sr            | Partial             |
+| Serbian (Latin)      | sr            | Partial             |
+| Slovenian            | sl            | None                |
+| Catalan              | ca            | None                |
+| Bulgarian            | bg            | None                |
+| Maltese              | mt            | None                |
+| Swahili              | sw            | None                |
+| Amharic              | am            | None                |
+| Oromo                | om            | None                |
+| Tigrinya             | ti            | None                |
+| Haitian Creole       | ht            | None                |
+| Latin                | la            | None                |
+| Lao                  | lo            | None                |
+| Malayalam            | ml            | None                |
+| Gujarati             | gu            | None                |
+| Thai                 | th            | None                |
+| Burmese              | my            | Partial             |
+| Tamil                | ta            | None                |
+| Telugu               | te            | None                |
+| Oriya                | or            | Partial             |
+| Armenian             | hy            | None                |
+| Mongolian (Cyrillic) | mn            | None                |
+| Georgian             | ka            | None                |
+| Khmer                | km            | None                |
+| Bosnian              | bs            | None                |
+| Luxembourgish        | lb            | None                |
+| Moldovan             | ro            | None                |
+| Moldovan (Cyrillic)  | ro            | None                |
+| Romansh              | rm            | None                |
+| Turkish              | tr            | None                |
+| Sinhala              | si            | None                |
+| Uzbek                | uz            | None                |
+| Kyrgyz               | ky            | None                |
+| Tajik                | tg            | None                |
+| Abkhazian            | ab            | None                |
+| Afar                 | aa            | None                |
+| Afrikaans            | af            | None                |
+| Akan                 | ak            | None                |
+| Aragonese            | an            | None                |
+| Avaric               | av            | None                |
+| Avestan              | ae            | None                |
+| Ewe                  | ee            | None                |
+| Aymara               | ay            | None                |
+| Ojibwa               | oj            | None                |
+| Occitan              | oc            | None                |
+| Oriya                | or            | None                |
+| Ossetian             | os            | None                |
+| Pali                 | pi            | None                |
+| Bashkir              | ba            | None                |
+| Basque               | eu            | None                |
+| Bambara              | bm            | None                |
+| Breton               | br            | None                |
+| Chamorro             | ch            | None                |
+| Chechen              | ce            | None                |
+| Chuvash              | cv            | None                |
+| Tswana               | tn            | None                |
+| Divehi               | dv            | None                |
+| Ndebele, South       | nr            | None                |
+| Ndonga               | ng            | None                |
+| Faroese              | fo            | None                |
+| Fijian               | fj            | None                |
+| Frisian, Western     | fy            | None                |
+| Fulah                | ff            | None                |
+| Ganda                | lg            | None                |
+| Kongo                | kg            | None                |
+| Kalaallisut          | kl            | None                |
+| Church Slavic        | cu            | None                |
+| Guarani              | gn            | None                |
+| Interlingua          | ia            | None                |
+| Hausa                | ha            | None                |
+| Herero               | hz            | None                |
+| Kikuyu               | ki            | None                |
+| Rundi                | rn            | None                |
+| Kinyarwanda          | rw            | None                |
+| Kirghiz              | ky            | None                |
+| Galician             | gl            | None                |
+| Kanuri               | kr            | None                |
+| Cornish              | kw            | None                |
+| Komi                 | kv            | None                |
+| Xhosa                | xh            | None                |
+| Corsican             | co            | None                |
+| Cree                 | cr            | None                |
+| Croatian             | hr            | None                |
+| Quechua              | qu            | None                |
+| Kurdish (Latin)      | ku            | None                |
+| Kuanyama             | kj            | None                |
+| Limburgan            | li            | None                |
+| Lingala              | ln            | None                |
+| Manx                 | gv            | None                |
+| Malagasy             | mg            | None                |
+| Marshallese          | mh            | None                |
+| Maori                | mi            | None                |
+

{babeldoc-0.3.31 → babeldoc-0.3.32}/mkdocs.yml

@@ -132,6 +132,7 @@ not_in_nav: |
 # Page tree
 nav:
   - Home: index.md
+  - Supported languages: supported_languages.md
   - API:
     - Async Translation API: ImplementationDetails/AsyncTranslate/AsyncTranslate.md
   - Implementation Details:

{babeldoc-0.3.31 → babeldoc-0.3.32}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.31"
+version = "0.3.32"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.31"
+current_version = "0.3.32"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.3.31 → babeldoc-0.3.32}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.3.31"
+version = "0.3.32"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.3.31/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.31"

{babeldoc-0.3.31 → babeldoc-0.3.32}/babeldoc/document_il/midend/paragraph_finder.py

@@ -135,15 +135,15 @@ class ParagraphFinder:
                 self.process_page(page)
                 pbar.advance()
 
-            if self.check_cid_paragraph(document):
-                raise ExtractTextError("The document contains too many CID paragraphs.")
-
             total_paragraph_count = 0
             for page in document.page:
                 total_paragraph_count += len(page.pdf_paragraph)
             if total_paragraph_count == 0:
                 raise ExtractTextError("The document contains no paragraphs.")
 
+            if self.check_cid_paragraph(document):
+                raise ExtractTextError("The document contains too many CID paragraphs.")
+
     def check_cid_paragraph(self, doc: Document):
         cid_para_count = 0
         para_total = 0