BabelDOC 0.5.0__tar.gz → 0.5.1__tar.gz

{babeldoc-0.5.0 → babeldoc-0.5.1}/.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@d9e0f98d3fc6adb07d1e3d37f3043649ddad06a1 # v6.5.0
         with:
           python-version: "3.12"
           enable-cache: true

{babeldoc-0.5.0 → babeldoc-0.5.1}/.github/workflows/publish-to-pypi.yml

@@ -35,7 +35,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           
       - name: Setup uv with Python 3.12
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@d9e0f98d3fc6adb07d1e3d37f3043649ddad06a1 # v6.5.0
         with:
           python-version: "3.12"
           enable-cache: true

{babeldoc-0.5.0 → babeldoc-0.5.1}/.github/workflows/test.yml

@@ -28,7 +28,7 @@ jobs:
           path: ~/.cache/babeldoc
           key: babeldoc-assets-${{ hashFiles('babeldoc/assets/embedding_assets_metadata.py') }}
       - name: Setup uv with Python ${{ matrix.python-version }}
-        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        uses: astral-sh/setup-uv@d9e0f98d3fc6adb07d1e3d37f3043649ddad06a1 # v6.5.0
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true

{babeldoc-0.5.0 → babeldoc-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.5.0
+Version: 0.5.1
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues

babeldoc-0.5.1/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.1"

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/format/pdf/document_il/backend/pdf_creater.py

@@ -1256,103 +1256,9 @@ class PDFCreater:
                 len(self.docs.page),
             ) as pbar:
                 for page in self.docs.page:
-                    assert page.cropbox is not None and page.cropbox.box is not None
-                    page_crop_box = page.cropbox.box
-                    ctm_for_ops = (
-                        1,
-                        0,
-                        0,
-                        1,
-                        -page_crop_box.x,
-                        -page_crop_box.y,
+                    self.update_page_content_stream(
+                        check_font_exists, page, pdf, translation_config
                     )
-                    ctm_for_ops = (
-                        f" {' '.join(f'{x:f}' for x in ctm_for_ops)} cm ".encode()
-                    )
-
-                    translation_config.raise_if_cancelled()
-                    xobj_available_fonts = {}
-                    xobj_draw_ops = {}
-                    xobj_encoding_length_map = {}
-                    available_font_list = self.get_available_font_list(pdf, page)
-                    page_encoding_length_map: dict[str | None, int | None] = {
-                        f.font_id: f.encoding_length for f in page.pdf_font
-                    }
-                    all_encoding_length_map = page_encoding_length_map.copy()
-
-                    for xobj in page.pdf_xobject:
-                        xobj_available_fonts[xobj.xobj_id] = available_font_list.copy()
-                        try:
-                            xobj_available_fonts[xobj.xobj_id].update(
-                                self.get_xobj_available_fonts(xobj.xref_id, pdf),
-                            )
-                        except Exception:
-                            pass
-                        xobj_encoding_length_map[xobj.xobj_id] = {
-                            f.font_id: f.encoding_length for f in xobj.pdf_font
-                        }
-                        all_encoding_length_map.update(
-                            xobj_encoding_length_map[xobj.xobj_id]
-                        )
-                        xobj_encoding_length_map[xobj.xobj_id].update(
-                            page_encoding_length_map
-                        )
-                        xobj_op = BitStream()
-                        base_op = xobj.base_operations.value
-                        base_op = zstd_decompress(base_op)
-                        xobj_op.append(base_op.encode())
-                        xobj_draw_ops[xobj.xobj_id] = xobj_op
-
-                    page_op = BitStream()
-                    # q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}
-                    # page_op.append(b"q ")
-                    # base_op = page.base_operations.value
-                    # base_op = zstd_decompress(base_op)
-                    # page_op.append(base_op.encode())
-                    # page_op.append(b" \n")
-                    page_op.append(ctm_for_ops)
-                    page_op.append(b" \n")
-
-                    # Create render context
-                    context = RenderContext(
-                        pdf_creator=self,
-                        page=page,
-                        available_font_list=available_font_list,
-                        page_encoding_length_map=page_encoding_length_map,
-                        all_encoding_length_map=all_encoding_length_map,
-                        xobj_available_fonts=xobj_available_fonts,
-                        xobj_encoding_length_map=xobj_encoding_length_map,
-                        ctm_for_ops=ctm_for_ops,
-                        check_font_exists=check_font_exists,
-                    )
-
-                    # Create render units for all renderable objects
-                    render_units = self.create_render_units_for_page(
-                        page, translation_config
-                    )
-
-                    # Render all units to their appropriate streams
-                    self.render_units_to_stream(
-                        render_units, context, page_op, xobj_draw_ops
-                    )
-
-                    # Update xobject streams
-                    for xobj in page.pdf_xobject:
-                        draw_op = xobj_draw_ops[xobj.xobj_id]
-                        try:
-                            pdf.update_stream(xobj.xref_id, draw_op.tobytes())
-                        except Exception:
-                            logger.warning(
-                                f"update xref {xobj.xref_id} stream fail, continue"
-                            )
-
-                    draw_op = page_op
-                    op_container = pdf.get_new_xref()
-                    # Since this is a draw instruction container,
-                    # no additional information is needed
-                    pdf.update_object(op_container, "<<>>")
-                    pdf.update_stream(op_container, draw_op.tobytes())
-                    pdf[page.page_number].set_contents(op_container)
                     pbar.advance()
             translation_config.raise_if_cancelled()
             gc_level = 1
@@ -1512,3 +1418,90 @@ class PDFCreater:
             if not check_font_exists:
                 return self.write(translation_config, True)
             raise
+
+    def update_page_content_stream(
+        self, check_font_exists, page, pdf, translation_config, skip_char: bool = False
+    ):
+        assert page.cropbox is not None and page.cropbox.box is not None
+        page_crop_box = page.cropbox.box
+        ctm_for_ops = (
+            1,
+            0,
+            0,
+            1,
+            -page_crop_box.x,
+            -page_crop_box.y,
+        )
+        ctm_for_ops = f" {' '.join(f'{x:f}' for x in ctm_for_ops)} cm ".encode()
+        translation_config.raise_if_cancelled()
+        xobj_available_fonts = {}
+        xobj_draw_ops = {}
+        xobj_encoding_length_map = {}
+        available_font_list = self.get_available_font_list(pdf, page)
+        page_encoding_length_map: dict[str | None, int | None] = {
+            f.font_id: f.encoding_length for f in page.pdf_font
+        }
+        all_encoding_length_map = page_encoding_length_map.copy()
+        for xobj in page.pdf_xobject:
+            xobj_available_fonts[xobj.xobj_id] = available_font_list.copy()
+            try:
+                xobj_available_fonts[xobj.xobj_id].update(
+                    self.get_xobj_available_fonts(xobj.xref_id, pdf),
+                )
+            except Exception:
+                pass
+            xobj_encoding_length_map[xobj.xobj_id] = {
+                f.font_id: f.encoding_length for f in xobj.pdf_font
+            }
+            all_encoding_length_map.update(xobj_encoding_length_map[xobj.xobj_id])
+            xobj_encoding_length_map[xobj.xobj_id].update(page_encoding_length_map)
+            xobj_op = BitStream()
+            base_op = xobj.base_operations.value
+            base_op = zstd_decompress(base_op)
+            xobj_op.append(base_op.encode())
+            xobj_draw_ops[xobj.xobj_id] = xobj_op
+        page_op = BitStream()
+        # q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}
+        # page_op.append(b"q ")
+        # base_op = page.base_operations.value
+        # base_op = zstd_decompress(base_op)
+        # page_op.append(base_op.encode())
+        # page_op.append(b" \n")
+        page_op.append(ctm_for_ops)
+        page_op.append(b" \n")
+        # Create render context
+        context = RenderContext(
+            pdf_creator=self,
+            page=page,
+            available_font_list=available_font_list,
+            page_encoding_length_map=page_encoding_length_map,
+            all_encoding_length_map=all_encoding_length_map,
+            xobj_available_fonts=xobj_available_fonts,
+            xobj_encoding_length_map=xobj_encoding_length_map,
+            ctm_for_ops=ctm_for_ops,
+            check_font_exists=check_font_exists,
+        )
+        # Create render units for all renderable objects
+        render_units = self.create_render_units_for_page(page, translation_config)
+        if skip_char:
+            render_units = [
+                unit
+                for unit in render_units
+                if not isinstance(unit, CharacterRenderUnit)
+            ]
+        # Render all units to their appropriate streams
+        self.render_units_to_stream(render_units, context, page_op, xobj_draw_ops)
+        # Update xobject streams
+        for xobj in page.pdf_xobject:
+            draw_op = xobj_draw_ops[xobj.xobj_id]
+            try:
+                pdf.update_stream(xobj.xref_id, draw_op.tobytes())
+            except Exception:
+                logger.warning(f"update xref {xobj.xref_id} stream fail, continue")
+        draw_op = page_op
+        op_container = pdf.get_new_xref()
+        # Since this is a draw instruction container,
+        # no additional information is needed
+        pdf.update_object(op_container, "<<>>")
+        pdf.update_stream(op_container, draw_op.tobytes())
+        pdf[page.page_number].set_contents(op_container)

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/format/pdf/document_il/frontend/il_creater.py

@@ -932,6 +932,7 @@ class ILCreater:
         )
         if self.translation_config.ocr_workaround:
             pdf_char.pdf_style.graphic_state = BLACK
+            pdf_char.render_order = None
         if pdf_style.font_size == 0.0:
             logger.warning(
                 "Font size is 0.0 for character %s. Skip it.",

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/format/pdf/document_il/midend/detect_scanned_file.py

@@ -8,8 +8,8 @@ from skimage.metrics import structural_similarity
 
 from babeldoc.babeldoc_exception.BabelDOCException import ScannedPDFError
 from babeldoc.format.pdf.document_il import il_version_1
+from babeldoc.format.pdf.document_il.backend.pdf_creater import PDFCreater
 from babeldoc.format.pdf.document_il.utils.style_helper import GREEN
-from babeldoc.format.pdf.document_il.utils.zstd_helper import zstd_decompress
 from babeldoc.format.pdf.translation_config import TranslationConfig
 
 logger = logging.getLogger(__name__)
@@ -80,9 +80,16 @@ class DetectScannedFile:
             return bool(sum(hit_list) > len(doc) * 0.8)
         return False
 
-    def process(self, docs: il_version_1.Document):
+    def process(
+        self, docs: il_version_1.Document, original_pdf_path, mediabox_data: dict
+    ):
         """Generate layouts for all pages that need to be translated."""
         # Get pages that need to be translated
+
+        pdf_creater = PDFCreater(
+            original_pdf_path, docs, self.translation_config, mediabox_data
+        )
+
         pages_to_translate = [
             page
             for page in docs.page
@@ -104,7 +111,7 @@ class DetectScannedFile:
             for page in pages_to_translate:
                 if scanned < threshold and non_scanned < non_scanned_threshold:
                     # Only continue detection if both counts are below thresholds
-                    is_scanned = self.detect_page_is_scanned(page, mupdf)
+                    is_scanned = self.detect_page_is_scanned(page, mupdf, pdf_creater)
                     if is_scanned:
                         scanned += 1
                     else:
@@ -123,6 +130,8 @@ class DetectScannedFile:
                 self.translation_config.shared_context_cross_split_part.auto_enabled_ocr_workaround = True
                 self.translation_config.ocr_workaround = True
                 self.translation_config.skip_scanned_detection = True
+                self.clean_render_order_for_chars(docs)
+                self.translation_config.remove_non_formula_lines = False
             else:
                 logger.warning(
                     f"Detected {scanned} scanned pages, which is more than 80% of the total pages. "
@@ -130,25 +139,24 @@ class DetectScannedFile:
                 )
                 raise ScannedPDFError("Scanned PDF detected.")
 
-    @staticmethod
-    def detect_page_is_scanned(page: il_version_1.Page, pdf: pymupdf.Document) -> bool:
+    def clean_render_order_for_chars(self, docs: il_version_1.Document):
+        for page in docs.page:
+            for char in page.pdf_character:
+                char.render_order = None
+
+    def detect_page_is_scanned(
+        self, page: il_version_1.Page, pdf: pymupdf.Document, pdf_creater: PDFCreater
+    ) -> bool:
         before_page_image = pdf[page.page_number].get_pixmap()
         before_page_image = np.frombuffer(before_page_image.samples, np.uint8).reshape(
             before_page_image.height,
             before_page_image.width,
             3,
         )[:, :, ::-1]
-        new_xref = pdf.get_new_xref()
-        pdf.update_object(new_xref, "<<>>")
-        baseop = page.base_operations.value
-        baseop = zstd_decompress(baseop)
-        pdf.update_stream(new_xref, baseop.encode("utf-8"))
-        pdf[page.page_number].set_contents(new_xref)
-
-        for xobj in page.pdf_xobject:
-            base_op = xobj.base_operations.value
-            base_op = zstd_decompress(base_op)
-            pdf.update_stream(xobj.xref_id, base_op.encode("utf-8"))
+
+        pdf_creater.update_page_content_stream(
+            False, page, pdf, self.translation_config, True
+        )
 
         after_page_image = pdf[page.page_number].get_pixmap()
         after_page_image = np.frombuffer(after_page_image.samples, np.uint8).reshape(

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/format/pdf/document_il/midend/styles_and_formulas.py

@@ -369,7 +369,8 @@ class StylesAndFormulas:
             self.process_page_offsets(page)
         self.process_translatable_formulas(page)
         self.update_all_formula_data(page)
-        self.collect_contained_elements(page)
+        if not self.translation_config.ocr_workaround:
+            self.collect_contained_elements(page)
 
         # Process remaining non-formula lines after formula assignment is complete
         if self.translation_config.remove_non_formula_lines:

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/format/pdf/high_level.py

@@ -877,7 +877,9 @@ def _do_translate_single(
         logger.debug("skipping scanned file detection")
     else:
         logger.debug("start detect scanned file")
-        DetectScannedFile(translation_config).process(docs)
+        DetectScannedFile(translation_config).process(
+            docs, temp_pdf_path, mediabox_data
+        )
         logger.debug("finish detect scanned file")
         if translation_config.debug:
             xml_converter.write_json(

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/format/pdf/translation_config.py

@@ -162,8 +162,8 @@ class TranslationConfig:
         skip_curve_render: bool = False,
         only_parse_generate_pdf: bool = False,
         remove_non_formula_lines: bool = False,
-        non_formula_line_iou_threshold: float = 0.2,
-        figure_table_protection_threshold: float = 0.3,
+        non_formula_line_iou_threshold: float = 0.9,
+        figure_table_protection_threshold: float = 0.9,
         skip_formula_offset_calculation: bool = False,
     ):
         self.translator = translator
@@ -303,6 +303,9 @@ class TranslationConfig:
         self.figure_table_protection_threshold = figure_table_protection_threshold
         self.skip_formula_offset_calculation = skip_formula_offset_calculation
 
+        if self.ocr_workaround:
+            self.remove_non_formula_lines = False
+
     def parse_pages(self, pages_str: str | None) -> list[tuple[int, int]] | None:
         """解析页码字符串，返回页码范围列表
 

{babeldoc-0.5.0 → babeldoc-0.5.1}/babeldoc/main.py

@@ -22,7 +22,7 @@ from babeldoc.translator.translator import OpenAITranslator
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 
 
 def create_parser():

{babeldoc-0.5.0 → babeldoc-0.5.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.5.0"
+version = "0.5.1"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -162,7 +162,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.5.0"
+current_version = "0.5.1"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

babeldoc-0.5.0/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.5.0"