BabelDOC 0.2.31__tar.gz → 0.2.33__tar.gz

{babeldoc-0.2.31 → babeldoc-0.2.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.2.31
+Version: 0.2.33
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -187,6 +187,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--max-pages-per-part`: Maximum number of pages per part for split translation. If not set, no splitting will be performed.
 - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
 - `--translate-table-text`: Translate table text (experimental, default: False)
+- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
 
 > [!TIP]
 > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -194,6 +195,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 > - However, using `--skip-clean` will result in larger file sizes
 > - If you encounter any compatibility issues, try using `--enhance-compatibility` first
 > - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
+> - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
 
 ### Translation Service Options
 
@@ -268,6 +270,7 @@ use-alternating-pages-dual = false
 watermark-output-mode = "watermarked"  # Choices: "watermarked", "no_watermark", "both"
 max-pages-per-part = 50  # Automatically split the document for translation and merge it back.
 # no-watermark = false  # DEPRECATED: Use watermark-output-mode instead
+skip-scanned-detection = false  # Skip scanned document detection for faster processing
 
 # Translation service
 openai = true

{babeldoc-0.2.31 → babeldoc-0.2.33}/README.md

@@ -141,6 +141,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 - `--max-pages-per-part`: Maximum number of pages per part for split translation. If not set, no splitting will be performed.
 - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
 - `--translate-table-text`: Translate table text (experimental, default: False)
+- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
 
 > [!TIP]
 > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -148,6 +149,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
 > - However, using `--skip-clean` will result in larger file sizes
 > - If you encounter any compatibility issues, try using `--enhance-compatibility` first
 > - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
+> - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
 
 ### Translation Service Options
 
@@ -222,6 +224,7 @@ use-alternating-pages-dual = false
 watermark-output-mode = "watermarked"  # Choices: "watermarked", "no_watermark", "both"
 max-pages-per-part = 50  # Automatically split the document for translation and merge it back.
 # no-watermark = false  # DEPRECATED: Use watermark-output-mode instead
+skip-scanned-detection = false  # Skip scanned document detection for faster processing
 
 # Translation service
 openai = true

babeldoc-0.2.33/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.33"

{babeldoc-0.2.31 → babeldoc-0.2.33}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.2.31"
+__version__ = "0.2.33"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.2.31 → babeldoc-0.2.33}/babeldoc/document_il/midend/styles_and_formulas.py

@@ -541,6 +541,8 @@ class StylesAndFormulas:
             r"|Times-Italic.*"
             r"|MeridienLTStd.*"
             r"|Calibri.*"
+            r"|STIXMathJax_Main-Regular.*"
+            r"|STIXMathJax_Main-Italic.*"
             r")$"
         )
         if self.translation_config.formular_font_pattern:

{babeldoc-0.2.31 → babeldoc-0.2.33}/babeldoc/high_level.py

@@ -49,19 +49,19 @@ from babeldoc.translation_config import WatermarkOutputMode
 logger = logging.getLogger(__name__)
 
 TRANSLATE_STAGES = [
-    (ILCreater.stage_name, 5.35),  # Parse PDF and Create IR
-    (DetectScannedFile.stage_name, 0.5),  # No historical data, estimated
-    (LayoutParser.stage_name, 6.42),  # Parse Page Layout
-    (TableParser.stage_name, 0.5),  # Parse Table
-    (ParagraphFinder.stage_name, 2.14),  # Parse Paragraphs
-    (StylesAndFormulas.stage_name, 1.12),  # Parse Formulas and Styles
+    (ILCreater.stage_name, 14.12),  # Parse PDF and Create IR
+    (DetectScannedFile.stage_name, 2.45),  # DetectScannedFile
+    (LayoutParser.stage_name, 14.03),  # Parse Page Layout
+    (TableParser.stage_name, 1.0),  # Parse Table
+    (ParagraphFinder.stage_name, 6.26),  # Parse Paragraphs
+    (StylesAndFormulas.stage_name, 1.66),  # Parse Formulas and Styles
     # (RemoveDescent.stage_name, 0.15),  # Remove Char Descent
-    (ILTranslator.stage_name, 75.16),  # Translate Paragraphs
-    (Typesetting.stage_name, 3.84),  # Typesetting
-    (FontMapper.stage_name, 0.65),  # Add Fonts
-    (PDFCreater.stage_name, 1.41),  # Generate drawing instructions
-    (SUBSET_FONT_STAGE_NAME, 1.29),  # Subset font
-    (SAVE_PDF_STAGE_NAME, 2.45),  # Save PDF
+    (ILTranslator.stage_name, 46.96),  # Translate Paragraphs
+    (Typesetting.stage_name, 4.71),  # Typesetting
+    (FontMapper.stage_name, 0.61),  # Add Fonts
+    (PDFCreater.stage_name, 1.96),  # Generate drawing instructions
+    (SUBSET_FONT_STAGE_NAME, 0.92),  # Subset font
+    (SAVE_PDF_STAGE_NAME, 6.34),  # Save PDF
 ]
 
 resfont_map = {
@@ -431,6 +431,10 @@ def do_translate(
                                     (x, x) for x in should_translate_pages
                                 ]
 
+                                # Only first part should do scanned detection if enabled
+                                if i > 0:
+                                    part_config.skip_scanned_detection = True
+
                                 part_config.working_dir = (
                                     translation_config.get_part_working_dir(i)
                                 )
@@ -576,14 +580,17 @@ def _do_translate_single(
     # [Previous implementation of do_translate continues here]
 
     # 检测是否为扫描文件
-    logger.debug("start detect scanned file")
-    DetectScannedFile(translation_config).process(docs)
-    logger.debug("finish detect scanned file")
-    if translation_config.debug:
-        xml_converter.write_json(
-            docs,
-            translation_config.get_working_file_path("detect_scanned_file.json"),
-        )
+    if translation_config.skip_scanned_detection:
+        logger.debug("skipping scanned file detection")
+    else:
+        logger.debug("start detect scanned file")
+        DetectScannedFile(translation_config).process(docs)
+        logger.debug("finish detect scanned file")
+        if translation_config.debug:
+            xml_converter.write_json(
+                docs,
+                translation_config.get_working_file_path("detect_scanned_file.json"),
+            )
 
     # Generate layouts for all pages
     logger.debug("start generating layouts")

{babeldoc-0.2.31 → babeldoc-0.2.33}/babeldoc/main.py

@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.2.31"
+__version__ = "0.2.33"
 
 
 def create_parser():
@@ -204,6 +204,12 @@ def create_parser():
         default=False,
         help="Show character box (debug only)",
     )
+    translation_group.add_argument(
+        "--skip-scanned-detection",
+        action="store_true",
+        default=False,
+        help="Skip scanned document detection (speeds up processing for non-scanned documents)",
+    )
     # service option argument group
     service_group = translation_group.add_mutually_exclusive_group()
     service_group.add_argument(
@@ -370,6 +376,7 @@ async def main():
             split_strategy=split_strategy,
             table_model=table_model,
             show_char_box=args.show_char_box,
+            skip_scanned_detection=args.skip_scanned_detection,
         )
 
         # Create progress handler

babeldoc-0.2.33/babeldoc/tools/italic_assistance.py

@@ -0,0 +1,205 @@
+import argparse
+import json
+import re
+from pathlib import Path
+
+import orjson
+from babeldoc.const import CACHE_FOLDER
+
+WORKING_FOLDER = Path(CACHE_FOLDER) / "working"
+
+
+def find_latest_il_json() -> Path | None:
+    """
+    Find the latest il_translated.json file in ~/.cache/babeldoc/ subdirectories.
+
+    Returns:
+        Path to the most recently modified il_translated.json file, or None if not found.
+    """
+    base_dir = Path(WORKING_FOLDER)
+    json_files = list(base_dir.glob("*/il_translated.json"))
+
+    if not json_files:
+        return None
+
+    # Sort by modification time (newest first)
+    json_files.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+    return json_files[0]
+
+
+def extract_fonts_from_paragraph(
+    paragraph: dict, page_font_map: dict[str, tuple[str, str]]
+) -> set[tuple[str, str]]:
+    """
+    Extract all font_ids and names used in a paragraph.
+
+    Args:
+        paragraph: The paragraph dictionary
+        page_font_map: Dictionary mapping font_id to (font_id, name) tuples
+
+    Returns:
+        Set of (font_id, name) tuples
+    """
+    fonts = set()
+
+    # Check if paragraph has a pdfStyle with font_id
+    if (
+        "pdf_style" in paragraph
+        and paragraph["pdf_style"]
+        and "font_id" in paragraph["pdf_style"]
+    ):
+        font_id = paragraph["pdf_style"]["font_id"]
+        if font_id in page_font_map:
+            fonts.add(page_font_map[font_id])
+
+    # Process paragraph compositions if present
+    if "pdf_paragraph_composition" in paragraph:
+        for comp in paragraph["pdf_paragraph_composition"]:
+            # Check different composition types that might contain font information
+
+            # Direct pdfCharacter in composition
+            if "pdf_character" in comp and comp["pdf_character"]:
+                char = comp["pdf_character"]
+                if "pdf_style" in char and "font_id" in char["pdf_style"]:
+                    font_id = char["pdf_style"]["font_id"]
+                    if font_id in page_font_map:
+                        fonts.add(page_font_map[font_id])
+
+            # PdfLine in composition
+            elif "pdf_line" in comp and comp["pdf_line"]:
+                line = comp["pdf_line"]
+                if "pdf_character" in line:
+                    for char in line["pdf_character"]:
+                        if "pdf_style" in char and "font_id" in char["pdf_style"]:
+                            font_id = char["pdf_style"]["font_id"]
+                            if font_id in page_font_map:
+                                fonts.add(page_font_map[font_id])
+
+            # PdfFormula in composition
+            elif "pdf_formula" in comp and comp["pdf_formula"]:
+                formula = comp["pdf_formula"]
+                if "pdf_character" in formula:
+                    for char in formula["pdf_character"]:
+                        if "pdf_style" in char and "font_id" in char["pdf_style"]:
+                            font_id = char["pdf_style"]["font_id"]
+                            if font_id in page_font_map:
+                                fonts.add(page_font_map[font_id])
+
+            # PdfSameStyleCharacters in composition
+            elif (
+                "pdf_same_style_characters" in comp
+                and comp["pdf_same_style_characters"]
+            ):
+                same_style = comp["pdf_same_style_characters"]
+                if "pdf_style" in same_style and "font_id" in same_style["pdf_style"]:
+                    font_id = same_style["pdf_style"]["font_id"]
+                    if font_id in page_font_map:
+                        fonts.add(page_font_map[font_id])
+
+            # PdfSameStyleUnicodeCharacters in composition
+            elif (
+                "pdf_same_style_unicode_characters" in comp
+                and comp["pdf_same_style_unicode_characters"]
+            ):
+                same_style_unicode = comp["pdf_same_style_unicode_characters"]
+                if (
+                    "pdf_style" in same_style_unicode
+                    and "font_id" in same_style_unicode["pdf_style"]
+                ):
+                    font_id = same_style_unicode["pdf_style"]["font_id"]
+                    if font_id in page_font_map:
+                        fonts.add(page_font_map[font_id])
+
+    return fonts
+
+
+def find_fonts_by_debug_id(json_path: Path, debug_id_regex: str) -> dict[str, str]:
+    """
+    Find all fonts used in paragraphs with matching debug_id.
+
+    Args:
+        json_path: Path to the il_translated.json file
+        debug_id_regex: Regular expression to match debug_id values
+
+    Returns:
+        Dictionary mapping font_ids to font names
+    """
+    # Load and parse JSON
+    with json_path.open("rb") as f:
+        doc_data = orjson.loads(f.read())
+
+    # Compile regex pattern (case insensitive)
+    pattern = re.compile(debug_id_regex.strip(" \"'"), re.IGNORECASE)
+
+    # Set to collect all found font information
+    found_fonts = set()
+
+    # Process each page
+    for page in doc_data.get("page", []):
+        # Create a mapping of font_id to (font_id, name) tuples for this page
+        page_font_map = {}
+        for font in page.get("pdf_font", []):
+            if "font_id" in font and "name" in font:
+                page_font_map[font["font_id"]] = (font["font_id"], font["name"])
+
+        # Check each paragraph
+        for paragraph in page.get("pdf_paragraph", []):
+            # Check if paragraph has debug_id and if it matches the pattern
+            debug_id = paragraph.get("debug_id")
+            if debug_id and pattern.search(debug_id):
+                # Get all fonts used in this paragraph
+                paragraph_fonts = extract_fonts_from_paragraph(paragraph, page_font_map)
+                found_fonts.update(paragraph_fonts)
+
+    # Convert set of tuples to dictionary
+    return dict(found_fonts)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract fonts from paragraphs with matching debug_id"
+    )
+    parser.add_argument(
+        "debug_id_regex", nargs="+", help="Regular expression to match debug_id values"
+    )
+    parser.add_argument(
+        "--json-path",
+        help="Path to il_translated.json (if not provided, will use the latest file)",
+    )
+
+    args = parser.parse_args()
+
+    # Determine JSON file path
+    json_path = None
+    if args.json_path:
+        json_path = Path(args.json_path)
+        if not json_path.exists():
+            print(f"Error: File not found: {json_path}")
+            return 1
+    else:
+        json_path = find_latest_il_json()
+        if not json_path:
+            print("Error: Could not find any il_translated.json file")
+            return 1
+
+    print(f"Using JSON file: {json_path}")
+
+    # Find fonts matching the debug_id pattern
+    fonts = find_fonts_by_debug_id(json_path, "|".join(args.debug_id_regex))
+
+    # Output the results
+    if fonts:
+        print(
+            f"Found {len(fonts)} fonts in paragraphs matching debug_id pattern: {args.debug_id_regex}"
+        )
+        print(json.dumps(fonts, indent=2, ensure_ascii=False))
+    else:
+        print(
+            f"No fonts found for paragraphs matching debug_id pattern: {args.debug_id_regex}"
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())

{babeldoc-0.2.31 → babeldoc-0.2.33}/babeldoc/translation_config.py

@@ -67,6 +67,7 @@ class TranslationConfig:
         split_strategy: BaseSplitStrategy | None = None,
         table_model=None,
         show_char_box: bool = False,
+        skip_scanned_detection: bool = False,
     ):
         self.translator = translator
 
@@ -97,6 +98,7 @@ class TranslationConfig:
         self.doc_layout_model = doc_layout_model
 
         self.skip_clean = skip_clean or enhance_compatibility
+        self.skip_scanned_detection = skip_scanned_detection
 
         self.dual_translate_first = dual_translate_first or enhance_compatibility
         self.disable_rich_text_translate = (

{babeldoc-0.2.31 → babeldoc-0.2.33}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.2.31"
+version = "0.2.33"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -148,7 +148,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.2.31"
+current_version = "0.2.33"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.2.31 → babeldoc-0.2.33}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.2.31"
+version = "0.2.33"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },

babeldoc-0.2.31/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.31"