BabelDOC 0.1.6__py3-none-any.whl

babeldoc/translation_config.py

@@ -0,0 +1,156 @@
+import contextlib
+import shutil
+import tempfile
+import threading
+from pathlib import Path
+
+from babeldoc.const import CACHE_FOLDER
+from babeldoc.docvision.doclayout import DocLayoutModel
+from babeldoc.progress_monitor import ProgressMonitor
+
+
+class TranslationConfig:
+    def __init__(
+        self,
+        input_file: str,
+        translator,
+        lang_in: str,
+        lang_out: str,
+        font: str | None = None,
+        pages: str | None = None,
+        output_dir: str | None = None,
+        debug: bool = False,
+        working_dir: str | None = None,
+        no_dual: bool = False,
+        no_mono: bool = False,
+        formular_font_pattern: str | None = None,
+        formular_char_pattern: str | None = None,
+        qps: int = 1,
+        split_short_lines: bool = False,  # 是否将比较短的行强制切分成不同段落，此功能可能会导致糟糕的排版&bug
+        short_line_split_factor: float = 0.8,  # 切分阈值系数。实际阈值为当前页所有行长度中位数*此系数
+        use_rich_pbar: bool = True,  # 是否使用 rich 进度条
+        progress_monitor: ProgressMonitor | None = None,  # progress_monitor
+        doc_layout_model=None,
+        skip_clean: bool = False,
+        dual_translate_first: bool = False,
+    ):
+        self.input_file = input_file
+        self.translator = translator
+        self.font = font
+        self.pages = pages
+        self.page_ranges = self._parse_pages(pages) if pages else None
+        self.debug = debug
+        self.lang_in = lang_in
+        self.lang_out = lang_out
+        self.no_dual = no_dual
+        self.no_mono = no_mono
+        self.formular_font_pattern = formular_font_pattern
+        self.formular_char_pattern = formular_char_pattern
+        self.qps = qps
+        self.split_short_lines = split_short_lines
+        self.short_line_split_factor = short_line_split_factor
+        self.use_rich_pbar = use_rich_pbar
+        self.progress_monitor = progress_monitor
+        self.skip_clean = skip_clean
+        self.dual_translate_first = dual_translate_first
+        if progress_monitor:
+            if progress_monitor.cancel_event is None:
+                progress_monitor.cancel_event = threading.Event()
+            if progress_monitor.finish_event is None:
+                progress_monitor.finish_event = threading.Event()
+
+        if working_dir is None:
+            if debug:
+                working_dir = (
+                    Path(CACHE_FOLDER) / "working" / Path(input_file).name.split(".")[0]
+                )
+            else:
+                working_dir = tempfile.mkdtemp()
+        self.working_dir = working_dir
+        self._is_temp_dir = not debug and working_dir.startswith(tempfile.gettempdir())
+
+        Path(working_dir).mkdir(parents=True, exist_ok=True)
+
+        if output_dir is None:
+            # output_dir = os.path.dirname(input_file)
+            output_dir = Path.cwd()
+        self.output_dir = output_dir
+
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+        if doc_layout_model is None:
+            doc_layout_model = DocLayoutModel.load_available()
+        self.doc_layout_model = doc_layout_model
+
+    def get_output_file_path(self, filename):
+        return Path(self.output_dir) / filename
+
+    def get_working_file_path(self, filename):
+        return Path(self.working_dir) / filename
+
+    def _parse_pages(self, pages_str: str | None) -> list[tuple[int, int]] | None:
+        """解析页码字符串，返回页码范围列表
+
+        Args:
+            pages_str: 形如 "1-,2,-3,4" 的页码字符串
+
+        Returns:
+            包含 (start, end) 元组的列表，其中 -1 表示无限制
+        """
+        if not pages_str:
+            return None
+
+        ranges = []
+        for part in pages_str.split(","):
+            part = part.strip()
+            if "-" in part:
+                start, end = part.split("-")
+                start = int(start) if start else 1
+                end = int(end) if end else -1
+                ranges.append((start, end))
+            else:
+                page = int(part)
+                ranges.append((page, page))
+        return ranges
+
+    def should_translate_page(self, page_number: int) -> bool:
+        """判断指定页码是否需要翻译
+
+        Args:
+            page_number: 页码
+
+        Returns:
+            是否需要翻译该页
+        """
+        if not self.page_ranges:
+            return True
+
+        for start, end in self.page_ranges:
+            if start <= page_number and (end == -1 or page_number <= end):
+                return True
+        return False
+
+    def raise_if_cancelled(self):
+        if self.progress_monitor:
+            self.progress_monitor.raise_if_cancelled()
+
+    def cancel_translation(self):
+        if self.progress_monitor:
+            self.progress_monitor.cancel()
+
+    def __del__(self):
+        """Clean up temporary directory if it was created."""
+        if hasattr(self, "_is_temp_dir") and self._is_temp_dir:
+            with contextlib.suppress(Exception):
+                shutil.rmtree(self.working_dir, ignore_errors=True)
+
+
+class TranslateResult:
+    original_pdf_path: str
+    total_seconds: float
+    mono_pdf_path: str | None
+    dual_pdf_path: str | None
+
+    def __init__(self, mono_pdf_path: str | None, dual_pdf_path: str | None):
+        self.mono_pdf_path = mono_pdf_path
+        self.dual_pdf_path = dual_pdf_path

babeldoc-0.1.6.dist-info/METADATA

@@ -0,0 +1,283 @@
+Metadata-Version: 2.4
+Name: BabelDOC
+Version: 0.1.6
+Summary: Yet Another Document Translator
+Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
+Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
+License-Expression: AGPL-3.0
+License-File: LICENSE
+Keywords: PDF
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: <3.13,>=3.10
+Requires-Dist: bitstring>=4.3.0
+Requires-Dist: configargparse>=1.7
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: huggingface-hub>=0.27.0
+Requires-Dist: msgpack>=1.1.0
+Requires-Dist: numpy>=2.0.2
+Requires-Dist: onnx>=1.17.0
+Requires-Dist: onnxruntime>=1.16.1
+Requires-Dist: openai>=1.59.3
+Requires-Dist: opencv-python>=4.10.0.84
+Requires-Dist: orjson>=3.10.14
+Requires-Dist: pdfminer-six>=20240706
+Requires-Dist: peewee>=3.17.8
+Requires-Dist: pymupdf>=1.25.1
+Requires-Dist: rich>=13.9.4
+Requires-Dist: toml>=0.10.2
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: xsdata[cli,lxml,soap]>=24.12
+Description-Content-Type: text/markdown
+
+<!-- # Yet Another Document Translator -->
+
+<div align="center">
+<img src="./docs/images/banner.png" width="320px"  alt="YADT"/>
+
+
+<h2 id="title">BabelDOC</h2>
+
+<p>
+  <!-- PyPI -->
+  <a href="https://pypi.org/project/BabelDOC/">
+    <img src="https://img.shields.io/pypi/v/BabelDOC"></a>
+  <a href="https://pepy.tech/projects/BabelDOC">
+    <img src="https://static.pepy.tech/badge/BabelDOC"></a>
+  <!-- <a href="https://github.com/funstory-ai/BabelDOC/pulls">
+    <img src="https://img.shields.io/badge/contributions-welcome-green"></a> -->
+  <!-- License -->
+  <a href="./LICENSE">
+    <img src="https://img.shields.io/github/license/funstory-ai/BabelDOC"></a>
+  <a href="https://t.me/+Z9_SgnxmsmA5NzBl">
+    <img src="https://img.shields.io/badge/Telegram-2CA5E0?style=flat-squeare&logo=telegram&logoColor=white"></a>
+</p>
+</div>
+
+PDF scientific paper translation and bilingual comparison library.
+
+- Provides a simple [command line interface](#getting-started).
+- Provides a [Python API](#python-api).
+- Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.
+
+## Preview
+
+<div align="center">
+<img src="./docs/images/preview.gif" width="80%"/>
+</div>
+
+## Getting Started
+
+### Install from PyPI
+
+We recommend using the Tool feature of [uv](https://github.com/astral-sh/uv) to install yadt.
+
+1. First, you need to refer to [uv installation](https://github.com/astral-sh/uv#installation) to install uv and set up the `PATH` environment variable as prompted.
+
+2. Use the following command to install yadt:
+
+```bash
+uv tool install --python 3.12 BabelDOC
+
+babeldoc --help
+```
+
+3. Use the `babeldoc` command. For example:
+
+```bash
+babeldoc --bing  --files example.pdf
+
+# multiple files
+babeldoc --bing  --files example1.pdf --files example2.pdf
+```
+
+### Install from Source
+
+We still recommend using [uv](https://github.com/astral-sh/uv) to manage virtual environments.
+
+1. First, you need to refer to [uv installation](https://github.com/astral-sh/uv#installation) to install uv and set up the `PATH` environment variable as prompted.
+
+2. Use the following command to install yadt:
+
+```bash
+# clone the project
+git clone https://github.com/funstory-ai/BabelDOC
+
+# enter the project directory
+cd BabelDOC
+
+# install dependencies and run babeldoc
+uv run babeldoc --help
+```
+
+3. Use the `uv run babeldoc` command. For example:
+
+```bash
+uv run babeldoc --bing --files example.pdf
+
+# multiple files
+uv run babeldoc --bing --files example.pdf --files example2.pdf
+```
+
+> [!TIP]
+> The absolute path is recommended.
+
+## Advanced Options
+
+### Language Options
+
+- `--lang-in`, `-li`: Source language code (default: en)
+- `--lang-out`, `-lo`: Target language code (default: zh)
+
+> [!TIP]
+> Currently, this project mainly focuses on English-to-Chinese translation, and other scenarios have not been tested yet.
+
+### PDF Processing Options
+
+- `--files`: One or more file paths to input PDF documents.
+- `--pages`, `-p`: Specify pages to translate (e.g., "1,2,1-,-3,3-5"). If not set, translate all pages
+- `--split-short-lines`: Force split short lines into different paragraphs (may cause poor typesetting & bugs)
+- `--short-line-split-factor`: Split threshold factor (default: 0.8). The actual threshold is the median length of all lines on the current page \* this factor
+- `--skip-clean`: Skip PDF cleaning step
+- `--dual-translate-first`: Put translated pages first in dual PDF mode (default: original pages first)
+
+> [!TIP]
+> - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
+> - However, using `--skip-clean` will result in larger file sizes
+
+### Translation Service Options
+
+- `--qps`: QPS (Queries Per Second) limit for translation service (default: 4)
+- `--ignore-cache`: Ignore translation cache and force retranslation
+- `--no-dual`: Do not output bilingual PDF files
+- `--no-mono`: Do not output monolingual PDF files
+- `--openai`: Use OpenAI for translation (default: False)
+- `--bing`: Use Bing for translation (default: False)
+- `--google`: Use Google Translate for translation (default: False)
+
+> [!TIP]
+>
+> 1. You must specify one translation service among `--openai`, `--bing`, `--google`.
+> 2. It is recommended to use models with strong compatibility with OpenAI, such as: `glm-4-flash`, `deepseek-chat`, etc.
+> 3. Currently, it has not been optimized for traditional translation engines like Bing/Google, it is recommended to use LLMs.
+
+### OpenAI Specific Options
+
+- `--openai-model`: OpenAI model to use (default: gpt-4o-mini)
+- `--openai-base-url`: Base URL for OpenAI API
+- `--openai-api-key`: API key for OpenAI service
+
+### Output Control
+
+- `--output`, `-o`: Output directory for translated files. If not set, use current working directory.
+- `--debug`, `-d`: Enable debug logging level and export detailed intermediate results in `~/.cache/yadt/working`.
+
+### Configuration File
+
+- `--config`, `-c`: Configuration file path. Use the TOML format.
+
+Example Configuration:
+
+```toml
+[babeldoc]
+debug = true
+lang-in = "en-US"
+lang-out = "zh-CN"
+qps = 20
+# this is a comment
+# pages = 4
+openai = true
+openai-model = "SOME_ALSOME_MODEL"
+openai-base-url = "https://example.example/v1"
+openai-api-key = "[KEY]"
+# All other options can also be set in the configuration file.
+```
+
+## Python API
+
+You can refer to the example in [main.py](https://github.com/funstory-ai/yadt/blob/main/babeldoc/main.py) to use BabelDOC's Python API.
+
+Please note:
+
+1. Make sure call `babeldoc.high_level.init()` before using the API
+
+2. The current `TranslationConfig` does not fully validate input parameters, so you need to ensure the validity of input parameters
+
+## Background
+
+There are a lot projects and teams working on to make document editing and translating easier like:
+
+- [mathpix](https://mathpix.com/)
+- [Doc2X](https://doc2x.noedgeai.com/)
+- [minerU](https://github.com/opendatalab/MinerU)
+- [PDFMathTranslate](https://github.com/funstory-ai/yadt)
+
+There are also some solutions to solve specific parts of the problem like:
+
+- [layoutreader](https://github.com/microsoft/unilm/tree/master/layoutreader): the read order of the text block in a pdf
+- [Surya](https://github.com/surya-is/surya): the structure of the pdf
+
+This project hopes to promote a standard pipeline and interface to solve the problem.
+
+In fact, there are two main stages of a PDF parser or translator:
+
+- **Parsing**: A stage of parsing means to get the structure of the pdf such as text blocks, images, tables, etc.
+- **Rendering**: A stage of rendering means to render the structure into a new pdf or other format.
+
+For a service like mathpix, it will parse the pdf into a structure may be in a XML format, and then render them using a single column reader order as [layoutreader](https://github.com/microsoft/unilm/tree/master/layoutreader) does. The bad news is that the original structure lost.
+
+Some people will use Adobe PDF Parser because it will generate a Word document and it keeps the original structure. But it is somewhat expensive.
+And you know, a pdf or word document is not a good format for reading in mobile devices.
+
+We offer an intermediate representation of the results from parser and can be rendered into a new pdf or other format. The pipeline is also a plugin-based system which everybody can add their new model, ocr, renderer, etc.
+
+## Roadmap
+
+- [ ] Add line support
+- [ ] Add table support
+- [ ] Add cross-page/cross-column paragraph support
+- [ ] More advanced typesetting features
+- [ ] Outline support
+- [ ] ...
+
+Our first 1.0 version goal is to finish a translation from [PDF Reference, Version 1.7](https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/pdfreference1.7old.pdf) to the following language version:
+
+- Simplified Chinese
+- Traditional Chinese
+- Japanese
+- Spanish
+
+And meet the following requirements:
+
+- layout error less than 1%
+- content loss less than 1%
+
+## Known Issues
+
+1. Parsing errors in the author and reference sections; they get merged into one paragraph after translation.
+2. Lines are not supported.
+3. Does not support drop caps.
+
+## How to Contribute
+
+We encourage you to contribute to YADT! Please check out the [CONTRIBUTING](https://github.com/funstory-ai/yadt/blob/main/docs/CONTRIBUTING.md) guide.
+
+Everyone interacting in YADT and its sub-projects' codebases, issue trackers, chat rooms, and mailing lists is expected to follow the YADT [Code of Conduct](https://github.com/funstory-ai/yadt/blob/main/docs/CODE_OF_CONDUCT.md).
+
+## Acknowledgements
+
+- [PDFMathTranslate](https://github.com/Byaidu/PDFMathTranslate)
+- [DocLayout-YOLO](https://github.com/opendatalab/DocLayout-YOLO)
+- [pdfminer](https://github.com/pdfminer/pdfminer.six)
+- [PyMuPDF](https://github.com/pymupdf/PyMuPDF)
+- [Asynchronize](https://github.com/multimeric/Asynchronize/tree/master?tab=readme-ov-file)
+
+<h2 id="star_hist">Star History</h2>
+
+<a href="https://star-history.com/#funstory-ai/babeldoc&Date">
+ <picture>
+   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=funstory-ai/babeldoc&type=Date&theme=dark" />
+   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=funstory-ai/babeldoc&type=Date" />
+   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=funstory-ai/babeldoc&type=Date"/>
+ </picture>
+</a>

babeldoc-0.1.6.dist-info/RECORD

@@ -0,0 +1,42 @@
+babeldoc/__init__.py,sha256=n3oM6B_EMz93NsTI18NNZd-jKFcUPzUkbIKj5VFK5ok,22
+babeldoc/const.py,sha256=u247IzFnLn3tb7W7XmXY8rFv5jIe66s2xZDc8CuEKlk,149
+babeldoc/converter.py,sha256=vVS6pOfYU1is2googvbc-VMyNYZTvj16edizgA_Bwwc,24391
+babeldoc/high_level.py,sha256=WqUvNb09f1_zmuG7CUhDQAllmHueq8YGg1_jGTMhMAA,17121
+babeldoc/main.py,sha256=EHhPhsPnzIPgnozgetVpJEjCUTae8glZ0XcawkAjPNg,14936
+babeldoc/pdfinterp.py,sha256=vXcfJ6FsFEFg9jNTK8UA96V3_pOp-BOI1j2TyT3zGGQ,18713
+babeldoc/progress_monitor.py,sha256=NE70TTBPlk9gQV6TIu88SQiKr_HR-bx3jbUoqF0wxWQ,4890
+babeldoc/translation_config.py,sha256=0yIostif5zTZ8w7ACl65Vw40R4R9tyXWwS19kur2XNU,5412
+babeldoc/asynchronize/__init__.py,sha256=W7KpbxIjg-AZ3epqOgjbs4uolW3RmzdqSiVIZbQxr0M,1802
+babeldoc/document_il/__init__.py,sha256=QGzQfV-GqFU3FWstqHP5okNTKf4JaW6TJBDXmS1xWLw,1569
+babeldoc/document_il/il_version_1.py,sha256=q8TD8h0dNsCFs2oRqrrmiydyFIUL84pveMAm2zcj0nY,16809
+babeldoc/document_il/il_version_1.rnc,sha256=Al2lgyQC1idvaRJnLeIAD7261F9ZrWrrt5_qEZOI9Sc,3603
+babeldoc/document_il/il_version_1.rng,sha256=xaGtcT7ch6JdZQ-o9ZNqP0Zl6BKqUDZCbYbWnq0RrJ8,9809
+babeldoc/document_il/il_version_1.xsd,sha256=yaVEsJohd4JjzIYOiWvFlQRooNRnpPISpyXaGai1QJI,8644
+babeldoc/document_il/xml_converter.py,sha256=7safmkEMU1FyAZwC36a6bRgz6--Ptu0UxjXBxhGzFSw,1775
+babeldoc/document_il/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+babeldoc/document_il/backend/pdf_creater.py,sha256=_AQJsIIDHIznaxubfFEw_P2u8sw6p0BpFzhVsS4kbwc,16893
+babeldoc/document_il/frontend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+babeldoc/document_il/frontend/il_creater.py,sha256=GCvnHVZspFsPlQNmTTwqbnXqm1qi-RyTfJl8OHU5VoY,12530
+babeldoc/document_il/midend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+babeldoc/document_il/midend/add_debug_information.py,sha256=SYgmSMGxWYaSWc_hoS1Zk1aO4MIOIdxEep9rqrr7JGg,4789
+babeldoc/document_il/midend/il_translator.py,sha256=vW9hud3h0bIpNtecpmD_Yw8DgIxPOQaGU56klRU4K5Q,20418
+babeldoc/document_il/midend/layout_parser.py,sha256=rZRX-wl_EG4bet-pivuztu3VaDrYzo08cfQhblRGgIg,6808
+babeldoc/document_il/midend/paragraph_finder.py,sha256=e09R94obb6HO6C603AQnEYZpGndCF8vZG-1LMbwsU8w,16850
+babeldoc/document_il/midend/remove_descent.py,sha256=K2PHPsISTvmU1vW-8Jzhrzc_kNiHl8GmZfToEbBU2O8,6809
+babeldoc/document_il/midend/styles_and_formulas.py,sha256=eK2Me4oSP7EQL-Vg8MDBs3ODJE0XL0xSsm8MHMF_PxM,28367
+babeldoc/document_il/midend/typesetting.py,sha256=ECVlixb7Hj3jLAxx5jSAa7Kwlz1xuBCumfajuHZtB1I,26486
+babeldoc/document_il/translator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+babeldoc/document_il/translator/cache.py,sha256=HIVj3Ec7WEmckUF8ELW6lFjfF42HAPiX3FfMt5OzpEs,4304
+babeldoc/document_il/translator/translator.py,sha256=TcQJboMDU94uRQ-PnutulJCxk0hqpM418t4WKXWQdOo,9232
+babeldoc/document_il/utils/fontmap.py,sha256=Z7D4uIY3Fd0jgYZXS32St8BkOn1WYOWNj976kuStBJI,9055
+babeldoc/document_il/utils/layout_helper.py,sha256=MEe2oVGFdjP2N5VelIUZYF5UFA9pcll9YvlUlPrRK9c,16778
+babeldoc/document_il/utils/style_helper.py,sha256=AHQnTd6O1JZyy4auNgTDxabynxX7p_H8rMf26Jt4PiY,2764
+babeldoc/docvision/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+babeldoc/docvision/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+babeldoc/docvision/doclayout.py,sha256=54xLHL6qsUJ9CsbOr0DaIQRQjEWn0eahXpU4NLYxYH8,9008
+babeldoc/docvision/rpc_doclayout.py,sha256=RsycJnpb850-T0_787qVC7ObS1NGApbMMOj82mlzEW4,5098
+babeldoc-0.1.6.dist-info/METADATA,sha256=m8Vs6O7qxcMzZjzYxuSlaqeHWcRvw-xCAyt4vKxNYYo,10388
+babeldoc-0.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+babeldoc-0.1.6.dist-info/entry_points.txt,sha256=65vKshdSbjvg1dh1fDsdNwtluWZB54OB6hXH9NihWuE,47
+babeldoc-0.1.6.dist-info/licenses/LICENSE,sha256=AJcaDjGuPz-cOS67fE_68gL7xlKbqWFWBYfhdMCaXsQ,34515
+babeldoc-0.1.6.dist-info/RECORD,,

babeldoc-0.1.6.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

babeldoc-0.1.6.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+babeldoc = babeldoc.main:cli