BabelDOC 0.3.43__tar.gz → 0.3.50__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- babeldoc-0.3.50/.github/PULL_REQUEST_TEMPLATE.md +65 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/test.yml +1 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/PKG-INFO +4 -3
- {babeldoc-0.3.43 → babeldoc-0.3.50}/README.md +1 -0
- babeldoc-0.3.50/babeldoc/__init__.py +1 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/const.py +1 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/backend/pdf_creater.py +11 -7
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/il_translator.py +1 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/il_translator_llm_only.py +104 -75
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/layout_parser.py +3 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/styles_and_formulas.py +4 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/table_parser.py +3 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/translator/translator.py +1 -0
- babeldoc-0.3.50/babeldoc/document_il/utils/mupdf_helper.py +10 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/doclayout.py +4 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/rpc_doclayout.py +3 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/table_detection/rapidocr.py +3 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/high_level.py +14 -7
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/main.py +23 -2
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/pdfinterp.py +7 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/translation_config.py +4 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/CONTRIBUTING.md +7 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/pyproject.toml +4 -4
- {babeldoc-0.3.43 → babeldoc-0.3.50}/uv.lock +538 -229
- babeldoc-0.3.43/babeldoc/__init__.py +0 -1
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.cursorignore +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/PULL_REQUEST_TEMPLATE/pr_form.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/dependabot.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/labels.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/release-drafter.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/codeql.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/docs.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/labeler.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/lint.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/pr-lint.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/publish-to-pypi.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.gitignore +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/.pre-commit-config.yaml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/LICENSE +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/assets/assets.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/assets/embedding_assets_metadata.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/asynchronize/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/converter.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/backend/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/frontend/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/frontend/il_creater.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.rnc +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.rng +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.xsd +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/add_debug_information.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/detect_scanned_file.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/paragraph_finder.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/remove_descent.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/typesetting.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/translator/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/translator/cache.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/atomic_integer.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/fontmap.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/layout_helper.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/paragraph_helper.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/priority_thread_pool_executor.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/style_helper.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/zstd_helper.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/xml_converter.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/README.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/format/office/__init__.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/progress_monitor.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/result_merger.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/split_manager.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/tools/generate_font_metadata.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/tools/italic_assistance.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/tools/italic_recognize_tool.py +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/CODE_OF_CONDUCT.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/CONTRIBUTOR_REWARD.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/ILTranslator/ILTranslator.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/PDFCreation/PDFCreation.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/PDFParsing/PDFParsing.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/README.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/StylesAndFormulas/StylesAndFormulas.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/Typesetting/Typesetting.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/README.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/deploy.sh +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-banner.png +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.png +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.svg +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-with-transparent-background.png +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-with-transparent-background.svg +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo.png +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-contributor_reward_example.png +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-preview.gif +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-small-logo-with-transparent-background.png +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-small-logo-with-transparent-background.svg +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-small-logo.png +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/index.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/intro-to-pdf-object.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/requirements.txt +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/supported_languages.md +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/mkdocs.yml +0 -0
- {babeldoc-0.3.43 → babeldoc-0.3.50}/tests/test_translation_config.py +0 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
### PR Title
|
|
2
|
+
|
|
3
|
+
<!-- Please fill in a concise and clear PR title below -->
|
|
4
|
+
[PR] <Your concise title here>
|
|
5
|
+
|
|
6
|
+
### Related Issue(s)
|
|
7
|
+
|
|
8
|
+
<!-- If this PR closes or is related to one or more issues, please list them here (e.g., #37) -->
|
|
9
|
+
<!-- e.g.: Closes #37, Relates to #42 -->
|
|
10
|
+
|
|
11
|
+
### Motivation and Context
|
|
12
|
+
|
|
13
|
+
<!-- Why is this change required? What problem does it solve? -->
|
|
14
|
+
<!-- If it fixes an open issue, please link to the issue here. -->
|
|
15
|
+
|
|
16
|
+
### Summary of Changes
|
|
17
|
+
|
|
18
|
+
<!-- What does this PR introduce or fix? Please describe concisely. -->
|
|
19
|
+
|
|
20
|
+
### PR Type
|
|
21
|
+
|
|
22
|
+
<!-- What kind of change is this? Please select one or more -->
|
|
23
|
+
- [ ] ✨ Enhancement
|
|
24
|
+
- [ ] 🐛 Bug Fix
|
|
25
|
+
- [ ] 📚 Documentation
|
|
26
|
+
- [ ] 🏗️ Refactor
|
|
27
|
+
- [ ] 🧪 Test
|
|
28
|
+
- [ ] 🧹 Chore
|
|
29
|
+
|
|
30
|
+
### Breaking Changes
|
|
31
|
+
|
|
32
|
+
<!-- Does this PR introduce any breaking changes? If so, please describe them. -->
|
|
33
|
+
<!-- - [ ] Yes, this PR introduces breaking changes.
|
|
34
|
+
<!-- - [ ] No, this PR does not introduce breaking changes. -->
|
|
35
|
+
<!-- Detailed description of breaking changes (if any): -->
|
|
36
|
+
|
|
37
|
+
### Contributor Checklist
|
|
38
|
+
|
|
39
|
+
- [ ] I have fully read and understood the **[CONTRIBUTING.md](https://funstory-ai.github.io/BabelDOC/CONTRIBUTING/)** guide.
|
|
40
|
+
- [ ] I have performed a self-review of my own code.
|
|
41
|
+
- [ ] My changes follow the project's code style and guidelines
|
|
42
|
+
- [ ] I have linked the related issue(s) in the description above (if applicable)
|
|
43
|
+
- [ ] I have updated relevant documentation (if applicable)
|
|
44
|
+
- [ ] I have added necessary tests that prove my fix is effective or that my feature works (if applicable)
|
|
45
|
+
- [ ] All new and existing tests passed locally with my changes
|
|
46
|
+
- [ ] My changes generate no new warnings or errors
|
|
47
|
+
- [ ] I understand that due to limited maintainer resources, only small PRs are accepted. Suggestions with proof-of-concept patches are appreciated, and my patch may be rewritten if necessary.
|
|
48
|
+
|
|
49
|
+
### Testing Instructions
|
|
50
|
+
|
|
51
|
+
<!-- Please provide clear and concise step-by-step instructions on how to test your changes. -->
|
|
52
|
+
<!-- e.g.: -->
|
|
53
|
+
<!-- 1. Check out this branch. -->
|
|
54
|
+
<!-- 2. Run `...` to install dependencies. -->
|
|
55
|
+
<!-- 3. Run `...` to start the application/run the script. -->
|
|
56
|
+
<!-- 4. Navigate to `...` or observe `...` -->
|
|
57
|
+
<!-- 5. Verify that `...` (expected outcome). -->
|
|
58
|
+
|
|
59
|
+
### Screenshots (if applicable)
|
|
60
|
+
|
|
61
|
+
<!-- If your changes include UI modifications, please add screenshots or GIFs to show the before and after. -->
|
|
62
|
+
|
|
63
|
+
### Additional Notes
|
|
64
|
+
|
|
65
|
+
<!-- Is there anything else the reviewer should know? For example, any dependencies, or potential impacts. -->
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: BabelDOC
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.50
|
|
4
4
|
Summary: Yet Another Document Translator
|
|
5
5
|
Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
|
|
6
6
|
Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
|
|
@@ -11,7 +11,7 @@ License-File: LICENSE
|
|
|
11
11
|
Keywords: PDF
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
14
|
-
Requires-Python: <3.
|
|
14
|
+
Requires-Python: <3.14,>=3.10
|
|
15
15
|
Requires-Dist: bitstring>=4.3.0
|
|
16
16
|
Requires-Dist: configargparse>=1.7
|
|
17
17
|
Requires-Dist: freetype-py>=2.5.1
|
|
@@ -19,7 +19,7 @@ Requires-Dist: httpx[socks]>=0.27.0
|
|
|
19
19
|
Requires-Dist: huggingface-hub>=0.27.0
|
|
20
20
|
Requires-Dist: msgpack>=1.1.0
|
|
21
21
|
Requires-Dist: numpy>=2.0.2
|
|
22
|
-
Requires-Dist: onnx>=1.
|
|
22
|
+
Requires-Dist: onnx>=1.18.0
|
|
23
23
|
Requires-Dist: onnxruntime>=1.16.1
|
|
24
24
|
Requires-Dist: openai>=1.59.3
|
|
25
25
|
Requires-Dist: opencv-python-headless>=4.10.0.84
|
|
@@ -205,6 +205,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
|
|
|
205
205
|
- `--translate-table-text`: Translate table text (experimental, default: False)
|
|
206
206
|
- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
|
|
207
207
|
- `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
|
|
208
|
+
- `--working-dir`: Working directory for translation. If not set, use temp directory.
|
|
208
209
|
|
|
209
210
|
> [!TIP]
|
|
210
211
|
> - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
|
|
@@ -156,6 +156,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
|
|
|
156
156
|
- `--translate-table-text`: Translate table text (experimental, default: False)
|
|
157
157
|
- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
|
|
158
158
|
- `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
|
|
159
|
+
- `--working-dir`: Working directory for translation. If not set, use temp directory.
|
|
159
160
|
|
|
160
161
|
> [!TIP]
|
|
161
162
|
> - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.50"
|
|
@@ -386,20 +386,22 @@ class PDFCreater:
|
|
|
386
386
|
# Get pages from both PDFs
|
|
387
387
|
orig_page = original_pdf[page_id]
|
|
388
388
|
trans_page = translated_pdf[page_id]
|
|
389
|
-
|
|
390
|
-
# Calculate total width and use max height
|
|
389
|
+
rotate_angle = orig_page.rotation
|
|
391
390
|
total_width = orig_page.rect.width + trans_page.rect.width
|
|
392
391
|
max_height = max(orig_page.rect.height, trans_page.rect.height)
|
|
393
|
-
|
|
394
|
-
# Create new page with combined width
|
|
395
|
-
dual_page = dual.new_page(width=total_width, height=max_height)
|
|
396
|
-
|
|
397
|
-
# Define rectangles for left and right sides
|
|
398
392
|
left_width = (
|
|
399
393
|
orig_page.rect.width
|
|
400
394
|
if not translation_config.dual_translate_first
|
|
401
395
|
else trans_page.rect.width
|
|
402
396
|
)
|
|
397
|
+
|
|
398
|
+
orig_page.set_rotation(0)
|
|
399
|
+
trans_page.set_rotation(0)
|
|
400
|
+
|
|
401
|
+
# Create new page with combined width
|
|
402
|
+
dual_page = dual.new_page(width=total_width, height=max_height)
|
|
403
|
+
|
|
404
|
+
# Define rectangles for left and right sides
|
|
403
405
|
rect_left = pymupdf.Rect(0, 0, left_width, max_height)
|
|
404
406
|
rect_right = pymupdf.Rect(left_width, 0, total_width, max_height)
|
|
405
407
|
|
|
@@ -414,6 +416,7 @@ class PDFCreater:
|
|
|
414
416
|
original_pdf,
|
|
415
417
|
page_id,
|
|
416
418
|
keep_proportion=True,
|
|
419
|
+
rotate=-rotate_angle,
|
|
417
420
|
)
|
|
418
421
|
except Exception as e:
|
|
419
422
|
logger.warning(
|
|
@@ -429,6 +432,7 @@ class PDFCreater:
|
|
|
429
432
|
translated_pdf,
|
|
430
433
|
page_id,
|
|
431
434
|
keep_proportion=True,
|
|
435
|
+
rotate=-rotate_angle,
|
|
432
436
|
)
|
|
433
437
|
except Exception as e:
|
|
434
438
|
logger.warning(
|
|
@@ -649,7 +649,7 @@ class ILTranslator:
|
|
|
649
649
|
and re.match(f"^{p.left_regex_pattern}", matched_text)
|
|
650
650
|
)
|
|
651
651
|
text = re.match(
|
|
652
|
-
f"^{placeholder.left_regex_pattern}(
|
|
652
|
+
f"^{placeholder.left_regex_pattern}(.*){placeholder.right_regex_pattern}$",
|
|
653
653
|
matched_text,
|
|
654
654
|
).group(1)
|
|
655
655
|
|
|
@@ -267,37 +267,25 @@ class ILTranslatorLLMOnly:
|
|
|
267
267
|
obj["formula_placeholders_hint"] = placeholders_hint
|
|
268
268
|
json_format_input.append(obj)
|
|
269
269
|
|
|
270
|
-
|
|
270
|
+
json_format_input_str = json.dumps(
|
|
271
271
|
json_format_input, ensure_ascii=False, indent=2
|
|
272
272
|
)
|
|
273
|
+
|
|
274
|
+
# Start building the new prompt
|
|
275
|
+
llm_prompt_parts = []
|
|
276
|
+
|
|
277
|
+
# 1. #role
|
|
278
|
+
llm_prompt_parts.append("#role")
|
|
273
279
|
if self.translation_config.custom_system_prompt:
|
|
274
|
-
|
|
280
|
+
llm_prompt_parts.append(self.translation_config.custom_system_prompt)
|
|
275
281
|
else:
|
|
276
|
-
|
|
282
|
+
llm_prompt_parts.append(
|
|
277
283
|
f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
|
|
278
|
-
]
|
|
279
|
-
|
|
280
|
-
llm_hint = []
|
|
281
|
-
if title_paragraph:
|
|
282
|
-
llm_hint.append(
|
|
283
|
-
f"The first title in the full text: {title_paragraph.unicode}"
|
|
284
|
-
)
|
|
285
|
-
if (
|
|
286
|
-
local_title_paragraph
|
|
287
|
-
and local_title_paragraph.debug_id != title_paragraph.debug_id
|
|
288
|
-
):
|
|
289
|
-
llm_hint.append(
|
|
290
|
-
f"The most similar title in the full text: {local_title_paragraph.unicode}"
|
|
291
284
|
)
|
|
292
285
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
)
|
|
297
|
-
for i, line in enumerate(llm_hint):
|
|
298
|
-
llm_input.append(f"{i}. {line}")
|
|
299
|
-
llm_input.append("When translating, please follow the following rules:")
|
|
300
|
-
# Create a structured prompt template for LLM translation
|
|
286
|
+
# 2. ##rules
|
|
287
|
+
llm_prompt_parts.append("\n##rules")
|
|
288
|
+
# Dynamically get placeholder examples
|
|
301
289
|
rich_text_left_placeholder = (
|
|
302
290
|
self.translate_engine.get_rich_text_left_placeholder(1)
|
|
303
291
|
)
|
|
@@ -308,66 +296,102 @@ class ILTranslatorLLMOnly:
|
|
|
308
296
|
)
|
|
309
297
|
if isinstance(rich_text_right_placeholder, tuple):
|
|
310
298
|
rich_text_right_placeholder = rich_text_right_placeholder[0]
|
|
311
|
-
llm_input.append(
|
|
312
|
-
f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
|
|
313
|
-
)
|
|
314
|
-
|
|
315
299
|
formula_placeholder = self.translate_engine.get_formular_placeholder(3)
|
|
316
300
|
if isinstance(formula_placeholder, tuple):
|
|
317
301
|
formula_placeholder = formula_placeholder[0]
|
|
318
|
-
|
|
302
|
+
|
|
303
|
+
llm_prompt_parts.append(
|
|
304
|
+
f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"'
|
|
305
|
+
)
|
|
306
|
+
llm_prompt_parts.append(
|
|
319
307
|
f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
|
|
320
308
|
)
|
|
321
|
-
|
|
309
|
+
llm_prompt_parts.append(
|
|
322
310
|
"3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
|
|
323
311
|
)
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
<example>
|
|
337
|
-
```json
|
|
338
|
-
Input:
|
|
339
|
-
{
|
|
340
|
-
"id": 1,
|
|
341
|
-
"input": "Source",
|
|
342
|
-
"layout_label": "plain text",
|
|
343
|
-
// this is optional
|
|
344
|
-
"formula_placeholders_hint": {
|
|
345
|
-
"placeholder1": "hint1",
|
|
346
|
-
"placeholder2": "hint2"
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
```
|
|
350
|
-
Output:
|
|
351
|
-
```json
|
|
352
|
-
{
|
|
353
|
-
"id": 1,
|
|
354
|
-
"output": "Translation"
|
|
355
|
-
}
|
|
356
|
-
```
|
|
357
|
-
</example>
|
|
358
|
-
|
|
359
|
-
Please return the translated json directly without wrapping ```json``` tag or include any additional information.
|
|
360
|
-
"""
|
|
312
|
+
|
|
313
|
+
# 3. ##Input
|
|
314
|
+
llm_prompt_parts.append("\n##Input")
|
|
315
|
+
llm_prompt_parts.append(
|
|
316
|
+
'You will be given a JSON formatted input containing entries with "id" and "input" fields.'
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
# 4. ##Output
|
|
320
|
+
llm_prompt_parts.append("\n##Output")
|
|
321
|
+
llm_prompt_parts.append(
|
|
322
|
+
f'For each entry in the JSON, translate the contents of the "input" field into {self.translation_config.lang_out}.'
|
|
361
323
|
)
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
f"""Here is the input:
|
|
365
|
-
|
|
366
|
-
```json
|
|
367
|
-
{json_format_input}
|
|
368
|
-
```"""
|
|
324
|
+
llm_prompt_parts.append(
|
|
325
|
+
'Write the translation back into the "output" field for that entry.'
|
|
369
326
|
)
|
|
370
|
-
|
|
327
|
+
llm_prompt_parts.append(
|
|
328
|
+
"Please return the translated json directly without wrapping ```json``` tag or include any additional information."
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
# 5. ##example
|
|
332
|
+
llm_prompt_parts.append("\n##example")
|
|
333
|
+
llm_prompt_parts.append("Here is an example of the expected format:")
|
|
334
|
+
llm_prompt_parts.append("") # Blank line
|
|
335
|
+
llm_prompt_parts.append("<example>")
|
|
336
|
+
llm_prompt_parts.append("```json")
|
|
337
|
+
llm_prompt_parts.append("Input:")
|
|
338
|
+
llm_prompt_parts.append("{")
|
|
339
|
+
llm_prompt_parts.append(' "id": 1,')
|
|
340
|
+
llm_prompt_parts.append(' "input": "Source",')
|
|
341
|
+
llm_prompt_parts.append(' "layout_label": "plain text",')
|
|
342
|
+
llm_prompt_parts.append(" // this is optional")
|
|
343
|
+
llm_prompt_parts.append(' "formula_placeholders_hint": {')
|
|
344
|
+
llm_prompt_parts.append(' "placeholder1": "hint1",')
|
|
345
|
+
llm_prompt_parts.append(' "placeholder2": "hint2"')
|
|
346
|
+
llm_prompt_parts.append(" }")
|
|
347
|
+
llm_prompt_parts.append("}")
|
|
348
|
+
llm_prompt_parts.append("```")
|
|
349
|
+
llm_prompt_parts.append("Output:")
|
|
350
|
+
llm_prompt_parts.append("```json")
|
|
351
|
+
llm_prompt_parts.append("{")
|
|
352
|
+
llm_prompt_parts.append(' "id": 1,')
|
|
353
|
+
llm_prompt_parts.append(' "output": "Translation"')
|
|
354
|
+
llm_prompt_parts.append("}")
|
|
355
|
+
llm_prompt_parts.append("```")
|
|
356
|
+
llm_prompt_parts.append("</example>")
|
|
357
|
+
|
|
358
|
+
# 6. ##Others (contextual hints)
|
|
359
|
+
other_hints = []
|
|
360
|
+
hint_idx = 0
|
|
361
|
+
if title_paragraph:
|
|
362
|
+
other_hints.append(
|
|
363
|
+
f"{hint_idx}. The first title in the full text: {title_paragraph.unicode}"
|
|
364
|
+
)
|
|
365
|
+
hint_idx += 1
|
|
366
|
+
|
|
367
|
+
if local_title_paragraph:
|
|
368
|
+
# Check if it's different from the global title_paragraph before adding
|
|
369
|
+
is_different_from_global = True
|
|
370
|
+
if title_paragraph:
|
|
371
|
+
if local_title_paragraph.debug_id == title_paragraph.debug_id:
|
|
372
|
+
is_different_from_global = False
|
|
373
|
+
|
|
374
|
+
if is_different_from_global:
|
|
375
|
+
other_hints.append(
|
|
376
|
+
f"{hint_idx}. The most similar title in the full text: {local_title_paragraph.unicode}"
|
|
377
|
+
)
|
|
378
|
+
hint_idx += 1
|
|
379
|
+
|
|
380
|
+
if other_hints:
|
|
381
|
+
llm_prompt_parts.append("\n##Others")
|
|
382
|
+
llm_prompt_parts.append(
|
|
383
|
+
"When translating, please refer to the following information to improve translation quality:"
|
|
384
|
+
)
|
|
385
|
+
llm_prompt_parts.extend(other_hints)
|
|
386
|
+
|
|
387
|
+
llm_prompt_parts.append("\n## Actual Input")
|
|
388
|
+
|
|
389
|
+
# Combine all parts for the main prompt
|
|
390
|
+
main_prompt_content = "\n".join(llm_prompt_parts)
|
|
391
|
+
|
|
392
|
+
# Append the actual JSON input string at the end, without markdown fence
|
|
393
|
+
final_input = main_prompt_content + "\n\n" + json_format_input_str
|
|
394
|
+
|
|
371
395
|
for llm_translate_tracker in llm_translate_trackers:
|
|
372
396
|
llm_translate_tracker.set_input(final_input)
|
|
373
397
|
llm_output = self.translate_engine.llm_translate(
|
|
@@ -382,6 +406,11 @@ class ILTranslatorLLMOnly:
|
|
|
382
406
|
|
|
383
407
|
parsed_output = json.loads(llm_output)
|
|
384
408
|
|
|
409
|
+
if isinstance(parsed_output, dict) and parsed_output.get(
|
|
410
|
+
"output", parsed_output.get("input", False)
|
|
411
|
+
):
|
|
412
|
+
parsed_output = [parsed_output]
|
|
413
|
+
|
|
385
414
|
translation_results = {
|
|
386
415
|
item["id"]: item.get("output", item.get("input"))
|
|
387
416
|
for item in parsed_output
|
|
@@ -6,6 +6,7 @@ import numpy as np
|
|
|
6
6
|
from pymupdf import Document
|
|
7
7
|
|
|
8
8
|
from babeldoc.document_il import il_version_1
|
|
9
|
+
from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
|
|
9
10
|
from babeldoc.document_il.utils.style_helper import GREEN
|
|
10
11
|
from babeldoc.translation_config import TranslationConfig
|
|
11
12
|
|
|
@@ -124,7 +125,8 @@ class LayoutParser:
|
|
|
124
125
|
# Convert coordinate system from picture to il
|
|
125
126
|
# system to the il coordinate system
|
|
126
127
|
x0, y0, x1, y1 = layout.xyxy
|
|
127
|
-
pix = mupdf_doc[page.page_number]
|
|
128
|
+
pix = get_no_rotation_img(mupdf_doc[page.page_number])
|
|
129
|
+
# pix = mupdf_doc[page.page_number].get_pixmap()
|
|
128
130
|
h, w = pix.height, pix.width
|
|
129
131
|
x0, y0, x1, y1 = (
|
|
130
132
|
np.clip(int(x0 - 1), 0, w - 1),
|
|
@@ -607,6 +607,9 @@ class StylesAndFormulas:
|
|
|
607
607
|
r"|.*Mincho.*"
|
|
608
608
|
r"|.*Amerigo.*"
|
|
609
609
|
r"|.*MSGloriolaIIStd.*"
|
|
610
|
+
r"|.*CMU.+"
|
|
611
|
+
r"|.*LinLibertine.*"
|
|
612
|
+
r"|.*txsys.*"
|
|
610
613
|
r")$"
|
|
611
614
|
)
|
|
612
615
|
precise_formula_font_pattern = (
|
|
@@ -644,7 +647,7 @@ class StylesAndFormulas:
|
|
|
644
647
|
r"|.*KpMath.*"
|
|
645
648
|
r"|.*Lete Sans Math.*"
|
|
646
649
|
r"|.*LeteSansMath.*"
|
|
647
|
-
r"|.*LinLibertineO.*"
|
|
650
|
+
# r"|.*LinLibertineO.*"
|
|
648
651
|
r"|.*Linux Libertine O.*"
|
|
649
652
|
r"|.*LibertinusMath-Regular.*"
|
|
650
653
|
r"|.*Libertinus Math.*"
|
|
@@ -6,6 +6,7 @@ import numpy as np
|
|
|
6
6
|
from pymupdf import Document
|
|
7
7
|
|
|
8
8
|
from babeldoc.document_il import il_version_1
|
|
9
|
+
from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
|
|
9
10
|
from babeldoc.document_il.utils.style_helper import GREEN
|
|
10
11
|
from babeldoc.translation_config import TranslationConfig
|
|
11
12
|
|
|
@@ -136,7 +137,8 @@ class TableParser:
|
|
|
136
137
|
# Convert coordinate system from picture to il
|
|
137
138
|
# system to the il coordinate system
|
|
138
139
|
x0, y0, x1, y1 = layout.xyxy
|
|
139
|
-
pix = mupdf_doc[page.page_number].get_pixmap()
|
|
140
|
+
# pix = mupdf_doc[page.page_number].get_pixmap()
|
|
141
|
+
pix = get_no_rotation_img(mupdf_doc[page.page_number])
|
|
140
142
|
h, w = pix.height, pix.width
|
|
141
143
|
x0, y0, x1, y1 = (
|
|
142
144
|
np.clip(int(x0 - 1), 0, w - 1),
|
|
@@ -9,6 +9,8 @@ from collections.abc import Generator
|
|
|
9
9
|
import cv2
|
|
10
10
|
import numpy as np
|
|
11
11
|
|
|
12
|
+
from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
|
|
13
|
+
|
|
12
14
|
try:
|
|
13
15
|
import onnx
|
|
14
16
|
import onnxruntime
|
|
@@ -282,7 +284,8 @@ class OnnxModel(DocLayoutModel):
|
|
|
282
284
|
for page in pages:
|
|
283
285
|
translate_config.raise_if_cancelled()
|
|
284
286
|
with self.lock:
|
|
285
|
-
pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
|
|
287
|
+
# pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
|
|
288
|
+
pix = get_no_rotation_img(mupdf_doc[page.page_number])
|
|
286
289
|
image = np.fromstring(pix.samples, np.uint8).reshape(
|
|
287
290
|
pix.height,
|
|
288
291
|
pix.width,
|
|
@@ -14,6 +14,7 @@ from tenacity import stop_after_attempt
|
|
|
14
14
|
from tenacity import wait_exponential
|
|
15
15
|
|
|
16
16
|
import babeldoc
|
|
17
|
+
from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
|
|
17
18
|
from babeldoc.docvision.doclayout import DocLayoutModel
|
|
18
19
|
from babeldoc.docvision.doclayout import YoloBox
|
|
19
20
|
from babeldoc.docvision.doclayout import YoloResult
|
|
@@ -256,7 +257,8 @@ class RpcDocLayoutModel(DocLayoutModel):
|
|
|
256
257
|
):
|
|
257
258
|
translate_config.raise_if_cancelled()
|
|
258
259
|
with self.lock:
|
|
259
|
-
pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
|
|
260
|
+
# pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
|
|
261
|
+
pix = get_no_rotation_img(mupdf_doc[page.page_number])
|
|
260
262
|
image = np.fromstring(pix.samples, np.uint8).reshape(
|
|
261
263
|
pix.height,
|
|
262
264
|
pix.width,
|
|
@@ -6,6 +6,7 @@ from collections.abc import Generator
|
|
|
6
6
|
import cv2
|
|
7
7
|
import numpy as np
|
|
8
8
|
from babeldoc.assets.assets import get_table_detection_rapidocr_model_path
|
|
9
|
+
from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
|
|
9
10
|
from babeldoc.docvision.doclayout import YoloBox
|
|
10
11
|
from babeldoc.docvision.doclayout import YoloResult
|
|
11
12
|
from rapidocr_onnxruntime import RapidOCR
|
|
@@ -236,7 +237,8 @@ class RapidOCRModel:
|
|
|
236
237
|
for page in pages:
|
|
237
238
|
translate_config.raise_if_cancelled()
|
|
238
239
|
with self.lock:
|
|
239
|
-
pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
|
|
240
|
+
# pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
|
|
241
|
+
pix = get_no_rotation_img(mupdf_doc[page.page_number])
|
|
240
242
|
image = np.fromstring(pix.samples, np.uint8).reshape(
|
|
241
243
|
pix.height,
|
|
242
244
|
pix.width,
|
|
@@ -423,9 +423,10 @@ def fix_filter(doc):
|
|
|
423
423
|
contents = page.get_contents()
|
|
424
424
|
if len(contents) > 1:
|
|
425
425
|
page_streams = [doc.xref_stream(i) for i in contents]
|
|
426
|
-
|
|
427
|
-
doc.
|
|
428
|
-
doc.
|
|
426
|
+
r = doc.get_new_xref()
|
|
427
|
+
doc.update_object(r, "<<>>")
|
|
428
|
+
doc.update_stream(r, b" ".join(page_streams))
|
|
429
|
+
doc.xref_set_key(page.xref, "Contents", f"{r} 0 R")
|
|
429
430
|
|
|
430
431
|
|
|
431
432
|
def do_translate(
|
|
@@ -643,8 +644,11 @@ def _do_translate_single(
|
|
|
643
644
|
"input.decompressed.pdf",
|
|
644
645
|
)
|
|
645
646
|
# Fix null xref in PDF file
|
|
646
|
-
|
|
647
|
-
|
|
647
|
+
try:
|
|
648
|
+
fix_filter(doc_input)
|
|
649
|
+
fix_null_xref(doc_input)
|
|
650
|
+
except Exception:
|
|
651
|
+
logger.exception("auto fix failed, please check the pdf file")
|
|
648
652
|
doc_input.save(output_path, expand=True, pretty=True)
|
|
649
653
|
del doc_input
|
|
650
654
|
|
|
@@ -654,8 +658,11 @@ def _do_translate_single(
|
|
|
654
658
|
resfont = "china-ss"
|
|
655
659
|
|
|
656
660
|
# Fix null xref in PDF file
|
|
657
|
-
|
|
658
|
-
|
|
661
|
+
try:
|
|
662
|
+
fix_filter(doc_pdf2zh)
|
|
663
|
+
fix_null_xref(doc_pdf2zh)
|
|
664
|
+
except Exception:
|
|
665
|
+
logger.exception("auto fix failed, please check the pdf file")
|
|
659
666
|
|
|
660
667
|
mediabox_data = fix_media_box(doc_pdf2zh)
|
|
661
668
|
|
|
@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
|
|
|
23
23
|
from babeldoc.translation_config import WatermarkOutputMode
|
|
24
24
|
|
|
25
25
|
logger = logging.getLogger(__name__)
|
|
26
|
-
__version__ = "0.3.
|
|
26
|
+
__version__ = "0.3.50"
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
def create_parser():
|
|
@@ -70,6 +70,11 @@ def create_parser():
|
|
|
70
70
|
default=None,
|
|
71
71
|
help="Restore offline assets package from the specified file",
|
|
72
72
|
)
|
|
73
|
+
parser.add_argument(
|
|
74
|
+
"--working-dir",
|
|
75
|
+
default=None,
|
|
76
|
+
help="Working directory for translation. If not set, use temp directory.",
|
|
77
|
+
)
|
|
73
78
|
# translation option argument group
|
|
74
79
|
translation_group = parser.add_argument_group(
|
|
75
80
|
"Translation",
|
|
@@ -320,7 +325,7 @@ async def main():
|
|
|
320
325
|
if not Path(file).exists():
|
|
321
326
|
logger.error(f"文件不存在:{file}")
|
|
322
327
|
exit(1)
|
|
323
|
-
if not file.endswith(".pdf"):
|
|
328
|
+
if not file.lower().endswith(".pdf"):
|
|
324
329
|
logger.error(f"文件不是 PDF 文件:{file}")
|
|
325
330
|
exit(1)
|
|
326
331
|
pending_files.append(file)
|
|
@@ -339,6 +344,21 @@ async def main():
|
|
|
339
344
|
else:
|
|
340
345
|
args.output = None
|
|
341
346
|
|
|
347
|
+
if args.working_dir:
|
|
348
|
+
working_dir = Path(args.working_dir)
|
|
349
|
+
if not working_dir.exists():
|
|
350
|
+
logger.info(f"工作目录不存在,创建:{working_dir}")
|
|
351
|
+
try:
|
|
352
|
+
working_dir.mkdir(parents=True, exist_ok=True)
|
|
353
|
+
except OSError:
|
|
354
|
+
logger.critical(
|
|
355
|
+
f"Failed to create working directory at {working_dir}",
|
|
356
|
+
exc_info=True,
|
|
357
|
+
)
|
|
358
|
+
exit(1)
|
|
359
|
+
else:
|
|
360
|
+
working_dir = None
|
|
361
|
+
|
|
342
362
|
watermark_output_mode = WatermarkOutputMode.Watermarked
|
|
343
363
|
if args.no_watermark:
|
|
344
364
|
watermark_output_mode = WatermarkOutputMode.NoWatermark
|
|
@@ -390,6 +410,7 @@ async def main():
|
|
|
390
410
|
skip_scanned_detection=args.skip_scanned_detection,
|
|
391
411
|
ocr_workaround=args.ocr_workaround,
|
|
392
412
|
custom_system_prompt=args.custom_system_prompt,
|
|
413
|
+
working_dir=working_dir,
|
|
393
414
|
)
|
|
394
415
|
|
|
395
416
|
# Create progress handler
|
|
@@ -377,6 +377,11 @@ class PDFPageInterpreterEx(PDFPageInterpreter):
|
|
|
377
377
|
ctm = (0, 1, -1, 0, y1, -x0)
|
|
378
378
|
else:
|
|
379
379
|
ctm = (1, 0, 0, 1, -x0, -y0)
|
|
380
|
+
# ctm_for_ops = copy.copy(ctm)
|
|
381
|
+
ctm_for_ops = (1, 0, 0, 1, -x0, -y0)
|
|
382
|
+
ctm = (1, 0, 0, 1, -x0, -y0)
|
|
383
|
+
if page.rotate == 90 or page.rotate == 270:
|
|
384
|
+
(x0, y0, x1, y1) = (y0, x1, y1, x0)
|
|
380
385
|
self.il_creater.on_page_start()
|
|
381
386
|
self.il_creater.on_page_crop_box(x0, y0, x1, y1)
|
|
382
387
|
self.device.begin_page(page, ctm)
|
|
@@ -391,7 +396,8 @@ class PDFPageInterpreterEx(PDFPageInterpreter):
|
|
|
391
396
|
# )
|
|
392
397
|
# for obj in page.contents:
|
|
393
398
|
# self.obj_patch[obj.objid] = ""
|
|
394
|
-
return f"q {ops_base} Q
|
|
399
|
+
return f"q {ops_base} Q {' '.join(f'{x:f}' for x in ctm_for_ops)} cm"
|
|
400
|
+
# return f"q {ops_base} Q 1 0 0 1 {x0} {y0} cm"
|
|
395
401
|
|
|
396
402
|
def render_contents(
|
|
397
403
|
self,
|