BabelDOC 0.3.43__tar.gz → 0.3.50__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. babeldoc-0.3.50/.github/PULL_REQUEST_TEMPLATE.md +65 -0
  2. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/test.yml +1 -1
  3. {babeldoc-0.3.43 → babeldoc-0.3.50}/PKG-INFO +4 -3
  4. {babeldoc-0.3.43 → babeldoc-0.3.50}/README.md +1 -0
  5. babeldoc-0.3.50/babeldoc/__init__.py +1 -0
  6. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/const.py +1 -1
  7. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/backend/pdf_creater.py +11 -7
  8. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/il_translator.py +1 -1
  9. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/il_translator_llm_only.py +104 -75
  10. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/layout_parser.py +3 -1
  11. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/styles_and_formulas.py +4 -1
  12. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/table_parser.py +3 -1
  13. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/translator/translator.py +1 -0
  14. babeldoc-0.3.50/babeldoc/document_il/utils/mupdf_helper.py +10 -0
  15. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/doclayout.py +4 -1
  16. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/rpc_doclayout.py +3 -1
  17. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/table_detection/rapidocr.py +3 -1
  18. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/high_level.py +14 -7
  19. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/main.py +23 -2
  20. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/pdfinterp.py +7 -1
  21. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/translation_config.py +4 -0
  22. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/CONTRIBUTING.md +7 -1
  23. {babeldoc-0.3.43 → babeldoc-0.3.50}/pyproject.toml +4 -4
  24. {babeldoc-0.3.43 → babeldoc-0.3.50}/uv.lock +538 -229
  25. babeldoc-0.3.43/babeldoc/__init__.py +0 -1
  26. {babeldoc-0.3.43 → babeldoc-0.3.50}/.cursorignore +0 -0
  27. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
  28. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
  29. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/PULL_REQUEST_TEMPLATE/pr_form.yml +0 -0
  30. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/dependabot.yml +0 -0
  31. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/labels.yml +0 -0
  32. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/release-drafter.yml +0 -0
  33. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/codeql.yml +0 -0
  34. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/docs.yml +0 -0
  35. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/labeler.yml +0 -0
  36. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/lint.yml +0 -0
  37. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/pr-lint.yml +0 -0
  38. {babeldoc-0.3.43 → babeldoc-0.3.50}/.github/workflows/publish-to-pypi.yml +0 -0
  39. {babeldoc-0.3.43 → babeldoc-0.3.50}/.gitignore +0 -0
  40. {babeldoc-0.3.43 → babeldoc-0.3.50}/.pre-commit-config.yaml +0 -0
  41. {babeldoc-0.3.43 → babeldoc-0.3.50}/LICENSE +0 -0
  42. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/assets/assets.py +0 -0
  43. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/assets/embedding_assets_metadata.py +0 -0
  44. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/asynchronize/__init__.py +0 -0
  45. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/converter.py +0 -0
  46. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/__init__.py +0 -0
  47. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py +0 -0
  48. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/backend/__init__.py +0 -0
  49. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/frontend/__init__.py +0 -0
  50. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/frontend/il_creater.py +0 -0
  51. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.py +0 -0
  52. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.rnc +0 -0
  53. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.rng +0 -0
  54. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/il_version_1.xsd +0 -0
  55. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/__init__.py +0 -0
  56. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/add_debug_information.py +0 -0
  57. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/detect_scanned_file.py +0 -0
  58. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/paragraph_finder.py +0 -0
  59. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/remove_descent.py +0 -0
  60. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/midend/typesetting.py +0 -0
  61. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/translator/__init__.py +0 -0
  62. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/translator/cache.py +0 -0
  63. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/atomic_integer.py +0 -0
  64. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/fontmap.py +0 -0
  65. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/layout_helper.py +0 -0
  66. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/paragraph_helper.py +0 -0
  67. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/priority_thread_pool_executor.py +0 -0
  68. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/style_helper.py +0 -0
  69. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/utils/zstd_helper.py +0 -0
  70. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/document_il/xml_converter.py +0 -0
  71. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/README.md +0 -0
  72. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/docvision/__init__.py +0 -0
  73. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/format/office/__init__.py +0 -0
  74. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/progress_monitor.py +0 -0
  75. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/result_merger.py +0 -0
  76. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/split_manager.py +0 -0
  77. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/tools/generate_font_metadata.py +0 -0
  78. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/tools/italic_assistance.py +0 -0
  79. {babeldoc-0.3.43 → babeldoc-0.3.50}/babeldoc/tools/italic_recognize_tool.py +0 -0
  80. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/CODE_OF_CONDUCT.md +0 -0
  81. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/CONTRIBUTOR_REWARD.md +0 -0
  82. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md +0 -0
  83. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/ILTranslator/ILTranslator.md +0 -0
  84. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/PDFCreation/PDFCreation.md +0 -0
  85. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/PDFParsing/PDFParsing.md +0 -0
  86. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md +0 -0
  87. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/README.md +0 -0
  88. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/StylesAndFormulas/StylesAndFormulas.md +0 -0
  89. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/ImplementationDetails/Typesetting/Typesetting.md +0 -0
  90. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/README.md +0 -0
  91. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/deploy.sh +0 -0
  92. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-banner.png +0 -0
  93. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.png +0 -0
  94. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.svg +0 -0
  95. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-with-transparent-background.png +0 -0
  96. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo-with-transparent-background.svg +0 -0
  97. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-big-logo.png +0 -0
  98. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-contributor_reward_example.png +0 -0
  99. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-preview.gif +0 -0
  100. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-small-logo-with-transparent-background.png +0 -0
  101. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-small-logo-with-transparent-background.svg +0 -0
  102. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/images/babeldoc-small-logo.png +0 -0
  103. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/index.md +0 -0
  104. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/intro-to-pdf-object.md +0 -0
  105. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/requirements.txt +0 -0
  106. {babeldoc-0.3.43 → babeldoc-0.3.50}/docs/supported_languages.md +0 -0
  107. {babeldoc-0.3.43 → babeldoc-0.3.50}/mkdocs.yml +0 -0
  108. {babeldoc-0.3.43 → babeldoc-0.3.50}/tests/test_translation_config.py +0 -0
@@ -0,0 +1,65 @@
1
+ ### PR Title
2
+
3
+ <!-- Please fill in a concise and clear PR title below -->
4
+ [PR] <Your concise title here>
5
+
6
+ ### Related Issue(s)
7
+
8
+ <!-- If this PR closes or is related to one or more issues, please list them here (e.g., #37) -->
9
+ <!-- e.g.: Closes #37, Relates to #42 -->
10
+
11
+ ### Motivation and Context
12
+
13
+ <!-- Why is this change required? What problem does it solve? -->
14
+ <!-- If it fixes an open issue, please link to the issue here. -->
15
+
16
+ ### Summary of Changes
17
+
18
+ <!-- What does this PR introduce or fix? Please describe concisely. -->
19
+
20
+ ### PR Type
21
+
22
+ <!-- What kind of change is this? Please select one or more -->
23
+ - [ ] ✨ Enhancement
24
+ - [ ] 🐛 Bug Fix
25
+ - [ ] 📚 Documentation
26
+ - [ ] 🏗️ Refactor
27
+ - [ ] 🧪 Test
28
+ - [ ] 🧹 Chore
29
+
30
+ ### Breaking Changes
31
+
32
+ <!-- Does this PR introduce any breaking changes? If so, please describe them. -->
33
+ <!-- - [ ] Yes, this PR introduces breaking changes.
34
+ <!-- - [ ] No, this PR does not introduce breaking changes. -->
35
+ <!-- Detailed description of breaking changes (if any): -->
36
+
37
+ ### Contributor Checklist
38
+
39
+ - [ ] I have fully read and understood the **[CONTRIBUTING.md](https://funstory-ai.github.io/BabelDOC/CONTRIBUTING/)** guide.
40
+ - [ ] I have performed a self-review of my own code.
41
+ - [ ] My changes follow the project's code style and guidelines
42
+ - [ ] I have linked the related issue(s) in the description above (if applicable)
43
+ - [ ] I have updated relevant documentation (if applicable)
44
+ - [ ] I have added necessary tests that prove my fix is effective or that my feature works (if applicable)
45
+ - [ ] All new and existing tests passed locally with my changes
46
+ - [ ] My changes generate no new warnings or errors
47
+ - [ ] I understand that due to limited maintainer resources, only small PRs are accepted. Suggestions with proof-of-concept patches are appreciated, and my patch may be rewritten if necessary.
48
+
49
+ ### Testing Instructions
50
+
51
+ <!-- Please provide clear and concise step-by-step instructions on how to test your changes. -->
52
+ <!-- e.g.: -->
53
+ <!-- 1. Check out this branch. -->
54
+ <!-- 2. Run `...` to install dependencies. -->
55
+ <!-- 3. Run `...` to start the application/run the script. -->
56
+ <!-- 4. Navigate to `...` or observe `...` -->
57
+ <!-- 5. Verify that `...` (expected outcome). -->
58
+
59
+ ### Screenshots (if applicable)
60
+
61
+ <!-- If your changes include UI modifications, please add screenshots or GIFs to show the before and after. -->
62
+
63
+ ### Additional Notes
64
+
65
+ <!-- Is there anything else the reviewer should know? For example, any dependencies, or potential impacts. -->
@@ -15,7 +15,7 @@ jobs:
15
15
  runs-on: ubuntu-latest
16
16
  strategy:
17
17
  matrix:
18
- python-version: ["3.10", "3.11", "3.12"]
18
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
19
19
 
20
20
  steps:
21
21
  - uses: actions/checkout@v4
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: BabelDOC
3
- Version: 0.3.43
3
+ Version: 0.3.50
4
4
  Summary: Yet Another Document Translator
5
5
  Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
6
6
  Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -11,7 +11,7 @@ License-File: LICENSE
11
11
  Keywords: PDF
12
12
  Classifier: Operating System :: OS Independent
13
13
  Classifier: Programming Language :: Python :: 3
14
- Requires-Python: <3.13,>=3.10
14
+ Requires-Python: <3.14,>=3.10
15
15
  Requires-Dist: bitstring>=4.3.0
16
16
  Requires-Dist: configargparse>=1.7
17
17
  Requires-Dist: freetype-py>=2.5.1
@@ -19,7 +19,7 @@ Requires-Dist: httpx[socks]>=0.27.0
19
19
  Requires-Dist: huggingface-hub>=0.27.0
20
20
  Requires-Dist: msgpack>=1.1.0
21
21
  Requires-Dist: numpy>=2.0.2
22
- Requires-Dist: onnx>=1.17.0
22
+ Requires-Dist: onnx>=1.18.0
23
23
  Requires-Dist: onnxruntime>=1.16.1
24
24
  Requires-Dist: openai>=1.59.3
25
25
  Requires-Dist: opencv-python-headless>=4.10.0.84
@@ -205,6 +205,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
205
205
  - `--translate-table-text`: Translate table text (experimental, default: False)
206
206
  - `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
207
207
  - `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
208
+ - `--working-dir`: Working directory for translation. If not set, use temp directory.
208
209
 
209
210
  > [!TIP]
210
211
  > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -156,6 +156,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
156
156
  - `--translate-table-text`: Translate table text (experimental, default: False)
157
157
  - `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
158
158
  - `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
159
+ - `--working-dir`: Working directory for translation. If not set, use temp directory.
159
160
 
160
161
  > [!TIP]
161
162
  > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -0,0 +1 @@
1
+ __version__ = "0.3.50"
@@ -3,7 +3,7 @@ import shutil
3
3
  import subprocess
4
4
  from pathlib import Path
5
5
 
6
- __version__ = "0.3.43"
6
+ __version__ = "0.3.50"
7
7
 
8
8
  CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
9
9
 
@@ -386,20 +386,22 @@ class PDFCreater:
386
386
  # Get pages from both PDFs
387
387
  orig_page = original_pdf[page_id]
388
388
  trans_page = translated_pdf[page_id]
389
-
390
- # Calculate total width and use max height
389
+ rotate_angle = orig_page.rotation
391
390
  total_width = orig_page.rect.width + trans_page.rect.width
392
391
  max_height = max(orig_page.rect.height, trans_page.rect.height)
393
-
394
- # Create new page with combined width
395
- dual_page = dual.new_page(width=total_width, height=max_height)
396
-
397
- # Define rectangles for left and right sides
398
392
  left_width = (
399
393
  orig_page.rect.width
400
394
  if not translation_config.dual_translate_first
401
395
  else trans_page.rect.width
402
396
  )
397
+
398
+ orig_page.set_rotation(0)
399
+ trans_page.set_rotation(0)
400
+
401
+ # Create new page with combined width
402
+ dual_page = dual.new_page(width=total_width, height=max_height)
403
+
404
+ # Define rectangles for left and right sides
403
405
  rect_left = pymupdf.Rect(0, 0, left_width, max_height)
404
406
  rect_right = pymupdf.Rect(left_width, 0, total_width, max_height)
405
407
 
@@ -414,6 +416,7 @@ class PDFCreater:
414
416
  original_pdf,
415
417
  page_id,
416
418
  keep_proportion=True,
419
+ rotate=-rotate_angle,
417
420
  )
418
421
  except Exception as e:
419
422
  logger.warning(
@@ -429,6 +432,7 @@ class PDFCreater:
429
432
  translated_pdf,
430
433
  page_id,
431
434
  keep_proportion=True,
435
+ rotate=-rotate_angle,
432
436
  )
433
437
  except Exception as e:
434
438
  logger.warning(
@@ -649,7 +649,7 @@ class ILTranslator:
649
649
  and re.match(f"^{p.left_regex_pattern}", matched_text)
650
650
  )
651
651
  text = re.match(
652
- f"^{placeholder.left_regex_pattern}(.+){placeholder.right_regex_pattern}$",
652
+ f"^{placeholder.left_regex_pattern}(.*){placeholder.right_regex_pattern}$",
653
653
  matched_text,
654
654
  ).group(1)
655
655
 
@@ -267,37 +267,25 @@ class ILTranslatorLLMOnly:
267
267
  obj["formula_placeholders_hint"] = placeholders_hint
268
268
  json_format_input.append(obj)
269
269
 
270
- json_format_input = json.dumps(
270
+ json_format_input_str = json.dumps(
271
271
  json_format_input, ensure_ascii=False, indent=2
272
272
  )
273
+
274
+ # Start building the new prompt
275
+ llm_prompt_parts = []
276
+
277
+ # 1. #role
278
+ llm_prompt_parts.append("#role")
273
279
  if self.translation_config.custom_system_prompt:
274
- llm_input = [self.translation_config.custom_system_prompt]
280
+ llm_prompt_parts.append(self.translation_config.custom_system_prompt)
275
281
  else:
276
- llm_input = [
282
+ llm_prompt_parts.append(
277
283
  f"You are a professional and reliable machine translation engine responsible for translating the input text into {self.translation_config.lang_out}."
278
- ]
279
-
280
- llm_hint = []
281
- if title_paragraph:
282
- llm_hint.append(
283
- f"The first title in the full text: {title_paragraph.unicode}"
284
- )
285
- if (
286
- local_title_paragraph
287
- and local_title_paragraph.debug_id != title_paragraph.debug_id
288
- ):
289
- llm_hint.append(
290
- f"The most similar title in the full text: {local_title_paragraph.unicode}"
291
284
  )
292
285
 
293
- if llm_hint:
294
- llm_input.append(
295
- "When translating, please refer to the following information to improve translation quality:"
296
- )
297
- for i, line in enumerate(llm_hint):
298
- llm_input.append(f"{i}. {line}")
299
- llm_input.append("When translating, please follow the following rules:")
300
- # Create a structured prompt template for LLM translation
286
+ # 2. ##rules
287
+ llm_prompt_parts.append("\n##rules")
288
+ # Dynamically get placeholder examples
301
289
  rich_text_left_placeholder = (
302
290
  self.translate_engine.get_rich_text_left_placeholder(1)
303
291
  )
@@ -308,66 +296,102 @@ class ILTranslatorLLMOnly:
308
296
  )
309
297
  if isinstance(rich_text_right_placeholder, tuple):
310
298
  rich_text_right_placeholder = rich_text_right_placeholder[0]
311
- llm_input.append(
312
- f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"!'
313
- )
314
-
315
299
  formula_placeholder = self.translate_engine.get_formular_placeholder(3)
316
300
  if isinstance(formula_placeholder, tuple):
317
301
  formula_placeholder = formula_placeholder[0]
318
- llm_input.append(
302
+
303
+ llm_prompt_parts.append(
304
+ f'1. Do not translate style tags, such as "{rich_text_left_placeholder}xxx{rich_text_right_placeholder}"'
305
+ )
306
+ llm_prompt_parts.append(
319
307
  f'2. Do not translate formula placeholders, such as "{formula_placeholder}". The system will automatically replace the placeholders with the corresponding formulas.'
320
308
  )
321
- llm_input.append(
309
+ llm_prompt_parts.append(
322
310
  "3. If there is no need to translate (such as proper nouns, codes, etc.), then return the original text."
323
311
  )
324
- # Create a structured prompt template for LLM translation
325
- prompt_template = (
326
- f"""
327
- 4. You will be given a JSON formatted input containing entries with "id" and "input" fields.
328
-
329
- For each entry in the JSON, translate the contents of the "input" field into {self.translation_config.lang_out}.
330
- Write the translation back into the "output" field for that entry.
331
-
332
- """
333
- + """
334
- Here is an example of the expected format:
335
-
336
- <example>
337
- ```json
338
- Input:
339
- {
340
- "id": 1,
341
- "input": "Source",
342
- "layout_label": "plain text",
343
- // this is optional
344
- "formula_placeholders_hint": {
345
- "placeholder1": "hint1",
346
- "placeholder2": "hint2"
347
- }
348
- }
349
- ```
350
- Output:
351
- ```json
352
- {
353
- "id": 1,
354
- "output": "Translation"
355
- }
356
- ```
357
- </example>
358
-
359
- Please return the translated json directly without wrapping ```json``` tag or include any additional information.
360
- """
312
+
313
+ # 3. ##Input
314
+ llm_prompt_parts.append("\n##Input")
315
+ llm_prompt_parts.append(
316
+ 'You will be given a JSON formatted input containing entries with "id" and "input" fields.'
317
+ )
318
+
319
+ # 4. ##Output
320
+ llm_prompt_parts.append("\n##Output")
321
+ llm_prompt_parts.append(
322
+ f'For each entry in the JSON, translate the contents of the "input" field into {self.translation_config.lang_out}.'
361
323
  )
362
- llm_input.append(prompt_template)
363
- llm_input.append(
364
- f"""Here is the input:
365
-
366
- ```json
367
- {json_format_input}
368
- ```"""
324
+ llm_prompt_parts.append(
325
+ 'Write the translation back into the "output" field for that entry.'
369
326
  )
370
- final_input = "\n".join(llm_input).strip()
327
+ llm_prompt_parts.append(
328
+ "Please return the translated json directly without wrapping ```json``` tag or include any additional information."
329
+ )
330
+
331
+ # 5. ##example
332
+ llm_prompt_parts.append("\n##example")
333
+ llm_prompt_parts.append("Here is an example of the expected format:")
334
+ llm_prompt_parts.append("") # Blank line
335
+ llm_prompt_parts.append("<example>")
336
+ llm_prompt_parts.append("```json")
337
+ llm_prompt_parts.append("Input:")
338
+ llm_prompt_parts.append("{")
339
+ llm_prompt_parts.append(' "id": 1,')
340
+ llm_prompt_parts.append(' "input": "Source",')
341
+ llm_prompt_parts.append(' "layout_label": "plain text",')
342
+ llm_prompt_parts.append(" // this is optional")
343
+ llm_prompt_parts.append(' "formula_placeholders_hint": {')
344
+ llm_prompt_parts.append(' "placeholder1": "hint1",')
345
+ llm_prompt_parts.append(' "placeholder2": "hint2"')
346
+ llm_prompt_parts.append(" }")
347
+ llm_prompt_parts.append("}")
348
+ llm_prompt_parts.append("```")
349
+ llm_prompt_parts.append("Output:")
350
+ llm_prompt_parts.append("```json")
351
+ llm_prompt_parts.append("{")
352
+ llm_prompt_parts.append(' "id": 1,')
353
+ llm_prompt_parts.append(' "output": "Translation"')
354
+ llm_prompt_parts.append("}")
355
+ llm_prompt_parts.append("```")
356
+ llm_prompt_parts.append("</example>")
357
+
358
+ # 6. ##Others (contextual hints)
359
+ other_hints = []
360
+ hint_idx = 0
361
+ if title_paragraph:
362
+ other_hints.append(
363
+ f"{hint_idx}. The first title in the full text: {title_paragraph.unicode}"
364
+ )
365
+ hint_idx += 1
366
+
367
+ if local_title_paragraph:
368
+ # Check if it's different from the global title_paragraph before adding
369
+ is_different_from_global = True
370
+ if title_paragraph:
371
+ if local_title_paragraph.debug_id == title_paragraph.debug_id:
372
+ is_different_from_global = False
373
+
374
+ if is_different_from_global:
375
+ other_hints.append(
376
+ f"{hint_idx}. The most similar title in the full text: {local_title_paragraph.unicode}"
377
+ )
378
+ hint_idx += 1
379
+
380
+ if other_hints:
381
+ llm_prompt_parts.append("\n##Others")
382
+ llm_prompt_parts.append(
383
+ "When translating, please refer to the following information to improve translation quality:"
384
+ )
385
+ llm_prompt_parts.extend(other_hints)
386
+
387
+ llm_prompt_parts.append("\n## Actual Input")
388
+
389
+ # Combine all parts for the main prompt
390
+ main_prompt_content = "\n".join(llm_prompt_parts)
391
+
392
+ # Append the actual JSON input string at the end, without markdown fence
393
+ final_input = main_prompt_content + "\n\n" + json_format_input_str
394
+
371
395
  for llm_translate_tracker in llm_translate_trackers:
372
396
  llm_translate_tracker.set_input(final_input)
373
397
  llm_output = self.translate_engine.llm_translate(
@@ -382,6 +406,11 @@ class ILTranslatorLLMOnly:
382
406
 
383
407
  parsed_output = json.loads(llm_output)
384
408
 
409
+ if isinstance(parsed_output, dict) and parsed_output.get(
410
+ "output", parsed_output.get("input", False)
411
+ ):
412
+ parsed_output = [parsed_output]
413
+
385
414
  translation_results = {
386
415
  item["id"]: item.get("output", item.get("input"))
387
416
  for item in parsed_output
@@ -6,6 +6,7 @@ import numpy as np
6
6
  from pymupdf import Document
7
7
 
8
8
  from babeldoc.document_il import il_version_1
9
+ from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
9
10
  from babeldoc.document_il.utils.style_helper import GREEN
10
11
  from babeldoc.translation_config import TranslationConfig
11
12
 
@@ -124,7 +125,8 @@ class LayoutParser:
124
125
  # Convert coordinate system from picture to il
125
126
  # system to the il coordinate system
126
127
  x0, y0, x1, y1 = layout.xyxy
127
- pix = mupdf_doc[page.page_number].get_pixmap()
128
+ pix = get_no_rotation_img(mupdf_doc[page.page_number])
129
+ # pix = mupdf_doc[page.page_number].get_pixmap()
128
130
  h, w = pix.height, pix.width
129
131
  x0, y0, x1, y1 = (
130
132
  np.clip(int(x0 - 1), 0, w - 1),
@@ -607,6 +607,9 @@ class StylesAndFormulas:
607
607
  r"|.*Mincho.*"
608
608
  r"|.*Amerigo.*"
609
609
  r"|.*MSGloriolaIIStd.*"
610
+ r"|.*CMU.+"
611
+ r"|.*LinLibertine.*"
612
+ r"|.*txsys.*"
610
613
  r")$"
611
614
  )
612
615
  precise_formula_font_pattern = (
@@ -644,7 +647,7 @@ class StylesAndFormulas:
644
647
  r"|.*KpMath.*"
645
648
  r"|.*Lete Sans Math.*"
646
649
  r"|.*LeteSansMath.*"
647
- r"|.*LinLibertineO.*"
650
+ # r"|.*LinLibertineO.*"
648
651
  r"|.*Linux Libertine O.*"
649
652
  r"|.*LibertinusMath-Regular.*"
650
653
  r"|.*Libertinus Math.*"
@@ -6,6 +6,7 @@ import numpy as np
6
6
  from pymupdf import Document
7
7
 
8
8
  from babeldoc.document_il import il_version_1
9
+ from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
9
10
  from babeldoc.document_il.utils.style_helper import GREEN
10
11
  from babeldoc.translation_config import TranslationConfig
11
12
 
@@ -136,7 +137,8 @@ class TableParser:
136
137
  # Convert coordinate system from picture to il
137
138
  # system to the il coordinate system
138
139
  x0, y0, x1, y1 = layout.xyxy
139
- pix = mupdf_doc[page.page_number].get_pixmap()
140
+ # pix = mupdf_doc[page.page_number].get_pixmap()
141
+ pix = get_no_rotation_img(mupdf_doc[page.page_number])
140
142
  h, w = pix.height, pix.width
141
143
  x0, y0, x1, y1 = (
142
144
  np.clip(int(x0 - 1), 0, w - 1),
@@ -255,6 +255,7 @@ class OpenAITranslator(BaseTranslator):
255
255
  response = self.client.chat.completions.create(
256
256
  model=self.model,
257
257
  **self.options,
258
+ max_tokens=2048,
258
259
  messages=[
259
260
  {
260
261
  "role": "user",
@@ -0,0 +1,10 @@
1
+ import pymupdf
2
+
3
+
4
+ def get_no_rotation_img(page: pymupdf.Page):
5
+ # return page.get_pixmap(dpi=72)
6
+ original_rotation = page.rotation
7
+ page.set_rotation(0)
8
+ pix = page.get_pixmap(dpi=72)
9
+ page.set_rotation(original_rotation)
10
+ return pix
@@ -9,6 +9,8 @@ from collections.abc import Generator
9
9
  import cv2
10
10
  import numpy as np
11
11
 
12
+ from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
13
+
12
14
  try:
13
15
  import onnx
14
16
  import onnxruntime
@@ -282,7 +284,8 @@ class OnnxModel(DocLayoutModel):
282
284
  for page in pages:
283
285
  translate_config.raise_if_cancelled()
284
286
  with self.lock:
285
- pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
287
+ # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
288
+ pix = get_no_rotation_img(mupdf_doc[page.page_number])
286
289
  image = np.fromstring(pix.samples, np.uint8).reshape(
287
290
  pix.height,
288
291
  pix.width,
@@ -14,6 +14,7 @@ from tenacity import stop_after_attempt
14
14
  from tenacity import wait_exponential
15
15
 
16
16
  import babeldoc
17
+ from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
17
18
  from babeldoc.docvision.doclayout import DocLayoutModel
18
19
  from babeldoc.docvision.doclayout import YoloBox
19
20
  from babeldoc.docvision.doclayout import YoloResult
@@ -256,7 +257,8 @@ class RpcDocLayoutModel(DocLayoutModel):
256
257
  ):
257
258
  translate_config.raise_if_cancelled()
258
259
  with self.lock:
259
- pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
260
+ # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
261
+ pix = get_no_rotation_img(mupdf_doc[page.page_number])
260
262
  image = np.fromstring(pix.samples, np.uint8).reshape(
261
263
  pix.height,
262
264
  pix.width,
@@ -6,6 +6,7 @@ from collections.abc import Generator
6
6
  import cv2
7
7
  import numpy as np
8
8
  from babeldoc.assets.assets import get_table_detection_rapidocr_model_path
9
+ from babeldoc.document_il.utils.mupdf_helper import get_no_rotation_img
9
10
  from babeldoc.docvision.doclayout import YoloBox
10
11
  from babeldoc.docvision.doclayout import YoloResult
11
12
  from rapidocr_onnxruntime import RapidOCR
@@ -236,7 +237,8 @@ class RapidOCRModel:
236
237
  for page in pages:
237
238
  translate_config.raise_if_cancelled()
238
239
  with self.lock:
239
- pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
240
+ # pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
241
+ pix = get_no_rotation_img(mupdf_doc[page.page_number])
240
242
  image = np.fromstring(pix.samples, np.uint8).reshape(
241
243
  pix.height,
242
244
  pix.width,
@@ -423,9 +423,10 @@ def fix_filter(doc):
423
423
  contents = page.get_contents()
424
424
  if len(contents) > 1:
425
425
  page_streams = [doc.xref_stream(i) for i in contents]
426
- head = contents[0]
427
- doc.update_stream(head, b" ".join(page_streams))
428
- doc.xref_set_key(page.xref, "Contents", f"{head} 0 R")
426
+ r = doc.get_new_xref()
427
+ doc.update_object(r, "<<>>")
428
+ doc.update_stream(r, b" ".join(page_streams))
429
+ doc.xref_set_key(page.xref, "Contents", f"{r} 0 R")
429
430
 
430
431
 
431
432
  def do_translate(
@@ -643,8 +644,11 @@ def _do_translate_single(
643
644
  "input.decompressed.pdf",
644
645
  )
645
646
  # Fix null xref in PDF file
646
- fix_filter(doc_input)
647
- fix_null_xref(doc_input)
647
+ try:
648
+ fix_filter(doc_input)
649
+ fix_null_xref(doc_input)
650
+ except Exception:
651
+ logger.exception("auto fix failed, please check the pdf file")
648
652
  doc_input.save(output_path, expand=True, pretty=True)
649
653
  del doc_input
650
654
 
@@ -654,8 +658,11 @@ def _do_translate_single(
654
658
  resfont = "china-ss"
655
659
 
656
660
  # Fix null xref in PDF file
657
- fix_filter(doc_pdf2zh)
658
- fix_null_xref(doc_pdf2zh)
661
+ try:
662
+ fix_filter(doc_pdf2zh)
663
+ fix_null_xref(doc_pdf2zh)
664
+ except Exception:
665
+ logger.exception("auto fix failed, please check the pdf file")
659
666
 
660
667
  mediabox_data = fix_media_box(doc_pdf2zh)
661
668
 
@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
23
23
  from babeldoc.translation_config import WatermarkOutputMode
24
24
 
25
25
  logger = logging.getLogger(__name__)
26
- __version__ = "0.3.43"
26
+ __version__ = "0.3.50"
27
27
 
28
28
 
29
29
  def create_parser():
@@ -70,6 +70,11 @@ def create_parser():
70
70
  default=None,
71
71
  help="Restore offline assets package from the specified file",
72
72
  )
73
+ parser.add_argument(
74
+ "--working-dir",
75
+ default=None,
76
+ help="Working directory for translation. If not set, use temp directory.",
77
+ )
73
78
  # translation option argument group
74
79
  translation_group = parser.add_argument_group(
75
80
  "Translation",
@@ -320,7 +325,7 @@ async def main():
320
325
  if not Path(file).exists():
321
326
  logger.error(f"文件不存在:{file}")
322
327
  exit(1)
323
- if not file.endswith(".pdf"):
328
+ if not file.lower().endswith(".pdf"):
324
329
  logger.error(f"文件不是 PDF 文件:{file}")
325
330
  exit(1)
326
331
  pending_files.append(file)
@@ -339,6 +344,21 @@ async def main():
339
344
  else:
340
345
  args.output = None
341
346
 
347
+ if args.working_dir:
348
+ working_dir = Path(args.working_dir)
349
+ if not working_dir.exists():
350
+ logger.info(f"工作目录不存在,创建:{working_dir}")
351
+ try:
352
+ working_dir.mkdir(parents=True, exist_ok=True)
353
+ except OSError:
354
+ logger.critical(
355
+ f"Failed to create working directory at {working_dir}",
356
+ exc_info=True,
357
+ )
358
+ exit(1)
359
+ else:
360
+ working_dir = None
361
+
342
362
  watermark_output_mode = WatermarkOutputMode.Watermarked
343
363
  if args.no_watermark:
344
364
  watermark_output_mode = WatermarkOutputMode.NoWatermark
@@ -390,6 +410,7 @@ async def main():
390
410
  skip_scanned_detection=args.skip_scanned_detection,
391
411
  ocr_workaround=args.ocr_workaround,
392
412
  custom_system_prompt=args.custom_system_prompt,
413
+ working_dir=working_dir,
393
414
  )
394
415
 
395
416
  # Create progress handler
@@ -377,6 +377,11 @@ class PDFPageInterpreterEx(PDFPageInterpreter):
377
377
  ctm = (0, 1, -1, 0, y1, -x0)
378
378
  else:
379
379
  ctm = (1, 0, 0, 1, -x0, -y0)
380
+ # ctm_for_ops = copy.copy(ctm)
381
+ ctm_for_ops = (1, 0, 0, 1, -x0, -y0)
382
+ ctm = (1, 0, 0, 1, -x0, -y0)
383
+ if page.rotate == 90 or page.rotate == 270:
384
+ (x0, y0, x1, y1) = (y0, x1, y1, x0)
380
385
  self.il_creater.on_page_start()
381
386
  self.il_creater.on_page_crop_box(x0, y0, x1, y1)
382
387
  self.device.begin_page(page, ctm)
@@ -391,7 +396,8 @@ class PDFPageInterpreterEx(PDFPageInterpreter):
391
396
  # )
392
397
  # for obj in page.contents:
393
398
  # self.obj_patch[obj.objid] = ""
394
- return f"q {ops_base} Q 1 0 0 1 {x0} {y0} cm"
399
+ return f"q {ops_base} Q {' '.join(f'{x:f}' for x in ctm_for_ops)} cm"
400
+ # return f"q {ops_base} Q 1 0 0 1 {x0} {y0} cm"
395
401
 
396
402
  def render_contents(
397
403
  self,