BabelDOC 0.3.38__tar.gz → 0.3.40__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/workflows/publish-to-pypi.yml +2 -2
  2. {babeldoc-0.3.38 → babeldoc-0.3.40}/PKG-INFO +4 -1
  3. {babeldoc-0.3.38 → babeldoc-0.3.40}/README.md +3 -0
  4. babeldoc-0.3.40/babeldoc/__init__.py +1 -0
  5. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/const.py +1 -1
  6. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/add_debug_information.py +1 -1
  7. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/detect_scanned_file.py +1 -1
  8. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/il_translator.py +113 -10
  9. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/il_translator_llm_only.py +64 -18
  10. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/layout_parser.py +1 -1
  11. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/styles_and_formulas.py +1 -0
  12. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/table_parser.py +1 -1
  13. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/translator/translator.py +1 -1
  14. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/utils/style_helper.py +1 -1
  15. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/high_level.py +2 -2
  16. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/main.py +7 -1
  17. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/translation_config.py +4 -0
  18. {babeldoc-0.3.38 → babeldoc-0.3.40}/pyproject.toml +2 -2
  19. {babeldoc-0.3.38 → babeldoc-0.3.40}/uv.lock +1 -1
  20. babeldoc-0.3.38/babeldoc/__init__.py +0 -1
  21. {babeldoc-0.3.38 → babeldoc-0.3.40}/.cursorignore +0 -0
  22. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
  23. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
  24. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/PULL_REQUEST_TEMPLATE/pr_form.yml +0 -0
  25. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/dependabot.yml +0 -0
  26. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/labels.yml +0 -0
  27. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/release-drafter.yml +0 -0
  28. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/workflows/codeql.yml +0 -0
  29. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/workflows/docs.yml +0 -0
  30. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/workflows/labeler.yml +0 -0
  31. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/workflows/lint.yml +0 -0
  32. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/workflows/pr-lint.yml +0 -0
  33. {babeldoc-0.3.38 → babeldoc-0.3.40}/.github/workflows/test.yml +0 -0
  34. {babeldoc-0.3.38 → babeldoc-0.3.40}/.gitignore +0 -0
  35. {babeldoc-0.3.38 → babeldoc-0.3.40}/.pre-commit-config.yaml +0 -0
  36. {babeldoc-0.3.38 → babeldoc-0.3.40}/LICENSE +0 -0
  37. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/assets/assets.py +0 -0
  38. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/assets/embedding_assets_metadata.py +0 -0
  39. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/asynchronize/__init__.py +0 -0
  40. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/converter.py +0 -0
  41. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/__init__.py +0 -0
  42. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py +0 -0
  43. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/backend/__init__.py +0 -0
  44. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/backend/pdf_creater.py +0 -0
  45. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/frontend/__init__.py +0 -0
  46. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/frontend/il_creater.py +0 -0
  47. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/il_version_1.py +0 -0
  48. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/il_version_1.rnc +0 -0
  49. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/il_version_1.rng +0 -0
  50. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/il_version_1.xsd +0 -0
  51. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/__init__.py +0 -0
  52. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/paragraph_finder.py +0 -0
  53. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/remove_descent.py +0 -0
  54. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/midend/typesetting.py +0 -0
  55. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/translator/__init__.py +0 -0
  56. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/translator/cache.py +0 -0
  57. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/utils/atomic_integer.py +0 -0
  58. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/utils/fontmap.py +0 -0
  59. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/utils/layout_helper.py +0 -0
  60. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/utils/paragraph_helper.py +0 -0
  61. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/utils/priority_thread_pool_executor.py +0 -0
  62. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/utils/zstd_helper.py +0 -0
  63. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/document_il/xml_converter.py +0 -0
  64. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/docvision/README.md +0 -0
  65. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/docvision/__init__.py +0 -0
  66. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/docvision/doclayout.py +0 -0
  67. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/docvision/rpc_doclayout.py +0 -0
  68. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/docvision/table_detection/rapidocr.py +0 -0
  69. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/format/office/__init__.py +0 -0
  70. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/pdfinterp.py +0 -0
  71. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/progress_monitor.py +0 -0
  72. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/result_merger.py +0 -0
  73. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/split_manager.py +0 -0
  74. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/tools/generate_font_metadata.py +0 -0
  75. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/tools/italic_assistance.py +0 -0
  76. {babeldoc-0.3.38 → babeldoc-0.3.40}/babeldoc/tools/italic_recognize_tool.py +0 -0
  77. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/CODE_OF_CONDUCT.md +0 -0
  78. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/CONTRIBUTING.md +0 -0
  79. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/CONTRIBUTOR_REWARD.md +0 -0
  80. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md +0 -0
  81. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/ILTranslator/ILTranslator.md +0 -0
  82. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/PDFCreation/PDFCreation.md +0 -0
  83. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/PDFParsing/PDFParsing.md +0 -0
  84. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md +0 -0
  85. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/README.md +0 -0
  86. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/StylesAndFormulas/StylesAndFormulas.md +0 -0
  87. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/ImplementationDetails/Typesetting/Typesetting.md +0 -0
  88. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/README.md +0 -0
  89. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/deploy.sh +0 -0
  90. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-banner.png +0 -0
  91. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.png +0 -0
  92. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.svg +0 -0
  93. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-big-logo-with-transparent-background.png +0 -0
  94. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-big-logo-with-transparent-background.svg +0 -0
  95. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-big-logo.png +0 -0
  96. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-contributor_reward_example.png +0 -0
  97. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-preview.gif +0 -0
  98. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-small-logo-with-transparent-background.png +0 -0
  99. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-small-logo-with-transparent-background.svg +0 -0
  100. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/images/babeldoc-small-logo.png +0 -0
  101. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/index.md +0 -0
  102. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/intro-to-pdf-object.md +0 -0
  103. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/requirements.txt +0 -0
  104. {babeldoc-0.3.38 → babeldoc-0.3.40}/docs/supported_languages.md +0 -0
  105. {babeldoc-0.3.38 → babeldoc-0.3.40}/mkdocs.yml +0 -0
  106. {babeldoc-0.3.38 → babeldoc-0.3.40}/tests/test_translation_config.py +0 -0
@@ -61,8 +61,8 @@ jobs:
61
61
  - name: Bump version for developmental release
62
62
  if: "! steps.check-version.outputs.tag"
63
63
  run: |
64
- version=$(bumpver update --patch --tag=final --dry 2>&1 | grep "New Version" | awk '{print $NF}') &&
65
- bumpver update --set-version $version.dev$(date +%s)
64
+ version=$(uv run bumpver update --patch --tag=final --dry 2>&1 | grep "New Version" | awk '{print $NF}') &&
65
+ uv run bumpver update --set-version $version.dev$(date +%s)
66
66
 
67
67
  - name: Build package
68
68
  run: "uv build"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: BabelDOC
3
- Version: 0.3.38
3
+ Version: 0.3.40
4
4
  Summary: Yet Another Document Translator
5
5
  Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
6
6
  Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -215,6 +215,8 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
215
215
  - `--no-mono`: Do not output monolingual PDF files
216
216
  - `--min-text-length`: Minimum text length to translate (default: 5)
217
217
  - `--openai`: Use OpenAI for translation (default: False)
218
+ - `--custom-system-prompt`: Custom system prompt for translation.
219
+ - `--add-formula-placehold-hint`: Add formula placeholder hint for translation. (Currently not recommended, it may affect translation quality, default: False)
218
220
 
219
221
  > [!TIP]
220
222
  >
@@ -222,6 +224,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
222
224
  > 2. It is recommended to use models with strong compatibility with OpenAI, such as: `glm-4-flash`, `deepseek-chat`, etc.
223
225
  > 3. Currently, it has not been optimized for traditional translation engines like Bing/Google, it is recommended to use LLMs.
224
226
  > 4. You can use [litellm](https://github.com/BerriAI/litellm) to access multiple models.
227
+ > 5. `--custom-system-prompt`: It is mainly used to add the `/no_think` instruction of Qwen 3 in the prompt. For example: `--custom-system-prompt "/no_think You are a professional, authentic machine translation engine."`
225
228
 
226
229
  ### OpenAI Specific Options
227
230
 
@@ -166,6 +166,8 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
166
166
  - `--no-mono`: Do not output monolingual PDF files
167
167
  - `--min-text-length`: Minimum text length to translate (default: 5)
168
168
  - `--openai`: Use OpenAI for translation (default: False)
169
+ - `--custom-system-prompt`: Custom system prompt for translation.
170
+ - `--add-formula-placehold-hint`: Add formula placeholder hint for translation. (Currently not recommended, it may affect translation quality, default: False)
169
171
 
170
172
  > [!TIP]
171
173
  >
@@ -173,6 +175,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
173
175
  > 2. It is recommended to use models with strong compatibility with OpenAI, such as: `glm-4-flash`, `deepseek-chat`, etc.
174
176
  > 3. Currently, it has not been optimized for traditional translation engines like Bing/Google, it is recommended to use LLMs.
175
177
  > 4. You can use [litellm](https://github.com/BerriAI/litellm) to access multiple models.
178
+ > 5. `--custom-system-prompt`: It is mainly used to add the `/no_think` instruction of Qwen 3 in the prompt. For example: `--custom-system-prompt "/no_think You are a professional, authentic machine translation engine."`
176
179
 
177
180
  ### OpenAI Specific Options
178
181
 
@@ -0,0 +1 @@
1
+ __version__ = "0.3.40"
@@ -3,7 +3,7 @@ import shutil
3
3
  import subprocess
4
4
  from pathlib import Path
5
5
 
6
- __version__ = "0.3.38"
6
+ __version__ = "0.3.40"
7
7
 
8
8
  CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
9
9
 
@@ -34,7 +34,7 @@ class AddDebugInformation:
34
34
 
35
35
  def _create_text(self, text: str, color: GraphicState, box: il_version_1.Box):
36
36
  style = il_version_1.PdfStyle(
37
- font_id="china-ss",
37
+ font_id="base",
38
38
  font_size=4,
39
39
  graphic_state=color,
40
40
  )
@@ -31,7 +31,7 @@ class DetectScannedFile:
31
31
  # Note: PDF coordinates are from bottom-left,
32
32
  # so we use y2 for top position
33
33
  style = il_version_1.PdfStyle(
34
- font_id="china-ss",
34
+ font_id="base",
35
35
  font_size=4,
36
36
  graphic_state=color,
37
37
  )
@@ -1,3 +1,6 @@
1
+ from __future__ import annotations
2
+
3
+ import copy
1
4
  import json
2
5
  import logging
3
6
  import re
@@ -78,6 +81,10 @@ class DocumentTranslateTracker:
78
81
  i_str = getattr(para, "input", None)
79
82
  o_str = getattr(para, "output", None)
80
83
  pdf_unicode = getattr(para, "pdf_unicode", None)
84
+ llm_translate_trackers = getattr(para, "llm_translate_trackers", None)
85
+ llm_translate_trackers_json = []
86
+ for tracker in llm_translate_trackers:
87
+ llm_translate_trackers_json.append(tracker.to_dict())
81
88
  if pdf_unicode is None or i_str is None:
82
89
  continue
83
90
  paragraphs.append(
@@ -85,6 +92,7 @@ class DocumentTranslateTracker:
85
92
  "input": i_str,
86
93
  "output": o_str,
87
94
  "pdf_unicode": pdf_unicode,
95
+ "llm_translate_trackers": llm_translate_trackers_json,
88
96
  },
89
97
  )
90
98
  pages.append({"paragraph": paragraphs})
@@ -103,7 +111,7 @@ class PageTranslateTracker:
103
111
 
104
112
  class ParagraphTranslateTracker:
105
113
  def __init__(self):
106
- pass
114
+ self.llm_translate_trackers = []
107
115
 
108
116
  def set_pdf_unicode(self, unicode: str):
109
117
  self.pdf_unicode = unicode
@@ -114,6 +122,52 @@ class ParagraphTranslateTracker:
114
122
  def set_output(self, output: str):
115
123
  self.output = output
116
124
 
125
+ def new_llm_translate_tracker(self) -> LLMTranslateTracker:
126
+ tracker = LLMTranslateTracker()
127
+ self.llm_translate_trackers.append(tracker)
128
+ return tracker
129
+
130
+ def last_llm_translate_tracker(self) -> LLMTranslateTracker | None:
131
+ if self.llm_translate_trackers:
132
+ return self.llm_translate_trackers[-1]
133
+ return None
134
+
135
+
136
+ class LLMTranslateTracker:
137
+ def __init__(self):
138
+ self.input = ""
139
+ self.output = ""
140
+ self.has_error = False
141
+ self.error_message = ""
142
+ self.placeholder_full_match = False
143
+ self.fallback_to_translate = False
144
+
145
+ def set_input(self, input_text: str):
146
+ self.input = input_text
147
+
148
+ def set_output(self, output_text: str):
149
+ self.output = output_text
150
+
151
+ def set_error_message(self, error_message: str):
152
+ self.has_error = True
153
+ self.error_message = error_message
154
+
155
+ def set_placeholder_full_match(self):
156
+ self.placeholder_full_match = True
157
+
158
+ def set_fallback_to_translate(self):
159
+ self.fallback_to_translate = True
160
+
161
+ def to_dict(self):
162
+ return {
163
+ "input": self.input,
164
+ "output": self.output,
165
+ "has_error": self.has_error,
166
+ "error_message": self.error_message,
167
+ "placeholder_full_match": self.placeholder_full_match,
168
+ "fallback_to_translate": self.fallback_to_translate,
169
+ }
170
+
117
171
 
118
172
  class ILTranslator:
119
173
  stage_name = "Translate Paragraphs"
@@ -156,9 +210,11 @@ class ILTranslator:
156
210
  # Try to find the first title paragraph
157
211
  title_paragraph = self.find_title_paragraph(docs)
158
212
  self.translation_config.shared_context_cross_split_part.first_paragraph = (
213
+ copy.deepcopy(title_paragraph)
214
+ )
215
+ self.translation_config.shared_context_cross_split_part.recent_title_paragraph = copy.deepcopy(
159
216
  title_paragraph
160
217
  )
161
- self.translation_config.shared_context_cross_split_part.recent_title_paragraph = title_paragraph
162
218
  if title_paragraph:
163
219
  logger.info(f"Found first title paragraph: {title_paragraph.unicode}")
164
220
 
@@ -220,7 +276,9 @@ class ILTranslator:
220
276
  # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
221
277
  paragraph_token_count = self.calc_token_count(paragraph.unicode)
222
278
  if paragraph.layout_label == "title":
223
- self.shared_context_cross_split_part.recent_title_paragraph = paragraph
279
+ self.shared_context_cross_split_part.recent_title_paragraph = (
280
+ copy.deepcopy(paragraph)
281
+ )
224
282
  executor.submit(
225
283
  self.translate_paragraph,
226
284
  paragraph,
@@ -245,6 +303,24 @@ class ILTranslator:
245
303
  self.placeholders = placeholders
246
304
  self.base_style = base_style
247
305
 
306
+ def get_placeholders_hint(self) -> dict[str, str] | None:
307
+ hint = {}
308
+ for placeholder in self.placeholders:
309
+ if isinstance(placeholder, FormulaPlaceholder):
310
+ cid_count = 0
311
+ for char in placeholder.formula.pdf_character:
312
+ if re.match(r"^\(cid:\d+\)$", char.char_unicode):
313
+ cid_count += 1
314
+ if cid_count > len(placeholder.formula.pdf_character) * 0.8:
315
+ continue
316
+
317
+ hint[placeholder.placeholder] = get_char_unicode_string(
318
+ placeholder.formula.pdf_character
319
+ )
320
+ if hint:
321
+ return hint
322
+ return None
323
+
248
324
  def create_formula_placeholder(
249
325
  self,
250
326
  formula: PdfFormula,
@@ -452,9 +528,8 @@ class ILTranslator:
452
528
  self,
453
529
  input_text: TranslateInput,
454
530
  output: str,
531
+ llm_translate_tracker: LLMTranslateTracker | None = None,
455
532
  ) -> [PdfParagraphComposition]:
456
- import re
457
-
458
533
  result = []
459
534
 
460
535
  # 如果没有占位符,直接返回整个文本
@@ -463,6 +538,8 @@ class ILTranslator:
463
538
  comp.pdf_same_style_unicode_characters = PdfSameStyleUnicodeCharacters()
464
539
  comp.pdf_same_style_unicode_characters.unicode = output
465
540
  comp.pdf_same_style_unicode_characters.pdf_style = input_text.base_style
541
+ if llm_translate_tracker:
542
+ llm_translate_tracker.set_placeholder_full_match()
466
543
  return [comp]
467
544
 
468
545
  # 构建正则表达式模式
@@ -484,7 +561,14 @@ class ILTranslator:
484
561
  placeholder_patterns.append(f"({left})")
485
562
  placeholder_patterns.append(f"({right})")
486
563
  placeholder_map[placeholder.left_placeholder] = placeholder
487
-
564
+ all_match = True
565
+ for pattern in patterns:
566
+ if not re.search(pattern, output, flags=re.IGNORECASE):
567
+ all_match = False
568
+ break
569
+ if all_match:
570
+ if llm_translate_tracker:
571
+ llm_translate_tracker.set_placeholder_full_match()
488
572
  # 合并所有模式
489
573
  combined_pattern = "|".join(patterns)
490
574
  combined_placeholder_pattern = "|".join(placeholder_patterns)
@@ -626,10 +710,12 @@ class ILTranslator:
626
710
  """Post-translation processing: update paragraph with translated text."""
627
711
  tracker.set_output(translated_text)
628
712
  if translated_text == translate_input.unicode:
713
+ if llm_translate_tracker := tracker.last_llm_translate_tracker():
714
+ llm_translate_tracker.set_placeholder_full_match()
629
715
  return False
630
716
  paragraph.unicode = translated_text
631
717
  paragraph.pdf_paragraph_composition = self.parse_translate_output(
632
- translate_input, translated_text
718
+ translate_input, translated_text, tracker.last_llm_translate_tracker()
633
719
  )
634
720
  for composition in paragraph.pdf_paragraph_composition:
635
721
  if (
@@ -646,8 +732,14 @@ class ILTranslator:
646
732
  text: str,
647
733
  title_paragraph: PdfParagraph | None = None,
648
734
  local_title_paragraph: PdfParagraph | None = None,
735
+ translate_input: TranslateInput | None = None,
649
736
  ):
650
- llm_input = ["You are a professional, authentic machine translation engine."]
737
+ if self.translation_config.custom_system_prompt:
738
+ llm_input = [self.translation_config.custom_system_prompt]
739
+ else:
740
+ llm_input = [
741
+ "You are a professional, authentic machine translation engine."
742
+ ]
651
743
 
652
744
  if title_paragraph:
653
745
  llm_input.append(
@@ -668,6 +760,15 @@ class ILTranslator:
668
760
  llm_input.append(
669
761
  f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
670
762
  )
763
+ if self.translation_config.add_formula_placehold_hint:
764
+ placeholders_hint = translate_input.get_placeholders_hint()
765
+ if placeholders_hint:
766
+ llm_input.append(
767
+ f"This is the formula placeholder hint: \n{placeholders_hint}"
768
+ )
769
+ llm_input.append(
770
+ "The system will automatically replace placeholders with corresponding formulas, please do not translate the placeholders!"
771
+ )
671
772
  prompt_template = f"""
672
773
  ;; Treat next line as plain text input and translate it into {self.translation_config.lang_out}, output translation ONLY. If translation is unnecessary (e.g. proper nouns, codes, {"{{1}}, etc. "}), return the original text. NO explanations. NO notes. Input:\n\n{text}
673
774
 
@@ -699,18 +800,20 @@ class ILTranslator:
699
800
  )
700
801
  if text is None:
701
802
  return
702
-
803
+ llm_translate_tracker = tracker.new_llm_translate_tracker()
703
804
  # Perform translation
704
805
  if self.support_llm_translate:
705
806
  llm_prompt = self.generate_prompt_for_llm(
706
- text, title_paragraph, local_title_paragraph
807
+ text, title_paragraph, local_title_paragraph, translate_input
707
808
  )
809
+ llm_translate_tracker.set_input(llm_prompt)
708
810
  translated_text = self.translate_engine.llm_translate(
709
811
  llm_prompt,
710
812
  rate_limit_params={
711
813
  "paragraph_token_count": paragraph_token_count
712
814
  },
713
815
  )
816
+ llm_translate_tracker.set_output(translated_text)
714
817
  else:
715
818
  translated_text = self.translate_engine.translate(
716
819
  text,
@@ -1,3 +1,4 @@
1
+ import copy
1
2
  import json
2
3
  import logging
3
4
  import re
@@ -11,6 +12,7 @@ from babeldoc.document_il import Document
11
12
  from babeldoc.document_il import Page
12
13
  from babeldoc.document_il import PdfFont
13
14
  from babeldoc.document_il import PdfParagraph
15
+ from babeldoc.document_il.midend import il_translator
14
16
  from babeldoc.document_il.midend.il_translator import DocumentTranslateTracker
15
17
  from babeldoc.document_il.midend.il_translator import ILTranslator
16
18
  from babeldoc.document_il.midend.il_translator import PageTranslateTracker
@@ -94,9 +96,11 @@ class ILTranslatorLLMOnly:
94
96
  # Try to find the first title paragraph
95
97
  title_paragraph = self.find_title_paragraph(docs)
96
98
  self.translation_config.shared_context_cross_split_part.first_paragraph = (
99
+ copy.deepcopy(title_paragraph)
100
+ )
101
+ self.translation_config.shared_context_cross_split_part.recent_title_paragraph = copy.deepcopy(
97
102
  title_paragraph
98
103
  )
99
- self.translation_config.shared_context_cross_split_part.recent_title_paragraph = title_paragraph
100
104
  if title_paragraph:
101
105
  logger.info(f"Found first title paragraph: {title_paragraph.unicode}")
102
106
 
@@ -169,7 +173,9 @@ class ILTranslatorLLMOnly:
169
173
  total_token_count += self.calc_token_count(paragraph.unicode)
170
174
  paragraphs.append(paragraph)
171
175
  if paragraph.layout_label == "title":
172
- self.shared_context_cross_split_part.recent_title_paragraph = paragraph
176
+ self.shared_context_cross_split_part.recent_title_paragraph = (
177
+ copy.deepcopy(paragraph)
178
+ )
173
179
 
174
180
  if total_token_count > 400 or len(paragraphs) > 5:
175
181
  executor.submit(
@@ -217,7 +223,7 @@ class ILTranslatorLLMOnly:
217
223
  should_translate_paragraph = []
218
224
  try:
219
225
  inputs = []
220
-
226
+ llm_translate_trackers = []
221
227
  for i in range(len(batch_paragraph.paragraphs)):
222
228
  paragraph = batch_paragraph.paragraphs[i]
223
229
  tracker = batch_paragraph.trackers[i]
@@ -227,26 +233,40 @@ class ILTranslatorLLMOnly:
227
233
  if text is None:
228
234
  pbar.advance(1)
229
235
  continue
236
+ llm_translate_tracker = tracker.new_llm_translate_tracker()
230
237
  should_translate_paragraph.append(i)
231
- inputs.append((text, translate_input, paragraph, tracker))
238
+ llm_translate_trackers.append(llm_translate_tracker)
239
+ inputs.append(
240
+ (text, translate_input, paragraph, tracker, llm_translate_tracker)
241
+ )
232
242
  if not inputs:
233
243
  return
234
244
  json_format_input = []
235
245
 
236
246
  for id_, input_text in enumerate(inputs):
237
- json_format_input.append(
238
- {
239
- "id": id_,
240
- "input": input_text[0],
241
- "layout_label": input_text[2].layout_label,
242
- }
243
- )
247
+ ti: il_translator.ILTranslator.TranslateInput = input_text[1]
248
+ placeholders_hint = ti.get_placeholders_hint()
249
+ obj = {
250
+ "id": id_,
251
+ "input": input_text[0],
252
+ "layout_label": input_text[2].layout_label,
253
+ }
254
+ if (
255
+ placeholders_hint
256
+ and self.translation_config.add_formula_placehold_hint
257
+ ):
258
+ obj["formula_placeholders_hint"] = placeholders_hint
259
+ json_format_input.append(obj)
260
+
244
261
  json_format_input = json.dumps(
245
262
  json_format_input, ensure_ascii=False, indent=2
246
263
  )
247
- llm_input = [
248
- "You are a professional, authentic machine translation engine."
249
- ]
264
+ if self.translation_config.custom_system_prompt:
265
+ llm_input = [self.translation_config.custom_system_prompt]
266
+ else:
267
+ llm_input = [
268
+ "You are a professional, authentic machine translation engine."
269
+ ]
250
270
 
251
271
  if title_paragraph:
252
272
  llm_input.append(
@@ -277,7 +297,12 @@ class ILTranslatorLLMOnly:
277
297
  {
278
298
  "id": 1,
279
299
  "input": "Source",
280
- "layout_label": "plain text"
300
+ "layout_label": "plain text",
301
+ // this is optional
302
+ "formula_placeholders_hint": {
303
+ "placeholder1": "hint1",
304
+ "placeholder2": "hint2"
305
+ }
281
306
  }
282
307
  ```
283
308
  Output:
@@ -296,10 +321,13 @@ class ILTranslatorLLMOnly:
296
321
  llm_input.append(
297
322
  f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
298
323
  )
299
-
300
324
  llm_input.append(
301
325
  f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
302
326
  )
327
+ if self.translation_config.add_formula_placehold_hint:
328
+ llm_input.append(
329
+ "The system will automatically replace placeholders with corresponding formulas, please do not translate the placeholders!"
330
+ )
303
331
  llm_input.append(
304
332
  f"""Here is the input:
305
333
 
@@ -308,10 +336,14 @@ class ILTranslatorLLMOnly:
308
336
  ```"""
309
337
  )
310
338
  final_input = "\n".join(llm_input).strip()
339
+ for llm_translate_tracker in llm_translate_trackers:
340
+ llm_translate_tracker.set_input(final_input)
311
341
  llm_output = self.translate_engine.llm_translate(
312
342
  final_input,
313
343
  rate_limit_params={"paragraph_token_count": paragraph_token_count},
314
344
  )
345
+ for llm_translate_tracker in llm_translate_trackers:
346
+ llm_translate_tracker.set_output(llm_output)
315
347
  llm_output = llm_output.strip()
316
348
 
317
349
  llm_output = self._clean_json_output(llm_output)
@@ -344,6 +376,7 @@ class ILTranslatorLLMOnly:
344
376
 
345
377
  # Get the original input for this translation
346
378
  translate_input = inputs[id_][1]
379
+ llm_translate_tracker = inputs[id_][4]
347
380
 
348
381
  input_unicode = inputs[id_][2].unicode
349
382
  output_unicode = translated_text
@@ -352,6 +385,9 @@ class ILTranslatorLLMOnly:
352
385
  output_token_count = self.calc_token_count(output_unicode)
353
386
 
354
387
  if not (0.3 < output_token_count / input_token_count < 3):
388
+ llm_translate_tracker.set_error_message(
389
+ f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
390
+ )
355
391
  logger.warning(
356
392
  f"Translation result is too long or too short. Input: {input_token_count}, Output: {output_token_count}"
357
393
  )
@@ -359,6 +395,9 @@ class ILTranslatorLLMOnly:
359
395
 
360
396
  edit_distance = Levenshtein.distance(input_unicode, output_unicode)
361
397
  if edit_distance < 5 and input_token_count > 20:
398
+ llm_translate_tracker.set_error_message(
399
+ f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
400
+ )
362
401
  logger.warning(
363
402
  f"Translation result edit distance is too small. distance: {edit_distance}, input: {input_unicode}, output: {output_unicode}"
364
403
  )
@@ -374,11 +413,15 @@ class ILTranslatorLLMOnly:
374
413
  if pbar:
375
414
  pbar.advance(1)
376
415
  except Exception as e:
377
- logger.exception(f"Error translating paragraph. Error: {e}.")
416
+ error_message = f"Error translating paragraph. Error: {e}."
417
+ logger.exception(error_message)
378
418
  # Ignore error and continue
419
+ for llm_translate_tracker in llm_translate_trackers:
420
+ llm_translate_tracker.set_error_message(error_message)
379
421
  continue
380
422
  finally:
381
423
  if should_fallback:
424
+ inputs[id_][4].set_fallback_to_translate()
382
425
  logger.warning(
383
426
  f"Fallback to simple translation. paragraph id: {inputs[id_][2].debug_id}"
384
427
  )
@@ -399,7 +442,10 @@ class ILTranslatorLLMOnly:
399
442
  )
400
443
 
401
444
  except Exception as e:
402
- logger.warning(f"Error {e} during translation. try fallback")
445
+ error_message = f"Error {e} during translation. try fallback"
446
+ logger.warning(error_message)
447
+ for llm_translate_tracker in llm_translate_trackers:
448
+ llm_translate_tracker.set_error_message(error_message)
403
449
  if not should_translate_paragraph:
404
450
  should_translate_paragraph = list(
405
451
  range(len(batch_paragraph.paragraphs))
@@ -78,7 +78,7 @@ class LayoutParser:
78
78
  # Note: PDF coordinates are from bottom-left,
79
79
  # so we use y2 for top position
80
80
  style = il_version_1.PdfStyle(
81
- font_id="china-ss",
81
+ font_id="base",
82
82
  font_size=4,
83
83
  graphic_state=color,
84
84
  )
@@ -606,6 +606,7 @@ class StylesAndFormulas:
606
606
  r"|.*NimbusSan.*"
607
607
  r"|.*Mincho.*"
608
608
  r"|.*Amerigo.*"
609
+ r"|.*MSGloriolaIIStd.*"
609
610
  r")$"
610
611
  )
611
612
  precise_formula_font_pattern = (
@@ -83,7 +83,7 @@ class TableParser:
83
83
  # Note: PDF coordinates are from bottom-left,
84
84
  # so we use y2 for top position
85
85
  style = il_version_1.PdfStyle(
86
- font_id="china-ss",
86
+ font_id="base",
87
87
  font_size=4,
88
88
  graphic_state=color,
89
89
  )
@@ -277,7 +277,7 @@ class OpenAITranslator(BaseTranslator):
277
277
  logger.exception("Error updating token count")
278
278
 
279
279
  def get_formular_placeholder(self, placeholder_id: int):
280
- return "{{v" + str(placeholder_id) + "}}"
280
+ return "{v" + str(placeholder_id) + "}"
281
281
  return "{{" + str(placeholder_id) + "}}"
282
282
 
283
283
  def get_rich_text_left_placeholder(self, placeholder_id: int):
@@ -1,7 +1,7 @@
1
1
  from babeldoc.document_il import il_version_1
2
2
 
3
3
 
4
- def create_pdf_style(r, g, b, font_id="china-ss", font_size=6):
4
+ def create_pdf_style(r, g, b, font_id="base", font_size=6):
5
5
  """
6
6
  Create a PdfStyle object from RGB values.
7
7
 
@@ -651,8 +651,8 @@ def _do_translate_single(
651
651
 
652
652
  mediabox_data = fix_media_box(doc_pdf2zh)
653
653
 
654
- for page in doc_pdf2zh:
655
- page.insert_font(resfont, None)
654
+ # for page in doc_pdf2zh:
655
+ # page.insert_font(resfont, None)
656
656
 
657
657
  resfont = None
658
658
  doc_pdf2zh.save(temp_pdf_path)
@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
23
23
  from babeldoc.translation_config import WatermarkOutputMode
24
24
 
25
25
  logger = logging.getLogger(__name__)
26
- __version__ = "0.3.38"
26
+ __version__ = "0.3.40"
27
27
 
28
28
 
29
29
  def create_parser():
@@ -216,6 +216,11 @@ def create_parser():
216
216
  default=False,
217
217
  help="Add text fill background (experimental)",
218
218
  )
219
+ translation_group.add_argument(
220
+ "--custom-system-prompt",
221
+ help="Custom system prompt for translation.",
222
+ default=None,
223
+ )
219
224
  # service option argument group
220
225
  service_group = translation_group.add_mutually_exclusive_group()
221
226
  service_group.add_argument(
@@ -384,6 +389,7 @@ async def main():
384
389
  show_char_box=args.show_char_box,
385
390
  skip_scanned_detection=args.skip_scanned_detection,
386
391
  ocr_workaround=args.ocr_workaround,
392
+ custom_system_prompt=args.custom_system_prompt,
387
393
  )
388
394
 
389
395
  # Create progress handler
@@ -69,6 +69,8 @@ class TranslationConfig:
69
69
  show_char_box: bool = False,
70
70
  skip_scanned_detection: bool = False,
71
71
  ocr_workaround: bool = False,
72
+ custom_system_prompt: str | None = None,
73
+ add_formula_placehold_hint: bool = False,
72
74
  ):
73
75
  self.translator = translator
74
76
 
@@ -153,6 +155,8 @@ class TranslationConfig:
153
155
 
154
156
  self.table_model = table_model
155
157
  self.show_char_box = show_char_box
158
+ self.custom_system_prompt = custom_system_prompt
159
+ self.add_formula_placehold_hint = add_formula_placehold_hint
156
160
 
157
161
  def parse_pages(self, pages_str: str | None) -> list[tuple[int, int]] | None:
158
162
  """解析页码字符串,返回页码范围列表
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "BabelDOC"
3
- version = "0.3.38"
3
+ version = "0.3.40"
4
4
  description = "Yet Another Document Translator"
5
5
  license = "AGPL-3.0"
6
6
  readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
150
150
  testpaths = ["tests"]
151
151
 
152
152
  [bumpver]
153
- current_version = "0.3.38"
153
+ current_version = "0.3.40"
154
154
  version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
155
155
 
156
156
  [bumpver.file_patterns]
@@ -48,7 +48,7 @@ wheels = [
48
48
 
49
49
  [[package]]
50
50
  name = "babeldoc"
51
- version = "0.3.38"
51
+ version = "0.3.40"
52
52
  source = { editable = "." }
53
53
  dependencies = [
54
54
  { name = "bitstring" },
@@ -1 +0,0 @@
1
- __version__ = "0.3.38"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes