BabelDOC 0.3.31__tar.gz → 0.3.33__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. {babeldoc-0.3.31 → babeldoc-0.3.33}/PKG-INFO +3 -1
  2. {babeldoc-0.3.31 → babeldoc-0.3.33}/README.md +2 -0
  3. babeldoc-0.3.33/babeldoc/__init__.py +1 -0
  4. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/const.py +1 -1
  5. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/backend/pdf_creater.py +1 -1
  6. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/frontend/il_creater.py +2 -2
  7. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/il_translator.py +107 -6
  8. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/il_translator_llm_only.py +18 -5
  9. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/styles_and_formulas.py +6 -4
  10. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/utils/paragraph_helper.py +2 -1
  11. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/high_level.py +13 -1
  12. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/main.py +1 -1
  13. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/pdfinterp.py +41 -1
  14. babeldoc-0.3.33/docs/supported_languages.md +140 -0
  15. {babeldoc-0.3.31 → babeldoc-0.3.33}/mkdocs.yml +1 -0
  16. {babeldoc-0.3.31 → babeldoc-0.3.33}/pyproject.toml +2 -2
  17. {babeldoc-0.3.31 → babeldoc-0.3.33}/uv.lock +1 -1
  18. babeldoc-0.3.31/babeldoc/__init__.py +0 -1
  19. {babeldoc-0.3.31 → babeldoc-0.3.33}/.cursorignore +0 -0
  20. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
  21. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
  22. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/PULL_REQUEST_TEMPLATE/pr_form.yml +0 -0
  23. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/dependabot.yml +0 -0
  24. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/labels.yml +0 -0
  25. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/release-drafter.yml +0 -0
  26. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/workflows/codeql.yml +0 -0
  27. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/workflows/docs.yml +0 -0
  28. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/workflows/labeler.yml +0 -0
  29. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/workflows/lint.yml +0 -0
  30. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/workflows/pr-lint.yml +0 -0
  31. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/workflows/publish-to-pypi.yml +0 -0
  32. {babeldoc-0.3.31 → babeldoc-0.3.33}/.github/workflows/test.yml +0 -0
  33. {babeldoc-0.3.31 → babeldoc-0.3.33}/.gitignore +0 -0
  34. {babeldoc-0.3.31 → babeldoc-0.3.33}/.pre-commit-config.yaml +0 -0
  35. {babeldoc-0.3.31 → babeldoc-0.3.33}/LICENSE +0 -0
  36. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/assets/assets.py +0 -0
  37. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/assets/embedding_assets_metadata.py +0 -0
  38. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/asynchronize/__init__.py +0 -0
  39. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/converter.py +0 -0
  40. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/__init__.py +0 -0
  41. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py +0 -0
  42. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/backend/__init__.py +0 -0
  43. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/frontend/__init__.py +0 -0
  44. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/il_version_1.py +0 -0
  45. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/il_version_1.rnc +0 -0
  46. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/il_version_1.rng +0 -0
  47. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/il_version_1.xsd +0 -0
  48. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/__init__.py +0 -0
  49. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/add_debug_information.py +0 -0
  50. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/detect_scanned_file.py +0 -0
  51. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/layout_parser.py +0 -0
  52. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/paragraph_finder.py +3 -3
  53. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/remove_descent.py +0 -0
  54. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/table_parser.py +0 -0
  55. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/midend/typesetting.py +0 -0
  56. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/translator/__init__.py +0 -0
  57. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/translator/cache.py +0 -0
  58. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/translator/translator.py +0 -0
  59. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/utils/atomic_integer.py +0 -0
  60. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/utils/fontmap.py +0 -0
  61. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/utils/layout_helper.py +0 -0
  62. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/utils/priority_thread_pool_executor.py +0 -0
  63. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/utils/style_helper.py +0 -0
  64. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/utils/zstd_helper.py +0 -0
  65. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/document_il/xml_converter.py +0 -0
  66. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/docvision/README.md +0 -0
  67. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/docvision/__init__.py +0 -0
  68. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/docvision/doclayout.py +0 -0
  69. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/docvision/rpc_doclayout.py +0 -0
  70. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/docvision/table_detection/rapidocr.py +0 -0
  71. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/format/office/__init__.py +0 -0
  72. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/progress_monitor.py +0 -0
  73. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/result_merger.py +0 -0
  74. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/split_manager.py +0 -0
  75. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/tools/generate_font_metadata.py +0 -0
  76. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/tools/italic_assistance.py +0 -0
  77. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/tools/italic_recognize_tool.py +0 -0
  78. {babeldoc-0.3.31 → babeldoc-0.3.33}/babeldoc/translation_config.py +0 -0
  79. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/CODE_OF_CONDUCT.md +0 -0
  80. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/CONTRIBUTING.md +0 -0
  81. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/CONTRIBUTOR_REWARD.md +0 -0
  82. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md +0 -0
  83. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/ILTranslator/ILTranslator.md +0 -0
  84. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/PDFCreation/PDFCreation.md +0 -0
  85. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/PDFParsing/PDFParsing.md +0 -0
  86. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md +0 -0
  87. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/README.md +0 -0
  88. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/StylesAndFormulas/StylesAndFormulas.md +0 -0
  89. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/ImplementationDetails/Typesetting/Typesetting.md +0 -0
  90. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/README.md +0 -0
  91. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/deploy.sh +0 -0
  92. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-banner.png +0 -0
  93. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.png +0 -0
  94. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.svg +0 -0
  95. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-big-logo-with-transparent-background.png +0 -0
  96. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-big-logo-with-transparent-background.svg +0 -0
  97. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-big-logo.png +0 -0
  98. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-contributor_reward_example.png +0 -0
  99. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-preview.gif +0 -0
  100. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-small-logo-with-transparent-background.png +0 -0
  101. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-small-logo-with-transparent-background.svg +0 -0
  102. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/images/babeldoc-small-logo.png +0 -0
  103. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/index.md +0 -0
  104. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/intro-to-pdf-object.md +0 -0
  105. {babeldoc-0.3.31 → babeldoc-0.3.33}/docs/requirements.txt +0 -0
  106. {babeldoc-0.3.31 → babeldoc-0.3.33}/tests/test_translation_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: BabelDOC
3
- Version: 0.3.31
3
+ Version: 0.3.33
4
4
  Summary: Yet Another Document Translator
5
5
  Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
6
6
  Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -88,6 +88,8 @@ PDF scientific paper translation and bilingual comparison library.
88
88
  - Provides a [Python API](#python-api).
89
89
  - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.
90
90
 
91
+ [Supported Language](https://funstory-ai.github.io/BabelDOC/supported_languages/)
92
+
91
93
  ## Preview
92
94
 
93
95
  <div align="center">
@@ -39,6 +39,8 @@ PDF scientific paper translation and bilingual comparison library.
39
39
  - Provides a [Python API](#python-api).
40
40
  - Mainly designed to be embedded into other programs, but can also be used directly for simple translation tasks.
41
41
 
42
+ [Supported Language](https://funstory-ai.github.io/BabelDOC/supported_languages/)
43
+
42
44
  ## Preview
43
45
 
44
46
  <div align="center">
@@ -0,0 +1 @@
1
+ __version__ = "0.3.33"
@@ -3,7 +3,7 @@ import shutil
3
3
  import subprocess
4
4
  from pathlib import Path
5
5
 
6
- __version__ = "0.3.31"
6
+ __version__ = "0.3.33"
7
7
 
8
8
  CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
9
9
 
@@ -782,7 +782,7 @@ class PDFCreater:
782
782
  try:
783
783
  doc.xref_set_key(doc[pageno].xref, name, box)
784
784
  except Exception:
785
- logger.error(f"Error restoring media box {name} from PDF")
785
+ logger.debug(f"Error restoring media box {name} from PDF")
786
786
 
787
787
  def write(
788
788
  self, translation_config: TranslationConfig, check_font_exists: bool = False
@@ -859,8 +859,8 @@ class ILCreater:
859
859
  char_bounding_box = None
860
860
 
861
861
  char_unicode = char.get_text()
862
- if "(cid:" not in char_unicode and len(char_unicode) > 1:
863
- return
862
+ # if "(cid:" not in char_unicode and len(char_unicode) > 1:
863
+ # return
864
864
  if space_regex.match(char_unicode):
865
865
  char_unicode = " "
866
866
  advance = char.adv
@@ -127,12 +127,22 @@ class ILTranslator:
127
127
  self.translate_engine = translate_engine
128
128
  self.translation_config = translation_config
129
129
  self.font_mapper = FontMapper(translation_config)
130
-
130
+ self.shared_context_cross_split_part = (
131
+ translation_config.shared_context_cross_split_part
132
+ )
131
133
  if tokenizer is None:
132
134
  self.tokenizer = tiktoken.encoding_for_model("gpt-4o")
133
135
  else:
134
136
  self.tokenizer = tokenizer
135
137
 
138
+ self.support_llm_translate = False
139
+ try:
140
+ if translate_engine and hasattr(translate_engine, "do_llm_translate"):
141
+ translate_engine.do_llm_translate(None)
142
+ self.support_llm_translate = True
143
+ except NotImplementedError:
144
+ self.support_llm_translate = False
145
+
136
146
  def calc_token_count(self, text: str) -> int:
137
147
  try:
138
148
  return len(self.tokenizer.encode(text, disallowed_special=()))
@@ -141,6 +151,17 @@ class ILTranslator:
141
151
 
142
152
  def translate(self, docs: Document):
143
153
  tracker = DocumentTranslateTracker()
154
+
155
+ if not self.translation_config.shared_context_cross_split_part.first_paragraph:
156
+ # Try to find the first title paragraph
157
+ title_paragraph = self.find_title_paragraph(docs)
158
+ self.translation_config.shared_context_cross_split_part.first_paragraph = (
159
+ title_paragraph
160
+ )
161
+ self.translation_config.shared_context_cross_split_part.recent_title_paragraph = title_paragraph
162
+ if title_paragraph:
163
+ logger.info(f"Found first title paragraph: {title_paragraph.unicode}")
164
+
144
165
  # count total paragraph
145
166
  total = sum(len(page.pdf_paragraph) for page in docs.page)
146
167
  with self.translation_config.progress_monitor.stage_start(
@@ -163,6 +184,22 @@ class ILTranslator:
163
184
  with Path(path).open("w", encoding="utf-8") as f:
164
185
  f.write(tracker.to_json())
165
186
 
187
+ def find_title_paragraph(self, docs: Document) -> PdfParagraph | None:
188
+ """Find the first paragraph with layout_label 'title' in the document.
189
+
190
+ Args:
191
+ docs: The document to search in
192
+
193
+ Returns:
194
+ The first title paragraph found, or None if no title paragraph exists
195
+ """
196
+ for page in docs.page:
197
+ for paragraph in page.pdf_paragraph:
198
+ if paragraph.layout_label == "title":
199
+ logger.info(f"Found title paragraph: {paragraph.unicode}")
200
+ return paragraph
201
+ return None
202
+
166
203
  def process_page(
167
204
  self,
168
205
  page: Page,
@@ -182,6 +219,8 @@ class ILTranslator:
182
219
  page_xobj_font_map[xobj.xobj_id][font.font_id] = font
183
220
  # self.translate_paragraph(paragraph, pbar,tracker.new_paragraph(), page_font_map, page_xobj_font_map)
184
221
  paragraph_token_count = self.calc_token_count(paragraph.unicode)
222
+ if paragraph.layout_label == "title":
223
+ self.shared_context_cross_split_part.recent_title_paragraph = paragraph
185
224
  executor.submit(
186
225
  self.translate_paragraph,
187
226
  paragraph,
@@ -191,6 +230,8 @@ class ILTranslator:
191
230
  page_xobj_font_map,
192
231
  priority=1048576 - paragraph_token_count,
193
232
  paragraph_token_count=paragraph_token_count,
233
+ title_paragraph=self.translation_config.shared_context_cross_split_part.first_paragraph,
234
+ local_title_paragraph=self.translation_config.shared_context_cross_split_part.recent_title_paragraph,
194
235
  )
195
236
 
196
237
  class TranslateInput:
@@ -555,7 +596,15 @@ class ILTranslator:
555
596
  tracker.set_pdf_unicode(paragraph.unicode)
556
597
  if paragraph.xobj_id in xobj_font_map:
557
598
  page_font_map = xobj_font_map[paragraph.xobj_id]
558
- translate_input = self.get_translate_input(paragraph, page_font_map)
599
+ disable_rich_text_translate = (
600
+ self.translation_config.disable_rich_text_translate
601
+ )
602
+ if not self.support_llm_translate:
603
+ disable_rich_text_translate = True
604
+
605
+ translate_input = self.get_translate_input(
606
+ paragraph, page_font_map, disable_rich_text_translate
607
+ )
559
608
  if not translate_input:
560
609
  return None, None
561
610
  tracker.set_input(translate_input.unicode)
@@ -592,6 +641,43 @@ class ILTranslator:
592
641
  )
593
642
  return True
594
643
 
644
+ def generate_prompt_for_llm(
645
+ self,
646
+ text: str,
647
+ title_paragraph: PdfParagraph | None = None,
648
+ local_title_paragraph: PdfParagraph | None = None,
649
+ ):
650
+ llm_input = ["You are a professional, authentic machine translation engine."]
651
+
652
+ if title_paragraph:
653
+ llm_input.append(
654
+ f"The first title in the full text: {title_paragraph.unicode}"
655
+ )
656
+ if (
657
+ local_title_paragraph
658
+ and local_title_paragraph.debug_id != title_paragraph.debug_id
659
+ ):
660
+ llm_input.append(
661
+ f"The most similar title in the full text: {local_title_paragraph.unicode}"
662
+ )
663
+ # Create a structured prompt template for LLM translation
664
+ llm_input.append(
665
+ f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
666
+ )
667
+
668
+ llm_input.append(
669
+ f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
670
+ )
671
+ prompt_template = f"""
672
+ ;; Treat next line as plain text input and translate it into {self.translation_config.lang_out}, output translation ONLY. If translation is unnecessary (e.g. proper nouns, codes, {"{{1}}, etc. "}), return the original text. NO explanations. NO notes. Input:\n\n{text}
673
+
674
+ """
675
+ llm_input.append(prompt_template)
676
+
677
+ final_input = "\n".join(llm_input).strip()
678
+
679
+ return final_input
680
+
595
681
  def translate_paragraph(
596
682
  self,
597
683
  paragraph: PdfParagraph,
@@ -600,6 +686,8 @@ class ILTranslator:
600
686
  page_font_map: dict[str, PdfFont] = None,
601
687
  xobj_font_map: dict[int, dict[str, PdfFont]] = None,
602
688
  paragraph_token_count: int = 0,
689
+ title_paragraph: PdfParagraph | None = None,
690
+ local_title_paragraph: PdfParagraph | None = None,
603
691
  ):
604
692
  """Translate a paragraph using pre and post processing functions."""
605
693
  self.translation_config.raise_if_cancelled()
@@ -613,10 +701,23 @@ class ILTranslator:
613
701
  return
614
702
 
615
703
  # Perform translation
616
- translated_text = self.translate_engine.translate(
617
- text,
618
- rate_limit_params={"paragraph_token_count": paragraph_token_count},
619
- )
704
+ if self.support_llm_translate:
705
+ llm_prompt = self.generate_prompt_for_llm(
706
+ text, title_paragraph, local_title_paragraph
707
+ )
708
+ translated_text = self.translate_engine.llm_translate(
709
+ llm_prompt,
710
+ rate_limit_params={
711
+ "paragraph_token_count": paragraph_token_count
712
+ },
713
+ )
714
+ else:
715
+ translated_text = self.translate_engine.translate(
716
+ text,
717
+ rate_limit_params={
718
+ "paragraph_token_count": paragraph_token_count
719
+ },
720
+ )
620
721
  translated_text = re.sub(r"[. 。…,]{20,}", ".", translated_text)
621
722
 
622
723
  # Post-translation processing
@@ -262,11 +262,7 @@ class ILTranslatorLLMOnly:
262
262
  # Create a structured prompt template for LLM translation
263
263
  prompt_template = (
264
264
  f"""
265
- You will be given a JSON formatted input containing entries with "id" and "input" fields. Here is the input:
266
-
267
- ```json
268
- {json_format_input}
269
- ```
265
+ You will be given a JSON formatted input containing entries with "id" and "input" fields.
270
266
 
271
267
  For each entry in the JSON, translate the contents of the "input" field into {self.translation_config.lang_out}.
272
268
  Write the translation back into the "output" field for that entry.
@@ -297,7 +293,20 @@ class ILTranslatorLLMOnly:
297
293
  """
298
294
  )
299
295
  llm_input.append(prompt_template)
296
+ llm_input.append(
297
+ f'Please do not translate style tags like "{self.translate_engine.get_rich_text_left_placeholder(1)}xxx{self.translate_engine.get_rich_text_right_placeholder(2)}"!'
298
+ )
300
299
 
300
+ llm_input.append(
301
+ f'Please do not translate formula placeholders like "{self.translate_engine.get_formular_placeholder(3)}"!'
302
+ )
303
+ llm_input.append(
304
+ f"""Here is the input:
305
+
306
+ ```json
307
+ {json_format_input}
308
+ ```"""
309
+ )
301
310
  final_input = "\n".join(llm_input).strip()
302
311
  llm_output = self.translate_engine.llm_translate(
303
312
  final_input,
@@ -385,6 +394,8 @@ class ILTranslatorLLMOnly:
385
394
  xobj_font_map,
386
395
  priority=1048576 - paragraph_token_count,
387
396
  paragraph_token_count=paragraph_token_count,
397
+ title_paragraph=title_paragraph,
398
+ local_title_paragraph=local_title_paragraph,
388
399
  )
389
400
 
390
401
  except Exception as e:
@@ -408,6 +419,8 @@ class ILTranslatorLLMOnly:
408
419
  xobj_font_map,
409
420
  priority=1048576 - paragraph_token_count,
410
421
  paragraph_token_count=paragraph_token_count,
422
+ title_paragraph=title_paragraph,
423
+ local_title_paragraph=local_title_paragraph,
411
424
  )
412
425
 
413
426
  def _clean_json_output(self, llm_output: str) -> str:
@@ -529,10 +529,10 @@ class StylesAndFormulas:
529
529
  pattern_text = (
530
530
  r"^("
531
531
  r"|Cambria.*"
532
- r"|EUAlbertina.+"
533
- r"|NimbusRomNo9L.+"
534
- r"|GlosaMath.+"
535
- r"|URWPalladioL.+"
532
+ r"|EUAlbertina.*"
533
+ r"|NimbusRomNo9L.*"
534
+ r"|GlosaMath.*"
535
+ r"|URWPalladioL.*"
536
536
  r"|CMSS.+"
537
537
  r"|Arial.*"
538
538
  r"|TimesNewRoman.*"
@@ -721,6 +721,8 @@ class StylesAndFormulas:
721
721
  if "(cid:" in char:
722
722
  return True
723
723
  if not self.font_mapper.has_char(char):
724
+ if len(char) > 1 and all(self.font_mapper.has_char(x) for x in char):
725
+ return False
724
726
  return True
725
727
  if self.translation_config.formular_char_pattern:
726
728
  pattern = self.translation_config.formular_char_pattern
@@ -14,7 +14,8 @@ def is_cid_paragraph(paragraph: il_version_1.PdfParagraph):
14
14
  elif composition.pdf_same_style_characters:
15
15
  chars.extend(composition.pdf_same_style_characters.pdf_character)
16
16
  elif composition.pdf_same_style_unicode_characters:
17
- chars.extend(composition.pdf_same_style_unicode_characters.unicode)
17
+ continue
18
+ # chars.extend(composition.pdf_same_style_unicode_characters.unicode)
18
19
  elif composition.pdf_formula:
19
20
  chars.extend(composition.pdf_formula.pdf_character)
20
21
  elif composition.pdf_character:
@@ -5,6 +5,7 @@ import io
5
5
  import logging
6
6
  import pathlib
7
7
  import shutil
8
+ import struct
8
9
  import threading
9
10
  import time
10
11
  from asyncio import CancelledError
@@ -13,6 +14,7 @@ from typing import Any
13
14
  from typing import BinaryIO
14
15
 
15
16
  import pymupdf
17
+ from pdfminer.cmapdb import IdentityCMap
16
18
  from pdfminer.pdfdocument import PDFDocument
17
19
  from pdfminer.pdfinterp import PDFResourceManager
18
20
  from pdfminer.pdfpage import PDFPage
@@ -49,6 +51,17 @@ from babeldoc.translation_config import TranslateResult
49
51
  from babeldoc.translation_config import TranslationConfig
50
52
  from babeldoc.translation_config import WatermarkOutputMode
51
53
 
54
+
55
+ def decode(_, code: bytes) -> tuple[int, ...]:
56
+ n = len(code) // 2
57
+ if n:
58
+ return struct.unpack_from(f">{n}H", code)
59
+ else:
60
+ return ()
61
+
62
+
63
+ IdentityCMap.decode = decode
64
+
52
65
  logger = logging.getLogger(__name__)
53
66
 
54
67
  TRANSLATE_STAGES = [
@@ -710,7 +723,6 @@ def _do_translate_single(
710
723
  support_llm_translate = True
711
724
  except NotImplementedError:
712
725
  support_llm_translate = False
713
- il_translator = None
714
726
  if support_llm_translate:
715
727
  il_translator = ILTranslatorLLMOnly(translate_engine, translation_config)
716
728
  else:
@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
23
23
  from babeldoc.translation_config import WatermarkOutputMode
24
24
 
25
25
  logger = logging.getLogger(__name__)
26
- __version__ = "0.3.31"
26
+ __version__ = "0.3.33"
27
27
 
28
28
 
29
29
  def create_parser():
@@ -19,12 +19,15 @@ from pdfminer.pdfinterp import PDFPageInterpreter
19
19
  from pdfminer.pdfinterp import PDFResourceManager
20
20
  from pdfminer.pdfinterp import PDFStackT
21
21
  from pdfminer.pdfpage import PDFPage
22
+ from pdfminer.pdftypes import LITERALS_ASCII85_DECODE
22
23
  from pdfminer.pdftypes import PDFObjRef
24
+ from pdfminer.pdftypes import PDFStream
23
25
  from pdfminer.pdftypes import dict_value
24
26
  from pdfminer.pdftypes import list_value
25
27
  from pdfminer.pdftypes import resolve1
26
28
  from pdfminer.pdftypes import stream_value
27
29
  from pdfminer.psexceptions import PSEOF
30
+ from pdfminer.psexceptions import PSTypeError
28
31
  from pdfminer.psparser import PSKeyword
29
32
  from pdfminer.psparser import PSLiteral
30
33
  from pdfminer.psparser import keyword_name
@@ -33,6 +36,7 @@ from pdfminer.utils import MATRIX_IDENTITY
33
36
  from pdfminer.utils import Matrix
34
37
  from pdfminer.utils import Rect
35
38
  from pdfminer.utils import apply_matrix_pt
39
+ from pdfminer.utils import choplist
36
40
  from pdfminer.utils import mult_matrix
37
41
 
38
42
  from babeldoc.document_il.frontend.il_creater import ILCreater
@@ -47,6 +51,42 @@ def safe_float(o: Any) -> float | None:
47
51
  return None
48
52
 
49
53
 
54
+ class PDFContentParserEx(PDFContentParser):
55
+ def __init__(self, streams: Sequence[object]) -> None:
56
+ super().__init__(streams)
57
+
58
+ def do_keyword(self, pos: int, token: PSKeyword) -> None:
59
+ if token is self.KEYWORD_BI:
60
+ # inline image within a content stream
61
+ self.start_type(pos, "inline")
62
+ elif token is self.KEYWORD_ID:
63
+ try:
64
+ (_, objs) = self.end_type("inline")
65
+ if len(objs) % 2 != 0:
66
+ error_msg = f"Invalid dictionary construct: {objs!r}"
67
+ raise PSTypeError(error_msg)
68
+ d = {literal_name(k): resolve1(v) for (k, v) in choplist(2, objs)}
69
+ eos = b"EI"
70
+ filter_ = d.get("F", None)
71
+ if filter_:
72
+ if isinstance(filter_, PSLiteral):
73
+ filter_ = [filter_]
74
+ if filter_[0] in LITERALS_ASCII85_DECODE:
75
+ eos = b"~>"
76
+ (pos, data) = self.get_inline_data(pos + len(b"ID "), target=eos)
77
+ if eos != b"EI": # it may be necessary for decoding
78
+ data += eos
79
+ obj = PDFStream(d, data)
80
+ self.push((pos, obj))
81
+ if eos == b"EI": # otherwise it is still in the stream
82
+ self.push((pos, self.KEYWORD_EI))
83
+ except PSTypeError:
84
+ if settings.STRICT:
85
+ raise
86
+ else:
87
+ self.push((pos, token))
88
+
89
+
50
90
  class PDFPageInterpreterEx(PDFPageInterpreter):
51
91
  """Processor for the content of a PDF page
52
92
 
@@ -409,7 +449,7 @@ class PDFPageInterpreterEx(PDFPageInterpreter):
409
449
  self.il_creater.on_new_stream()
410
450
  # 重载返回指令流
411
451
  try:
412
- parser = PDFContentParser([stream])
452
+ parser = PDFContentParserEx([stream])
413
453
  except PSEOF:
414
454
  # empty page
415
455
  return
@@ -0,0 +1,140 @@
1
+ # Supported Languages
2
+
3
+ For languages in the table below that do not rely on ligature support, BabelDOC provides good support. For languages that partially rely on ligatures, BabelDOC's translation results can generally meet self-reading needs. For languages that completely rely on ligatures (such as some Indian languages), BabelDOC does not currently support them.
4
+
5
+ We are working hard to develop support for ligatures as soon as possible.
6
+
7
+ | Language | Language Code | Ligature Dependency |
8
+ | :------------------- | :------------ | :------------------ |
9
+ | Simplified Chinese | zh-CN | None |
10
+ | Traditional Chinese | zh-TW | None |
11
+ | Japanese | JP | None |
12
+ | Korean | KR | None |
13
+ | Polish | PL | Partial |
14
+ | Russian | RU | None |
15
+ | Spanish | es | None |
16
+ | Portuguese | pt | None |
17
+ | French | fr | Partial |
18
+ | Malay | ms | None |
19
+ | Indonesian | id | None |
20
+ | Turkmen | tk | None |
21
+ | Filipino (Tagalog) | tl | None |
22
+ | Vietnamese | vi | None |
23
+ | Azerbaijani | az | None |
24
+ | Kazakh (Latin) | kk | None |
25
+ | Kazakh (Cyrillic) | kk | None |
26
+ | German | de | None |
27
+ | Dutch | nl | None |
28
+ | Irish | ga | None |
29
+ | Italian | it | None |
30
+ | Greek | el | None |
31
+ | Swedish | sv | None |
32
+ | Danish | da | None |
33
+ | Norwegian | no | None |
34
+ | Icelandic | is | None |
35
+ | Finnish | fi | None |
36
+ | Ukrainian | uk | None |
37
+ | Czech | cs | None |
38
+ | Romanian | ro | None |
39
+ | Hungarian | hu | None |
40
+ | Slovak | sk | None |
41
+ | Croatian | hr | None |
42
+ | Estonian | et | None |
43
+ | Latvian | lv | None |
44
+ | Lithuanian | lt | None |
45
+ | Belarusian | be | None |
46
+ | Macedonian | mk | None |
47
+ | Albanian | sq | None |
48
+ | Serbian (Cyrillic) | sr | Partial |
49
+ | Serbian (Latin) | sr | Partial |
50
+ | Slovenian | sl | None |
51
+ | Catalan | ca | None |
52
+ | Bulgarian | bg | None |
53
+ | Maltese | mt | None |
54
+ | Swahili | sw | None |
55
+ | Amharic | am | None |
56
+ | Oromo | om | None |
57
+ | Tigrinya | ti | None |
58
+ | Haitian Creole | ht | None |
59
+ | Latin | la | None |
60
+ | Lao | lo | None |
61
+ | Malayalam | ml | None |
62
+ | Gujarati | gu | None |
63
+ | Thai | th | None |
64
+ | Burmese | my | Partial |
65
+ | Tamil | ta | None |
66
+ | Telugu | te | None |
67
+ | Oriya | or | Partial |
68
+ | Armenian | hy | None |
69
+ | Mongolian (Cyrillic) | mn | None |
70
+ | Georgian | ka | None |
71
+ | Khmer | km | None |
72
+ | Bosnian | bs | None |
73
+ | Luxembourgish | lb | None |
74
+ | Moldovan | ro | None |
75
+ | Moldovan (Cyrillic) | ro | None |
76
+ | Romansh | rm | None |
77
+ | Turkish | tr | None |
78
+ | Sinhala | si | None |
79
+ | Uzbek | uz | None |
80
+ | Kyrgyz | ky | None |
81
+ | Tajik | tg | None |
82
+ | Abkhazian | ab | None |
83
+ | Afar | aa | None |
84
+ | Afrikaans | af | None |
85
+ | Akan | ak | None |
86
+ | Aragonese | an | None |
87
+ | Avaric | av | None |
88
+ | Avestan | ae | None |
89
+ | Ewe | ee | None |
90
+ | Aymara | ay | None |
91
+ | Ojibwa | oj | None |
92
+ | Occitan | oc | None |
93
+ | Oriya | or | None |
94
+ | Ossetian | os | None |
95
+ | Pali | pi | None |
96
+ | Bashkir | ba | None |
97
+ | Basque | eu | None |
98
+ | Bambara | bm | None |
99
+ | Breton | br | None |
100
+ | Chamorro | ch | None |
101
+ | Chechen | ce | None |
102
+ | Chuvash | cv | None |
103
+ | Tswana | tn | None |
104
+ | Divehi | dv | None |
105
+ | Ndebele, South | nr | None |
106
+ | Ndonga | ng | None |
107
+ | Faroese | fo | None |
108
+ | Fijian | fj | None |
109
+ | Frisian, Western | fy | None |
110
+ | Fulah | ff | None |
111
+ | Ganda | lg | None |
112
+ | Kongo | kg | None |
113
+ | Kalaallisut | kl | None |
114
+ | Church Slavic | cu | None |
115
+ | Guarani | gn | None |
116
+ | Interlingua | ia | None |
117
+ | Hausa | ha | None |
118
+ | Herero | hz | None |
119
+ | Kikuyu | ki | None |
120
+ | Rundi | rn | None |
121
+ | Kinyarwanda | rw | None |
122
+ | Kirghiz | ky | None |
123
+ | Galician | gl | None |
124
+ | Kanuri | kr | None |
125
+ | Cornish | kw | None |
126
+ | Komi | kv | None |
127
+ | Xhosa | xh | None |
128
+ | Corsican | co | None |
129
+ | Cree | cr | None |
130
+ | Croatian | hr | None |
131
+ | Quechua | qu | None |
132
+ | Kurdish (Latin) | ku | None |
133
+ | Kuanyama | kj | None |
134
+ | Limburgan | li | None |
135
+ | Lingala | ln | None |
136
+ | Manx | gv | None |
137
+ | Malagasy | mg | None |
138
+ | Marshallese | mh | None |
139
+ | Maori | mi | None |
140
+
@@ -132,6 +132,7 @@ not_in_nav: |
132
132
  # Page tree
133
133
  nav:
134
134
  - Home: index.md
135
+ - Supported languages: supported_languages.md
135
136
  - API:
136
137
  - Async Translation API: ImplementationDetails/AsyncTranslate/AsyncTranslate.md
137
138
  - Implementation Details:
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "BabelDOC"
3
- version = "0.3.31"
3
+ version = "0.3.33"
4
4
  description = "Yet Another Document Translator"
5
5
  license = "AGPL-3.0"
6
6
  readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
150
150
  testpaths = ["tests"]
151
151
 
152
152
  [bumpver]
153
- current_version = "0.3.31"
153
+ current_version = "0.3.33"
154
154
  version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
155
155
 
156
156
  [bumpver.file_patterns]
@@ -48,7 +48,7 @@ wheels = [
48
48
 
49
49
  [[package]]
50
50
  name = "babeldoc"
51
- version = "0.3.31"
51
+ version = "0.3.33"
52
52
  source = { editable = "." }
53
53
  dependencies = [
54
54
  { name = "bitstring" },
@@ -1 +0,0 @@
1
- __version__ = "0.3.31"
File without changes
File without changes
File without changes
File without changes
@@ -135,15 +135,15 @@ class ParagraphFinder:
135
135
  self.process_page(page)
136
136
  pbar.advance()
137
137
 
138
- if self.check_cid_paragraph(document):
139
- raise ExtractTextError("The document contains too many CID paragraphs.")
140
-
141
138
  total_paragraph_count = 0
142
139
  for page in document.page:
143
140
  total_paragraph_count += len(page.pdf_paragraph)
144
141
  if total_paragraph_count == 0:
145
142
  raise ExtractTextError("The document contains no paragraphs.")
146
143
 
144
+ if self.check_cid_paragraph(document):
145
+ raise ExtractTextError("The document contains too many CID paragraphs.")
146
+
147
147
  def check_cid_paragraph(self, doc: Document):
148
148
  cid_para_count = 0
149
149
  para_total = 0
File without changes
File without changes
File without changes