BabelDOC 0.3.15__tar.gz → 0.3.17__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. {babeldoc-0.3.15 → babeldoc-0.3.17}/PKG-INFO +3 -1
  2. {babeldoc-0.3.15 → babeldoc-0.3.17}/README.md +2 -0
  3. babeldoc-0.3.17/babeldoc/__init__.py +1 -0
  4. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/const.py +1 -1
  5. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/converter.py +9 -1
  6. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/backend/pdf_creater.py +33 -18
  7. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/frontend/il_creater.py +3 -0
  8. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.py +18 -0
  9. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.rnc +4 -1
  10. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/paragraph_finder.py +48 -0
  11. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/styles_and_formulas.py +5 -0
  12. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/typesetting.py +1 -0
  13. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/utils/style_helper.py +4 -0
  14. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/docvision/doclayout.py +4 -1
  15. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/docvision/rpc_doclayout.py +4 -1
  16. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/docvision/table_detection/rapidocr.py +4 -1
  17. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/main.py +8 -1
  18. babeldoc-0.3.17/babeldoc/tools/italic_recognize_tool.py +79 -0
  19. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/translation_config.py +13 -5
  20. {babeldoc-0.3.15 → babeldoc-0.3.17}/pyproject.toml +2 -2
  21. {babeldoc-0.3.15 → babeldoc-0.3.17}/uv.lock +68 -61
  22. babeldoc-0.3.15/babeldoc/__init__.py +0 -1
  23. {babeldoc-0.3.15 → babeldoc-0.3.17}/.cursorignore +0 -0
  24. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
  25. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
  26. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/PULL_REQUEST_TEMPLATE/pr_form.yml +0 -0
  27. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/dependabot.yml +0 -0
  28. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/labels.yml +0 -0
  29. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/release-drafter.yml +0 -0
  30. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/workflows/codeql.yml +0 -0
  31. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/workflows/docs.yml +0 -0
  32. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/workflows/labeler.yml +0 -0
  33. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/workflows/publish-to-pypi.yml +0 -0
  34. {babeldoc-0.3.15 → babeldoc-0.3.17}/.github/workflows/test.yml +0 -0
  35. {babeldoc-0.3.15 → babeldoc-0.3.17}/.gitignore +0 -0
  36. {babeldoc-0.3.15 → babeldoc-0.3.17}/.pre-commit-config.yaml +0 -0
  37. {babeldoc-0.3.15 → babeldoc-0.3.17}/LICENSE +0 -0
  38. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/assets/assets.py +0 -0
  39. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/assets/embedding_assets_metadata.py +0 -0
  40. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/asynchronize/__init__.py +0 -0
  41. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/__init__.py +0 -0
  42. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py +0 -0
  43. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/backend/__init__.py +0 -0
  44. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/frontend/__init__.py +0 -0
  45. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.rng +0 -0
  46. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.xsd +0 -0
  47. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/__init__.py +0 -0
  48. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/add_debug_information.py +0 -0
  49. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/detect_scanned_file.py +0 -0
  50. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/il_translator.py +0 -0
  51. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/il_translator_llm_only.py +0 -0
  52. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/layout_parser.py +0 -0
  53. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/remove_descent.py +0 -0
  54. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/midend/table_parser.py +0 -0
  55. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/translator/__init__.py +0 -0
  56. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/translator/cache.py +0 -0
  57. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/translator/translator.py +0 -0
  58. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/utils/atomic_integer.py +0 -0
  59. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/utils/fontmap.py +0 -0
  60. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/utils/layout_helper.py +0 -0
  61. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/utils/priority_thread_pool_executor.py +0 -0
  62. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/document_il/xml_converter.py +0 -0
  63. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/docvision/README.md +0 -0
  64. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/docvision/__init__.py +0 -0
  65. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/high_level.py +0 -0
  66. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/pdfinterp.py +0 -0
  67. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/progress_monitor.py +0 -0
  68. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/result_merger.py +0 -0
  69. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/split_manager.py +0 -0
  70. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/tools/generate_font_metadata.py +0 -0
  71. {babeldoc-0.3.15 → babeldoc-0.3.17}/babeldoc/tools/italic_assistance.py +0 -0
  72. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/CODE_OF_CONDUCT.md +0 -0
  73. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/CONTRIBUTING.md +0 -0
  74. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/CONTRIBUTOR_REWARD.md +0 -0
  75. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md +0 -0
  76. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/ILTranslator/ILTranslator.md +0 -0
  77. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/PDFCreation/PDFCreation.md +0 -0
  78. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/PDFParsing/PDFParsing.md +0 -0
  79. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md +0 -0
  80. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/README.md +0 -0
  81. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/StylesAndFormulas/StylesAndFormulas.md +0 -0
  82. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/ImplementationDetails/Typesetting/Typesetting.md +0 -0
  83. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/README.md +0 -0
  84. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/deploy.sh +0 -0
  85. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-banner.png +0 -0
  86. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.png +0 -0
  87. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.svg +0 -0
  88. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-with-transparent-background.png +0 -0
  89. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-with-transparent-background.svg +0 -0
  90. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo.png +0 -0
  91. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-contributor_reward_example.png +0 -0
  92. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-preview.gif +0 -0
  93. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo-with-transparent-background.png +0 -0
  94. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo-with-transparent-background.svg +0 -0
  95. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo.png +0 -0
  96. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/index.md +0 -0
  97. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/intro-to-pdf-object.md +0 -0
  98. {babeldoc-0.3.15 → babeldoc-0.3.17}/docs/requirements.txt +0 -0
  99. {babeldoc-0.3.15 → babeldoc-0.3.17}/mkdocs.yml +0 -0
  100. {babeldoc-0.3.15 → babeldoc-0.3.17}/tests/test_translation_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: BabelDOC
3
- Version: 0.3.15
3
+ Version: 0.3.17
4
4
  Summary: Yet Another Document Translator
5
5
  Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
6
6
  Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -191,6 +191,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
191
191
  - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
192
192
  - `--translate-table-text`: Translate table text (experimental, default: False)
193
193
  - `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
194
+ - `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
194
195
 
195
196
  > [!TIP]
196
197
  > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -199,6 +200,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
199
200
  > - If you encounter any compatibility issues, try using `--enhance-compatibility` first
200
201
  > - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
201
202
  > - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
203
+ > - Use `--ocr-workaround` to fill background for scanned PDF. (Current assumption: background is pure white, text is pure black, this option will also auto enable `--skip-scanned-detection`)
202
204
 
203
205
  ### Translation Service Options
204
206
 
@@ -145,6 +145,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
145
145
  - `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
146
146
  - `--translate-table-text`: Translate table text (experimental, default: False)
147
147
  - `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
148
+ - `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
148
149
 
149
150
  > [!TIP]
150
151
  > - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
@@ -153,6 +154,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
153
154
  > - If you encounter any compatibility issues, try using `--enhance-compatibility` first
154
155
  > - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
155
156
  > - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
157
+ > - Use `--ocr-workaround` to fill background for scanned PDF. (Current assumption: background is pure white, text is pure black, this option will also auto enable `--skip-scanned-detection`)
156
158
 
157
159
  ### Translation Service Options
158
160
 
@@ -0,0 +1 @@
1
+ __version__ = "0.3.17"
@@ -3,7 +3,7 @@ import shutil
3
3
  import subprocess
4
4
  from pathlib import Path
5
5
 
6
- __version__ = "0.3.15"
6
+ __version__ = "0.3.17"
7
7
 
8
8
  CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
9
9
 
@@ -92,7 +92,15 @@ class PDFConverterEx(PDFConverter):
92
92
  textwidth = font.char_width(cid)
93
93
  textdisp = font.char_disp(cid)
94
94
 
95
- font_id = self.il_creater.current_page_font_name_id_map.get(font.xobj_id, None)
95
+ if not hasattr(font, "xobj_id"):
96
+ log.debug(
97
+ f"Font {font.fontname} does not have xobj_id attribute.",
98
+ )
99
+ font_id = "UNKNOW"
100
+ else:
101
+ font_id = self.il_creater.current_page_font_name_id_map.get(
102
+ font.xobj_id, None
103
+ )
96
104
 
97
105
  item = AWLTChar(
98
106
  matrix,
@@ -185,21 +185,25 @@ class PDFCreater:
185
185
  except Exception:
186
186
  return set()
187
187
 
188
- def _debug_render_rectangle(
188
+ def _render_rectangle(
189
189
  self,
190
190
  draw_op: BitStream,
191
191
  rectangle: il_version_1.PdfRectangle,
192
+ line_width: float = 1,
192
193
  ):
193
- """Draw a debug rectangle in PDF for visualization purposes.
194
+ """Draw a rectangle in PDF for visualization purposes.
194
195
 
195
196
  Args:
196
197
  draw_op: BitStream to append PDF drawing operations
197
198
  rectangle: Rectangle object containing position information
199
+ line_width: Line width
198
200
  """
199
201
  x1 = rectangle.box.x
200
202
  y1 = rectangle.box.y
201
203
  x2 = rectangle.box.x2
202
204
  y2 = rectangle.box.y2
205
+ width = x2 - x1
206
+ height = y2 - y1
203
207
  # Save graphics state
204
208
  draw_op.append(b"q ")
205
209
 
@@ -207,17 +211,13 @@ class PDFCreater:
207
211
  draw_op.append(
208
212
  rectangle.graphic_state.passthrough_per_char_instruction.encode(),
209
213
  ) # Green stroke
210
- draw_op.append(b" 1 w ") # Line width
211
-
212
- # Draw four lines manually
213
- # Bottom line
214
- draw_op.append(f"{x1} {y1} m {x2} {y1} l S ".encode())
215
- # Right line
216
- draw_op.append(f"{x2} {y1} m {x2} {y2} l S ".encode())
217
- # Top line
218
- draw_op.append(f"{x2} {y2} m {x1} {y2} l S ".encode())
219
- # Left line
220
- draw_op.append(f"{x1} {y2} m {x1} {y1} l S ".encode())
214
+ if line_width > 0:
215
+ draw_op.append(f" {line_width} w ".encode()) # Line width
216
+ draw_op.append(f"{x1} {y1} {width} {height} re ".encode())
217
+ if rectangle.fill_background:
218
+ draw_op.append(b" f ")
219
+ else:
220
+ draw_op.append(b" S ")
221
221
 
222
222
  # Restore graphics state
223
223
  draw_op.append(b"Q\n")
@@ -410,7 +410,7 @@ class PDFCreater:
410
410
  for rect in page.pdf_rectangle:
411
411
  if not rect.debug_info:
412
412
  continue
413
- self._debug_render_rectangle(page_op, rect)
413
+ self._render_rectangle(page_op, rect)
414
414
  draw_op = page_op
415
415
  # Since this is a draw instruction container,
416
416
  # no additional information is needed
@@ -711,7 +711,17 @@ class PDFCreater:
711
711
  # 然后添加段落中的字符
712
712
  for paragraph in page.pdf_paragraph:
713
713
  chars.extend(self.render_paragraph_to_char(paragraph))
714
-
714
+ for rect in page.pdf_rectangle:
715
+ if (
716
+ translation_config.ocr_workaround
717
+ and not rect.debug_info
718
+ and rect.fill_background
719
+ ):
720
+ if rect.xobj_id in xobj_available_fonts:
721
+ draw_op = xobj_draw_ops[rect.xobj_id]
722
+ else:
723
+ draw_op = page_op
724
+ self._render_rectangle(draw_op, rect, line_width=0.1)
715
725
  # 渲染所有字符
716
726
  for char in chars:
717
727
  if char.char_unicode == "\n":
@@ -773,7 +783,9 @@ class PDFCreater:
773
783
  )
774
784
  # pdf.update_stream(xobj.xref_id, b'')
775
785
  for rect in page.pdf_rectangle:
776
- self._debug_render_rectangle(page_op, rect)
786
+ if translation_config.debug and rect.debug_info:
787
+ self._render_rectangle(page_op, rect)
788
+
777
789
  draw_op = page_op
778
790
  op_container = pdf.get_new_xref()
779
791
  # Since this is a draw instruction container,
@@ -783,6 +795,9 @@ class PDFCreater:
783
795
  pdf[page.page_number].set_contents(op_container)
784
796
  pbar.advance()
785
797
  translation_config.raise_if_cancelled()
798
+ gc_level = 1
799
+ if self.translation_config.ocr_workaround:
800
+ gc_level = 4
786
801
  with self.translation_config.progress_monitor.stage_start(
787
802
  SUBSET_FONT_STAGE_NAME,
788
803
  1,
@@ -811,7 +826,7 @@ class PDFCreater:
811
826
  pdf,
812
827
  mono_out_path,
813
828
  translation_config,
814
- garbage=1,
829
+ garbage=gc_level,
815
830
  deflate=True,
816
831
  clean=not translation_config.skip_clean,
817
832
  deflate_fonts=True,
@@ -864,7 +879,7 @@ class PDFCreater:
864
879
  dual,
865
880
  dual_out_path,
866
881
  translation_config,
867
- garbage=1,
882
+ garbage=gc_level,
868
883
  deflate=True,
869
884
  clean=not translation_config.skip_clean,
870
885
  deflate_fonts=True,
@@ -19,6 +19,7 @@ from pdfminer.pdftypes import resolve1 as pdftypes_resolve1
19
19
  from pdfminer.psparser import PSLiteral
20
20
 
21
21
  from babeldoc.document_il import il_version_1
22
+ from babeldoc.document_il.utils.style_helper import BLACK
22
23
  from babeldoc.document_il.utils.style_helper import YELLOW
23
24
  from babeldoc.translation_config import TranslationConfig
24
25
 
@@ -887,6 +888,8 @@ class ILCreater:
887
888
  xobj_id=char.xobj_id,
888
889
  visual_bbox=visual_bbox,
889
890
  )
891
+ if self.translation_config.ocr_workaround:
892
+ pdf_char.pdf_style.graphic_state = BLACK
890
893
  if pdf_style.font_size == 0.0:
891
894
  logger.warning(
892
895
  "Font size is 0.0 for character %s. Skip it.",
@@ -364,6 +364,18 @@ class PdfRectangle:
364
364
  "type": "Attribute",
365
365
  },
366
366
  )
367
+ fill_background: bool | None = field(
368
+ default=None,
369
+ metadata={
370
+ "type": "Attribute",
371
+ },
372
+ )
373
+ xobj_id: int | None = field(
374
+ default=None,
375
+ metadata={
376
+ "type": "Attribute",
377
+ },
378
+ )
367
379
 
368
380
 
369
381
  @dataclass
@@ -756,6 +768,12 @@ class PdfParagraph:
756
768
  "type": "Attribute",
757
769
  },
758
770
  )
771
+ layout_id: int | None = field(
772
+ default=None,
773
+ metadata={
774
+ "type": "Attribute",
775
+ },
776
+ )
759
777
 
760
778
 
761
779
  @dataclass
@@ -115,6 +115,7 @@ PDFParagraph =
115
115
  attribute FirstLineIndent { xsd:boolean }?,
116
116
  attribute debug_id { xsd:string }?,
117
117
  attribute layout_label { xsd:string }?,
118
+ attribute layout_id { xsd:int }?,
118
119
  Box,
119
120
  PDFStyle,
120
121
  PDFParagraphComposition*
@@ -148,5 +149,7 @@ PDFRectangle =
148
149
  element pdfRectangle {
149
150
  Box,
150
151
  GraphicState,
151
- attribute debug_info { xsd:boolean }?
152
+ attribute debug_info { xsd:boolean }?,
153
+ attribute fill_background { xsd:boolean }?,
154
+ attribute xobjId { xsd:int }?
152
155
  }
@@ -8,9 +8,11 @@ from babeldoc.document_il import PdfCharacter
8
8
  from babeldoc.document_il import PdfLine
9
9
  from babeldoc.document_il import PdfParagraph
10
10
  from babeldoc.document_il import PdfParagraphComposition
11
+ from babeldoc.document_il import PdfRectangle
11
12
  from babeldoc.document_il.utils.layout_helper import Layout
12
13
  from babeldoc.document_il.utils.layout_helper import add_space_dummy_chars
13
14
  from babeldoc.document_il.utils.layout_helper import get_char_unicode_string
15
+ from babeldoc.document_il.utils.style_helper import WHITE
14
16
  from babeldoc.translation_config import TranslationConfig
15
17
 
16
18
  logger = logging.getLogger(__name__)
@@ -35,6 +37,41 @@ class ParagraphFinder:
35
37
  def __init__(self, translation_config: TranslationConfig):
36
38
  self.translation_config = translation_config
37
39
 
40
+ def add_text_fill_background(self, page: Page):
41
+ layout_map = {layout.id: layout for layout in page.page_layout}
42
+ for paragraph in page.pdf_paragraph:
43
+ layout_id = paragraph.layout_id
44
+ if layout_id is None:
45
+ continue
46
+ layout = layout_map[layout_id]
47
+ if paragraph.box is None:
48
+ continue
49
+ x1, y1, x2, y2 = (
50
+ paragraph.box.x,
51
+ paragraph.box.y,
52
+ paragraph.box.x2,
53
+ paragraph.box.y2,
54
+ )
55
+ layout_box = layout.box
56
+ if layout_box.x < x1:
57
+ x1 = layout_box.x
58
+ if layout_box.y < y1:
59
+ y1 = layout_box.y
60
+ if layout_box.x2 > x2:
61
+ x2 = layout_box.x2
62
+ if layout_box.y2 > y2:
63
+ y2 = layout_box.y2
64
+ assert x2 > x1 and y2 > y1
65
+ page.pdf_rectangle.append(
66
+ PdfRectangle(
67
+ box=Box(x1, y1, x2, y2),
68
+ fill_background=True,
69
+ graphic_state=WHITE,
70
+ debug_info=False,
71
+ xobj_id=paragraph.xobj_id,
72
+ )
73
+ )
74
+
38
75
  def update_paragraph_data(self, paragraph: PdfParagraph, update_unicode=False):
39
76
  if not paragraph.pdf_paragraph_composition:
40
77
  return
@@ -126,6 +163,12 @@ class ParagraphFinder:
126
163
  for paragraph in paragraphs:
127
164
  self.update_paragraph_data(paragraph, update_unicode=True)
128
165
 
166
+ if self.translation_config.ocr_workaround:
167
+ self.add_text_fill_background(page)
168
+ # since this is ocr file,
169
+ # image characters are not needed
170
+ page.pdf_character = []
171
+
129
172
  def is_isolated_formula(self, char: PdfCharacter):
130
173
  return char.char_unicode in (
131
174
  "(cid:122)",
@@ -176,6 +219,7 @@ class ParagraphFinder:
176
219
  if current_paragraph is None:
177
220
  current_paragraph = PdfParagraph(
178
221
  pdf_paragraph_composition=[line],
222
+ layout_id=char_layout.id,
179
223
  debug_id=generate_base58_id(),
180
224
  layout_label=char_layout.name
181
225
  if not current_layout
@@ -214,6 +258,7 @@ class ParagraphFinder:
214
258
  else:
215
259
  current_paragraph = PdfParagraph(
216
260
  pdf_paragraph_composition=[line],
261
+ layout_id=current_layout.id,
217
262
  debug_id=generate_base58_id(),
218
263
  layout_label=current_layout.name,
219
264
  )
@@ -231,6 +276,7 @@ class ParagraphFinder:
231
276
  if current_paragraph is None:
232
277
  current_paragraph = PdfParagraph(
233
278
  pdf_paragraph_composition=[line],
279
+ layout_id=current_layout.id,
234
280
  debug_id=generate_base58_id(),
235
281
  layout_label=current_layout.name,
236
282
  )
@@ -420,6 +466,7 @@ class ParagraphFinder:
420
466
  unicode="",
421
467
  debug_id=generate_base58_id(),
422
468
  layout_label=paragraph.layout_label,
469
+ layout_id=paragraph.layout_id,
423
470
  )
424
471
  # 更新原段落
425
472
  paragraph.pdf_paragraph_composition = (
@@ -449,6 +496,7 @@ class ParagraphFinder:
449
496
  unicode="",
450
497
  debug_id=generate_base58_id(),
451
498
  layout_label=paragraph.layout_label,
499
+ layout_id=paragraph.layout_id,
452
500
  )
453
501
  # 更新原段落
454
502
  paragraph.pdf_paragraph_composition = (
@@ -554,6 +554,11 @@ class StylesAndFormulas:
554
554
  r"|.*DJ5EscrowCond.*"
555
555
  r"|.*ExchangeBook.*"
556
556
  r"|.*DJ5Exchange.*"
557
+ r"|.*Times-Italic.*"
558
+ r"|.*Times-BoldItalic.*"
559
+ r"|.*CronosPro.*"
560
+ r"|.*ACaslon.*"
561
+ r"|.*Frutiger.*"
557
562
  r")$"
558
563
  )
559
564
  if self.translation_config.formular_font_pattern:
@@ -112,6 +112,7 @@ class TypesettingUnit:
112
112
  r"\u0250-\u02A0" # IPA Extensions
113
113
  r"\u0400-\u04FF" # Cyrillic
114
114
  r"'"
115
+ r"-" # Hyphen
115
116
  r"]+$",
116
117
  unicode,
117
118
  ):
@@ -25,6 +25,10 @@ def create_pdf_style(r, g, b, font_id="china-ss", font_size=6):
25
25
  )
26
26
 
27
27
 
28
+ BLACK = il_version_1.GraphicState(passthrough_per_char_instruction="0 g 0 G")
29
+
30
+ WHITE = il_version_1.GraphicState(passthrough_per_char_instruction="1 g 1 G")
31
+
28
32
  # Generate all color styles
29
33
  RED = il_version_1.GraphicState(
30
34
  passthrough_per_char_instruction="1.0000000000 0.2313725490 0.1882352941 rg "
@@ -3,6 +3,7 @@ import ast
3
3
  import logging
4
4
  import platform
5
5
  import re
6
+ import threading
6
7
  from collections.abc import Generator
7
8
 
8
9
  import cv2
@@ -121,6 +122,7 @@ class OnnxModel(DocLayoutModel):
121
122
  model.SerializeToString(),
122
123
  providers=providers,
123
124
  )
125
+ self.lock = threading.Lock()
124
126
 
125
127
  @staticmethod
126
128
  def from_pretrained():
@@ -279,7 +281,8 @@ class OnnxModel(DocLayoutModel):
279
281
  ]:
280
282
  for page in pages:
281
283
  translate_config.raise_if_cancelled()
282
- pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
284
+ with self.lock:
285
+ pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
283
286
  image = np.fromstring(pix.samples, np.uint8).reshape(
284
287
  pix.height,
285
288
  pix.width,
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ import threading
2
3
  from concurrent.futures import ThreadPoolExecutor
3
4
  from pathlib import Path
4
5
 
@@ -125,6 +126,7 @@ class RpcDocLayoutModel(DocLayoutModel):
125
126
  self.host = host
126
127
  self._stride = 32 # Default stride value
127
128
  self._names = ["text", "title", "list", "table", "figure"]
129
+ self.lock = threading.Lock()
128
130
 
129
131
  @property
130
132
  def stride(self) -> int:
@@ -253,7 +255,8 @@ class RpcDocLayoutModel(DocLayoutModel):
253
255
  self, page, mupdf_doc: pymupdf.Document, translate_config, save_debug_image
254
256
  ):
255
257
  translate_config.raise_if_cancelled()
256
- pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
258
+ with self.lock:
259
+ pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
257
260
  image = np.fromstring(pix.samples, np.uint8).reshape(
258
261
  pix.height,
259
262
  pix.width,
@@ -1,5 +1,6 @@
1
1
  import logging
2
2
  import re
3
+ import threading
3
4
  from collections.abc import Generator
4
5
 
5
6
  import cv2
@@ -97,6 +98,7 @@ class RapidOCRModel:
97
98
  det_use_dml=self.use_dml,
98
99
  )
99
100
  self.names = {0: "table_text"}
101
+ self.lock = threading.Lock()
100
102
 
101
103
  @property
102
104
  def stride(self):
@@ -233,7 +235,8 @@ class RapidOCRModel:
233
235
  ]:
234
236
  for page in pages:
235
237
  translate_config.raise_if_cancelled()
236
- pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
238
+ with self.lock:
239
+ pix = mupdf_doc[page.page_number].get_pixmap(dpi=72)
237
240
  image = np.fromstring(pix.samples, np.uint8).reshape(
238
241
  pix.height,
239
242
  pix.width,
@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
23
23
  from babeldoc.translation_config import WatermarkOutputMode
24
24
 
25
25
  logger = logging.getLogger(__name__)
26
- __version__ = "0.3.15"
26
+ __version__ = "0.3.17"
27
27
 
28
28
 
29
29
  def create_parser():
@@ -210,6 +210,12 @@ def create_parser():
210
210
  default=False,
211
211
  help="Skip scanned document detection (speeds up processing for non-scanned documents)",
212
212
  )
213
+ translation_group.add_argument(
214
+ "--ocr-workaround",
215
+ action="store_true",
216
+ default=False,
217
+ help="Add text fill background (experimental)",
218
+ )
213
219
  # service option argument group
214
220
  service_group = translation_group.add_mutually_exclusive_group()
215
221
  service_group.add_argument(
@@ -377,6 +383,7 @@ async def main():
377
383
  table_model=table_model,
378
384
  show_char_box=args.show_char_box,
379
385
  skip_scanned_detection=args.skip_scanned_detection,
386
+ ocr_workaround=args.ocr_workaround,
380
387
  )
381
388
 
382
389
  # Create progress handler
@@ -0,0 +1,79 @@
1
+ # Identify non-formula italic fonts that were incorrectly classified as formulas in BableDOC translation results (intermediate)
2
+
3
+ import json
4
+
5
+ import babeldoc.tools.italic_assistance as italic_assistance
6
+ from babeldoc.document_il.midend.styles_and_formulas import StylesAndFormulas
7
+ from babeldoc.translation_config import TranslationConfig
8
+ from rich.console import Console
9
+ from rich.table import Table
10
+
11
+ console = Console()
12
+
13
+ json_path = italic_assistance.find_latest_il_json()
14
+
15
+ fonts = []
16
+
17
+ # Read intermediate representation
18
+ with json_path.open(encoding="utf-8") as f:
19
+ pdf_data = json.load(f)
20
+
21
+ for page_index, page in enumerate(pdf_data["page"]):
22
+ for paragraph_index, paragraph_content in enumerate(page["pdf_paragraph"]):
23
+ font_debug_id = paragraph_content["debug_id"]
24
+ if font_debug_id:
25
+ # Create page font mapping
26
+ page_font_map = {}
27
+ for font in page["pdf_font"]:
28
+ if "font_id" in font and "name" in font:
29
+ page_font_map[font["font_id"]] = (font["font_id"], font["name"])
30
+
31
+ # Extract fonts from paragraph
32
+ name_list = []
33
+ paragraph_fonts = italic_assistance.extract_fonts_from_paragraph(
34
+ paragraph_content, page_font_map
35
+ )
36
+ for _font_id, font_name in paragraph_fonts:
37
+ name_list.append(font_name)
38
+
39
+ font_list = []
40
+ for each in fonts:
41
+ font_list.append(each[1])
42
+
43
+ for each_name in name_list:
44
+ if each_name not in font_list:
45
+ fonts.append(
46
+ (page_index, each_name, paragraph_index, font_debug_id)
47
+ )
48
+
49
+ # Initialize checker
50
+ translation_config = TranslationConfig(
51
+ *[None for _ in range(3)], lang_out="zh_cn", doc_layout_model=1
52
+ )
53
+ checker = StylesAndFormulas(translation_config)
54
+
55
+ # Create table
56
+ table = Table(title="Font Recognition Results")
57
+ table.add_column("Page #", justify="center", style="cyan")
58
+ table.add_column("Paragraph #", justify="center", style="cyan")
59
+ table.add_column("DEBUG_ID", justify="center", style="cyan")
60
+ table.add_column("Font Name", style="magenta")
61
+ table.add_column("Recognition Result", justify="center")
62
+
63
+ # Output results
64
+ for each_font in fonts:
65
+ page_index, font_name, paragraph_index, font_debug_id = each_font
66
+
67
+ if checker.is_formulas_font(font_name):
68
+ table.add_row(
69
+ str(page_index),
70
+ str(paragraph_index),
71
+ str(font_debug_id),
72
+ font_name,
73
+ "[bold red]Formula Font[/bold red]",
74
+ )
75
+ # else:
76
+ # table.add_row(str(page_index), font_name, "[bold blue]Non-Formula Font[/bold blue]")
77
+
78
+ # Print table
79
+ console.print(table)
@@ -68,6 +68,7 @@ class TranslationConfig:
68
68
  table_model=None,
69
69
  show_char_box: bool = False,
70
70
  skip_scanned_detection: bool = False,
71
+ ocr_workaround: bool = False,
71
72
  ):
72
73
  self.translator = translator
73
74
 
@@ -108,6 +109,10 @@ class TranslationConfig:
108
109
  self.report_interval = report_interval
109
110
  self.min_text_length = min_text_length
110
111
  self.use_alternating_pages_dual = use_alternating_pages_dual
112
+ self.ocr_workaround = ocr_workaround
113
+
114
+ if self.ocr_workaround:
115
+ self.skip_scanned_detection = True
111
116
 
112
117
  # for backward compatibility
113
118
  if use_side_by_side_dual is False and use_alternating_pages_dual is False:
@@ -234,11 +239,14 @@ class TranslationConfig:
234
239
 
235
240
  def cleanup_temp_files(self):
236
241
  """Clean up all temporary files including part working directories"""
237
- for part_index in list(self._part_working_dirs.keys()):
238
- self.cleanup_part_working_dir(part_index)
239
- if self._is_temp_dir:
240
- logger.info(f"cleanup temp files: {self.working_dir}")
241
- shutil.rmtree(self.working_dir)
242
+ try:
243
+ for part_index in list(self._part_working_dirs.keys()):
244
+ self.cleanup_part_working_dir(part_index)
245
+ if self._is_temp_dir:
246
+ logger.info(f"cleanup temp files: {self.working_dir}")
247
+ shutil.rmtree(self.working_dir)
248
+ except Exception:
249
+ logger.exception("Error cleaning up temporary files")
242
250
 
243
251
  def raise_if_cancelled(self):
244
252
  if self.progress_monitor is not None:
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "BabelDOC"
3
- version = "0.3.15"
3
+ version = "0.3.17"
4
4
  description = "Yet Another Document Translator"
5
5
  license = "AGPL-3.0"
6
6
  readme = "README.md"
@@ -148,7 +148,7 @@ pythonpath = [".", "src"]
148
148
  testpaths = ["tests"]
149
149
 
150
150
  [bumpver]
151
- current_version = "0.3.15"
151
+ current_version = "0.3.17"
152
152
  version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
153
153
 
154
154
  [bumpver.file_patterns]