BabelDOC 0.3.16__tar.gz → 0.3.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {babeldoc-0.3.16 → babeldoc-0.3.17}/PKG-INFO +3 -1
- {babeldoc-0.3.16 → babeldoc-0.3.17}/README.md +2 -0
- babeldoc-0.3.17/babeldoc/__init__.py +1 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/const.py +1 -1
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/converter.py +9 -1
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/backend/pdf_creater.py +33 -18
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/frontend/il_creater.py +3 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.py +18 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.rnc +4 -1
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/paragraph_finder.py +48 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/styles_and_formulas.py +3 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/utils/style_helper.py +4 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/main.py +8 -1
- babeldoc-0.3.17/babeldoc/tools/italic_recognize_tool.py +79 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/translation_config.py +13 -5
- {babeldoc-0.3.16 → babeldoc-0.3.17}/pyproject.toml +2 -2
- {babeldoc-0.3.16 → babeldoc-0.3.17}/uv.lock +2 -2
- babeldoc-0.3.16/babeldoc/__init__.py +0 -1
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.cursorignore +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/PULL_REQUEST_TEMPLATE/pr_form.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/dependabot.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/labels.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/release-drafter.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/workflows/codeql.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/workflows/docs.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/workflows/labeler.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/workflows/publish-to-pypi.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.github/workflows/test.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.gitignore +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/.pre-commit-config.yaml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/LICENSE +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/assets/assets.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/assets/embedding_assets_metadata.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/asynchronize/__init__.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/__init__.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/backend/__init__.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/frontend/__init__.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.rng +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/il_version_1.xsd +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/__init__.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/add_debug_information.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/detect_scanned_file.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/il_translator.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/il_translator_llm_only.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/layout_parser.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/remove_descent.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/table_parser.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/midend/typesetting.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/translator/__init__.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/translator/cache.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/translator/translator.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/utils/atomic_integer.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/utils/fontmap.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/utils/layout_helper.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/utils/priority_thread_pool_executor.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/xml_converter.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/docvision/README.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/docvision/__init__.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/docvision/doclayout.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/docvision/rpc_doclayout.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/docvision/table_detection/rapidocr.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/high_level.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/pdfinterp.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/progress_monitor.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/result_merger.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/split_manager.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/tools/generate_font_metadata.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/tools/italic_assistance.py +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/CODE_OF_CONDUCT.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/CONTRIBUTING.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/CONTRIBUTOR_REWARD.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/ILTranslator/ILTranslator.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/PDFCreation/PDFCreation.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/PDFParsing/PDFParsing.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/README.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/StylesAndFormulas/StylesAndFormulas.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/Typesetting/Typesetting.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/README.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/deploy.sh +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-banner.png +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.png +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.svg +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-with-transparent-background.png +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-with-transparent-background.svg +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo.png +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-contributor_reward_example.png +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-preview.gif +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo-with-transparent-background.png +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo-with-transparent-background.svg +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo.png +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/index.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/intro-to-pdf-object.md +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/docs/requirements.txt +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/mkdocs.yml +0 -0
- {babeldoc-0.3.16 → babeldoc-0.3.17}/tests/test_translation_config.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: BabelDOC
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.17
|
|
4
4
|
Summary: Yet Another Document Translator
|
|
5
5
|
Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
|
|
6
6
|
Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
|
|
@@ -191,6 +191,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
|
|
|
191
191
|
- `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
|
|
192
192
|
- `--translate-table-text`: Translate table text (experimental, default: False)
|
|
193
193
|
- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
|
|
194
|
+
- `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
|
|
194
195
|
|
|
195
196
|
> [!TIP]
|
|
196
197
|
> - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
|
|
@@ -199,6 +200,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
|
|
|
199
200
|
> - If you encounter any compatibility issues, try using `--enhance-compatibility` first
|
|
200
201
|
> - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
|
|
201
202
|
> - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
|
|
203
|
+
> - Use `--ocr-workaround` to fill background for scanned PDF. (Current assumption: background is pure white, text is pure black, this option will also auto enable `--skip-scanned-detection`)
|
|
202
204
|
|
|
203
205
|
### Translation Service Options
|
|
204
206
|
|
|
@@ -145,6 +145,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
|
|
|
145
145
|
- `--no-watermark`: [DEPRECATED] Use --watermark-output-mode=no_watermark instead.
|
|
146
146
|
- `--translate-table-text`: Translate table text (experimental, default: False)
|
|
147
147
|
- `--skip-scanned-detection`: Skip scanned document detection (default: False). When using split translation, only the first part performs detection if not skipped.
|
|
148
|
+
- `--ocr-workaround`: Use OCR workaround (default: False). When enabled, the tool will use OCR to detect text and fill background for scanned PDF.
|
|
148
149
|
|
|
149
150
|
> [!TIP]
|
|
150
151
|
> - Both `--skip-clean` and `--dual-translate-first` may help improve compatibility with some PDF readers
|
|
@@ -153,6 +154,7 @@ uv run babeldoc --files example.pdf --files example2.pdf --openai --openai-model
|
|
|
153
154
|
> - If you encounter any compatibility issues, try using `--enhance-compatibility` first
|
|
154
155
|
> - Use `--max-pages-per-part` for large documents to split them into smaller parts for translation and automatically merge them back.
|
|
155
156
|
> - Use `--skip-scanned-detection` to speed up processing when you know your document is not a scanned PDF.
|
|
157
|
+
> - Use `--ocr-workaround` to fill background for scanned PDF. (Current assumption: background is pure white, text is pure black, this option will also auto enable `--skip-scanned-detection`)
|
|
156
158
|
|
|
157
159
|
### Translation Service Options
|
|
158
160
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.17"
|
|
@@ -92,7 +92,15 @@ class PDFConverterEx(PDFConverter):
|
|
|
92
92
|
textwidth = font.char_width(cid)
|
|
93
93
|
textdisp = font.char_disp(cid)
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
if not hasattr(font, "xobj_id"):
|
|
96
|
+
log.debug(
|
|
97
|
+
f"Font {font.fontname} does not have xobj_id attribute.",
|
|
98
|
+
)
|
|
99
|
+
font_id = "UNKNOW"
|
|
100
|
+
else:
|
|
101
|
+
font_id = self.il_creater.current_page_font_name_id_map.get(
|
|
102
|
+
font.xobj_id, None
|
|
103
|
+
)
|
|
96
104
|
|
|
97
105
|
item = AWLTChar(
|
|
98
106
|
matrix,
|
|
@@ -185,21 +185,25 @@ class PDFCreater:
|
|
|
185
185
|
except Exception:
|
|
186
186
|
return set()
|
|
187
187
|
|
|
188
|
-
def
|
|
188
|
+
def _render_rectangle(
|
|
189
189
|
self,
|
|
190
190
|
draw_op: BitStream,
|
|
191
191
|
rectangle: il_version_1.PdfRectangle,
|
|
192
|
+
line_width: float = 1,
|
|
192
193
|
):
|
|
193
|
-
"""Draw a
|
|
194
|
+
"""Draw a rectangle in PDF for visualization purposes.
|
|
194
195
|
|
|
195
196
|
Args:
|
|
196
197
|
draw_op: BitStream to append PDF drawing operations
|
|
197
198
|
rectangle: Rectangle object containing position information
|
|
199
|
+
line_width: Line width
|
|
198
200
|
"""
|
|
199
201
|
x1 = rectangle.box.x
|
|
200
202
|
y1 = rectangle.box.y
|
|
201
203
|
x2 = rectangle.box.x2
|
|
202
204
|
y2 = rectangle.box.y2
|
|
205
|
+
width = x2 - x1
|
|
206
|
+
height = y2 - y1
|
|
203
207
|
# Save graphics state
|
|
204
208
|
draw_op.append(b"q ")
|
|
205
209
|
|
|
@@ -207,17 +211,13 @@ class PDFCreater:
|
|
|
207
211
|
draw_op.append(
|
|
208
212
|
rectangle.graphic_state.passthrough_per_char_instruction.encode(),
|
|
209
213
|
) # Green stroke
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
# Top line
|
|
218
|
-
draw_op.append(f"{x2} {y2} m {x1} {y2} l S ".encode())
|
|
219
|
-
# Left line
|
|
220
|
-
draw_op.append(f"{x1} {y2} m {x1} {y1} l S ".encode())
|
|
214
|
+
if line_width > 0:
|
|
215
|
+
draw_op.append(f" {line_width} w ".encode()) # Line width
|
|
216
|
+
draw_op.append(f"{x1} {y1} {width} {height} re ".encode())
|
|
217
|
+
if rectangle.fill_background:
|
|
218
|
+
draw_op.append(b" f ")
|
|
219
|
+
else:
|
|
220
|
+
draw_op.append(b" S ")
|
|
221
221
|
|
|
222
222
|
# Restore graphics state
|
|
223
223
|
draw_op.append(b"Q\n")
|
|
@@ -410,7 +410,7 @@ class PDFCreater:
|
|
|
410
410
|
for rect in page.pdf_rectangle:
|
|
411
411
|
if not rect.debug_info:
|
|
412
412
|
continue
|
|
413
|
-
self.
|
|
413
|
+
self._render_rectangle(page_op, rect)
|
|
414
414
|
draw_op = page_op
|
|
415
415
|
# Since this is a draw instruction container,
|
|
416
416
|
# no additional information is needed
|
|
@@ -711,7 +711,17 @@ class PDFCreater:
|
|
|
711
711
|
# 然后添加段落中的字符
|
|
712
712
|
for paragraph in page.pdf_paragraph:
|
|
713
713
|
chars.extend(self.render_paragraph_to_char(paragraph))
|
|
714
|
-
|
|
714
|
+
for rect in page.pdf_rectangle:
|
|
715
|
+
if (
|
|
716
|
+
translation_config.ocr_workaround
|
|
717
|
+
and not rect.debug_info
|
|
718
|
+
and rect.fill_background
|
|
719
|
+
):
|
|
720
|
+
if rect.xobj_id in xobj_available_fonts:
|
|
721
|
+
draw_op = xobj_draw_ops[rect.xobj_id]
|
|
722
|
+
else:
|
|
723
|
+
draw_op = page_op
|
|
724
|
+
self._render_rectangle(draw_op, rect, line_width=0.1)
|
|
715
725
|
# 渲染所有字符
|
|
716
726
|
for char in chars:
|
|
717
727
|
if char.char_unicode == "\n":
|
|
@@ -773,7 +783,9 @@ class PDFCreater:
|
|
|
773
783
|
)
|
|
774
784
|
# pdf.update_stream(xobj.xref_id, b'')
|
|
775
785
|
for rect in page.pdf_rectangle:
|
|
776
|
-
|
|
786
|
+
if translation_config.debug and rect.debug_info:
|
|
787
|
+
self._render_rectangle(page_op, rect)
|
|
788
|
+
|
|
777
789
|
draw_op = page_op
|
|
778
790
|
op_container = pdf.get_new_xref()
|
|
779
791
|
# Since this is a draw instruction container,
|
|
@@ -783,6 +795,9 @@ class PDFCreater:
|
|
|
783
795
|
pdf[page.page_number].set_contents(op_container)
|
|
784
796
|
pbar.advance()
|
|
785
797
|
translation_config.raise_if_cancelled()
|
|
798
|
+
gc_level = 1
|
|
799
|
+
if self.translation_config.ocr_workaround:
|
|
800
|
+
gc_level = 4
|
|
786
801
|
with self.translation_config.progress_monitor.stage_start(
|
|
787
802
|
SUBSET_FONT_STAGE_NAME,
|
|
788
803
|
1,
|
|
@@ -811,7 +826,7 @@ class PDFCreater:
|
|
|
811
826
|
pdf,
|
|
812
827
|
mono_out_path,
|
|
813
828
|
translation_config,
|
|
814
|
-
garbage=
|
|
829
|
+
garbage=gc_level,
|
|
815
830
|
deflate=True,
|
|
816
831
|
clean=not translation_config.skip_clean,
|
|
817
832
|
deflate_fonts=True,
|
|
@@ -864,7 +879,7 @@ class PDFCreater:
|
|
|
864
879
|
dual,
|
|
865
880
|
dual_out_path,
|
|
866
881
|
translation_config,
|
|
867
|
-
garbage=
|
|
882
|
+
garbage=gc_level,
|
|
868
883
|
deflate=True,
|
|
869
884
|
clean=not translation_config.skip_clean,
|
|
870
885
|
deflate_fonts=True,
|
|
@@ -19,6 +19,7 @@ from pdfminer.pdftypes import resolve1 as pdftypes_resolve1
|
|
|
19
19
|
from pdfminer.psparser import PSLiteral
|
|
20
20
|
|
|
21
21
|
from babeldoc.document_il import il_version_1
|
|
22
|
+
from babeldoc.document_il.utils.style_helper import BLACK
|
|
22
23
|
from babeldoc.document_il.utils.style_helper import YELLOW
|
|
23
24
|
from babeldoc.translation_config import TranslationConfig
|
|
24
25
|
|
|
@@ -887,6 +888,8 @@ class ILCreater:
|
|
|
887
888
|
xobj_id=char.xobj_id,
|
|
888
889
|
visual_bbox=visual_bbox,
|
|
889
890
|
)
|
|
891
|
+
if self.translation_config.ocr_workaround:
|
|
892
|
+
pdf_char.pdf_style.graphic_state = BLACK
|
|
890
893
|
if pdf_style.font_size == 0.0:
|
|
891
894
|
logger.warning(
|
|
892
895
|
"Font size is 0.0 for character %s. Skip it.",
|
|
@@ -364,6 +364,18 @@ class PdfRectangle:
|
|
|
364
364
|
"type": "Attribute",
|
|
365
365
|
},
|
|
366
366
|
)
|
|
367
|
+
fill_background: bool | None = field(
|
|
368
|
+
default=None,
|
|
369
|
+
metadata={
|
|
370
|
+
"type": "Attribute",
|
|
371
|
+
},
|
|
372
|
+
)
|
|
373
|
+
xobj_id: int | None = field(
|
|
374
|
+
default=None,
|
|
375
|
+
metadata={
|
|
376
|
+
"type": "Attribute",
|
|
377
|
+
},
|
|
378
|
+
)
|
|
367
379
|
|
|
368
380
|
|
|
369
381
|
@dataclass
|
|
@@ -756,6 +768,12 @@ class PdfParagraph:
|
|
|
756
768
|
"type": "Attribute",
|
|
757
769
|
},
|
|
758
770
|
)
|
|
771
|
+
layout_id: int | None = field(
|
|
772
|
+
default=None,
|
|
773
|
+
metadata={
|
|
774
|
+
"type": "Attribute",
|
|
775
|
+
},
|
|
776
|
+
)
|
|
759
777
|
|
|
760
778
|
|
|
761
779
|
@dataclass
|
|
@@ -115,6 +115,7 @@ PDFParagraph =
|
|
|
115
115
|
attribute FirstLineIndent { xsd:boolean }?,
|
|
116
116
|
attribute debug_id { xsd:string }?,
|
|
117
117
|
attribute layout_label { xsd:string }?,
|
|
118
|
+
attribute layout_id { xsd:int }?,
|
|
118
119
|
Box,
|
|
119
120
|
PDFStyle,
|
|
120
121
|
PDFParagraphComposition*
|
|
@@ -148,5 +149,7 @@ PDFRectangle =
|
|
|
148
149
|
element pdfRectangle {
|
|
149
150
|
Box,
|
|
150
151
|
GraphicState,
|
|
151
|
-
attribute debug_info { xsd:boolean }
|
|
152
|
+
attribute debug_info { xsd:boolean }?,
|
|
153
|
+
attribute fill_background { xsd:boolean }?,
|
|
154
|
+
attribute xobjId { xsd:int }?
|
|
152
155
|
}
|
|
@@ -8,9 +8,11 @@ from babeldoc.document_il import PdfCharacter
|
|
|
8
8
|
from babeldoc.document_il import PdfLine
|
|
9
9
|
from babeldoc.document_il import PdfParagraph
|
|
10
10
|
from babeldoc.document_il import PdfParagraphComposition
|
|
11
|
+
from babeldoc.document_il import PdfRectangle
|
|
11
12
|
from babeldoc.document_il.utils.layout_helper import Layout
|
|
12
13
|
from babeldoc.document_il.utils.layout_helper import add_space_dummy_chars
|
|
13
14
|
from babeldoc.document_il.utils.layout_helper import get_char_unicode_string
|
|
15
|
+
from babeldoc.document_il.utils.style_helper import WHITE
|
|
14
16
|
from babeldoc.translation_config import TranslationConfig
|
|
15
17
|
|
|
16
18
|
logger = logging.getLogger(__name__)
|
|
@@ -35,6 +37,41 @@ class ParagraphFinder:
|
|
|
35
37
|
def __init__(self, translation_config: TranslationConfig):
|
|
36
38
|
self.translation_config = translation_config
|
|
37
39
|
|
|
40
|
+
def add_text_fill_background(self, page: Page):
|
|
41
|
+
layout_map = {layout.id: layout for layout in page.page_layout}
|
|
42
|
+
for paragraph in page.pdf_paragraph:
|
|
43
|
+
layout_id = paragraph.layout_id
|
|
44
|
+
if layout_id is None:
|
|
45
|
+
continue
|
|
46
|
+
layout = layout_map[layout_id]
|
|
47
|
+
if paragraph.box is None:
|
|
48
|
+
continue
|
|
49
|
+
x1, y1, x2, y2 = (
|
|
50
|
+
paragraph.box.x,
|
|
51
|
+
paragraph.box.y,
|
|
52
|
+
paragraph.box.x2,
|
|
53
|
+
paragraph.box.y2,
|
|
54
|
+
)
|
|
55
|
+
layout_box = layout.box
|
|
56
|
+
if layout_box.x < x1:
|
|
57
|
+
x1 = layout_box.x
|
|
58
|
+
if layout_box.y < y1:
|
|
59
|
+
y1 = layout_box.y
|
|
60
|
+
if layout_box.x2 > x2:
|
|
61
|
+
x2 = layout_box.x2
|
|
62
|
+
if layout_box.y2 > y2:
|
|
63
|
+
y2 = layout_box.y2
|
|
64
|
+
assert x2 > x1 and y2 > y1
|
|
65
|
+
page.pdf_rectangle.append(
|
|
66
|
+
PdfRectangle(
|
|
67
|
+
box=Box(x1, y1, x2, y2),
|
|
68
|
+
fill_background=True,
|
|
69
|
+
graphic_state=WHITE,
|
|
70
|
+
debug_info=False,
|
|
71
|
+
xobj_id=paragraph.xobj_id,
|
|
72
|
+
)
|
|
73
|
+
)
|
|
74
|
+
|
|
38
75
|
def update_paragraph_data(self, paragraph: PdfParagraph, update_unicode=False):
|
|
39
76
|
if not paragraph.pdf_paragraph_composition:
|
|
40
77
|
return
|
|
@@ -126,6 +163,12 @@ class ParagraphFinder:
|
|
|
126
163
|
for paragraph in paragraphs:
|
|
127
164
|
self.update_paragraph_data(paragraph, update_unicode=True)
|
|
128
165
|
|
|
166
|
+
if self.translation_config.ocr_workaround:
|
|
167
|
+
self.add_text_fill_background(page)
|
|
168
|
+
# since this is ocr file,
|
|
169
|
+
# image characters are not needed
|
|
170
|
+
page.pdf_character = []
|
|
171
|
+
|
|
129
172
|
def is_isolated_formula(self, char: PdfCharacter):
|
|
130
173
|
return char.char_unicode in (
|
|
131
174
|
"(cid:122)",
|
|
@@ -176,6 +219,7 @@ class ParagraphFinder:
|
|
|
176
219
|
if current_paragraph is None:
|
|
177
220
|
current_paragraph = PdfParagraph(
|
|
178
221
|
pdf_paragraph_composition=[line],
|
|
222
|
+
layout_id=char_layout.id,
|
|
179
223
|
debug_id=generate_base58_id(),
|
|
180
224
|
layout_label=char_layout.name
|
|
181
225
|
if not current_layout
|
|
@@ -214,6 +258,7 @@ class ParagraphFinder:
|
|
|
214
258
|
else:
|
|
215
259
|
current_paragraph = PdfParagraph(
|
|
216
260
|
pdf_paragraph_composition=[line],
|
|
261
|
+
layout_id=current_layout.id,
|
|
217
262
|
debug_id=generate_base58_id(),
|
|
218
263
|
layout_label=current_layout.name,
|
|
219
264
|
)
|
|
@@ -231,6 +276,7 @@ class ParagraphFinder:
|
|
|
231
276
|
if current_paragraph is None:
|
|
232
277
|
current_paragraph = PdfParagraph(
|
|
233
278
|
pdf_paragraph_composition=[line],
|
|
279
|
+
layout_id=current_layout.id,
|
|
234
280
|
debug_id=generate_base58_id(),
|
|
235
281
|
layout_label=current_layout.name,
|
|
236
282
|
)
|
|
@@ -420,6 +466,7 @@ class ParagraphFinder:
|
|
|
420
466
|
unicode="",
|
|
421
467
|
debug_id=generate_base58_id(),
|
|
422
468
|
layout_label=paragraph.layout_label,
|
|
469
|
+
layout_id=paragraph.layout_id,
|
|
423
470
|
)
|
|
424
471
|
# 更新原段落
|
|
425
472
|
paragraph.pdf_paragraph_composition = (
|
|
@@ -449,6 +496,7 @@ class ParagraphFinder:
|
|
|
449
496
|
unicode="",
|
|
450
497
|
debug_id=generate_base58_id(),
|
|
451
498
|
layout_label=paragraph.layout_label,
|
|
499
|
+
layout_id=paragraph.layout_id,
|
|
452
500
|
)
|
|
453
501
|
# 更新原段落
|
|
454
502
|
paragraph.pdf_paragraph_composition = (
|
|
@@ -25,6 +25,10 @@ def create_pdf_style(r, g, b, font_id="china-ss", font_size=6):
|
|
|
25
25
|
)
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
BLACK = il_version_1.GraphicState(passthrough_per_char_instruction="0 g 0 G")
|
|
29
|
+
|
|
30
|
+
WHITE = il_version_1.GraphicState(passthrough_per_char_instruction="1 g 1 G")
|
|
31
|
+
|
|
28
32
|
# Generate all color styles
|
|
29
33
|
RED = il_version_1.GraphicState(
|
|
30
34
|
passthrough_per_char_instruction="1.0000000000 0.2313725490 0.1882352941 rg "
|
|
@@ -23,7 +23,7 @@ from babeldoc.translation_config import TranslationConfig
|
|
|
23
23
|
from babeldoc.translation_config import WatermarkOutputMode
|
|
24
24
|
|
|
25
25
|
logger = logging.getLogger(__name__)
|
|
26
|
-
__version__ = "0.3.
|
|
26
|
+
__version__ = "0.3.17"
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
def create_parser():
|
|
@@ -210,6 +210,12 @@ def create_parser():
|
|
|
210
210
|
default=False,
|
|
211
211
|
help="Skip scanned document detection (speeds up processing for non-scanned documents)",
|
|
212
212
|
)
|
|
213
|
+
translation_group.add_argument(
|
|
214
|
+
"--ocr-workaround",
|
|
215
|
+
action="store_true",
|
|
216
|
+
default=False,
|
|
217
|
+
help="Add text fill background (experimental)",
|
|
218
|
+
)
|
|
213
219
|
# service option argument group
|
|
214
220
|
service_group = translation_group.add_mutually_exclusive_group()
|
|
215
221
|
service_group.add_argument(
|
|
@@ -377,6 +383,7 @@ async def main():
|
|
|
377
383
|
table_model=table_model,
|
|
378
384
|
show_char_box=args.show_char_box,
|
|
379
385
|
skip_scanned_detection=args.skip_scanned_detection,
|
|
386
|
+
ocr_workaround=args.ocr_workaround,
|
|
380
387
|
)
|
|
381
388
|
|
|
382
389
|
# Create progress handler
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Identify non-formula italic fonts that were incorrectly classified as formulas in BableDOC translation results (intermediate)
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
import babeldoc.tools.italic_assistance as italic_assistance
|
|
6
|
+
from babeldoc.document_il.midend.styles_and_formulas import StylesAndFormulas
|
|
7
|
+
from babeldoc.translation_config import TranslationConfig
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
console = Console()
|
|
12
|
+
|
|
13
|
+
json_path = italic_assistance.find_latest_il_json()
|
|
14
|
+
|
|
15
|
+
fonts = []
|
|
16
|
+
|
|
17
|
+
# Read intermediate representation
|
|
18
|
+
with json_path.open(encoding="utf-8") as f:
|
|
19
|
+
pdf_data = json.load(f)
|
|
20
|
+
|
|
21
|
+
for page_index, page in enumerate(pdf_data["page"]):
|
|
22
|
+
for paragraph_index, paragraph_content in enumerate(page["pdf_paragraph"]):
|
|
23
|
+
font_debug_id = paragraph_content["debug_id"]
|
|
24
|
+
if font_debug_id:
|
|
25
|
+
# Create page font mapping
|
|
26
|
+
page_font_map = {}
|
|
27
|
+
for font in page["pdf_font"]:
|
|
28
|
+
if "font_id" in font and "name" in font:
|
|
29
|
+
page_font_map[font["font_id"]] = (font["font_id"], font["name"])
|
|
30
|
+
|
|
31
|
+
# Extract fonts from paragraph
|
|
32
|
+
name_list = []
|
|
33
|
+
paragraph_fonts = italic_assistance.extract_fonts_from_paragraph(
|
|
34
|
+
paragraph_content, page_font_map
|
|
35
|
+
)
|
|
36
|
+
for _font_id, font_name in paragraph_fonts:
|
|
37
|
+
name_list.append(font_name)
|
|
38
|
+
|
|
39
|
+
font_list = []
|
|
40
|
+
for each in fonts:
|
|
41
|
+
font_list.append(each[1])
|
|
42
|
+
|
|
43
|
+
for each_name in name_list:
|
|
44
|
+
if each_name not in font_list:
|
|
45
|
+
fonts.append(
|
|
46
|
+
(page_index, each_name, paragraph_index, font_debug_id)
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# Initialize checker
|
|
50
|
+
translation_config = TranslationConfig(
|
|
51
|
+
*[None for _ in range(3)], lang_out="zh_cn", doc_layout_model=1
|
|
52
|
+
)
|
|
53
|
+
checker = StylesAndFormulas(translation_config)
|
|
54
|
+
|
|
55
|
+
# Create table
|
|
56
|
+
table = Table(title="Font Recognition Results")
|
|
57
|
+
table.add_column("Page #", justify="center", style="cyan")
|
|
58
|
+
table.add_column("Paragraph #", justify="center", style="cyan")
|
|
59
|
+
table.add_column("DEBUG_ID", justify="center", style="cyan")
|
|
60
|
+
table.add_column("Font Name", style="magenta")
|
|
61
|
+
table.add_column("Recognition Result", justify="center")
|
|
62
|
+
|
|
63
|
+
# Output results
|
|
64
|
+
for each_font in fonts:
|
|
65
|
+
page_index, font_name, paragraph_index, font_debug_id = each_font
|
|
66
|
+
|
|
67
|
+
if checker.is_formulas_font(font_name):
|
|
68
|
+
table.add_row(
|
|
69
|
+
str(page_index),
|
|
70
|
+
str(paragraph_index),
|
|
71
|
+
str(font_debug_id),
|
|
72
|
+
font_name,
|
|
73
|
+
"[bold red]Formula Font[/bold red]",
|
|
74
|
+
)
|
|
75
|
+
# else:
|
|
76
|
+
# table.add_row(str(page_index), font_name, "[bold blue]Non-Formula Font[/bold blue]")
|
|
77
|
+
|
|
78
|
+
# Print table
|
|
79
|
+
console.print(table)
|
|
@@ -68,6 +68,7 @@ class TranslationConfig:
|
|
|
68
68
|
table_model=None,
|
|
69
69
|
show_char_box: bool = False,
|
|
70
70
|
skip_scanned_detection: bool = False,
|
|
71
|
+
ocr_workaround: bool = False,
|
|
71
72
|
):
|
|
72
73
|
self.translator = translator
|
|
73
74
|
|
|
@@ -108,6 +109,10 @@ class TranslationConfig:
|
|
|
108
109
|
self.report_interval = report_interval
|
|
109
110
|
self.min_text_length = min_text_length
|
|
110
111
|
self.use_alternating_pages_dual = use_alternating_pages_dual
|
|
112
|
+
self.ocr_workaround = ocr_workaround
|
|
113
|
+
|
|
114
|
+
if self.ocr_workaround:
|
|
115
|
+
self.skip_scanned_detection = True
|
|
111
116
|
|
|
112
117
|
# for backward compatibility
|
|
113
118
|
if use_side_by_side_dual is False and use_alternating_pages_dual is False:
|
|
@@ -234,11 +239,14 @@ class TranslationConfig:
|
|
|
234
239
|
|
|
235
240
|
def cleanup_temp_files(self):
|
|
236
241
|
"""Clean up all temporary files including part working directories"""
|
|
237
|
-
|
|
238
|
-
self.
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
+
try:
|
|
243
|
+
for part_index in list(self._part_working_dirs.keys()):
|
|
244
|
+
self.cleanup_part_working_dir(part_index)
|
|
245
|
+
if self._is_temp_dir:
|
|
246
|
+
logger.info(f"cleanup temp files: {self.working_dir}")
|
|
247
|
+
shutil.rmtree(self.working_dir)
|
|
248
|
+
except Exception:
|
|
249
|
+
logger.exception("Error cleaning up temporary files")
|
|
242
250
|
|
|
243
251
|
def raise_if_cancelled(self):
|
|
244
252
|
if self.progress_monitor is not None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "BabelDOC"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.17"
|
|
4
4
|
description = "Yet Another Document Translator"
|
|
5
5
|
license = "AGPL-3.0"
|
|
6
6
|
readme = "README.md"
|
|
@@ -148,7 +148,7 @@ pythonpath = [".", "src"]
|
|
|
148
148
|
testpaths = ["tests"]
|
|
149
149
|
|
|
150
150
|
[bumpver]
|
|
151
|
-
current_version = "0.3.
|
|
151
|
+
current_version = "0.3.17"
|
|
152
152
|
version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
|
|
153
153
|
|
|
154
154
|
[bumpver.file_patterns]
|
|
@@ -48,7 +48,7 @@ wheels = [
|
|
|
48
48
|
|
|
49
49
|
[[package]]
|
|
50
50
|
name = "babeldoc"
|
|
51
|
-
version = "0.3.
|
|
51
|
+
version = "0.3.17"
|
|
52
52
|
source = { editable = "." }
|
|
53
53
|
dependencies = [
|
|
54
54
|
{ name = "bitstring" },
|
|
@@ -889,7 +889,7 @@ wheels = [
|
|
|
889
889
|
{ url = "https://files.pythonhosted.org/packages/8c/de/8eb6fffecd9c5f129461edcdd7e1ac944f9de15783e3d89c84ed6e0374bc/lxml-5.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aa837e6ee9534de8d63bc4c1249e83882a7ac22bd24523f83fad68e6ffdf41ae", size = 5652903 },
|
|
890
890
|
{ url = "https://files.pythonhosted.org/packages/95/79/80f4102a08495c100014593680f3f0f7bd7c1333b13520aed855fc993326/lxml-5.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:da4c9223319400b97a2acdfb10926b807e51b69eb7eb80aad4942c0516934858", size = 5491813 },
|
|
891
891
|
{ url = "https://files.pythonhosted.org/packages/15/f5/9b1f7edf6565ee31e4300edb1bcc61eaebe50a3cff4053c0206d8dc772f2/lxml-5.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dc0e9bdb3aa4d1de703a437576007d366b54f52c9897cae1a3716bb44fc1fc85", size = 5227837 },
|
|
892
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
892
|
+
{ url = "https://files.pythonhosted.org/packages/dd/53/a187c4ccfcd5fbfca01e6c96da39499d8b801ab5dcf57717db95d7a968a8/lxml-5.3.2-cp310-cp310-win32.win32.whl", hash = "sha256:dd755a0a78dd0b2c43f972e7b51a43be518ebc130c9f1a7c4480cf08b4385486", size = 3477533 },
|
|
893
893
|
{ url = "https://files.pythonhosted.org/packages/f2/2c/397c5a9d76a7a0faf9e5b13143ae1a7e223e71d2197a45da71c21aacb3d4/lxml-5.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:d64ea1686474074b38da13ae218d9fde0d1dc6525266976808f41ac98d9d7980", size = 3805160 },
|
|
894
894
|
{ url = "https://files.pythonhosted.org/packages/84/b8/2b727f5a90902f7cc5548349f563b60911ca05f3b92e35dfa751349f265f/lxml-5.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9d61a7d0d208ace43986a92b111e035881c4ed45b1f5b7a270070acae8b0bfb4", size = 8163457 },
|
|
895
895
|
{ url = "https://files.pythonhosted.org/packages/91/84/23135b2dc72b3440d68c8f39ace2bb00fe78e3a2255f7c74f7e76f22498e/lxml-5.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856dfd7eda0b75c29ac80a31a6411ca12209183e866c33faf46e77ace3ce8a79", size = 4433445 },
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.3.16"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/babeldoc/document_il/utils/priority_thread_pool_executor.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-with-transparent-background.png
RENAMED
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-big-logo-with-transparent-background.svg
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo-with-transparent-background.png
RENAMED
|
File without changes
|
{babeldoc-0.3.16 → babeldoc-0.3.17}/docs/images/babeldoc-small-logo-with-transparent-background.svg
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|