BabelDOC 0.3.55__tar.gz → 0.3.56__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. {babeldoc-0.3.55 → babeldoc-0.3.56}/PKG-INFO +2 -1
  2. babeldoc-0.3.56/babeldoc/__init__.py +1 -0
  3. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/const.py +1 -1
  4. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/automatic_term_extractor.py +2 -1
  5. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/translator/translator.py +5 -9
  6. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/glossary.py +80 -39
  7. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/main.py +1 -1
  8. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/result_merger.py +71 -36
  9. {babeldoc-0.3.55 → babeldoc-0.3.56}/pyproject.toml +3 -2
  10. {babeldoc-0.3.55 → babeldoc-0.3.56}/uv.lock +41 -1
  11. babeldoc-0.3.55/babeldoc/__init__.py +0 -1
  12. {babeldoc-0.3.55 → babeldoc-0.3.56}/.cursorignore +0 -0
  13. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
  14. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
  15. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/PULL_REQUEST_TEMPLATE/pr_form.yml +0 -0
  16. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  17. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/dependabot.yml +0 -0
  18. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/labels.yml +0 -0
  19. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/release-drafter.yml +0 -0
  20. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/workflows/codeql.yml +0 -0
  21. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/workflows/docs.yml +0 -0
  22. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/workflows/labeler.yml +0 -0
  23. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/workflows/lint.yml +0 -0
  24. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/workflows/pr-lint.yml +0 -0
  25. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/workflows/publish-to-pypi.yml +0 -0
  26. {babeldoc-0.3.55 → babeldoc-0.3.56}/.github/workflows/test.yml +0 -0
  27. {babeldoc-0.3.55 → babeldoc-0.3.56}/.gitignore +0 -0
  28. {babeldoc-0.3.55 → babeldoc-0.3.56}/.pre-commit-config.yaml +0 -0
  29. {babeldoc-0.3.55 → babeldoc-0.3.56}/LICENSE +0 -0
  30. {babeldoc-0.3.55 → babeldoc-0.3.56}/README.md +0 -0
  31. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/assets/assets.py +0 -0
  32. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/assets/embedding_assets_metadata.py +0 -0
  33. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/asynchronize/__init__.py +0 -0
  34. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/converter.py +0 -0
  35. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/__init__.py +0 -0
  36. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/babeldoc_exception/BabelDOCException.py +0 -0
  37. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/backend/__init__.py +0 -0
  38. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/backend/pdf_creater.py +0 -0
  39. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/frontend/__init__.py +0 -0
  40. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/frontend/il_creater.py +0 -0
  41. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/il_version_1.py +0 -0
  42. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/il_version_1.rnc +0 -0
  43. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/il_version_1.rng +0 -0
  44. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/il_version_1.xsd +0 -0
  45. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/__init__.py +0 -0
  46. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/add_debug_information.py +0 -0
  47. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/detect_scanned_file.py +0 -0
  48. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/il_translator.py +0 -0
  49. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/il_translator_llm_only.py +0 -0
  50. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/layout_parser.py +0 -0
  51. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/paragraph_finder.py +0 -0
  52. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/remove_descent.py +0 -0
  53. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/styles_and_formulas.py +0 -0
  54. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/table_parser.py +0 -0
  55. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/midend/typesetting.py +0 -0
  56. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/translator/__init__.py +0 -0
  57. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/translator/cache.py +0 -0
  58. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/atomic_integer.py +0 -0
  59. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/fontmap.py +0 -0
  60. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/layout_helper.py +0 -0
  61. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/mupdf_helper.py +0 -0
  62. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/paragraph_helper.py +0 -0
  63. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/priority_thread_pool_executor.py +0 -0
  64. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/style_helper.py +0 -0
  65. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/utils/zstd_helper.py +0 -0
  66. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/document_il/xml_converter.py +0 -0
  67. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/docvision/README.md +0 -0
  68. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/docvision/__init__.py +0 -0
  69. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/docvision/doclayout.py +0 -0
  70. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/docvision/rpc_doclayout.py +0 -0
  71. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/docvision/table_detection/rapidocr.py +0 -0
  72. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/format/office/__init__.py +0 -0
  73. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/high_level.py +0 -0
  74. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/pdfinterp.py +0 -0
  75. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/progress_monitor.py +0 -0
  76. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/split_manager.py +0 -0
  77. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/tools/generate_font_metadata.py +0 -0
  78. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/tools/italic_assistance.py +0 -0
  79. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/tools/italic_recognize_tool.py +0 -0
  80. {babeldoc-0.3.55 → babeldoc-0.3.56}/babeldoc/translation_config.py +0 -0
  81. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/CODE_OF_CONDUCT.md +0 -0
  82. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/CONTRIBUTING.md +0 -0
  83. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/CONTRIBUTOR_REWARD.md +0 -0
  84. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/AsyncTranslate/AsyncTranslate.md +0 -0
  85. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/ILTranslator/ILTranslator.md +0 -0
  86. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/PDFCreation/PDFCreation.md +0 -0
  87. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/PDFParsing/PDFParsing.md +0 -0
  88. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/ParagraphFinding/ParagraphFinding.md +0 -0
  89. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/README.md +0 -0
  90. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/StylesAndFormulas/StylesAndFormulas.md +0 -0
  91. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/ImplementationDetails/Typesetting/Typesetting.md +0 -0
  92. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/README.md +0 -0
  93. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/deploy.sh +0 -0
  94. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/example/demo_glossary.csv +0 -0
  95. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-banner.png +0 -0
  96. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.png +0 -0
  97. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-big-logo-darkmode-with-transparent-background.svg +0 -0
  98. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-big-logo-with-transparent-background.png +0 -0
  99. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-big-logo-with-transparent-background.svg +0 -0
  100. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-big-logo.png +0 -0
  101. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-contributor_reward_example.png +0 -0
  102. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-preview.gif +0 -0
  103. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-small-logo-with-transparent-background.png +0 -0
  104. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-small-logo-with-transparent-background.svg +0 -0
  105. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/images/babeldoc-small-logo.png +0 -0
  106. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/index.md +0 -0
  107. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/intro-to-pdf-object.md +0 -0
  108. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/requirements.txt +0 -0
  109. {babeldoc-0.3.55 → babeldoc-0.3.56}/docs/supported_languages.md +0 -0
  110. {babeldoc-0.3.55 → babeldoc-0.3.56}/mkdocs.yml +0 -0
  111. {babeldoc-0.3.55 → babeldoc-0.3.56}/tests/test_translation_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: BabelDOC
3
- Version: 0.3.55
3
+ Version: 0.3.56
4
4
  Summary: Yet Another Document Translator
5
5
  Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
6
6
  Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
@@ -17,6 +17,7 @@ Requires-Dist: configargparse>=1.7
17
17
  Requires-Dist: freetype-py>=2.5.1
18
18
  Requires-Dist: httpx[socks]>=0.27.0
19
19
  Requires-Dist: huggingface-hub>=0.27.0
20
+ Requires-Dist: hyperscan>=0.7.13
20
21
  Requires-Dist: msgpack>=1.1.0
21
22
  Requires-Dist: numpy>=2.0.2
22
23
  Requires-Dist: onnx>=1.18.0
@@ -0,0 +1 @@
1
+ __version__ = "0.3.56"
@@ -3,7 +3,7 @@ import shutil
3
3
  import subprocess
4
4
  from pathlib import Path
5
5
 
6
- __version__ = "0.3.55"
6
+ __version__ = "0.3.56"
7
7
 
8
8
  CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
9
9
 
@@ -227,7 +227,6 @@ class AutomaticTermExtractor:
227
227
  for u in inputs:
228
228
  tracker.append_paragraph_unicode(u)
229
229
  if not inputs:
230
- pbar.advance(len(paragraphs.paragraphs))
231
230
  return
232
231
  prompt = LLM_PROMPT_TEMPLATE.format(
233
232
  target_language=self.translation_config.lang_out,
@@ -256,6 +255,8 @@ class AutomaticTermExtractor:
256
255
  except Exception as e:
257
256
  logger.error(f"Error preparing LLM prompt for automatic terms extract: {e}")
258
257
  return
258
+ finally:
259
+ pbar.advance(len(paragraphs.paragraphs))
259
260
 
260
261
  def procress(self, doc_il: ILDocument):
261
262
  logger.info(f"{self.stage_name}: Starting term extraction for document.")
@@ -8,6 +8,7 @@ from abc import abstractmethod
8
8
 
9
9
  import httpx
10
10
  import openai
11
+ from tenacity import before_sleep_log
11
12
  from tenacity import retry
12
13
  from tenacity import retry_if_exception_type
13
14
  from tenacity import stop_after_attempt
@@ -198,7 +199,8 @@ class OpenAITranslator(BaseTranslator):
198
199
  http_client=httpx.Client(
199
200
  limits=httpx.Limits(
200
201
  max_connections=None, max_keepalive_connections=None
201
- )
202
+ ),
203
+ timeout=60, # Set a reasonable timeout
202
204
  ),
203
205
  )
204
206
  self.add_cache_impact_parameters("temperature", self.options["temperature"])
@@ -213,10 +215,7 @@ class OpenAITranslator(BaseTranslator):
213
215
  retry=retry_if_exception_type(openai.RateLimitError),
214
216
  stop=stop_after_attempt(100),
215
217
  wait=wait_exponential(multiplier=1, min=1, max=15),
216
- before_sleep=lambda retry_state: logger.warning(
217
- f"RateLimitError, retrying in {retry_state.next_action.sleep} seconds... "
218
- f"(Attempt {retry_state.attempt_number}/100)"
219
- ),
218
+ before_sleep=before_sleep_log(logger, logging.WARNING),
220
219
  )
221
220
  def do_translate(self, text, rate_limit_params: dict = None) -> str:
222
221
  response = self.client.chat.completions.create(
@@ -243,10 +242,7 @@ class OpenAITranslator(BaseTranslator):
243
242
  retry=retry_if_exception_type(openai.RateLimitError),
244
243
  stop=stop_after_attempt(100),
245
244
  wait=wait_exponential(multiplier=1, min=1, max=15),
246
- before_sleep=lambda retry_state: logger.warning(
247
- f"RateLimitError, retrying in {retry_state.next_action.sleep} seconds... "
248
- f"(Attempt {retry_state.attempt_number}/100)"
249
- ),
245
+ before_sleep=before_sleep_log(logger, logging.WARNING),
250
246
  )
251
247
  def do_llm_translate(self, text, rate_limit_params: dict = None):
252
248
  if text is None:
@@ -1,8 +1,16 @@
1
1
  import csv
2
2
  import io
3
+ import itertools
4
+ import logging
3
5
  import re
6
+ import time
4
7
  from pathlib import Path
5
8
 
9
+ import hyperscan
10
+ import regex
11
+
12
+ logger = logging.getLogger(__name__)
13
+
6
14
 
7
15
  class GlossaryEntry:
8
16
  def __init__(self, source: str, target: str, target_language: str | None = None):
@@ -14,58 +22,92 @@ class GlossaryEntry:
14
22
  return f"GlossaryEntry(source='{self.source}', target='{self.target}', target_language='{self.target_language}')"
15
23
 
16
24
 
25
+ def batched(iterable, n, *, strict=False):
26
+ # batched('ABCDEFG', 3) → ABC DEF G
27
+ if n < 1:
28
+ raise ValueError("n must be at least one")
29
+ iterator = iter(iterable)
30
+ while batch := tuple(itertools.islice(iterator, n)):
31
+ if strict and len(batch) != n:
32
+ raise ValueError("batched(): incomplete batch")
33
+ yield batch
34
+
35
+
36
+ TERM_NORM_PATTERN = re.compile(r"\s+", regex.UNICODE)
37
+
38
+
17
39
  class Glossary:
18
40
  def __init__(self, name: str, entries: list[GlossaryEntry]):
19
41
  self.name = name
20
42
  self.entries = entries
21
- self.source_terms_regex: re.Pattern | None = None
22
43
  self.normalized_lookup: dict[str, tuple[str, str]] = {}
44
+ self.id_lookup: list[tuple[str, str]] = []
45
+ self.hs_dbs: list[hyperscan.Database] | None = None
23
46
  self._build_regex_and_lookup()
24
47
 
25
48
  @staticmethod
26
49
  def normalize_source(source_term: str) -> str:
27
50
  """Normalizes a source term by lowercasing and standardizing whitespace."""
28
51
  term = source_term.lower()
29
- term = re.sub(
30
- r"\s+", " ", term
52
+ term = TERM_NORM_PATTERN.sub(
53
+ " ", term
31
54
  ) # Replace multiple whitespace with single space
32
55
  return term.strip()
33
56
 
34
57
  def _build_regex_and_lookup(self):
58
+ logger.debug(
59
+ f"start build regex for glossary {self.name} with {len(self.entries)} entries"
60
+ )
35
61
  """
36
62
  Builds a combined regex for all source terms and a lookup dictionary
37
63
  from normalized source terms to (original_source, original_target).
38
64
  Regex patterns are sorted by length in descending order to prioritize longer matches.
39
65
  """
40
66
  self.normalized_lookup = {}
41
- regex_patterns_for_or: list[str] = []
42
67
 
43
68
  if not self.entries:
44
69
  self.source_terms_regex = None
45
70
  return
46
71
 
47
- for entry in self.entries:
72
+ self.hs_dbs = []
73
+ hs_pattern = []
74
+ start = time.time()
75
+ for idx, entry in enumerate(self.entries):
48
76
  normalized_key = self.normalize_source(entry.source)
49
77
  self.normalized_lookup[normalized_key] = (entry.source, entry.target)
78
+ self.id_lookup.append((entry.source, entry.target))
50
79
 
51
- # Create a regex pattern for the source term, replacing spaces with \s+
52
- # and escaping other special characters.
53
- escaped_parts = [re.escape(part) for part in entry.source.split()]
54
- if escaped_parts: # Ensure there are parts to join
55
- pattern_for_entry = r"\s+".join(escaped_parts)
56
- regex_patterns_for_or.append(pattern_for_entry)
57
-
58
- if regex_patterns_for_or:
59
- # Sort patterns by length, longest first, to help with overlapping terms
60
- regex_patterns_for_or.sort(key=len, reverse=True)
61
- combined_regex_str = "|".join(
62
- f"({pattern})" for pattern in regex_patterns_for_or
63
- ) # Add capturing groups
64
- self.source_terms_regex = re.compile(
65
- combined_regex_str, flags=re.IGNORECASE
80
+ hs_pattern.append((re.escape(entry.source).encode("utf-8"), idx))
81
+
82
+ chunk_size = 20000
83
+ for i, pattern_chunk in enumerate(
84
+ batched(hs_pattern, chunk_size, strict=False)
85
+ ):
86
+ logger.debug(
87
+ f"building hs_db chunk {i + 1} / {len(self.entries) // chunk_size + 1}"
66
88
  )
67
- else:
68
- self.source_terms_regex = None
89
+ expressions, ids = zip(*pattern_chunk, strict=False)
90
+
91
+ hs_db = hyperscan.Database()
92
+ hs_db.compile(
93
+ expressions=expressions,
94
+ ids=ids,
95
+ elements=len(pattern_chunk),
96
+ flags=hyperscan.HS_FLAG_CASELESS | hyperscan.HS_FLAG_SINGLEMATCH,
97
+ # | hyperscan.HS_FLAG_UTF8
98
+ # | hyperscan.HS_FLAG_UCP,
99
+ )
100
+ self.hs_dbs.append(hs_db)
101
+
102
+ end = time.time()
103
+ logger.debug(
104
+ f"finished building regex for glossary {self.name} in {end - start:.2f} seconds"
105
+ )
106
+ logger.debug(
107
+ f"build hs database for glossary {self.name} with {len(self.entries)} entries, hs_info: {self.hs_dbs[0].info()}"
108
+ )
109
+ if not self.hs_dbs:
110
+ self.hs_dbs = None
69
111
 
70
112
  @classmethod
71
113
  def from_csv(cls, file_path: Path, target_lang_out: str) -> "Glossary":
@@ -136,24 +178,23 @@ class Glossary:
136
178
 
137
179
  def get_active_entries_for_text(self, text: str) -> list[tuple[str, str]]:
138
180
  """Returns a list of (original_source, target_text) tuples for terms found in the given text."""
139
- if not self.source_terms_regex or not text:
181
+ if not self.hs_dbs or not text:
182
+ return []
183
+
184
+ text = TERM_NORM_PATTERN.sub(" ", text) # Normalize whitespace in the text
185
+ if not text:
140
186
  return []
141
187
 
142
188
  active_entries = []
143
- unique_added_original_sources = set()
144
-
145
- # Find all non-overlapping matches for the combined regex
146
- # The regex is constructed with capturing groups for each original pattern
147
- # so findall will return tuples of all groups. We need to find which group matched.
148
- # A simpler way with finditer:
149
- for match in self.source_terms_regex.finditer(text):
150
- # The matched text itself
151
- matched_fragment = match.group(0)
152
- normalized_frag = self.normalize_source(matched_fragment)
153
-
154
- if normalized_frag in self.normalized_lookup:
155
- original_source, target_text = self.normalized_lookup[normalized_frag]
156
- if original_source not in unique_added_original_sources:
157
- active_entries.append((original_source, target_text))
158
- unique_added_original_sources.add(original_source)
189
+
190
+ def on_match(
191
+ idx: int, _from: int, _to: int, _flags: int, _context=None
192
+ ) -> bool | None:
193
+ active_entries.append(self.id_lookup[idx])
194
+ return False
195
+
196
+ for hs_db in self.hs_dbs:
197
+ # Scan the text with the hyperscan database
198
+ scratch = hyperscan.Scratch(hs_db)
199
+ hs_db.scan(text.encode("utf-8"), on_match, scratch=scratch)
159
200
  return active_entries
@@ -24,7 +24,7 @@ from babeldoc.translation_config import TranslationConfig
24
24
  from babeldoc.translation_config import WatermarkOutputMode
25
25
 
26
26
  logger = logging.getLogger(__name__)
27
- __version__ = "0.3.55"
27
+ __version__ = "0.3.56"
28
28
 
29
29
 
30
30
  def create_parser():
@@ -45,50 +45,85 @@ class ResultMerger:
45
45
  merged_dual_path = None
46
46
  merged_no_watermark_mono_path = None
47
47
  merged_no_watermark_dual_path = None
48
-
49
- # Merge monolingual PDFs if they exist
50
- if any(r.mono_pdf_path for r in results.values()):
51
- merged_mono_path = self._merge_pdfs(
52
- [r.mono_pdf_path for r in sorted_results.values() if r.mono_pdf_path],
53
- mono_file_name,
54
- tag="merged_mono",
55
- )
56
-
57
- # Merge dual-language PDFs if they exist
58
- if any(r.dual_pdf_path for r in results.values()):
59
- merged_dual_path = self._merge_pdfs(
60
- [r.dual_pdf_path for r in sorted_results.values() if r.dual_pdf_path],
61
- dual_file_name,
62
- tag="merged_dual",
63
- )
64
-
65
- if any(
66
- r.dual_pdf_path != r.no_watermark_dual_pdf_path
67
- or r.mono_pdf_path != r.no_watermark_mono_pdf_path
68
- for r in results.values()
69
- ):
70
- # Merge no-watermark PDFs if they exist
71
- if any(r.no_watermark_mono_pdf_path for r in results.values()):
72
- merged_no_watermark_mono_path = self._merge_pdfs(
48
+ try:
49
+ # Merge monolingual PDFs if they exist
50
+ if (
51
+ any(r.mono_pdf_path for r in results.values())
52
+ and not self.config.no_mono
53
+ ):
54
+ merged_mono_path = self._merge_pdfs(
73
55
  [
74
- r.no_watermark_mono_pdf_path
56
+ r.mono_pdf_path
75
57
  for r in sorted_results.values()
76
- if r.no_watermark_mono_pdf_path
58
+ if r.mono_pdf_path
77
59
  ],
78
- mono_file_name_no_watermark,
79
- tag="merged_no_watermark_mono",
60
+ mono_file_name,
61
+ tag="merged_mono",
80
62
  )
81
-
82
- if any(r.no_watermark_dual_pdf_path for r in results.values()):
83
- merged_no_watermark_dual_path = self._merge_pdfs(
63
+ except Exception as e:
64
+ logger.error(f"Error merging monolingual PDFs: {e}")
65
+ merged_mono_path = None
66
+
67
+ try:
68
+ # Merge dual-language PDFs if they exist
69
+ if (
70
+ any(r.dual_pdf_path for r in results.values())
71
+ and not self.config.no_dual
72
+ ):
73
+ merged_dual_path = self._merge_pdfs(
84
74
  [
85
- r.no_watermark_dual_pdf_path
75
+ r.dual_pdf_path
86
76
  for r in sorted_results.values()
87
- if r.no_watermark_dual_pdf_path
77
+ if r.dual_pdf_path
88
78
  ],
89
- "merged_no_watermark_dual.pdf",
90
- tag="merged_no_watermark_dual",
79
+ dual_file_name,
80
+ tag="merged_dual",
91
81
  )
82
+ except Exception as e:
83
+ logger.error(f"Error merging dual-language PDFs: {e}")
84
+ merged_dual_path = None
85
+
86
+ if any(
87
+ r.dual_pdf_path != r.no_watermark_dual_pdf_path
88
+ or r.mono_pdf_path != r.no_watermark_mono_pdf_path
89
+ for r in results.values()
90
+ ):
91
+ try:
92
+ # Merge no-watermark PDFs if they exist
93
+ if (
94
+ any(r.no_watermark_mono_pdf_path for r in results.values())
95
+ and not self.config.no_mono
96
+ ):
97
+ merged_no_watermark_mono_path = self._merge_pdfs(
98
+ [
99
+ r.no_watermark_mono_pdf_path
100
+ for r in sorted_results.values()
101
+ if r.no_watermark_mono_pdf_path
102
+ ],
103
+ mono_file_name_no_watermark,
104
+ tag="merged_no_watermark_mono",
105
+ )
106
+ except Exception as e:
107
+ logger.error(f"Error merging no-watermark PDFs: {e}")
108
+ merged_no_watermark_mono_path = None
109
+
110
+ try:
111
+ if (
112
+ any(r.no_watermark_dual_pdf_path for r in results.values())
113
+ and not self.config.no_dual
114
+ ):
115
+ merged_no_watermark_dual_path = self._merge_pdfs(
116
+ [
117
+ r.no_watermark_dual_pdf_path
118
+ for r in sorted_results.values()
119
+ if r.no_watermark_dual_pdf_path
120
+ ],
121
+ "merged_no_watermark_dual.pdf",
122
+ tag="merged_no_watermark_dual",
123
+ )
124
+ except Exception as e:
125
+ logger.error(f"Error merging no-watermark PDFs: {e}")
126
+ merged_no_watermark_dual_path = None
92
127
 
93
128
  # Create merged result
94
129
  merged_result = TranslateResult(
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "BabelDOC"
3
- version = "0.3.55"
3
+ version = "0.3.56"
4
4
  description = "Yet Another Document Translator"
5
5
  license = "AGPL-3.0"
6
6
  readme = "README.md"
@@ -44,6 +44,7 @@ dependencies = [
44
44
  "opencv-python-headless>=4.10.0.84",
45
45
  "rapidocr-onnxruntime>=1.4.4",
46
46
  "pyzstd>=0.17.0",
47
+ "hyperscan>=0.7.13",
47
48
  ]
48
49
 
49
50
  [project.optional-dependencies]
@@ -150,7 +151,7 @@ pythonpath = [".", "src"]
150
151
  testpaths = ["tests"]
151
152
 
152
153
  [bumpver]
153
- current_version = "0.3.55"
154
+ current_version = "0.3.56"
154
155
  version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
155
156
 
156
157
  [bumpver.file_patterns]
@@ -48,7 +48,7 @@ wheels = [
48
48
 
49
49
  [[package]]
50
50
  name = "babeldoc"
51
- version = "0.3.55"
51
+ version = "0.3.56"
52
52
  source = { editable = "." }
53
53
  dependencies = [
54
54
  { name = "bitstring" },
@@ -56,6 +56,7 @@ dependencies = [
56
56
  { name = "freetype-py" },
57
57
  { name = "httpx", extra = ["socks"] },
58
58
  { name = "huggingface-hub" },
59
+ { name = "hyperscan" },
59
60
  { name = "msgpack" },
60
61
  { name = "numpy" },
61
62
  { name = "onnx" },
@@ -113,6 +114,7 @@ requires-dist = [
113
114
  { name = "freetype-py", specifier = ">=2.5.1" },
114
115
  { name = "httpx", extras = ["socks"], specifier = ">=0.27.0" },
115
116
  { name = "huggingface-hub", specifier = ">=0.27.0" },
117
+ { name = "hyperscan", specifier = ">=0.7.13" },
116
118
  { name = "memray", marker = "extra == 'memray'", specifier = ">=1.17.1" },
117
119
  { name = "msgpack", specifier = ">=1.1.0" },
118
120
  { name = "numpy", specifier = ">=2.0.2" },
@@ -727,6 +729,44 @@ wheels = [
727
729
  { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" },
728
730
  ]
729
731
 
732
+ [[package]]
733
+ name = "hyperscan"
734
+ version = "0.7.13"
735
+ source = { registry = "https://pypi.org/simple" }
736
+ sdist = { url = "https://files.pythonhosted.org/packages/63/c3/2f0647459d1d6dec8ac8af2b8ffd4b2a81ed4088cab4ed623b0889699816/hyperscan-0.7.13.tar.gz", hash = "sha256:ca3743286b4dacf7f6e9dcab0fa8d883663f76a72c4adaf4b5d2e806edc07615", size = 93802, upload-time = "2025-05-29T01:04:02.688Z" }
737
+ wheels = [
738
+ { url = "https://files.pythonhosted.org/packages/65/1e/94a54d73d7d5ca94532d3a5a22a2374bd3215f5afc74b9dcba120526896a/hyperscan-0.7.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1b0a7ed8d2742c694826a969cbff3fabac8ae8e679fb0db5606879cba13ecdaf", size = 2170503, upload-time = "2025-05-29T01:03:08.246Z" },
739
+ { url = "https://files.pythonhosted.org/packages/fe/84/8e5172f1f02fa87955d876952d482241a37d16548f7b5f0d8772b465db15/hyperscan-0.7.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c81ce07b314f28e2e15a89bef5afd06e792fa672112e7c079c1385291e9a6f8", size = 1913046, upload-time = "2025-05-29T01:03:10.411Z" },
740
+ { url = "https://files.pythonhosted.org/packages/e8/24/c1f18e8d82485fa8ccde7cb9558679dd20bb969cda9af2dbf3dec604a6e2/hyperscan-0.7.13-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64c5e9ae721b5bdeffc0150082de2bf324daf43bf46245b08c6691ab8ffb8c76", size = 2745465, upload-time = "2025-05-29T01:03:12.098Z" },
741
+ { url = "https://files.pythonhosted.org/packages/cf/61/39003aaeb925b67df64742e2dbce2d9bfd79f93180dfdd2e9dc0e2a63659/hyperscan-0.7.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:caa648fecb8b25d0cca322eae1386c8859cf38e90f664d903379a46d6814e6fc", size = 2904367, upload-time = "2025-05-29T01:03:14.013Z" },
742
+ { url = "https://files.pythonhosted.org/packages/7e/57/b918b06a120e23f7813c14f9ebffc6e950d1cf88da2f512b46a726102582/hyperscan-0.7.13-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:12b4b598683dd24bb399d217d09929533dab715b04b787f2405ec9fb68b33237", size = 2958330, upload-time = "2025-05-29T01:03:16.039Z" },
743
+ { url = "https://files.pythonhosted.org/packages/88/49/650f56d9474d42993c5a2abdec4d45fc7e018143478b9efbcaacabfc42a0/hyperscan-0.7.13-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f93178b1532b6aae0e7f75918896894fc6d1d3642a4cec4c594354a56fd30e02", size = 3224724, upload-time = "2025-05-29T01:03:18.168Z" },
744
+ { url = "https://files.pythonhosted.org/packages/4f/41/f2c3910b4d0be6e19b90515db67c21c757a1ca6878e50f190d556f34912a/hyperscan-0.7.13-cp310-cp310-win_amd64.whl", hash = "sha256:ad569db2c83160c48e1e36a7770224575bf55ca77b153f16ea23ecfc6d979ff9", size = 1968443, upload-time = "2025-05-29T01:03:20.189Z" },
745
+ { url = "https://files.pythonhosted.org/packages/d3/d0/6e0cdd76ad0bf4b36d969dd7b0bb4917dfdf61d8f978f850e9e213c7976e/hyperscan-0.7.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a947c206c13f119b5b65a786129cfa85c76865c25534351c85eaf4c1a161fe43", size = 2170505, upload-time = "2025-05-29T01:03:21.672Z" },
746
+ { url = "https://files.pythonhosted.org/packages/1c/6a/770cdd9294d09f28ead148ad46de037c663c5e4dc19ef6dde8e9b2422d64/hyperscan-0.7.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9ae2e015def327e3fdda7dc9b2baa83f93e34bfafe1435e62d8f6276c6f246d2", size = 1912893, upload-time = "2025-05-29T01:03:23.557Z" },
747
+ { url = "https://files.pythonhosted.org/packages/fa/81/b266b2cd41c7b7a3b008706fb0086162778db5796c666036d758e9fa8173/hyperscan-0.7.13-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17c84504f651a9e8326d62d02737b03f2fc9cb0e880881d844a343b1d4342dd3", size = 2745465, upload-time = "2025-05-29T01:03:25.449Z" },
748
+ { url = "https://files.pythonhosted.org/packages/6e/05/2aa902dac18599839b2fabfb25b58f131282a9bdfc8faa0128b0355e5757/hyperscan-0.7.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac0421e812376eaf6d731addf22676d47b364796f3cbe289c234a744f2831a0", size = 2904213, upload-time = "2025-05-29T01:03:26.961Z" },
749
+ { url = "https://files.pythonhosted.org/packages/ed/0a/81d6591e1bb642744b908ec43c2ec92de2caf15cd83dc1a6a41893c3ff93/hyperscan-0.7.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:df902bf4a6a37160ef7c2cb4a8825297a433c0ab9430e68f7048ebde0d15a5cb", size = 2958329, upload-time = "2025-05-29T01:03:28.97Z" },
750
+ { url = "https://files.pythonhosted.org/packages/09/98/c7d7570c27e556de6afaf197420764535155d02def341199501eb5da19f7/hyperscan-0.7.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:366e654d03b65b7a6a401705b7d6a99f0bb587ee347959217bce1f4330cb839d", size = 3224729, upload-time = "2025-05-29T01:03:30.46Z" },
751
+ { url = "https://files.pythonhosted.org/packages/ac/3d/8db2cdf5c0c39f7de6598ddc3b37dd93c687c245c651eab0f61931a51724/hyperscan-0.7.13-cp311-cp311-win_amd64.whl", hash = "sha256:9685bfb664c80675e9084bc28f3137b08e3a0ec42bf2b7c09b8feb0b1cd57a16", size = 1968444, upload-time = "2025-05-29T01:03:32.832Z" },
752
+ { url = "https://files.pythonhosted.org/packages/72/b0/24b6d9337023f98d4b2d4aeab6e8adb51b7c8a7e71c6dd98e96fc1cf8ac4/hyperscan-0.7.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d87d580ba647ee4efa80f50089e3a05dc61470d73aac5a9d99bc2a1a9c168967", size = 2170605, upload-time = "2025-05-29T01:03:34.74Z" },
753
+ { url = "https://files.pythonhosted.org/packages/95/9d/01398890af1eed810113c08331f4d2eda7021cc231254c1d2724bc2e87de/hyperscan-0.7.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af422c4a6b3933f0711a928c92fab17d192d6871ec66e88ba2736216b336eba5", size = 1912920, upload-time = "2025-05-29T01:03:36.251Z" },
754
+ { url = "https://files.pythonhosted.org/packages/f1/35/f58661355881ed4c4517fcc4e6181f9499da6d70e44edce85442ccf9a0f6/hyperscan-0.7.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff5716d5a137e7bd6f49268e902528b6ea94b03549857578def6546bdc1ed070", size = 2745340, upload-time = "2025-05-29T01:03:37.718Z" },
755
+ { url = "https://files.pythonhosted.org/packages/96/8d/d6b81e31e7a30a35d9a51b092e761cdcbb83701887ba479e787086ff31c2/hyperscan-0.7.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:914f51ca7ee2fbf2f25c3a769c324338c4b6878f36ebf61b655db4d180036b8a", size = 2904145, upload-time = "2025-05-29T01:03:39.804Z" },
756
+ { url = "https://files.pythonhosted.org/packages/5e/f7/5a963b1eb4cef5b6eac6f6994aec0d067df0fc7acfc647efe282e293a7c2/hyperscan-0.7.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:84f5b66e2e7a7afce4ad861a9f7a6b0218242fb4a296b6fed3ee56a9b1b80238", size = 2958515, upload-time = "2025-05-29T01:03:41.385Z" },
757
+ { url = "https://files.pythonhosted.org/packages/28/29/bf2cff1ae6c8382900e3455466f8f52d1cdc226336e2b73bbf292f12dfcb/hyperscan-0.7.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:75ed7956f3a517844df5213afb94dcdfa223c6cce8195f9500096e0320b45e8a", size = 3225011, upload-time = "2025-05-29T01:03:43.771Z" },
758
+ { url = "https://files.pythonhosted.org/packages/63/be/0f7500d37c3663feae94ddf6f8cd82cd1ac56d9568bdb6a5598c03db8074/hyperscan-0.7.13-cp312-cp312-win_amd64.whl", hash = "sha256:063a8016d4c3fd5293aa1e67c488eea6c6b2c19a73f1f9a7f5c6f89e50135598", size = 1968429, upload-time = "2025-05-29T01:03:45.85Z" },
759
+ { url = "https://files.pythonhosted.org/packages/7c/fe/22bc51617a865030955cb7dae4a0eb5d15d4282b518e104525962804d79d/hyperscan-0.7.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1eb4db1c48222db5ed5d987d89e09d3dde1d3bb7484dafe7615a90b046b7586a", size = 2170601, upload-time = "2025-05-29T01:03:47.292Z" },
760
+ { url = "https://files.pythonhosted.org/packages/1a/de/6f3fd5a4ba79dbb5997385effa022d4b4e6d66667e71b1313a80918f4154/hyperscan-0.7.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:267c09256916086a58b01e595688aff4dd1a5c1950497f6cde04e4cf8a3f5637", size = 1913234, upload-time = "2025-05-29T01:03:48.727Z" },
761
+ { url = "https://files.pythonhosted.org/packages/88/c8/409ce4b5f6eb12f145f1e497b4ae7581ff89bfff63722924ce2484262e57/hyperscan-0.7.13-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a008c9de788d7720163abd9b4a606f4dd5459104f36c06a9da934a1e4f23315b", size = 2745558, upload-time = "2025-05-29T01:03:50.234Z" },
762
+ { url = "https://files.pythonhosted.org/packages/97/ec/eb02523b78c5392f41f37f2a6e7ee53e6c871c29d5bce4429c4b8d3fe4f1/hyperscan-0.7.13-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b68c2d98043c6497b9aae0d0368d53b0449fc247506a66bba3448dca5c2d1b7", size = 2904150, upload-time = "2025-05-29T01:03:52.17Z" },
763
+ { url = "https://files.pythonhosted.org/packages/25/e9/1744104e2a7b1efad0a531486754f08772b0e0c42045fcf8479999e224c1/hyperscan-0.7.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d95ca8d21ff28b6f20e1a8c9f2fc1e660f4aedd452e440600f7b28438fb865ba", size = 2958466, upload-time = "2025-05-29T01:03:53.659Z" },
764
+ { url = "https://files.pythonhosted.org/packages/0d/60/b21520411d2d917871b1606927f23b198f86c4bd18b35bcfdaee56e3bff2/hyperscan-0.7.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9eab88bfd9031d07f47bc6f31562f0a2d4052f3141e0edbc94dd3cc521ab9c73", size = 3224990, upload-time = "2025-05-29T01:03:55.727Z" },
765
+ { url = "https://files.pythonhosted.org/packages/c5/8b/9ce271f0f1d93d2bd823b04b9942dc4043d055c11c14eeaadd63669639a0/hyperscan-0.7.13-cp313-cp313-win_amd64.whl", hash = "sha256:9e33df21dba84f5817e97192ea4d1d6b2f30376581801a32de2d622eb2b789af", size = 1968418, upload-time = "2025-05-29T01:03:57.274Z" },
766
+ { url = "https://files.pythonhosted.org/packages/79/e3/f26287afccbfde82f865c52135aadd559108ab0c987dcfe86be4e7b58db2/hyperscan-0.7.13-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89e98aac914a6cf40231380f1325a94c9cac1a1c797bd4395bc3ed90156ee5f6", size = 2745676, upload-time = "2025-05-29T01:03:59.401Z" },
767
+ { url = "https://files.pythonhosted.org/packages/0a/25/3a849e16a76ac8fbedbb557010ded0867ce0ebf60afb8f793cf01009ef61/hyperscan-0.7.13-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d59d0bd095e16af258f836634cfd16efda425366aa2011a06b7a0289c8e8cd03", size = 2904700, upload-time = "2025-05-29T01:04:01.234Z" },
768
+ ]
769
+
730
770
  [[package]]
731
771
  name = "identify"
732
772
  version = "2.6.10"
@@ -1 +0,0 @@
1
- __version__ = "0.3.55"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes