BabelDOC 0.2.7__tar.gz → 0.2.8__tar.gz

{babeldoc-0.2.7 → babeldoc-0.2.8}/PKG-INFO

@@ -1,9 +1,11 @@
 Metadata-Version: 2.4
 Name: BabelDOC
-Version: 0.2.7
+Version: 0.2.8
 Summary: Yet Another Document Translator
 Project-URL: Homepage, https://github.com/funstory-ai/BabelDOC
 Project-URL: Issues, https://github.com/funstory-ai/BabelDOC/issues
+Author-email: awwaawwa <aw@funstory.ai>
+Maintainer-email: awwaawwa <aw@funstory.ai>
 License-Expression: AGPL-3.0
 License-File: LICENSE
 Keywords: PDF
@@ -20,7 +22,7 @@ Requires-Dist: numpy>=2.0.2
 Requires-Dist: onnx>=1.17.0
 Requires-Dist: onnxruntime>=1.16.1
 Requires-Dist: openai>=1.59.3
-Requires-Dist: opencv-python>=4.10.0.84
+Requires-Dist: opencv-python-headless>=4.10.0.84
 Requires-Dist: orjson>=3.10.14
 Requires-Dist: pdfminer-six>=20240706
 Requires-Dist: peewee>=3.17.8
@@ -293,6 +295,12 @@ report-interval = 0.5
 
 ## Python API
 
+> [!TIP]
+>
+> 1. Before pdf2zh 2.0 is released, you can temporarily use BabelDOC's Python API. However, after pdf2zh 2.0 is released, please directly use pdf2zh's Python API.
+>
+> 2. This project's Python API does not guarantee any compatibility. However, the Python API from pdf2zh will guarantee a certain level of compatibility.
+
 You can refer to the example in [main.py](https://github.com/funstory-ai/yadt/blob/main/babeldoc/main.py) to use BabelDOC's Python API.
 
 Please note:

{babeldoc-0.2.7 → babeldoc-0.2.8}/README.md

@@ -255,6 +255,12 @@ report-interval = 0.5
 
 ## Python API
 
+> [!TIP]
+>
+> 1. Before pdf2zh 2.0 is released, you can temporarily use BabelDOC's Python API. However, after pdf2zh 2.0 is released, please directly use pdf2zh's Python API.
+>
+> 2. This project's Python API does not guarantee any compatibility. However, the Python API from pdf2zh will guarantee a certain level of compatibility.
+
 You can refer to the example in [main.py](https://github.com/funstory-ai/yadt/blob/main/babeldoc/main.py) to use BabelDOC's Python API.
 
 Please note:

babeldoc-0.2.8/babeldoc/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.8"

{babeldoc-0.2.7 → babeldoc-0.2.8}/babeldoc/const.py

@@ -3,7 +3,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
-__version__ = "0.2.7"
+__version__ = "0.2.8"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

{babeldoc-0.2.7 → babeldoc-0.2.8}/babeldoc/document_il/midend/il_translator.py

@@ -603,7 +603,7 @@ class ILTranslator:
                 )
             except Exception as e:
                 logger.exception(
-                    f"Error translating paragraph. Paragraph: {paragraph}. Error: {e}. ",
+                    f"Error translating paragraph. Paragraph: {paragraph.debug_id} ({paragraph.unicode}). Error: {e}. ",
                 )
                 # ignore error and continue
                 return

{babeldoc-0.2.7 → babeldoc-0.2.8}/babeldoc/document_il/translator/translator.py

@@ -62,6 +62,31 @@ def set_translate_rate_limiter(max_qps):
     _translate_rate_limiter.set_max_qps(max_qps)
 
 
+class AtomicInteger:
+    def __init__(self, value=0):
+        self._value = int(value)
+        self._lock = threading.Lock()
+
+    def inc(self, d=1):
+        with self._lock:
+            self._value += int(d)
+            return self._value
+
+    def dec(self, d=1):
+        return self.inc(-d)
+
+    @property
+    def value(self):
+        with self._lock:
+            return self._value
+
+    @value.setter
+    def value(self, v):
+        with self._lock:
+            self._value = int(v)
+            return self._value
+
+
 class BaseTranslator(ABC):
     # Due to cache limitations, name should be within 20 characters.
     # cache.py: translate_engine = CharField(max_length=20)
@@ -195,6 +220,9 @@ class OpenAITranslator(BaseTranslator):
         self.model = model
         self.add_cache_impact_parameters("model", self.model)
         self.add_cache_impact_parameters("prompt", self.prompt(""))
+        self.token_count = AtomicInteger()
+        self.prompt_token_count = AtomicInteger()
+        self.completion_token_count = AtomicInteger()
 
     @retry(
         retry=retry_if_exception_type(openai.RateLimitError),
@@ -211,6 +239,9 @@ class OpenAITranslator(BaseTranslator):
             **self.options,
             messages=self.prompt(text),
         )
+        self.token_count.inc(response.usage.total_tokens)
+        self.prompt_token_count.inc(response.usage.prompt_tokens)
+        self.completion_token_count.inc(response.usage.completion_tokens)
         return response.choices[0].message.content.strip()
 
     def prompt(self, text):
@@ -248,6 +279,9 @@ class OpenAITranslator(BaseTranslator):
                 },
             ],
         )
+        self.token_count.inc(response.usage.total_tokens)
+        self.prompt_token_count.inc(response.usage.prompt_tokens)
+        self.completion_token_count.inc(response.usage.completion_tokens)
         return response.choices[0].message.content.strip()
 
     def get_formular_placeholder(self, placeholder_id: int):

{babeldoc-0.2.7 → babeldoc-0.2.8}/babeldoc/docvision/doclayout.py

@@ -6,8 +6,17 @@ from collections.abc import Generator
 
 import cv2
 import numpy as np
-import onnx
-import onnxruntime
+
+try:
+    import onnx
+    import onnxruntime
+except ImportError as e:
+    if "DLL load failed" in str(e):
+        raise OSError(
+            "Microsoft Visual C++ Redistributable is not installed. "
+            "Download it at https://aka.ms/vs/17/release/vc_redist.x64.exe"
+        ) from e
+    raise
 import pymupdf
 
 import babeldoc.document_il.il_version_1

{babeldoc-0.2.7 → babeldoc-0.2.8}/babeldoc/main.py

@@ -22,7 +22,7 @@ from babeldoc.translation_config import TranslationConfig
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.2.7"
+__version__ = "0.2.8"
 
 
 def create_parser():
@@ -353,6 +353,9 @@ async def main():
                     result = event["translate_result"]
                     logger.info(str(result))
                     break
+    logger.info(f"Total tokens: {translator.token_count.value}")
+    logger.info(f"Prompt tokens: {translator.prompt_token_count.value}")
+    logger.info(f"Completion tokens: {translator.completion_token_count.value}")
 
 
 def create_progress_handler(translation_config: TranslationConfig):

{babeldoc-0.2.7 → babeldoc-0.2.8}/pyproject.toml

@@ -1,10 +1,16 @@
 [project]
 name = "BabelDOC"
-version = "0.2.7"
+version = "0.2.8"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
 requires-python = ">=3.10,<3.13"
+authors = [
+    { name = "awwaawwa", email = "aw@funstory.ai" }
+]
+maintainers = [
+    { name = "awwaawwa", email = "aw@funstory.ai" }
+]
 classifiers = [
     "Programming Language :: Python :: 3",
     "Operating System :: OS Independent",
@@ -19,7 +25,6 @@ dependencies = [
     "onnx>=1.17.0",
     "onnxruntime>=1.16.1",
     "openai>=1.59.3",
-    "opencv-python>=4.10.0.84",
     "orjson>=3.10.14",
     "pdfminer-six>=20240706",
     "peewee>=3.17.8",
@@ -35,6 +40,7 @@ dependencies = [
     "freetype-py>=2.5.1",
     "tiktoken>=0.9.0",
     "python-levenshtein>=0.27.1",
+    "opencv-python-headless>=4.10.0.84",
 ]
 
 [project.urls]
@@ -136,7 +142,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.2.7"
+current_version = "0.2.8"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]

{babeldoc-0.2.7 → babeldoc-0.2.8}/uv.lock

@@ -48,7 +48,7 @@ wheels = [
 
 [[package]]
 name = "babeldoc"
-version = "0.2.7"
+version = "0.2.8"
 source = { editable = "." }
 dependencies = [
     { name = "bitstring" },
@@ -61,7 +61,7 @@ dependencies = [
     { name = "onnx" },
     { name = "onnxruntime" },
     { name = "openai" },
-    { name = "opencv-python" },
+    { name = "opencv-python-headless" },
     { name = "orjson" },
     { name = "pdfminer-six" },
     { name = "peewee" },
@@ -104,7 +104,7 @@ requires-dist = [
     { name = "onnx", specifier = ">=1.17.0" },
     { name = "onnxruntime", specifier = ">=1.16.1" },
     { name = "openai", specifier = ">=1.59.3" },
-    { name = "opencv-python", specifier = ">=4.10.0.84" },
+    { name = "opencv-python-headless", specifier = ">=4.10.0.84" },
     { name = "orjson", specifier = ">=3.10.14" },
     { name = "pdfminer-six", specifier = ">=20240706" },
     { name = "peewee", specifier = ">=3.17.8" },
@@ -1368,20 +1368,20 @@ wheels = [
 ]
 
 [[package]]
-name = "opencv-python"
+name = "opencv-python-headless"
 version = "4.11.0.86"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "numpy" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956 }
+sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/05/4d/53b30a2a3ac1f75f65a59eb29cf2ee7207ce64867db47036ad61743d5a23/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:432f67c223f1dc2824f5e73cdfcd9db0efc8710647d4e813012195dc9122a52a", size = 37326322 },
-    { url = "https://files.pythonhosted.org/packages/3b/84/0a67490741867eacdfa37bc18df96e08a9d579583b419010d7f3da8ff503/opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:9d05ef13d23fe97f575153558653e2d6e87103995d54e6a35db3f282fe1f9c66", size = 56723197 },
-    { url = "https://files.pythonhosted.org/packages/f3/bd/29c126788da65c1fb2b5fb621b7fed0ed5f9122aa22a0868c5e2c15c6d23/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b92ae2c8852208817e6776ba1ea0d6b1e0a1b5431e971a2a0ddd2a8cc398202", size = 42230439 },
-    { url = "https://files.pythonhosted.org/packages/2c/8b/90eb44a40476fa0e71e05a0283947cfd74a5d36121a11d926ad6f3193cc4/opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b02611523803495003bd87362db3e1d2a0454a6a63025dc6658a9830570aa0d", size = 62986597 },
-    { url = "https://files.pythonhosted.org/packages/fb/d7/1d5941a9dde095468b288d989ff6539dd69cd429dbf1b9e839013d21b6f0/opencv_python-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:810549cb2a4aedaa84ad9a1c92fbfdfc14090e2749cedf2c1589ad8359aa169b", size = 29384337 },
-    { url = "https://files.pythonhosted.org/packages/a4/7d/f1c30a92854540bf789e9cd5dde7ef49bbe63f855b85a2e6b3db8135c591/opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec", size = 39488044 },
+    { url = "https://files.pythonhosted.org/packages/dc/53/2c50afa0b1e05ecdb4603818e85f7d174e683d874ef63a6abe3ac92220c8/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:48128188ade4a7e517237c8e1e11a9cdf5c282761473383e77beb875bb1e61ca", size = 37326460 },
+    { url = "https://files.pythonhosted.org/packages/3b/43/68555327df94bb9b59a1fd645f63fafb0762515344d2046698762fc19d58/opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81", size = 56723330 },
+    { url = "https://files.pythonhosted.org/packages/45/be/1438ce43ebe65317344a87e4b150865c5585f4c0db880a34cdae5ac46881/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6efabcaa9df731f29e5ea9051776715b1bdd1845d7c9530065c7951d2a2899eb", size = 29487060 },
+    { url = "https://files.pythonhosted.org/packages/dd/5c/c139a7876099916879609372bfa513b7f1257f7f1a908b0bdc1c2328241b/opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b", size = 49969856 },
+    { url = "https://files.pythonhosted.org/packages/95/dd/ed1191c9dc91abcc9f752b499b7928aacabf10567bb2c2535944d848af18/opencv_python_headless-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b", size = 29324425 },
+    { url = "https://files.pythonhosted.org/packages/86/8a/69176a64335aed183529207ba8bc3d329c2999d852b4f3818027203f50e6/opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca", size = 39402386 },
 ]
 
 [[package]]

babeldoc-0.2.7/babeldoc/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.7"

{babeldoc-0.2.7 → babeldoc-0.2.8}/babeldoc/document_il/midend/il_translator_llm_only.py

@@ -328,9 +328,9 @@ class ILTranslatorLLMOnly:
                         translate_input,
                         translated_text,
                     )
+                    should_fallback = False
                     if pbar:
                         pbar.advance(1)
-                    should_fallback = False
                 except Exception as e:
                     logger.exception(f"Error translating paragraph. Error: {e}.")
                     # Ignore error and continue