epub-translator 0.1.7__py3-none-any.whl → 0.1.8__py3-none-any.whl

epub_translator/llm/core.py

@@ -13,6 +13,7 @@ from ..template import create_env
 from .context import LLMContext
 from .executor import LLMExecutor
 from .increasable import Increasable
+from .statistics import Statistics
 from .types import Message
 
 # Global state for logger filename generation
@@ -44,7 +45,7 @@ class LLM:
         self._temperature: Increasable = Increasable(temperature)
         self._cache_path: Path | None = self._ensure_dir_path(cache_path)
         self._logger_save_path: Path | None = self._ensure_dir_path(log_dir_path)
-
+        self._statistics = Statistics()
         self._executor = LLMExecutor(
             url=url,
             model=model,
@@ -53,12 +54,29 @@ class LLM:
             retry_times=retry_times,
             retry_interval_seconds=retry_interval_seconds,
             create_logger=self._create_logger,
+            statistics=self._statistics,
         )
 
     @property
     def encoding(self) -> Encoding:
         return self._encoding
 
+    @property
+    def total_tokens(self) -> int:
+        return self._statistics.total_tokens
+
+    @property
+    def input_tokens(self) -> int:
+        return self._statistics.input_tokens
+
+    @property
+    def input_cache_tokens(self) -> int:
+        return self._statistics.input_cache_tokens
+
+    @property
+    def output_tokens(self) -> int:
+        return self._statistics.output_tokens
+
     def context(self, cache_seed_content: str | None = None) -> LLMContext:
         return LLMContext(
             executor=self._executor,

epub_translator/llm/executor.py

@@ -7,6 +7,7 @@ from openai import OpenAI
 from openai.types.chat import ChatCompletionMessageParam
 
 from .error import is_retry_error
+from .statistics import Statistics
 from .types import Message, MessageRole
 
 
@@ -20,12 +21,14 @@ class LLMExecutor:
         retry_times: int,
         retry_interval_seconds: float,
         create_logger: Callable[[], Logger | None],
+        statistics: Statistics,
     ) -> None:
         self._model_name: str = model
         self._timeout: float | None = timeout
         self._retry_times: int = retry_times
         self._retry_interval_seconds: float = retry_interval_seconds
         self._create_logger: Callable[[], Logger | None] = create_logger
+        self._statistics = statistics
         self._client = OpenAI(
             api_key=api_key,
             base_url=url,
@@ -156,6 +159,7 @@ class LLMExecutor:
             model=self._model_name,
             messages=messages,
             stream=True,
+            stream_options={"include_usage": True},
             top_p=top_p,
             temperature=temperature,
             max_tokens=max_tokens,
@@ -164,4 +168,5 @@ class LLMExecutor:
         for chunk in stream:
             if chunk.choices and chunk.choices[0].delta.content:
                 buffer.write(chunk.choices[0].delta.content)
+            self._statistics.submit_usage(chunk.usage)
         return buffer.getvalue()

epub_translator/llm/statistics.py

@@ -0,0 +1,25 @@
+from threading import Lock
+
+from openai.types import CompletionUsage
+
+
+class Statistics:
+    def __init__(self) -> None:
+        self._lock = Lock()
+        self.total_tokens = 0
+        self.input_tokens = 0
+        self.input_cache_tokens = 0
+        self.output_tokens = 0
+
+    def submit_usage(self, usage: CompletionUsage | None) -> None:
+        if usage is None:
+            return
+        with self._lock:
+            if usage.total_tokens:
+                self.total_tokens += usage.total_tokens
+            if usage.prompt_tokens:
+                self.input_tokens += usage.prompt_tokens
+            if usage.prompt_tokens_details and usage.prompt_tokens_details.cached_tokens:
+                self.input_cache_tokens += usage.prompt_tokens_details.cached_tokens
+            if usage.completion_tokens:
+                self.output_tokens += usage.completion_tokens

epub_translator/segment/text_segment.py

@@ -4,7 +4,12 @@ from enum import Enum, auto
 from typing import Self
 from xml.etree.ElementTree import Element
 
-from ..xml import expand_left_element_texts, expand_right_element_texts, is_inline_element, normalize_text_in_element
+from ..xml import (
+    expand_left_element_texts,
+    expand_right_element_texts,
+    is_inline_element,
+    normalize_text_in_element,
+)
 
 
 class TextPosition(Enum):

epub_translator/translation/xml_interrupter.py

@@ -6,8 +6,8 @@ from bs4 import BeautifulSoup
 from mathml2latex.mathml import process_mathml
 
 from ..segment import TextSegment, combine_text_segments, find_block_depth
-from ..utils import ensure_list
-from ..xml import clone_element
+from ..utils import ensure_list, normalize_whitespace
+from ..xml import DISPLAY_ATTRIBUTE, clone_element, is_inline_element
 
 _ID_KEY = "__XML_INTERRUPTER_ID"
 _MATH_TAG = "math"
@@ -87,9 +87,9 @@ class XMLInterrupter:
                     _ID_KEY: cast(str, interrupted_element.get(_ID_KEY)),
                 },
             )
-            interrupted_display = interrupted_element.get("display", None)
+            interrupted_display = interrupted_element.get(DISPLAY_ATTRIBUTE, None)
             if interrupted_display is not None:
-                placeholder_element.set("display", interrupted_display)
+                placeholder_element.set(DISPLAY_ATTRIBUTE, interrupted_display)
 
             raw_parent_stack = text_segment.parent_stack[:interrupted_index]
             parent_stack = raw_parent_stack + [placeholder_element]
@@ -159,10 +159,13 @@ class XMLInterrupter:
 
         if latex is None:
             latex = "".join(t.text for t in text_segments)
-        elif math_element.get("display", None) == "inline":
-            latex = f"${latex}$"
+            latex = normalize_whitespace(latex).strip()
         else:
-            latex = f"$${latex}$$"
+            latex = normalize_whitespace(latex).strip()
+            if is_inline_element(math_element):
+                latex = f"${latex}$"
+            else:
+                latex = f"$${latex}$$"
 
         return f" {latex} "
 

epub_translator/xml/const.py

@@ -1 +1,2 @@
 ID_KEY: str = "id"
+DISPLAY_ATTRIBUTE: str = "display"

epub_translator/xml/inline.py

@@ -1,5 +1,7 @@
 from xml.etree.ElementTree import Element
 
+from .const import DISPLAY_ATTRIBUTE
+
 # HTML inline-level elements
 # Reference: https://developer.mozilla.org/en-US/docs/Web/HTML/Inline_elements
 # Reference: https://developer.mozilla.org/en-US/docs/Glossary/Inline-level_content
@@ -105,9 +107,14 @@ _HTML_INLINE_TAGS = frozenset(
 
 
 def is_inline_element(element: Element) -> bool:
-    if element.tag.lower() in _HTML_INLINE_TAGS:
+    tag = element.tag.lower()
+    if tag in _HTML_INLINE_TAGS:
         return True
-    display = element.get("display", None)
-    if display is not None and display.lower() == "inline":
+    display = element.get(DISPLAY_ATTRIBUTE, None)
+    if display is not None:
+        display = display.lower()
+        if display == "inline":
+            return True
+    if tag == "math" and display != "block":
         return True
     return False

{epub_translator-0.1.7.dist-info → epub_translator-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: epub-translator
-Version: 0.1.7
+Version: 0.1.8
 Summary: Translate the epub book using LLM. The translated book will retain the original text and list the translated text side by side with the original text.
 License: MIT
 Keywords: epub,llm,translation,translator
@@ -422,6 +422,105 @@ translate(
 
 When using `concurrency > 1`, ensure that any custom callback functions (`on_progress`, `on_fill_failed`) are thread-safe. Built-in callbacks are thread-safe by default.
 
+### Token Usage Monitoring
+
+Track token consumption during translation to monitor API costs and usage:
+
+```python
+from epub_translator import LLM, translate, language, SubmitKind
+
+llm = LLM(
+    key="your-api-key",
+    url="https://api.openai.com/v1",
+    model="gpt-4",
+    token_encoding="o200k_base",
+)
+
+translate(
+    source_path="source.epub",
+    target_path="translated.epub",
+    target_language=language.ENGLISH,
+    submit=SubmitKind.APPEND_BLOCK,
+    llm=llm,
+)
+
+# Access token statistics after translation
+print(f"Total tokens: {llm.total_tokens}")
+print(f"Input tokens: {llm.input_tokens}")
+print(f"Input cache tokens: {llm.input_cache_tokens}")
+print(f"Output tokens: {llm.output_tokens}")
+```
+
+**Available Statistics:**
+
+- `total_tokens` - Total number of tokens used (input + output)
+- `input_tokens` - Number of prompt/input tokens
+- `input_cache_tokens` - Number of cached input tokens (when using prompt caching)
+- `output_tokens` - Number of generated/completion tokens
+
+**Real-time Monitoring:**
+
+You can also monitor token usage in real-time during translation:
+
+```python
+from tqdm import tqdm
+import time
+
+with tqdm(total=100, desc="Translating", unit="%") as pbar:
+    last_progress = 0.0
+    start_time = time.time()
+
+    def on_progress(progress: float):
+        nonlocal last_progress
+        increment = (progress - last_progress) * 100
+        pbar.update(increment)
+        last_progress = progress
+
+        # Update token stats in progress bar
+        pbar.set_postfix({
+            'tokens': llm.total_tokens,
+            'cost_est': f'${llm.total_tokens * 0.00001:.4f}'  # Estimate based on your pricing
+        })
+
+    translate(
+        source_path="source.epub",
+        target_path="translated.epub",
+        target_language=language.ENGLISH,
+        submit=SubmitKind.APPEND_BLOCK,
+        llm=llm,
+        on_progress=on_progress,
+    )
+
+    elapsed = time.time() - start_time
+    print(f"\nTranslation completed in {elapsed:.1f}s")
+    print(f"Total tokens used: {llm.total_tokens:,}")
+    print(f"Average tokens/second: {llm.total_tokens/elapsed:.1f}")
+```
+
+**Dual-LLM Token Tracking:**
+
+When using separate LLMs for translation and filling, each LLM tracks its own statistics:
+
+```python
+translation_llm = LLM(key="...", url="...", model="gpt-4", token_encoding="o200k_base")
+fill_llm = LLM(key="...", url="...", model="gpt-4", token_encoding="o200k_base")
+
+translate(
+    source_path="source.epub",
+    target_path="translated.epub",
+    target_language=language.ENGLISH,
+    submit=SubmitKind.APPEND_BLOCK,
+    translation_llm=translation_llm,
+    fill_llm=fill_llm,
+)
+
+print(f"Translation tokens: {translation_llm.total_tokens}")
+print(f"Fill tokens: {fill_llm.total_tokens}")
+print(f"Combined total: {translation_llm.total_tokens + fill_llm.total_tokens}")
+```
+
+**Note:** Token statistics are cumulative across all API calls made by the LLM instance. The counts only increase and are thread-safe when using concurrent translation.
+
 ## Related Projects
 
 ### PDF Craft

{epub_translator-0.1.7.dist-info → epub_translator-0.1.8.dist-info}/RECORD

@@ -10,16 +10,17 @@ epub_translator/epub/toc.py,sha256=TKJfyDT4svFkXd6JCNZk2ZEYc9q-5DXnV3zY2UKo8nE,1
 epub_translator/epub/zip.py,sha256=-3LI8f-ksgU8xCy28NjBOKyQPE8PhPEUPqIKZE1p8dw,2364
 epub_translator/llm/__init__.py,sha256=YcFYYnXmXyX0RUyC-PDbj5k7Woygp_XOpTI3vDiNSPM,75
 epub_translator/llm/context.py,sha256=8-0UnrZIaNshR_imy_ed_UpOK7H1a6dOsG-boaYOX8k,4186
-epub_translator/llm/core.py,sha256=wQwt6oG68ZN_iQOaytBiPXOC7sI62XII_A4dOHdAt_s,5979
+epub_translator/llm/core.py,sha256=MnToX8Zhr_r4sj9B3s54bclesojQEFarzl0VqHGDKlo,6488
 epub_translator/llm/error.py,sha256=4efAIQL14DFSvAnSTUfgdAbZRqaWBqOfUGsSfvxa5zM,1503
-epub_translator/llm/executor.py,sha256=A0IjQ-s9wBJuhAZAAydneb9zBXWnu2J9inR2Q8F-GDE,5533
+epub_translator/llm/executor.py,sha256=wxgFwWaLmuqAvctO3lcQX4U52aiw7EdaFw9Ut0v-ZzU,5745
 epub_translator/llm/increasable.py,sha256=8XkKeI1hiHlpMHj8dQ4fW0BkViSx4hH8QfbQsy-5SDw,1297
+epub_translator/llm/statistics.py,sha256=BX75qVWJ9aWbMoFtaQzoE8oVCLh7wiHoR06dX-AAl3E,875
 epub_translator/llm/types.py,sha256=c-dMAIvlG4R3la3mUTWEw5xei-sIYKmQeBja7mirxcI,219
 epub_translator/segment/__init__.py,sha256=nCHNaHASElKTbC8HEAQkI1Y12m6kEdX5uJVvVvHKtFg,595
 epub_translator/segment/block_segment.py,sha256=psNKA_HMIcwZtoug8AtnAcV9_mQ2WXLnXqFsekHzt2g,4570
 epub_translator/segment/common.py,sha256=gGWYQaJ0tGnWCuF1me9TOo-Q_DrZVakCu2patyFIOs0,714
 epub_translator/segment/inline_segment.py,sha256=nrRKoJ-vblsNITJeixrCgIOkVQyUXrchMg0XYU_8pLo,14563
-epub_translator/segment/text_segment.py,sha256=DHv8bkBpVYVUI01hOIf9Jdc7D3v7SdbLD1MeL4MUh4U,6267
+epub_translator/segment/text_segment.py,sha256=E_qgPI09sCV_-PsJtgwcloTa0tpOP3wl0pw5gV9dDNY,6288
 epub_translator/segment/utils.py,sha256=_tlIA1I7rYz9_q-oQ5cPZWPmhTObCXjksQzRtX3beXY,636
 epub_translator/serial/__init__.py,sha256=b3IMVmWcUwEqHKcGmey88b057pyz5ct946CaUZi4LB4,67
 epub_translator/serial/chunk.py,sha256=FrTaHikVOd6bLYumnEriTaAQ_DIDLjHm16gh-wBVR9k,1495
@@ -31,10 +32,10 @@ epub_translator/translation/epub_transcode.py,sha256=_pRzmQgDrlfsibalkUogVi0F0Qy
 epub_translator/translation/language.py,sha256=88osG0JNYxOkxBjg5Pm-P0Mhiyxf6GqdxoPW12HW0PE,493
 epub_translator/translation/punctuation.py,sha256=TPCGjEmlAyN3G11VuXdHn-pvUkuWDwWqbTNzw-ij60E,813
 epub_translator/translation/translator.py,sha256=WC4Yqx-ffhxBhqzMAujE_NQG7BsDwgn95UMNG7OkUSo,6487
-epub_translator/translation/xml_interrupter.py,sha256=c0wwfZzGpOkfKJ5v5p-lNgYlcqD0B6i2e6cQq-Tl0lI,8426
+epub_translator/translation/xml_interrupter.py,sha256=7TRGskn_OxRZT5mvKfjL0VMtU2VCgl1d9ElmfhFG0pM,8628
 epub_translator/utils.py,sha256=BfZWrYjzDNQ4cFrgvRNzd4i1CKLtPxS8Z4LBHhqEV78,914
 epub_translator/xml/__init__.py,sha256=qluFTfZYlPmOie8nR2C5O0tZ3UbCQEoEoR-Fq-__79c,160
-epub_translator/xml/const.py,sha256=Re2TYmpwG7-jVVgSq3R_K-uYhvAYzcXcRmLFkwCPD9Y,19
+epub_translator/xml/const.py,sha256=tCdeJfGwH5xgS4uOmR-pXSfyWXGxOHMJyZKE46BVkJU,54
 epub_translator/xml/deduplication.py,sha256=TaMbzeA70VvUQV0X1wcQFVbuMEPJUtj9Hq6iWlUmtAQ,1152
 epub_translator/xml/friendly/__init__.py,sha256=I5jhnhFWoHvojLsYXH4jfR4Gi8lKFZ3yQ56ze5hEe1M,74
 epub_translator/xml/friendly/decoder.py,sha256=xRQ5LnSunmYbba_0oT39oUr86-sLYAHYMUGmlseIu2U,2467
@@ -42,7 +43,7 @@ epub_translator/xml/friendly/encoder.py,sha256=evjvw6oE-oCud44IsJ-YZVHn6dtUzjNYX
 epub_translator/xml/friendly/parser.py,sha256=QlMHA0nfPJbNyx6IwRFrYVw7okuvzDB42NXCauIFV-o,6560
 epub_translator/xml/friendly/tag.py,sha256=ahaGoYttuAlnFxLFFgTV51KUZSpUiHho-COZX14nxN8,3308
 epub_translator/xml/friendly/transform.py,sha256=5tG1MJmzrXIR_Z5gmRxwcoKvXBzJBVH0ELeaRsG-8w0,1201
-epub_translator/xml/inline.py,sha256=fT_zm2NqHraJEwYXBHyqo9XjBFQUWRJO7YHB1rerkAc,2945
+epub_translator/xml/inline.py,sha256=VcaNEF2ebVl2fogVk2yV3f4vOP4rePsPTV_qU3fJCE0,3108
 epub_translator/xml/self_closing.py,sha256=41ofGUdss9yU51IVwI4It6hKfzh8YcxIR_j-ohD19LE,5240
 epub_translator/xml/utils.py,sha256=7tQ6L5P0_JXhxONeG64hEeeL5mKjA6NKS1H1Q9B1Cac,1062
 epub_translator/xml/xml.py,sha256=qQ5Wk1-KVVHE4TX25zGOR7fINsGkXnoq-qyKKNl5no4,1675
@@ -57,7 +58,7 @@ epub_translator/xml_translator/stream_mapper.py,sha256=nk8iRUHAUQA2B35_y-JOCo6il
 epub_translator/xml_translator/submitter.py,sha256=_ic2_JBPdEd2nMSu2mtQ5OzqpGv0zGrvYaicVUXAiUQ,14159
 epub_translator/xml_translator/translator.py,sha256=7Ja1jFbmjIgHcmI9V6gg_K0t7qb6in9mhRn54a7qhZ8,9497
 epub_translator/xml_translator/validation.py,sha256=-OKlSZuD__sjAiEpGAO93YQme4ZDSPmoPjRsAMOCEjc,16668
-epub_translator-0.1.7.dist-info/LICENSE,sha256=5RF32sL3LtMOJIErdDKp1ZEYPGXS8WPpsiSz_jMBnGI,1066
-epub_translator-0.1.7.dist-info/METADATA,sha256=-ySr_REjXY7qG5dZslElcQeIQ_U8r-ggkVtKytZBYMI,15684
-epub_translator-0.1.7.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-epub_translator-0.1.7.dist-info/RECORD,,
+epub_translator-0.1.8.dist-info/LICENSE,sha256=5RF32sL3LtMOJIErdDKp1ZEYPGXS8WPpsiSz_jMBnGI,1066
+epub_translator-0.1.8.dist-info/METADATA,sha256=DTipkbLL2pnijg7XIXSHogZXJzI009K7ZTkGUMy06d8,18663
+epub_translator-0.1.8.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+epub_translator-0.1.8.dist-info/RECORD,,