epub-translator 0.1.7__tar.gz → 0.1.9__tar.gz

{epub_translator-0.1.7 → epub_translator-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: epub-translator
-Version: 0.1.7
+Version: 0.1.9
 Summary: Translate the epub book using LLM. The translated book will retain the original text and list the translated text side by side with the original text.
 License: MIT
 Keywords: epub,llm,translation,translator
@@ -46,26 +46,17 @@ Description-Content-Type: text/markdown
 </div>
 
 
-Translate EPUB books using Large Language Models while preserving the original text. The translated content is displayed side-by-side with the original, creating bilingual books perfect for language learning and cross-reference reading.
+Want to read a book in a foreign language without losing the original context? EPUB Translator transforms any EPUB into a bilingual edition with AI-powered translations displayed side-by-side with the original text.
 
-![Translation Effect](./docs/images/translation.png)
+Whether you're learning a new language, conducting academic research, or simply enjoying foreign literature, you get both versions in one book - preserving all formatting, images, and structure.
 
-## Features
+![Translation Effect](./docs/images/translation.png)
 
-- **Bilingual Output**: Preserves original text alongside translations for easy comparison
-- **LLM-Powered**: Leverages large language models for high-quality, context-aware translations
-- **Format Preservation**: Maintains EPUB structure, styles, images, and formatting
-- **Complete Translation**: Translates chapter content, table of contents, and metadata
-- **Progress Tracking**: Monitor translation progress with built-in callbacks
-- **Flexible LLM Support**: Works with any OpenAI-compatible API endpoint
-- **Caching**: Built-in caching for progress recovery when translation fails
+### Online Demo
 
-## Use Cases
+We provide an [online demo platform](https://hub.oomol.com/package/books-translator) where you can try EPUB Translator's bilingual translation capabilities without any installation. Simply upload your EPUB file and get a translated bilingual edition.
 
-- **Language Learning**: Read books in their original language with side-by-side translations
-- **Academic Research**: Access foreign literature with bilingual references
-- **Content Localization**: Prepare books for international audiences
-- **Cross-Cultural Reading**: Enjoy literature while understanding cultural nuances
+[![EPUB Translator Online Demo](docs/images/online-en.png)](https://hub.oomol.com/package/books-translator)
 
 ## Installation
 
@@ -422,6 +413,105 @@ translate(
 
 When using `concurrency > 1`, ensure that any custom callback functions (`on_progress`, `on_fill_failed`) are thread-safe. Built-in callbacks are thread-safe by default.
 
+### Token Usage Monitoring
+
+Track token consumption during translation to monitor API costs and usage:
+
+```python
+from epub_translator import LLM, translate, language, SubmitKind
+
+llm = LLM(
+    key="your-api-key",
+    url="https://api.openai.com/v1",
+    model="gpt-4",
+    token_encoding="o200k_base",
+)
+
+translate(
+    source_path="source.epub",
+    target_path="translated.epub",
+    target_language=language.ENGLISH,
+    submit=SubmitKind.APPEND_BLOCK,
+    llm=llm,
+)
+
+# Access token statistics after translation
+print(f"Total tokens: {llm.total_tokens}")
+print(f"Input tokens: {llm.input_tokens}")
+print(f"Input cache tokens: {llm.input_cache_tokens}")
+print(f"Output tokens: {llm.output_tokens}")
+```
+
+**Available Statistics:**
+
+- `total_tokens` - Total number of tokens used (input + output)
+- `input_tokens` - Number of prompt/input tokens
+- `input_cache_tokens` - Number of cached input tokens (when using prompt caching)
+- `output_tokens` - Number of generated/completion tokens
+
+**Real-time Monitoring:**
+
+You can also monitor token usage in real-time during translation:
+
+```python
+from tqdm import tqdm
+import time
+
+with tqdm(total=100, desc="Translating", unit="%") as pbar:
+    last_progress = 0.0
+    start_time = time.time()
+
+    def on_progress(progress: float):
+        nonlocal last_progress
+        increment = (progress - last_progress) * 100
+        pbar.update(increment)
+        last_progress = progress
+
+        # Update token stats in progress bar
+        pbar.set_postfix({
+            'tokens': llm.total_tokens,
+            'cost_est': f'${llm.total_tokens * 0.00001:.4f}'  # Estimate based on your pricing
+        })
+
+    translate(
+        source_path="source.epub",
+        target_path="translated.epub",
+        target_language=language.ENGLISH,
+        submit=SubmitKind.APPEND_BLOCK,
+        llm=llm,
+        on_progress=on_progress,
+    )
+
+    elapsed = time.time() - start_time
+    print(f"\nTranslation completed in {elapsed:.1f}s")
+    print(f"Total tokens used: {llm.total_tokens:,}")
+    print(f"Average tokens/second: {llm.total_tokens/elapsed:.1f}")
+```
+
+**Dual-LLM Token Tracking:**
+
+When using separate LLMs for translation and filling, each LLM tracks its own statistics:
+
+```python
+translation_llm = LLM(key="...", url="...", model="gpt-4", token_encoding="o200k_base")
+fill_llm = LLM(key="...", url="...", model="gpt-4", token_encoding="o200k_base")
+
+translate(
+    source_path="source.epub",
+    target_path="translated.epub",
+    target_language=language.ENGLISH,
+    submit=SubmitKind.APPEND_BLOCK,
+    translation_llm=translation_llm,
+    fill_llm=fill_llm,
+)
+
+print(f"Translation tokens: {translation_llm.total_tokens}")
+print(f"Fill tokens: {fill_llm.total_tokens}")
+print(f"Combined total: {translation_llm.total_tokens + fill_llm.total_tokens}")
+```
+
+**Note:** Token statistics are cumulative across all API calls made by the LLM instance. The counts only increase and are thread-safe when using concurrent translation.
+
 ## Related Projects
 
 ### PDF Craft

{epub_translator-0.1.7 → epub_translator-0.1.9}/README.md

@@ -12,26 +12,17 @@
 </div>
 
 
-Translate EPUB books using Large Language Models while preserving the original text. The translated content is displayed side-by-side with the original, creating bilingual books perfect for language learning and cross-reference reading.
+Want to read a book in a foreign language without losing the original context? EPUB Translator transforms any EPUB into a bilingual edition with AI-powered translations displayed side-by-side with the original text.
 
-![Translation Effect](./docs/images/translation.png)
+Whether you're learning a new language, conducting academic research, or simply enjoying foreign literature, you get both versions in one book - preserving all formatting, images, and structure.
 
-## Features
+![Translation Effect](./docs/images/translation.png)
 
-- **Bilingual Output**: Preserves original text alongside translations for easy comparison
-- **LLM-Powered**: Leverages large language models for high-quality, context-aware translations
-- **Format Preservation**: Maintains EPUB structure, styles, images, and formatting
-- **Complete Translation**: Translates chapter content, table of contents, and metadata
-- **Progress Tracking**: Monitor translation progress with built-in callbacks
-- **Flexible LLM Support**: Works with any OpenAI-compatible API endpoint
-- **Caching**: Built-in caching for progress recovery when translation fails
+### Online Demo
 
-## Use Cases
+We provide an [online demo platform](https://hub.oomol.com/package/books-translator) where you can try EPUB Translator's bilingual translation capabilities without any installation. Simply upload your EPUB file and get a translated bilingual edition.
 
-- **Language Learning**: Read books in their original language with side-by-side translations
-- **Academic Research**: Access foreign literature with bilingual references
-- **Content Localization**: Prepare books for international audiences
-- **Cross-Cultural Reading**: Enjoy literature while understanding cultural nuances
+[![EPUB Translator Online Demo](docs/images/online-en.png)](https://hub.oomol.com/package/books-translator)
 
 ## Installation
 
@@ -388,6 +379,105 @@ translate(
 
 When using `concurrency > 1`, ensure that any custom callback functions (`on_progress`, `on_fill_failed`) are thread-safe. Built-in callbacks are thread-safe by default.
 
+### Token Usage Monitoring
+
+Track token consumption during translation to monitor API costs and usage:
+
+```python
+from epub_translator import LLM, translate, language, SubmitKind
+
+llm = LLM(
+    key="your-api-key",
+    url="https://api.openai.com/v1",
+    model="gpt-4",
+    token_encoding="o200k_base",
+)
+
+translate(
+    source_path="source.epub",
+    target_path="translated.epub",
+    target_language=language.ENGLISH,
+    submit=SubmitKind.APPEND_BLOCK,
+    llm=llm,
+)
+
+# Access token statistics after translation
+print(f"Total tokens: {llm.total_tokens}")
+print(f"Input tokens: {llm.input_tokens}")
+print(f"Input cache tokens: {llm.input_cache_tokens}")
+print(f"Output tokens: {llm.output_tokens}")
+```
+
+**Available Statistics:**
+
+- `total_tokens` - Total number of tokens used (input + output)
+- `input_tokens` - Number of prompt/input tokens
+- `input_cache_tokens` - Number of cached input tokens (when using prompt caching)
+- `output_tokens` - Number of generated/completion tokens
+
+**Real-time Monitoring:**
+
+You can also monitor token usage in real-time during translation:
+
+```python
+from tqdm import tqdm
+import time
+
+with tqdm(total=100, desc="Translating", unit="%") as pbar:
+    last_progress = 0.0
+    start_time = time.time()
+
+    def on_progress(progress: float):
+        nonlocal last_progress
+        increment = (progress - last_progress) * 100
+        pbar.update(increment)
+        last_progress = progress
+
+        # Update token stats in progress bar
+        pbar.set_postfix({
+            'tokens': llm.total_tokens,
+            'cost_est': f'${llm.total_tokens * 0.00001:.4f}'  # Estimate based on your pricing
+        })
+
+    translate(
+        source_path="source.epub",
+        target_path="translated.epub",
+        target_language=language.ENGLISH,
+        submit=SubmitKind.APPEND_BLOCK,
+        llm=llm,
+        on_progress=on_progress,
+    )
+
+    elapsed = time.time() - start_time
+    print(f"\nTranslation completed in {elapsed:.1f}s")
+    print(f"Total tokens used: {llm.total_tokens:,}")
+    print(f"Average tokens/second: {llm.total_tokens/elapsed:.1f}")
+```
+
+**Dual-LLM Token Tracking:**
+
+When using separate LLMs for translation and filling, each LLM tracks its own statistics:
+
+```python
+translation_llm = LLM(key="...", url="...", model="gpt-4", token_encoding="o200k_base")
+fill_llm = LLM(key="...", url="...", model="gpt-4", token_encoding="o200k_base")
+
+translate(
+    source_path="source.epub",
+    target_path="translated.epub",
+    target_language=language.ENGLISH,
+    submit=SubmitKind.APPEND_BLOCK,
+    translation_llm=translation_llm,
+    fill_llm=fill_llm,
+)
+
+print(f"Translation tokens: {translation_llm.total_tokens}")
+print(f"Fill tokens: {fill_llm.total_tokens}")
+print(f"Combined total: {translation_llm.total_tokens + fill_llm.total_tokens}")
+```
+
+**Note:** Token statistics are cumulative across all API calls made by the LLM instance. The counts only increase and are thread-safe when using concurrent translation.
+
 ## Related Projects
 
 ### PDF Craft

epub_translator-0.1.9/epub_translator/epub/__init__.py

@@ -0,0 +1,4 @@
+from .metadata import MetadataContext, read_metadata, write_metadata
+from .spines import search_spine_paths
+from .toc import Toc, TocContext, read_toc, write_toc
+from .zip import Zip

epub_translator-0.1.9/epub_translator/epub/metadata.py

@@ -0,0 +1,85 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..xml import XMLLikeNode
+from .common import find_opf_path
+from .zip import Zip
+
+
+@dataclass
+class MetadataField:
+    tag_name: str
+    text: str
+
+
+@dataclass
+class MetadataContext:
+    opf_path: Path  # OPF 文件路径
+    xml_node: XMLLikeNode  # XMLLikeNode 对象，保留原始文件信息
+
+
+SKIP_FIELDS = frozenset(
+    (
+        "language",
+        "identifier",
+        "date",
+        "meta",
+        "contributor",  # Usually technical information
+    )
+)
+
+
+def read_metadata(zip: Zip) -> tuple[list[MetadataField], MetadataContext]:
+    opf_path = find_opf_path(zip)
+
+    with zip.read(opf_path) as f:
+        xml_node = XMLLikeNode(f, is_html_like=False)
+
+    metadata_elem = None
+    for child in xml_node.element:
+        if child.tag.endswith("metadata"):
+            metadata_elem = child
+            break
+
+    if metadata_elem is None:
+        context = MetadataContext(opf_path=opf_path, xml_node=xml_node)
+        return [], context
+
+    fields: list[MetadataField] = []
+    for elem in metadata_elem:
+        tag_name = elem.tag
+        if elem.text and elem.text.strip() and tag_name not in SKIP_FIELDS:
+            fields.append(MetadataField(tag_name=tag_name, text=elem.text.strip()))
+
+    context = MetadataContext(opf_path=opf_path, xml_node=xml_node)
+    return fields, context
+
+
+def write_metadata(zip: Zip, fields: list[MetadataField], context: MetadataContext) -> None:
+    metadata_elem = None
+    for child in context.xml_node.element:
+        if child.tag.endswith("metadata"):
+            metadata_elem = child
+            break
+
+    if metadata_elem is None:
+        return
+
+    fields_by_tag: dict[str, list[str]] = {}
+    for field in fields:
+        if field.tag_name not in fields_by_tag:
+            fields_by_tag[field.tag_name] = []
+        fields_by_tag[field.tag_name].append(field.text)
+
+    tag_counters: dict[str, int] = {tag: 0 for tag in fields_by_tag}
+
+    for elem in metadata_elem:
+        tag_name = elem.tag
+        if tag_name in fields_by_tag and elem.text and elem.text.strip():
+            counter = tag_counters[tag_name]
+            if counter < len(fields_by_tag[tag_name]):
+                elem.text = fields_by_tag[tag_name][counter]
+                tag_counters[tag_name] += 1
+
+    with zip.replace(context.opf_path) as f:
+        context.xml_node.save(f)

{epub_translator-0.1.7 → epub_translator-0.1.9}/epub_translator/epub/toc.py

@@ -3,8 +3,8 @@ from pathlib import Path
 from xml.etree import ElementTree as ET
 from xml.etree.ElementTree import Element
 
-from ..xml.xml import plain_text
-from .common import extract_namespace, find_opf_path, strip_namespace
+from ..xml import XMLLikeNode, plain_text
+from .common import find_opf_path, strip_namespace
 from .zip import Zip
 
 
@@ -41,30 +41,40 @@ class Toc:
         return self.href
 
 
-def read_toc(zip: Zip) -> list[Toc]:
+@dataclass
+class TocContext:
+    version: int
+    toc_path: Path
+    xml_node: XMLLikeNode
+
+
+def read_toc(zip: Zip) -> tuple[list[Toc], TocContext]:
     version = _detect_epub_version(zip)
     toc_path = _find_toc_path(zip, version)
 
     if toc_path is None:
-        return []
+        raise ValueError("Cannot find TOC file in EPUB")
 
-    if version == 2:
-        return _read_ncx_toc(zip, toc_path)
-    else:
-        return _read_nav_toc(zip, toc_path)
+    with zip.read(toc_path) as f:
+        xml_node = XMLLikeNode(f, is_html_like=False)
 
+    if version == 3:
+        toc_list = _read_nav_toc(xml_node.element)
+    else:
+        toc_list = _read_ncx_toc(xml_node.element)
 
-def write_toc(zip: Zip, toc: list[Toc]) -> None:
-    version = _detect_epub_version(zip)
-    toc_path = _find_toc_path(zip, version)
+    context = TocContext(version=version, toc_path=toc_path, xml_node=xml_node)
+    return toc_list, context
 
-    if toc_path is None:
-        raise ValueError("Cannot find TOC file in EPUB")
 
-    if version == 2:
-        _write_ncx_toc(zip, toc_path, toc)
+def write_toc(zip: Zip, toc: list[Toc], context: TocContext) -> None:
+    if context.version == 2:
+        _update_ncx_toc(context.xml_node.element, toc)
     else:
-        _write_nav_toc(zip, toc_path, toc)
+        _update_nav_toc(context.xml_node.element, toc)
+
+    with zip.replace(context.toc_path) as f:
+        context.xml_node.save(f)
 
 
 def _detect_epub_version(zip: Zip) -> int:
@@ -72,8 +82,6 @@ def _detect_epub_version(zip: Zip) -> int:
     with zip.read(opf_path) as f:
         content = f.read()
         root = ET.fromstring(content)
-
-        # 检查 package 元素的 version 属性
         version_str = root.get("version", "2.0")
 
         if version_str.startswith("3"):
@@ -89,7 +97,7 @@ def _find_toc_path(zip: Zip, version: int) -> Path | None:
     with zip.read(opf_path) as f:
         content = f.read()
         root = ET.fromstring(content)
-        strip_namespace(root)  # 移除命名空间前缀以简化 XPath
+        strip_namespace(root)
 
         manifest = root.find(".//manifest")
         if manifest is None:
@@ -115,23 +123,18 @@ def _find_toc_path(zip: Zip, version: int) -> Path | None:
         return None
 
 
-def _read_ncx_toc(zip: Zip, ncx_path: Path) -> list[Toc]:
-    with zip.read(ncx_path) as f:
-        content = f.read()
-        root = ET.fromstring(content)
-        strip_namespace(root)  # 移除命名空间前缀以简化 XPath
-
-        nav_map = root.find(".//navMap")
-        if nav_map is None:
-            return []
+def _read_ncx_toc(root: Element) -> list[Toc]:
+    nav_map = root.find(".//navMap")
+    if nav_map is None:
+        return []
 
-        result = []
-        for nav_point in nav_map.findall("navPoint"):
-            toc_item = _parse_nav_point(nav_point)
-            if toc_item:
-                result.append(toc_item)
+    result = []
+    for nav_point in nav_map.findall("navPoint"):
+        toc_item = _parse_nav_point(nav_point)
+        if toc_item:
+            result.append(toc_item)
 
-        return result
+    return result
 
 
 def _parse_nav_point(nav_point: Element) -> Toc | None:
@@ -172,18 +175,11 @@ def _parse_nav_point(nav_point: Element) -> Toc | None:
     )
 
 
-def _write_ncx_toc(zip: Zip, ncx_path: Path, toc_list: list[Toc]) -> None:
-    with zip.read(ncx_path) as f:
-        content = f.read()
-        root = ET.fromstring(content)
-        ns = extract_namespace(root.tag)
-        nav_map = root.find(f".//{{{ns}}}navMap" if ns else ".//navMap")
-        if nav_map is None:
-            raise ValueError("Cannot find navMap in NCX file")
-        _update_nav_points(nav_map, toc_list, ns)
-        tree = ET.ElementTree(root)
-        with zip.replace(ncx_path) as out:
-            tree.write(out, encoding="utf-8", xml_declaration=True)
+def _update_ncx_toc(root: Element, toc_list: list[Toc]) -> None:
+    nav_map = root.find(".//navMap")
+    if nav_map is None:
+        raise ValueError("Cannot find navMap in NCX file")
+    _update_nav_points(nav_map, toc_list, None)
 
 
 def _update_nav_points(parent: Element, toc_list: list[Toc], ns: str | None, start_play_order: int = 1) -> int:
@@ -255,34 +251,28 @@ def _create_nav_point(toc: Toc, ns: str | None, play_order: int) -> Element:
     return nav_point
 
 
-def _read_nav_toc(zip: Zip, nav_path: Path) -> list[Toc]:
-    with zip.read(nav_path) as f:
-        content = f.read()
-        root = ET.fromstring(content)
-
-        strip_namespace(root)
-
-        nav_elem = None
-        for nav in root.findall(".//nav"):
-            epub_type = nav.get("{http://www.idpf.org/2007/ops}type") or nav.get("type")
-            if epub_type == "toc":
-                nav_elem = nav
-                break
+def _read_nav_toc(root: Element) -> list[Toc]:
+    nav_elem = None
+    for nav in root.findall(".//nav"):
+        epub_type = nav.get("type")
+        if epub_type == "toc":
+            nav_elem = nav
+            break
 
-        if nav_elem is None:
-            return []
+    if nav_elem is None:
+        return []
 
-        ol = nav_elem.find(".//ol")
-        if ol is None:
-            return []
+    ol = nav_elem.find(".//ol")
+    if ol is None:
+        return []
 
-        result = []
-        for li in ol.findall("li"):
-            toc_item = _parse_nav_li(li)
-            if toc_item:
-                result.append(toc_item)
+    result = []
+    for li in ol.findall("li"):
+        toc_item = _parse_nav_li(li)
+        if toc_item:
+            result.append(toc_item)
 
-        return result
+    return result
 
 
 def _parse_nav_li(li: Element) -> Toc | None:
@@ -331,30 +321,22 @@ def _parse_nav_li(li: Element) -> Toc | None:
     )
 
 
-def _write_nav_toc(zip: Zip, nav_path: Path, toc_list: list[Toc]) -> None:
-    with zip.read(nav_path) as f:
-        content = f.read()
-        root = ET.fromstring(content)
-        ns = extract_namespace(root.tag)
-        nav_elem = None
-        for nav in root.findall(f".//{{{ns}}}nav" if ns else ".//nav"):
-            epub_type = nav.get("{http://www.idpf.org/2007/ops}type") or nav.get("type") or nav.get(f"{{{ns}}}type")
-            if epub_type == "toc":
-                nav_elem = nav
-                break
-
-        if nav_elem is None:
-            raise ValueError("Cannot find nav element with type='toc'")
-
-        ol = nav_elem.find(f".//{{{ns}}}ol" if ns else ".//ol")
-        if ol is None:
-            raise ValueError("Cannot find ol in nav element")
-
-        _update_nav_lis(ol, toc_list, ns)
-
-        tree = ET.ElementTree(root)
-        with zip.replace(nav_path) as out:
-            tree.write(out, encoding="utf-8", xml_declaration=True)
+def _update_nav_toc(root: Element, toc_list: list[Toc]) -> None:
+    nav_elem = None
+    for nav in root.findall(".//nav"):
+        epub_type = nav.get("type")
+        if epub_type == "toc":
+            nav_elem = nav
+            break
+
+    if nav_elem is None:
+        raise ValueError("Cannot find nav element with type='toc'")
+
+    ol = nav_elem.find(".//ol")
+    if ol is None:
+        raise ValueError("Cannot find ol in nav element")
+
+    _update_nav_lis(ol, toc_list, None)
 
 
 def _update_nav_lis(ol: Element, toc_list: list[Toc], ns: str | None) -> None:

{epub_translator-0.1.7 → epub_translator-0.1.9}/epub_translator/llm/core.py

@@ -13,6 +13,7 @@ from ..template import create_env
 from .context import LLMContext
 from .executor import LLMExecutor
 from .increasable import Increasable
+from .statistics import Statistics
 from .types import Message
 
 # Global state for logger filename generation
@@ -44,7 +45,7 @@ class LLM:
         self._temperature: Increasable = Increasable(temperature)
         self._cache_path: Path | None = self._ensure_dir_path(cache_path)
         self._logger_save_path: Path | None = self._ensure_dir_path(log_dir_path)
-
+        self._statistics = Statistics()
         self._executor = LLMExecutor(
             url=url,
             model=model,
@@ -53,12 +54,29 @@ class LLM:
             retry_times=retry_times,
             retry_interval_seconds=retry_interval_seconds,
             create_logger=self._create_logger,
+            statistics=self._statistics,
         )
 
     @property
     def encoding(self) -> Encoding:
         return self._encoding
 
+    @property
+    def total_tokens(self) -> int:
+        return self._statistics.total_tokens
+
+    @property
+    def input_tokens(self) -> int:
+        return self._statistics.input_tokens
+
+    @property
+    def input_cache_tokens(self) -> int:
+        return self._statistics.input_cache_tokens
+
+    @property
+    def output_tokens(self) -> int:
+        return self._statistics.output_tokens
+
     def context(self, cache_seed_content: str | None = None) -> LLMContext:
         return LLMContext(
             executor=self._executor,