epub-translator 0.0.3__tar.gz → 0.0.5__tar.gz

{epub_translator-0.0.3 → epub_translator-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: epub-translator
-Version: 0.0.3
+Version: 0.0.5
 Summary: Translate the epub book using LLM. The translated book will retain the original text and list the translated text side by side with the original text.
 License: MIT
 Keywords: epub,llm,translation,translator

{epub_translator-0.0.3 → epub_translator-0.0.5}/epub_translator/llm/executor.py

@@ -38,7 +38,7 @@ class LLMExecutor:
       timeout=timeout,
     )
 
-  def request(self, input: LanguageModelInput, parser: Callable[[str], Any]) -> Any:
+  def request(self, input: LanguageModelInput, parser: Callable[[str], Any], max_tokens: int | None) -> Any:
     result: Any | None = None
     last_error: Exception | None = None
     did_success = False
@@ -56,6 +56,7 @@ class LLMExecutor:
             input=input,
             top_p=top_p.current,
             temperature=temperature.current,
+            max_tokens=max_tokens,
           )
           if logger is not None:
             logger.debug(f"[[Response]]:\n{response}\n")
@@ -133,12 +134,14 @@ class LLMExecutor:
         input: LanguageModelInput,
         top_p: float | None,
         temperature: float | None,
+        max_tokens: int | None,
       ):
     stream = self._model.stream(
       input=input,
       timeout=self._timeout,
       top_p=top_p,
       temperature=temperature,
+      max_tokens=max_tokens,
     )
     buffer = StringIO()
     for chunk in stream:

{epub_translator-0.0.3 → epub_translator-0.0.5}/epub_translator/llm/node.py

@@ -80,6 +80,7 @@ class LLM:
       text_tag: str,
       user_data: Element | str,
       parser: Callable[[str], R],
+      max_tokens: int | None = None,
       params: dict[str, Any] | None = None,
     ) -> R:
 
@@ -95,6 +96,7 @@ class LLM:
     return self._executor.request(
       input=self._create_input(template_name, user_data, params),
       parser=parse_response,
+      max_tokens=max_tokens,
     )
 
   def request_xml(
@@ -102,6 +104,7 @@ class LLM:
         template_name: str,
         user_data: Element | str,
         parser: Callable[[Element], R],
+        max_tokens: int | None = None,
         params: dict[str, Any] | None = None,
       ) -> R:
 
@@ -117,6 +120,7 @@ class LLM:
     return self._executor.request(
       input=self._create_input(template_name, user_data, params),
       parser=parse_response,
+      max_tokens=max_tokens,
     )
 
   def _create_input(self, template_name: str, user_data: Element | str, params: dict[str, Any]):

epub_translator-0.0.5/epub_translator/translation/__init__.py

@@ -0,0 +1,2 @@
+from .types import Fragment, Language
+from .translation import translate, ProgressReporter

{epub_translator-0.0.3 → epub_translator-0.0.5}/epub_translator/translation/chunk.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from typing import Iterator, Iterable, Generator
 from hashlib import sha512
 from ..llm import LLM
-from .types import Fragment
+from .types import Fragment, Language
 
 
 @dataclass
@@ -30,6 +30,7 @@ class ChunkRange:
 
 def match_fragments(
         llm: LLM,
+        target_language: Language,
         chunk_ranges_iter: Iterator[ChunkRange],
         fragments_iter: Iterator[Fragment],
       ) -> Generator[Chunk, None, None]:
@@ -44,7 +45,7 @@ def match_fragments(
     body = texts[head_length:head_length + body_length]
     tail = texts[head_length + body_length:]
 
-    hash = _hash_texts_list((head, body, tail))
+    hash = _hash_texts_list(target_language, (head, body, tail))
     head = _crop_extra_texts(llm, head, True, range.head_remain_tokens)
     tail = _crop_extra_texts(llm, tail, False, range.tail_remain_tokens)
 
@@ -88,15 +89,12 @@ def _match_range_and_texts(
 
   yield from matched_chunk_ranges
 
-def _hash_texts_list(texts_iterable: Iterable[list[str]]) -> bytes:
-  is_first = True
+def _hash_texts_list(target_language: Language, texts_iterable: Iterable[list[str]]) -> bytes:
   m = sha512()
+  m.update(target_language.value.encode("utf-8"))
   for texts in texts_iterable:
     for text in texts:
-      if is_first:
-        is_first = False
-      else:
-        m.update(b"\x00")
+      m.update(b"\x00")
       m.update(text.encode("utf-8"))
   return m.digest()
 

{epub_translator-0.0.3 → epub_translator-0.0.5}/epub_translator/translation/store.py

@@ -1,7 +1,6 @@
 from shutil import rmtree
 from pathlib import Path
 from typing import Iterator
-from .utils import clean_spaces
 
 
 class Store:
@@ -13,7 +12,7 @@ class Store:
     if not file_path.exists() or not file_path.is_file():
       return None
     with file_path.open("r", encoding="utf-8") as file:
-      return list(line for line in file if line.strip())
+      return file.read().split("\n")
 
   def put(self, chunk_hash: bytes, lines_iter: Iterator[str]):
     file_path = self._file_path(chunk_hash)
@@ -31,7 +30,7 @@ class Store:
           is_first_line = False
         else:
           file.write("\n")
-        file.write(clean_spaces(line))
+        file.write(line)
 
   def _file_path(self, chunk_hash: bytes) -> Path:
     return self._directory / f"{chunk_hash.hex()}.chunk"

{epub_translator-0.0.3 → epub_translator-0.0.5}/epub_translator/translation/translation.py

@@ -1,3 +1,4 @@
+from math import ceil
 from typing import Callable, Iterator, Generator
 from pathlib import Path
 from concurrent.futures import as_completed, ThreadPoolExecutor
@@ -6,7 +7,7 @@ from xml.etree.ElementTree import Element
 from ..llm import LLM
 from ..xml import encode_friendly
 
-from .types import Fragment, Language
+from .types import language_chinese_name, Fragment, Language
 from .store import Store
 from .splitter import split_into_chunks
 from .chunk import match_fragments, Chunk
@@ -46,12 +47,23 @@ def translate(
       )))
       for chunk in match_fragments(
         llm=llm,
+        target_language=target_language,
         chunk_ranges_iter=iter(chunk_ranges),
         fragments_iter=gen_fragments_iter(),
       )
     ]
+    def _generate_chunks_from_futures():
+      try:
+        for future in as_completed(futures):
+          yield future.result()
+      except Exception as err:
+        for future in futures:
+          if not future.done():
+            future.cancel()
+        raise err
+
     yield from _sort_translated_texts_by_chunk(
-      target=(f.result() for f in as_completed(futures)),
+      target=_generate_chunks_from_futures(),
       total_tokens_count=sum(chunk.tokens_count for chunk in chunk_ranges),
       report_progress=report_progress,
     )
@@ -96,27 +108,40 @@ def _translate_chunk(
     ) -> list[str]:
 
     translated_texts: list[str] | None = None
+    source_texts = chunk.head + chunk.body + chunk.tail
     if store is not None:
       translated_texts = store.get(chunk.hash)
+      if translated_texts is not None and \
+         len(source_texts) != len(translated_texts):
+        translated_texts = None
+        print(f"Warning: Mismatched lengths in cached translation for chunk: {chunk.hash.hex()}",)
 
     if translated_texts is None:
-      translated_texts = _translate_texts(
-        llm=llm,
-        texts=chunk.head + chunk.body + chunk.tail,
-        target_language=target_language,
-        user_prompt=user_prompt,
-      )
-    if store is not None:
-      store.put(chunk.hash, translated_texts)
+      translated_texts = [
+        clean_spaces(text)
+        for text in _translate_texts(
+          llm=llm,
+          texts=source_texts,
+          texts_tokens=chunk.tokens_count,
+          target_language=target_language,
+          user_prompt=user_prompt,
+        )
+      ]
+      if store is not None:
+        store.put(chunk.hash, translated_texts)
 
     head_length = len(chunk.head)
     translated_texts = translated_texts[head_length:head_length + len(chunk.body)]
 
     return translated_texts
 
+_PLAIN_TEXT_SCALE = 2.0
+_XML_TEXT_SCALE = 2.5
+
 def _translate_texts(
       llm: LLM,
       texts: list[str],
+      texts_tokens: int,
       target_language: Language,
       user_prompt: str | None,
     ) -> list[str]:
@@ -134,8 +159,9 @@ def _translate_texts(
     text_tag="TXT",
     user_data=user_data,
     parser=lambda r: r,
+    max_tokens=ceil(texts_tokens * _PLAIN_TEXT_SCALE),
     params={
-      "target_language": target_language.value,
+      "target_language": language_chinese_name(target_language),
       "user_prompt": user_prompt,
     },
   )
@@ -154,12 +180,15 @@ def _translate_texts(
   return llm.request_xml(
     template_name="format",
     user_data=request_text,
-    params={ "target_language": target_language.value },
+    max_tokens=ceil(texts_tokens * _XML_TEXT_SCALE),
     parser=lambda r: _parse_translated_response(r, len(texts)),
+    params={
+      "target_language": language_chinese_name(target_language),
+    },
   )
 
 def _parse_translated_response(resp_element: Element, sources_count: int) -> list[str]:
-  translated_fragments = [""] * sources_count
+  fragments: list[str | None] = [None] * sources_count
   for fragment_element in resp_element:
     if fragment_element.text is None:
       continue
@@ -167,11 +196,21 @@ def _parse_translated_response(resp_element: Element, sources_count: int) -> lis
     if id is None:
       continue
     index = int(id) - 1
-    if index < 0 or index >= len(translated_fragments):
+    if index < 0 or index >= len(fragments):
       raise ValueError(f"invalid fragment id: {id}")
-    translated_fragments[index] = fragment_element.text.strip()
-
-  return translated_fragments
+    fragments[index] = fragment_element.text.strip()
+
+  # 有时 LLM 会将多段融合在一起，这里尽可能让译文靠后，将空白段留在前面。
+  # 这样看起来一大段的译文对应若干小段原文，观感更好。
+  for i in range(len(fragments)):
+    fragment = fragments[i]
+    if fragment is not None and i < len(fragments) - 1:
+      next_fragment = fragments[i + 1]
+      if next_fragment is None:
+        fragments[i] = None
+        fragments[i + 1] = fragment
+
+  return [f or "" for f in fragments]
 
 def _normalize_user_input(user_lines: list[str]) -> str | None:
   empty_lines_count: int = 0

epub_translator-0.0.5/epub_translator/translation/types.py

@@ -0,0 +1,40 @@
+from enum import Enum
+from dataclasses import dataclass
+from resource_segmentation import Incision
+
+
+@dataclass
+class Fragment:
+  text: str
+  start_incision: Incision
+  end_incision: Incision
+
+class Language(Enum):
+  SIMPLIFIED_CHINESE = "zh-Hans"
+  TRADITIONAL_CHINESE = "zh-Hant"
+  ENGLISH = "en"
+  FRENCH = "fr"
+  GERMAN = "de"
+  SPANISH = "es"
+  RUSSIAN = "ru"
+  ITALIAN = "it"
+  PORTUGUESE = "pt"
+  JAPANESE = "ja"
+  KOREAN = "ko"
+
+_LANGUAGE_NAMES = {
+  Language.SIMPLIFIED_CHINESE: "简体中文",
+  Language.TRADITIONAL_CHINESE: "繁体中文",
+  Language.ENGLISH: "英语",
+  Language.FRENCH: "法语",
+  Language.GERMAN: "德语",
+  Language.SPANISH: "西班牙语",
+  Language.RUSSIAN: "俄语",
+  Language.ITALIAN: "意大利语",
+  Language.PORTUGUESE: "葡萄牙语",
+  Language.JAPANESE: "日语",
+  Language.KOREAN: "韩语",
+}
+
+def language_chinese_name(language: Language) -> str:
+  return _LANGUAGE_NAMES[language]

{epub_translator-0.0.3 → epub_translator-0.0.5}/epub_translator/translator.py

@@ -2,11 +2,12 @@ from os import PathLike
 from pathlib import Path
 from tempfile import mkdtemp
 from shutil import rmtree
+from resource_segmentation import Incision
 
 from .llm import LLM
 from .epub import HTMLFile
 from .zip_context import ZipContext
-from .translation import translate as _translate, Fragment, Incision, Language, ProgressReporter
+from .translation import translate as _translate, Fragment, Language, ProgressReporter
 
 
 def translate(
@@ -107,7 +108,7 @@ class _Translator:
 
   def _translate_spine(self, context: ZipContext, cache_path: Path, report_progress: ProgressReporter):
     spine_paths_iter = iter(list(context.search_spine_paths()))
-    spine_file: HTMLFile | None = None
+    spine: tuple[Path, HTMLFile] | None = None
     translated_texts: list[str] = []
     translated_count: int = 0
 
@@ -123,32 +124,37 @@ class _Translator:
     ):
       did_touch_end = False
 
-      if spine_file is not None and \
-        translated_count >= len(translated_texts):
+      if spine and translated_count >= len(translated_texts):
+        spine_path, spine_file = spine
         spine_file.write_texts(translated_texts)
-        spine_file = None
+        context.write_spine_file(spine_path, spine_file)
+        spine = None
 
-      while spine_file is None:
+      while not spine:
         spine_path = next(spine_paths_iter, None)
         if spine_path is None:
+          spine = None
           did_touch_end = True
           break
         spine_file = context.read_spine_file(spine_path)
         if spine_file.texts_length == 0:
-          spine_file = None
           continue
+        spine = (spine_path, spine_file)
         translated_texts = [""] * spine_file.texts_length
         translated_count = 0
+        break
 
       translated_texts[translated_count] = translated_text
       translated_count += 1
 
       if did_touch_end:
         break
-    if spine_file and translated_count > 0:
-      spine_file.write_texts(translated_texts)
 
-    context.write_spine_file(spine_path, spine_file)
+    if spine:
+      spine_path, spine_file = spine
+      if translated_count > 0:
+        spine_file.write_texts(translated_texts)
+      context.write_spine_file(spine_path, spine_file)
 
 def _gen_fragments(context: ZipContext):
   for spine_path in context.search_spine_paths():

{epub_translator-0.0.3 → epub_translator-0.0.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "epub-translator"
-version = "0.0.3"
+version = "0.0.5"
 description = "Translate the epub book using LLM. The translated book will retain the original text and list the translated text side by side with the original text."
 keywords=["epub", "llm", "translation", "translator"]
 authors = [

epub_translator-0.0.3/epub_translator/translation/__init__.py

@@ -1,2 +0,0 @@
-from .types import *
-from .translation import translate, ProgressReporter

epub_translator-0.0.3/epub_translator/translation/types.py

@@ -1,23 +0,0 @@
-from enum import Enum
-from dataclasses import dataclass
-from resource_segmentation import Incision
-
-
-@dataclass
-class Fragment:
-  text: str
-  start_incision: Incision
-  end_incision: Incision
-
-class Language(Enum):
-  SIMPLIFIED_CHINESE = "简体中文"
-  TRADITIONAL_CHINESE = "繁体中文"
-  ENGLISH = "英语"
-  FRENCH = "法语"
-  GERMAN = "德语"
-  SPANISH = "西班牙语"
-  RUSSIAN = "俄语"
-  ITALIAN = "意大利语"
-  PORTUGUESE = "葡萄牙语"
-  JAPANESE = "日语"
-  KOREAN = "韩语"