epub-translator 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

epub_translator/llm/executor.py

@@ -38,7 +38,7 @@ class LLMExecutor:
       timeout=timeout,
     )
 
-  def request(self, input: LanguageModelInput, parser: Callable[[str], Any]) -> Any:
+  def request(self, input: LanguageModelInput, parser: Callable[[str], Any], max_tokens: int | None) -> Any:
     result: Any | None = None
     last_error: Exception | None = None
     did_success = False
@@ -56,6 +56,7 @@ class LLMExecutor:
             input=input,
             top_p=top_p.current,
             temperature=temperature.current,
+            max_tokens=max_tokens,
           )
           if logger is not None:
             logger.debug(f"[[Response]]:\n{response}\n")
@@ -133,12 +134,14 @@ class LLMExecutor:
         input: LanguageModelInput,
         top_p: float | None,
         temperature: float | None,
+        max_tokens: int | None,
       ):
     stream = self._model.stream(
       input=input,
       timeout=self._timeout,
       top_p=top_p,
       temperature=temperature,
+      max_tokens=max_tokens,
     )
     buffer = StringIO()
     for chunk in stream:

epub_translator/llm/node.py

@@ -80,6 +80,7 @@ class LLM:
       text_tag: str,
       user_data: Element | str,
       parser: Callable[[str], R],
+      max_tokens: int | None = None,
       params: dict[str, Any] | None = None,
     ) -> R:
 
@@ -95,6 +96,7 @@ class LLM:
     return self._executor.request(
       input=self._create_input(template_name, user_data, params),
       parser=parse_response,
+      max_tokens=max_tokens,
     )
 
   def request_xml(
@@ -102,6 +104,7 @@ class LLM:
         template_name: str,
         user_data: Element | str,
         parser: Callable[[Element], R],
+        max_tokens: int | None = None,
         params: dict[str, Any] | None = None,
       ) -> R:
 
@@ -117,6 +120,7 @@ class LLM:
     return self._executor.request(
       input=self._create_input(template_name, user_data, params),
       parser=parse_response,
+      max_tokens=max_tokens,
     )
 
   def _create_input(self, template_name: str, user_data: Element | str, params: dict[str, Any]):

epub_translator/translation/__init__.py

@@ -1,2 +1,2 @@
-from .types import *
+from .types import Fragment, Language
 from .translation import translate, ProgressReporter

epub_translator/translation/chunk.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from typing import Iterator, Iterable, Generator
 from hashlib import sha512
 from ..llm import LLM
-from .types import Fragment
+from .types import Fragment, Language
 
 
 @dataclass
@@ -30,6 +30,7 @@ class ChunkRange:
 
 def match_fragments(
         llm: LLM,
+        target_language: Language,
         chunk_ranges_iter: Iterator[ChunkRange],
         fragments_iter: Iterator[Fragment],
       ) -> Generator[Chunk, None, None]:
@@ -44,7 +45,7 @@ def match_fragments(
     body = texts[head_length:head_length + body_length]
     tail = texts[head_length + body_length:]
 
-    hash = _hash_texts_list((head, body, tail))
+    hash = _hash_texts_list(target_language, (head, body, tail))
     head = _crop_extra_texts(llm, head, True, range.head_remain_tokens)
     tail = _crop_extra_texts(llm, tail, False, range.tail_remain_tokens)
 
@@ -88,15 +89,12 @@ def _match_range_and_texts(
 
   yield from matched_chunk_ranges
 
-def _hash_texts_list(texts_iterable: Iterable[list[str]]) -> bytes:
-  is_first = True
+def _hash_texts_list(target_language: Language, texts_iterable: Iterable[list[str]]) -> bytes:
   m = sha512()
+  m.update(target_language.value.encode("utf-8"))
   for texts in texts_iterable:
     for text in texts:
-      if is_first:
-        is_first = False
-      else:
-        m.update(b"\x00")
+      m.update(b"\x00")
       m.update(text.encode("utf-8"))
   return m.digest()
 

epub_translator/translation/store.py

@@ -1,7 +1,6 @@
 from shutil import rmtree
 from pathlib import Path
 from typing import Iterator
-from .utils import clean_spaces
 
 
 class Store:
@@ -13,7 +12,7 @@ class Store:
     if not file_path.exists() or not file_path.is_file():
       return None
     with file_path.open("r", encoding="utf-8") as file:
-      return list(line for line in file if line.strip())
+      return file.read().split("\n")
 
   def put(self, chunk_hash: bytes, lines_iter: Iterator[str]):
     file_path = self._file_path(chunk_hash)
@@ -31,7 +30,7 @@ class Store:
           is_first_line = False
         else:
           file.write("\n")
-        file.write(clean_spaces(line))
+        file.write(line)
 
   def _file_path(self, chunk_hash: bytes) -> Path:
     return self._directory / f"{chunk_hash.hex()}.chunk"

epub_translator/translation/translation.py

@@ -6,7 +6,7 @@ from xml.etree.ElementTree import Element
 from ..llm import LLM
 from ..xml import encode_friendly
 
-from .types import Fragment, Language
+from .types import language_chinese_name, Fragment, Language
 from .store import Store
 from .splitter import split_into_chunks
 from .chunk import match_fragments, Chunk
@@ -46,12 +46,23 @@ def translate(
       )))
       for chunk in match_fragments(
         llm=llm,
+        target_language=target_language,
         chunk_ranges_iter=iter(chunk_ranges),
         fragments_iter=gen_fragments_iter(),
       )
     ]
+    def _generate_chunks_from_futures():
+      try:
+        for future in as_completed(futures):
+          yield future.result()
+      except Exception as err:
+        for future in futures:
+          if not future.done():
+            future.cancel()
+        raise err
+
     yield from _sort_translated_texts_by_chunk(
-      target=(f.result() for f in as_completed(futures)),
+      target=_generate_chunks_from_futures(),
       total_tokens_count=sum(chunk.tokens_count for chunk in chunk_ranges),
       report_progress=report_progress,
     )
@@ -96,27 +107,40 @@ def _translate_chunk(
     ) -> list[str]:
 
     translated_texts: list[str] | None = None
+    source_texts = chunk.head + chunk.body + chunk.tail
     if store is not None:
       translated_texts = store.get(chunk.hash)
+      if translated_texts is not None and \
+         len(source_texts) != len(translated_texts):
+        translated_texts = None
+        print(f"Warning: Mismatched lengths in cached translation for chunk: {chunk.hash.hex()}",)
 
     if translated_texts is None:
-      translated_texts = _translate_texts(
-        llm=llm,
-        texts=chunk.head + chunk.body + chunk.tail,
-        target_language=target_language,
-        user_prompt=user_prompt,
-      )
-    if store is not None:
-      store.put(chunk.hash, translated_texts)
+      translated_texts = [
+        clean_spaces(text)
+        for text in _translate_texts(
+          llm=llm,
+          texts=source_texts,
+          texts_tokens=chunk.tokens_count,
+          target_language=target_language,
+          user_prompt=user_prompt,
+        )
+      ]
+      if store is not None:
+        store.put(chunk.hash, translated_texts)
 
     head_length = len(chunk.head)
     translated_texts = translated_texts[head_length:head_length + len(chunk.body)]
 
     return translated_texts
 
+_PLAIN_TEXT_SCALE = 2.0
+_XML_TEXT_SCALE = 2.5
+
 def _translate_texts(
       llm: LLM,
       texts: list[str],
+      texts_tokens: int,
       target_language: Language,
       user_prompt: str | None,
     ) -> list[str]:
@@ -134,8 +158,9 @@ def _translate_texts(
     text_tag="TXT",
     user_data=user_data,
     parser=lambda r: r,
+    max_tokens=texts_tokens * _PLAIN_TEXT_SCALE,
     params={
-      "target_language": target_language.value,
+      "target_language": language_chinese_name(target_language),
       "user_prompt": user_prompt,
     },
   )
@@ -154,12 +179,15 @@ def _translate_texts(
   return llm.request_xml(
     template_name="format",
     user_data=request_text,
-    params={ "target_language": target_language.value },
+    max_tokens=texts_tokens * _XML_TEXT_SCALE,
     parser=lambda r: _parse_translated_response(r, len(texts)),
+    params={
+      "target_language": language_chinese_name(target_language),
+    },
   )
 
 def _parse_translated_response(resp_element: Element, sources_count: int) -> list[str]:
-  translated_fragments = [""] * sources_count
+  fragments: list[str | None] = [None] * sources_count
   for fragment_element in resp_element:
     if fragment_element.text is None:
       continue
@@ -167,11 +195,21 @@ def _parse_translated_response(resp_element: Element, sources_count: int) -> lis
     if id is None:
       continue
     index = int(id) - 1
-    if index < 0 or index >= len(translated_fragments):
+    if index < 0 or index >= len(fragments):
       raise ValueError(f"invalid fragment id: {id}")
-    translated_fragments[index] = fragment_element.text.strip()
-
-  return translated_fragments
+    fragments[index] = fragment_element.text.strip()
+
+  # 有时 LLM 会将多段融合在一起，这里尽可能让译文靠后，将空白段留在前面。
+  # 这样看起来一大段的译文对应若干小段原文，观感更好。
+  for i in range(len(fragments)):
+    fragment = fragments[i]
+    if fragment is not None and i < len(fragments) - 1:
+      next_fragment = fragments[i + 1]
+      if next_fragment is None:
+        fragments[i] = None
+        fragments[i + 1] = fragment
+
+  return [f or "" for f in fragments]
 
 def _normalize_user_input(user_lines: list[str]) -> str | None:
   empty_lines_count: int = 0

epub_translator/translation/types.py

@@ -10,14 +10,40 @@ class Fragment:
   end_incision: Incision
 
 class Language(Enum):
-  SIMPLIFIED_CHINESE = "简体中文"
-  TRADITIONAL_CHINESE = "繁体中文"
-  ENGLISH = "英语"
-  FRENCH = "法语"
-  GERMAN = "德语"
-  SPANISH = "西班牙语"
-  RUSSIAN = "俄语"
-  ITALIAN = "意大利语"
-  PORTUGUESE = "葡萄牙语"
-  JAPANESE = "日语"
-  KOREAN = "韩语"
+  SIMPLIFIED_CHINESE = "zh-Hans"
+  TRADITIONAL_CHINESE = "zh-Hant"
+  ENGLISH = "en"
+  FRENCH = "fr"
+  GERMAN = "de"
+  SPANISH = "es"
+  RUSSIAN = "ru"
+  ITALIAN = "it"
+  PORTUGUESE = "pt"
+  JAPANESE = "ja"
+  KOREAN = "ko"
+
+def language_chinese_name(language: Language) -> str:
+  if language == Language.SIMPLIFIED_CHINESE:
+    return "简体中文"
+  elif language == Language.TRADITIONAL_CHINESE:
+    return "繁体中文"
+  elif language == Language.ENGLISH:
+    return "英语"
+  elif language == Language.FRENCH:
+    return "法语"
+  elif language == Language.GERMAN:
+    return "德语"
+  elif language == Language.SPANISH:
+    return "西班牙语"
+  elif language == Language.RUSSIAN:
+    return "俄语"
+  elif language == Language.ITALIAN:
+    return "意大利语"
+  elif language == Language.PORTUGUESE:
+    return "葡萄牙语"
+  elif language == Language.JAPANESE:
+    return "日语"
+  elif language == Language.KOREAN:
+    return "韩语"
+  else:
+    raise ValueError(f"Unknown language: {language}")

epub_translator/translator.py

@@ -2,11 +2,12 @@ from os import PathLike
 from pathlib import Path
 from tempfile import mkdtemp
 from shutil import rmtree
+from resource_segmentation import Incision
 
 from .llm import LLM
 from .epub import HTMLFile
 from .zip_context import ZipContext
-from .translation import translate as _translate, Fragment, Incision, Language, ProgressReporter
+from .translation import translate as _translate, Fragment, Language, ProgressReporter
 
 
 def translate(
@@ -107,7 +108,7 @@ class _Translator:
 
   def _translate_spine(self, context: ZipContext, cache_path: Path, report_progress: ProgressReporter):
     spine_paths_iter = iter(list(context.search_spine_paths()))
-    spine_file: HTMLFile | None = None
+    spine: tuple[Path, HTMLFile] | None = None
     translated_texts: list[str] = []
     translated_count: int = 0
 
@@ -123,32 +124,37 @@ class _Translator:
     ):
       did_touch_end = False
 
-      if spine_file is not None and \
-        translated_count >= len(translated_texts):
+      if spine and translated_count >= len(translated_texts):
+        spine_path, spine_file = spine
         spine_file.write_texts(translated_texts)
-        spine_file = None
+        context.write_spine_file(spine_path, spine_file)
+        spine = None
 
-      while spine_file is None:
+      while not spine:
         spine_path = next(spine_paths_iter, None)
         if spine_path is None:
+          spine = None
           did_touch_end = True
           break
         spine_file = context.read_spine_file(spine_path)
         if spine_file.texts_length == 0:
-          spine_file = None
           continue
+        spine = (spine_path, spine_file)
         translated_texts = [""] * spine_file.texts_length
         translated_count = 0
+        break
 
       translated_texts[translated_count] = translated_text
       translated_count += 1
 
       if did_touch_end:
         break
-    if spine_file and translated_count > 0:
-      spine_file.write_texts(translated_texts)
 
-    context.write_spine_file(spine_path, spine_file)
+    if spine:
+      spine_path, spine_file = spine
+      if translated_count > 0:
+        spine_file.write_texts(translated_texts)
+      context.write_spine_file(spine_path, spine_file)
 
 def _gen_fragments(context: ZipContext):
   for spine_path in context.search_spine_paths():

{epub_translator-0.0.3.dist-info → epub_translator-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: epub-translator
-Version: 0.0.3
+Version: 0.0.4
 Summary: Translate the epub book using LLM. The translated book will retain the original text and list the translated text side by side with the original text.
 License: MIT
 Keywords: epub,llm,translation,translator

{epub_translator-0.0.3.dist-info → epub_translator-0.0.4.dist-info}/RECORD

@@ -10,18 +10,18 @@ epub_translator/epub/html/file.py,sha256=AqUV-Tmptk5J2EYmw3oRVsLjGSqEMNz5rItnoRb
 epub_translator/epub/html/texts_searcher.py,sha256=vamO99pki6_sX2PeKCJk7mPwHdApZq1sOgSYDTPckx8,1376
 epub_translator/llm/__init__.py,sha256=wMBWLgh5iLNQBioniSOmWC83NS7RLM41hIs1V1uZiWI,21
 epub_translator/llm/error.py,sha256=fG0A3z69YoSNu0MNVWVFMtHCB_4fpOvAEb0Kajn9OHc,1401
-epub_translator/llm/executor.py,sha256=vwHqtlvCDHjDXLcvvKstlcQ5MfAGNPz1RKbq8W6WwKs,4378
+epub_translator/llm/executor.py,sha256=Z8mpsTAEEfG80E43Pv5VJJzaDKrrClyUhr8GFA2yVxA,4498
 epub_translator/llm/increasable.py,sha256=Dpu5z4JK5h1OtLorZgsOAdRFeTH2LOkdroasgmCWAIo,1136
-epub_translator/llm/node.py,sha256=IKgdWoBwiejHOJ7akv8AiXUpyFlv2U4fGllt7ZLE3M0,5970
+epub_translator/llm/node.py,sha256=qdkBY06OOt0mq8tnLgMJZS5dc3WFColzpeHmyWLE7KU,6104
 epub_translator/template.py,sha256=GdV3QnypProKFCMH1kBNfdt6wiShygP_-xGnE5EOUwU,1460
-epub_translator/translation/__init__.py,sha256=mudXLDVSIG0XTLoHUIos0-wtQCnL9ZreuHsTHcVKjnE,73
-epub_translator/translation/chunk.py,sha256=obrkx_yCeGMeikinfIx0NRvMo2kQBwXVbCdJbeT-ERA,3576
+epub_translator/translation/__init__.py,sha256=QvHULHbBc7NKDlV6d0hyXAFXcAWI1tctsubzJD89tvw,90
+epub_translator/translation/chunk.py,sha256=ERIDvhM7kZB_ZqWGw4UQfKgjIHn7TDWsxa-RuTpxRs0,3639
 epub_translator/translation/splitter.py,sha256=xOaP1p3lqY95CR0vDXdeGUMHYObiqs3y093EUAxJ-jI,2676
-epub_translator/translation/store.py,sha256=1FmksPAUj0mt3tN8Jdb_L1ovaI1p_5OhTWgxbIDl0SI,1133
-epub_translator/translation/translation.py,sha256=_qiw6s_z_Tv4VmIP1U-_YhDYNiKEDNfigHIIGKa41fU,5734
-epub_translator/translation/types.py,sha256=vDW5bVqYwngW_YUgf0SgfZ5zIFWUxcbBGO1U9Dsxc0o,499
+epub_translator/translation/store.py,sha256=4sR3DYZuU56IaNcw3xIMf8ZQP19kcoP5MZUaM8j4gxw,1067
+epub_translator/translation/translation.py,sha256=1ETNgP1zDEbJnUEPaMIWbw1vlh2jDVHmFEJ4fqDe8Ic,7078
+epub_translator/translation/types.py,sha256=OUxqgdyvrDEUFz21b0tv28D_oIqrP77Yv_xKAoXI-7c,1231
 epub_translator/translation/utils.py,sha256=G6Gqq6mot3lgFA-jqUD0UqtDS0GC1wrb9DnK7rTxJNs,223
-epub_translator/translator.py,sha256=oh7PdCijUSGU6f72hzc6doJdMWnAqg6zHRFIj3aeTjc,5332
+epub_translator/translator.py,sha256=U0ZBBpyRNdDEvrAM7AA3b3TatdUwlhM_MSTOt5Bclsw,5520
 epub_translator/xml/__init__.py,sha256=o2_qwUYU_MUcyfmfKkiOQ-cKUQyl4PiRL8YHVzCTAZU,106
 epub_translator/xml/decoder.py,sha256=UlqgmEKQDzxt3lvBeNGHgZP6jznmnq_1HLJuAe5X0C4,2181
 epub_translator/xml/encoder.py,sha256=p4A7GRSOM2i0WOh1lLtEdTTg2gXSQrxDdzMgUqbiV18,2428
@@ -30,7 +30,7 @@ epub_translator/xml/tag.py,sha256=QLZImF0PtYyiASI7swrB8DL_qUwcYpU6cL68jEXDnvg,23
 epub_translator/xml/transform.py,sha256=vS_a4d_o2Qqf9B6k2CovQVLUknp6TyUi3FyLOu21Vio,1126
 epub_translator/xml/utils.py,sha256=KDNGWHwaIiFKS27sjZF0e-bBSjeTxzceae_aeuj4wzI,384
 epub_translator/zip_context.py,sha256=7_05kycmADb4-vxHkw_DX__vkKOxT4zo9pr2a8F4L_U,2409
-epub_translator-0.0.3.dist-info/LICENSE,sha256=5RF32sL3LtMOJIErdDKp1ZEYPGXS8WPpsiSz_jMBnGI,1066
-epub_translator-0.0.3.dist-info/METADATA,sha256=HuD7ogzPi96e8ZKjAJyTARhSeR4xmu3vZt0Q3Vmg0Hk,2342
-epub_translator-0.0.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-epub_translator-0.0.3.dist-info/RECORD,,
+epub_translator-0.0.4.dist-info/LICENSE,sha256=5RF32sL3LtMOJIErdDKp1ZEYPGXS8WPpsiSz_jMBnGI,1066
+epub_translator-0.0.4.dist-info/METADATA,sha256=rM-6yy977tIiJvpAXHCJYFGYTgjTuOxKLJhbSbYwJNs,2342
+epub_translator-0.0.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+epub_translator-0.0.4.dist-info/RECORD,,