epub-translator 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

epub_translator/__init__.py

@@ -1,2 +1,3 @@
-from .translator import Translator
-from .file import translate_epub_file, ProgressReporter
+from .llm import LLM
+from .translator import translate
+from .translation import Language, ProgressReporter

epub_translator/data/format.jinja

@@ -0,0 +1,33 @@
+你是一个校对员，需要帮助用户校对译文与原文的对应关系。用户随后会提交一段文本，先是一段表示原文的XML格式内容，接下来是一段纯文本的{{ target_language }}译文。原文XML文本以<request>标签作为根，<fragment>标签作为子元素。原文将按顺序拆分再各个<fragment>标签中，每个<fragment>标签将包含一个id属性，以唯一标识该片段。
+
+你要将{{ target_language }}译文正确分割成片段，并与原文一一对应。并模仿用户提交的格式，将根节点由<request>替换成<response>节点，再将<fragment>的内容由原文替换成{{ target_language }}译文，但保留id不变。最终将整个XML格式内容输出。你的输出必须满足如下规则：
+- 分割后的片段对应标准是语义一致。即对应片段的原文与译文互相翻译后，是完全相同的内容。
+- 替换后的译文必须严格与用户提交的译文对应，不得增加、修改、删除内容。
+- 将你输出<fragment>中的译文单独提取出来按顺序读出来，应与用户提交的{{ target_language }}译文一字不差。
+- 译文必须是其对应的原文的直接翻译。
+- 绝大部分情况下，译文<fragment>的id能与原文<fragment>的id一一配对，不会出现错位、新增、遗漏的情况。但若发现无论如何都无法一一对应，应该尽可能将文字更多的片段对应对应上，跳过哪些由几个字或几个单词构成的短小、碎片化的片段。此时你输出的<fragment>的id可能不连续，也是没关系的，通过跳过短小片段来保证整体对应关系完整，是可接受的。决不可接受的是，因为遗漏短小片段，导致后面大段大段内容直接错位。
+
+特别注意，用户提交的译文也会分自然段，这个自然段与原文的<fragment>**没有任何关系**。这个分段是翻译家仅看过原文文本，但不知其<fragment>划分的情况下自行作出了。你绝对不可参考这个译文的分段，更不要被它误导。匹配标准只有一条，就是语义一致。
+
+这里举个例子，假设用户提交的原文是英文，译文是中文。用户提交的内容如下：
+```XML
+<request>
+  <fragment id="1">Although fermentation was an idea dear to the heart of many an alchemist, the particular notion of fermenting water in order to produce the specified materials of the world perceived by the senses is at heart Helmontian.</fragment>
+  <fragment id="2">In the following it will therefore be useful to give a brief overview of van Helmont’s matter-theory.</fragment>
+  <fragment id="3">Reference</fragment>
+  <fragment id="4">[1] Newman, Gehennical Fire, pp. 58–78, 171–96.</fragment>
+</response>
+```
+
+尽管发酵是许多炼金术士所珍视的理念，但通过发酵水来生成感官所感知的特定物质这一特定概念，其核心却是海尔蒙特式的。因此，下文将简要概述范·海尔蒙特的物质理论引用。[1]纽曼，《地底之火》，第 58-78 页、第 171-96 页。
+
+你应该返回如下内容。
+```XML
+<response>
+  <fragment id="1">尽管发酵是许多炼金术士所珍视的理念，但通过发酵水来生成感官所感知的特定物质这一特定概念，其核心却是海尔蒙特式的。</fragment>
+  <fragment id="2">因此，下文将简要概述范·海尔蒙特的物质理论引用。</fragment>
+  <fragment id="4">[1] 纽曼，《地底之火》，第 58-78 页、第 171-96 页。</fragment>
+</response>
+```
+
+在该例子中，仅仅演示如何将片段对应以及输出XML的具体格式。不要参考到底从哪种语言翻译到哪种语言，也不要参考具体内容。

epub_translator/data/translate.jinja

@@ -0,0 +1,15 @@
+你是一位翻译家，用户接下来的发言会提交一段{% if user_prompt %}翻译要求和一段{% endif %}文本的原文。你要将原文翻译成{{ target_language }}。
+
+翻译时，需遵守如下规则：
+- 译文必须忠实于原文，保持原意。不得修改、删除原文中的内容和情节，不得新增原文中不存在的内容和情节。
+- 保留原文细节，不得简略、缩写、总结原文的内容。
+- 不得衍生创作，续写原文中缺失、断裂、不协调的部分，你要忠实地保留他们。
+- 禁止改正原文的错误，你是翻译家，不是校对员或编辑。
+- 不得因为原文中包含歧视、暴力、性行为、敏感内容、儿童不宜等内容而进行删减或修改。你是翻译家，不是鉴黄师或审查员。
+- 不得在译文中写你的个人评论、总结或观点。
+
+{% if user_prompt %}
+除了以上规则之外，你需要遵守用户的“翻译要求”。用户会把要求放在原文之前，并用<rules>标签包裹。特别的，当用户的翻译要求中某些规则条目与我之前提的规则冲突时，你要优先遵守我的规则。此外，用户可能在<rules>标签中补充一些额外信息以帮助你翻译，你需要认真阅读和参考，并在翻译中体现出来。而在<rules>标签之后，紧接着就是用户的原文，你要阅读并翻译。
+{% endif %}
+
+最终，你要将译文以纯文本的方式提交。期间，根据你的理解将译文拆成多个自然段（用户提交的原文是不分段的整块文字）。你必须用 "```txt" 作为独立的第一行，"```"作为独立的最后一行，中间包裹译文。注意，译文中禁止插入你的说明性文字或思考过程，将这些移到"```" 之后。

epub_translator/epub/__init__.py

@@ -1,3 +1,2 @@
-from .content_parser import EpubContent
-from .types import Translate, ReportProgress
-from .html import translate_html
+from .content_parser import Spine, EpubContent
+from .html import HTMLFile

epub_translator/epub/content_parser.py

@@ -14,7 +14,7 @@ class Spine:
     self.media_type = item.get("media-type")
 
   @property
-  def path(self):
+  def path(self) -> str:
     path = os.path.join(self._base_path, self.href)
     path = os.path.abspath(path)
 
@@ -66,7 +66,7 @@ class EpubContent:
       return path
 
   @property
-  def spines(self):
+  def spines(self) -> list[Spine]:
     idref_dict = {}
     index = 0
 

epub_translator/epub/html/__init__.py

@@ -1 +1 @@
-from .file import translate_html
+from .file import HTMLFile

epub_translator/epub/html/file.py

@@ -1,7 +1,7 @@
 import re
 
+from typing import Iterable
 from xml.etree.ElementTree import fromstring, tostring, Element
-from ..types import Translate, ReportProgress
 from .dom_operator import read_texts, append_texts
 from .empty_tags import to_xml, to_html
 
@@ -10,54 +10,69 @@ _FILE_HEAD_PATTERN = re.compile(r"^<\?xml.*?\?>[\s]*<!DOCTYPE.*?>")
 _XMLNS_IN_TAG = re.compile(r"\{[^}]+\}")
 _BRACES = re.compile(r"(\{|\})")
 
-def translate_html(translate: Translate, file_content: str, report_progress: ReportProgress) -> str:
-  match = re.match(_FILE_HEAD_PATTERN, file_content)
-  head = match.group() if match else None
-  xml_content = re.sub(_FILE_HEAD_PATTERN, "", to_xml(file_content))
+class HTMLFile:
+  def __init__(self, file_content: str):
+    match = re.match(_FILE_HEAD_PATTERN, file_content)
+    xml_content = re.sub(_FILE_HEAD_PATTERN, "", to_xml(file_content))
+    self._head: str = match.group() if match else None
+    self._root: Element = fromstring(xml_content)
+    self._xmlns: str | None = self._extract_xmlns(self._root)
+    self._texts_length: int | None = None
 
-  root = fromstring(xml_content)
-  root_attrib = {**root.attrib}
-  xmlns = _extract_xmlns(root)
+  def _extract_xmlns(self, root: Element) -> str | None:
+    root_xmlns: str | None = None
+    for i, element in enumerate(_all_elements(root)):
+      need_clean_xmlns = True
+      match = re.match(_XMLNS_IN_TAG, element.tag)
 
-  source_texts = list(read_texts(root))
-  target_texts = translate(source_texts, report_progress)
-  append_texts(root, target_texts)
+      if match:
+        xmlns = re.sub(_BRACES, "", match.group())
+        if i == 0:
+          root_xmlns = xmlns
+        elif root_xmlns != xmlns:
+          need_clean_xmlns = False
+      if need_clean_xmlns:
+        element.tag = re.sub(_XMLNS_IN_TAG, "", element.tag)
 
-  if xmlns is not None:
-    root_attrib["xmlns"] = xmlns
-  root.attrib = root_attrib
+    return root_xmlns
 
-  if xmlns is None:
-    file_content = tostring(root, encoding="unicode")
-    file_content = to_html(file_content)
-  else:
-    # XHTML disable <tag/> (we need replace them with <tag></tag>)
-    for element in _all_elements(root):
-      if element.text is None:
-        element.text = ""
-    file_content = tostring(root, encoding="unicode")
+  def read_texts(self) -> list[str]:
+    texts = list(read_texts(self._root))
+    self._texts_length = len(texts)
+    return texts
 
-  if head is not None:
-    file_content = head + file_content
+  def write_texts(self, texts: Iterable[str]):
+    append_texts(self._root, texts)
 
-  return file_content
+  @property
+  def texts_length(self) -> int:
+    if self._texts_length is None:
+      self._texts_length = 0
+      for _ in read_texts(self._root):
+        self._texts_length += 1
+    return self._texts_length
 
-def _extract_xmlns(root: Element) -> str | None:
-  root_xmlns: str | None = None
-  for i, element in enumerate(_all_elements(root)):
-    need_clean_xmlns = True
-    match = re.match(_XMLNS_IN_TAG, element.tag)
+  @property
+  def file_content(self) -> str:
+    file_content: str
+    if self._xmlns is None:
+      file_content = tostring(self._root, encoding="unicode")
+      file_content = to_html(file_content)
+    else:
+      root = Element(
+        self._root.tag,
+        attrib={**self._root.attrib, "xmlns": self._xmlns},
+      )
+      root.extend(self._root)
+      # XHTML disable <tag/> (we need replace them with <tag></tag>)
+      for element in _all_elements(root):
+        if element.text is None:
+          element.text = ""
+      file_content = tostring(root, encoding="unicode")
 
-    if match:
-      xmlns = re.sub(_BRACES, "", match.group())
-      if i == 0:
-        root_xmlns = xmlns
-      elif root_xmlns != xmlns:
-        need_clean_xmlns = False
-    if need_clean_xmlns:
-      element.tag = re.sub(_XMLNS_IN_TAG, "", element.tag)
-
-  return root_xmlns
+    if self._head is not None:
+      file_content = self._head + file_content
+    return file_content
 
 def _all_elements(parent: Element):
   yield parent

epub_translator/epub/html/texts_searcher.py

@@ -12,7 +12,8 @@ class TextPosition(Enum):
 TextDescription = tuple[Element, TextPosition, Element | None]
 
 _IGNORE_TAGS = (
-  "title", "link", "style", "css", "img", "script", "metadata"
+  "title", "link", "style", "css", "img", "script", "metadata",
+  "{http://www.w3.org/1998/Math/MathML}math", # TODO: 公式是正文，也要读进去，暂时忽略避免扰乱得了。
 )
 
 _TEXT_LEAF_TAGS = (

epub_translator/llm/__init__.py

@@ -0,0 +1 @@
+from .node import LLM

epub_translator/llm/error.py

@@ -0,0 +1,49 @@
+import openai
+import httpx
+import requests
+
+
+def is_retry_error(err: Exception) -> bool:
+  if _is_openai_retry_error(err):
+    return True
+  if _is_httpx_retry_error(err):
+    return True
+  if _is_request_retry_error(err):
+    return True
+  return False
+
+# https://help.openai.com/en/articles/6897213-openai-library-error-types-guidance
+def _is_openai_retry_error(err: Exception) -> bool:
+  if isinstance(err, openai.Timeout):
+    return True
+  if isinstance(err, openai.APIConnectionError):
+    return True
+  if isinstance(err, openai.InternalServerError):
+    return err.status_code in (502, 503, 504)
+  return False
+
+# https://www.python-httpx.org/exceptions/
+def _is_httpx_retry_error(err: Exception) -> bool:
+  if isinstance(err, httpx.RemoteProtocolError):
+    return True
+  if isinstance(err, httpx.StreamError):
+    return True
+  if isinstance(err, httpx.TimeoutException):
+    return True
+  if isinstance(err, httpx.NetworkError):
+    return True
+  if isinstance(err, httpx.ProtocolError):
+    return True
+  return False
+
+# https://requests.readthedocs.io/en/latest/api/#exceptions
+def _is_request_retry_error(err: Exception) -> bool:
+  if isinstance(err, requests.ConnectionError):
+    return True
+  if isinstance(err, requests.ConnectTimeout):
+    return True
+  if isinstance(err, requests.ReadTimeout):
+    return True
+  if isinstance(err, requests.Timeout):
+    return True
+  return False

epub_translator/llm/executor.py

@@ -0,0 +1,147 @@
+from typing import cast, Any, Callable
+from io import StringIO
+from time import sleep
+from pydantic import SecretStr
+from logging import Logger
+from langchain_core.messages import SystemMessage, HumanMessage, AIMessage
+from langchain_core.language_models import LanguageModelInput
+from langchain_openai import ChatOpenAI
+
+from .increasable import Increasable, Increaser
+from .error import is_retry_error
+
+
+class LLMExecutor:
+  def __init__(
+    self,
+    api_key: SecretStr,
+    url: str,
+    model: str,
+    timeout: float | None,
+    top_p: Increasable,
+    temperature: Increasable,
+    retry_times: int,
+    retry_interval_seconds: float,
+    create_logger: Callable[[], Logger | None],
+  ) -> None:
+
+    self._timeout: float | None = timeout
+    self._top_p: Increasable = top_p
+    self._temperature: Increasable = temperature
+    self._retry_times: int = retry_times
+    self._retry_interval_seconds: float = retry_interval_seconds
+    self._create_logger: Callable[[], Logger | None] = create_logger
+    self._model = ChatOpenAI(
+      api_key=cast(SecretStr, api_key),
+      base_url=url,
+      model=model,
+      timeout=timeout,
+    )
+
+  def request(self, input: LanguageModelInput, parser: Callable[[str], Any]) -> Any:
+    result: Any | None = None
+    last_error: Exception | None = None
+    did_success = False
+    top_p: Increaser = self._top_p.context()
+    temperature: Increaser = self._temperature.context()
+    logger = self._create_logger()
+
+    if logger is not None:
+      logger.debug(f"[[Request]]:\n{self._input2str(input)}\n")
+
+    try:
+      for i in range(self._retry_times + 1):
+        try:
+          response = self._invoke_model(
+            input=input,
+            top_p=top_p.current,
+            temperature=temperature.current,
+          )
+          if logger is not None:
+            logger.debug(f"[[Response]]:\n{response}\n")
+
+        except Exception as err:
+          last_error = err
+          if not is_retry_error(err):
+            raise err
+          if logger is not None:
+            logger.warning(f"request failed with connection error, retrying... ({i + 1} times)")
+          if self._retry_interval_seconds > 0.0 and \
+            i < self._retry_times:
+            sleep(self._retry_interval_seconds)
+          continue
+
+        try:
+          result = parser(response)
+          did_success = True
+          break
+
+        except Exception as err:
+          last_error = err
+          warn_message = f"request failed with parsing error, retrying... ({i + 1} times)"
+          if logger is not None:
+            logger.warning(warn_message)
+          print(warn_message)
+          top_p.increase()
+          temperature.increase()
+          if self._retry_interval_seconds > 0.0 and \
+            i < self._retry_times:
+            sleep(self._retry_interval_seconds)
+          continue
+
+    except KeyboardInterrupt as err:
+      if last_error is not None and logger is not None:
+        logger.debug(f"[[Error]]:\n{last_error}\n")
+      raise err
+
+    if not did_success:
+      if last_error is None:
+        raise RuntimeError("Request failed with unknown error")
+      else:
+        raise last_error
+
+    return result
+
+  def _input2str(self, input: LanguageModelInput) -> str:
+    if isinstance(input, str):
+      return input
+    if not isinstance(input, list):
+      raise ValueError(f"Unsupported input type: {type(input)}")
+
+    buffer = StringIO()
+    is_first = True
+    for message in input:
+      if not is_first:
+        buffer.write("\n\n")
+      if isinstance(message, SystemMessage):
+        buffer.write("System:\n")
+        buffer.write(message.content)
+      elif isinstance(message, HumanMessage):
+        buffer.write("User:\n")
+        buffer.write(message.content)
+      elif isinstance(message, AIMessage):
+        buffer.write("Assistant:\n")
+        buffer.write(message.content)
+      else:
+        buffer.write(str(message))
+      is_first = False
+
+    return buffer.getvalue()
+
+  def _invoke_model(
+        self,
+        input: LanguageModelInput,
+        top_p: float | None,
+        temperature: float | None,
+      ):
+    stream = self._model.stream(
+      input=input,
+      timeout=self._timeout,
+      top_p=top_p,
+      temperature=temperature,
+    )
+    buffer = StringIO()
+    for chunk in stream:
+      data = str(chunk.content)
+      buffer.write(data)
+    return buffer.getvalue()

epub_translator/llm/increasable.py

@@ -0,0 +1,35 @@
+class Increaser:
+  def __init__(self, value_range: tuple[float, float] | None):
+    self._value_range: tuple[float, float] | None = value_range
+    self._current: float | None = value_range[0] if value_range is not None else None
+
+  @property
+  def current(self) -> float | None:
+    return self._current
+
+  def increase(self):
+    if self._value_range is None:
+      return
+    _, end_value = self._value_range
+    self._current = self._current + 0.5 * (end_value - self._current)
+
+class Increasable:
+  def __init__(self, param: float | tuple[float, float] | None):
+    self._value_range: tuple[float, float] | None = None
+
+    if isinstance(param, int):
+      param = float(param)
+    if isinstance(param, float):
+      param = (param, param)
+    if isinstance(param, tuple):
+      if len(param) != 2:
+        raise ValueError(f"Expected a tuple of length 2, got {len(param)}")
+      begin, end = param
+      if isinstance(begin, int):
+        begin = float(begin)
+      if isinstance(end, int):
+        end = float(end)
+      self._value_range = (begin, end)
+
+  def context(self) -> Increaser:
+    return Increaser(self._value_range)

epub_translator/llm/node.py

@@ -0,0 +1,197 @@
+import datetime
+
+from os import PathLike
+from pathlib import Path
+from typing import cast, Any, TypeVar, Generator, Sequence, Callable
+from importlib.resources import files
+from jinja2 import Environment, Template
+from xml.etree.ElementTree import Element
+from pydantic import SecretStr
+from logging import getLogger, DEBUG, Formatter, Logger, FileHandler
+from tiktoken import get_encoding, Encoding
+from langchain_core.messages import SystemMessage, HumanMessage
+
+from ..template import create_env
+from ..xml import decode_friendly, encode_friendly
+from .increasable import Increasable
+from .executor import LLMExecutor
+
+
+R = TypeVar("R")
+
+class LLM:
+  def __init__(
+      self,
+      key: str,
+      url: str,
+      model: str,
+      token_encoding: str,
+      timeout: float | None = None,
+      top_p: float | tuple[float, float] | None = None,
+      temperature: float | tuple[float, float] | None = None,
+      retry_times: int = 5,
+      retry_interval_seconds: float = 6.0,
+      log_dir_path: PathLike | None = None,
+    ):
+    prompts_path = files("epub_translator") / "data"
+    self._templates: dict[str, Template] = {}
+    self._encoding: Encoding = get_encoding(token_encoding)
+    self._env: Environment = create_env(prompts_path)
+    self._logger_save_path: Path | None = None
+
+    if log_dir_path is not None:
+      self._logger_save_path = Path(log_dir_path)
+      if not self._logger_save_path.exists():
+        self._logger_save_path.mkdir(parents=True, exist_ok=True)
+      elif not self._logger_save_path.is_dir():
+        self._logger_save_path = None
+
+    self._executor = LLMExecutor(
+      url=url,
+      model=model,
+      api_key=cast(SecretStr, key),
+      timeout=timeout,
+      top_p=Increasable(top_p),
+      temperature=Increasable(temperature),
+      retry_times=retry_times,
+      retry_interval_seconds=retry_interval_seconds,
+      create_logger=self._create_logger,
+    )
+
+  def _create_logger(self) -> Logger | None:
+    if self._logger_save_path is None:
+      return None
+
+    now = datetime.datetime.now(datetime.timezone.utc)
+    timestamp = now.strftime("%Y-%m-%d %H-%M-%S %f")
+    file_path = self._logger_save_path / f"request {timestamp}.log"
+    logger = getLogger(f"LLM Request {timestamp}")
+    logger.setLevel(DEBUG)
+    handler = FileHandler(file_path, encoding="utf-8")
+    handler.setLevel(DEBUG)
+    handler.setFormatter(Formatter("%(asctime)s    %(message)s", "%H:%M:%S"))
+    logger.addHandler(handler)
+
+    return logger
+
+  def request_text(
+      self,
+      template_name: str,
+      text_tag: str,
+      user_data: Element | str,
+      parser: Callable[[str], R],
+      params: dict[str, Any] | None = None,
+    ) -> R:
+
+    if params is None:
+      params = {}
+
+    def parse_response(response: str) -> R:
+      text = next(self._search_quotes(text_tag.lower(), response), None)
+      if text is None:
+        raise ValueError(f"No valid {text_tag} response found")
+      return parser(text)
+
+    return self._executor.request(
+      input=self._create_input(template_name, user_data, params),
+      parser=parse_response,
+    )
+
+  def request_xml(
+        self,
+        template_name: str,
+        user_data: Element | str,
+        parser: Callable[[Element], R],
+        params: dict[str, Any] | None = None,
+      ) -> R:
+
+    if params is None:
+      params = {}
+
+    def parse_response(response: str) -> R:
+      element = next(decode_friendly(response, "response"), None)
+      if element is None:
+        raise ValueError("No valid XML response found")
+      return parser(element)
+
+    return self._executor.request(
+      input=self._create_input(template_name, user_data, params),
+      parser=parse_response,
+    )
+
+  def _create_input(self, template_name: str, user_data: Element | str, params: dict[str, Any]):
+    data: str
+    if isinstance(user_data, Element):
+      data = encode_friendly(user_data)
+      data = f"```XML\n{data}\n```"
+    else:
+      data = user_data
+
+    template = self._template(template_name)
+    prompt = template.render(**params)
+    return [
+      SystemMessage(content=prompt),
+      HumanMessage(content=data)
+    ]
+
+  def prompt_tokens_count(self, template_name: str, params: dict[str, Any]) -> int:
+    template = self._template(template_name)
+    prompt = template.render(**params)
+    return len(self._encoding.encode(prompt))
+
+  def encode_tokens(self, text: str) -> list[int]:
+    return self._encoding.encode(text)
+
+  def decode_tokens(self, tokens: Sequence[int]) -> str:
+    return self._encoding.decode(tokens)
+
+  def count_tokens_count(self, text: str) -> int:
+    return len(self._encoding.encode(text))
+
+  def _template(self, template_name: str) -> Template:
+    template = self._templates.get(template_name, None)
+    if template is None:
+      template = self._env.get_template(template_name)
+      self._templates[template_name] = template
+    return template
+
+  def _search_quotes(self, kind: str, response: str) -> Generator[str, None, None]:
+    start_marker = f"```{kind}"
+    end_marker = "```"
+    start_index = 0
+
+    while True:
+      start_index = self._find_ignore_case(
+        raw=response,
+        sub=start_marker,
+        start=start_index,
+      )
+      if start_index == -1:
+        break
+
+      end_index = self._find_ignore_case(
+        raw=response,
+        sub=end_marker,
+        start=start_index + len(start_marker),
+      )
+      if end_index == -1:
+        break
+
+      extracted_text = response[start_index + len(start_marker):end_index].strip()
+      yield extracted_text
+      start_index = end_index + len(end_marker)
+
+  def _find_ignore_case(self, raw: str, sub: str, start: int = 0):
+    if not sub:
+      return 0 if 0 >= start else -1
+
+    raw_len, sub_len = len(raw), len(sub)
+    for i in range(start, raw_len - sub_len + 1):
+      match = True
+      for j in range(sub_len):
+        if raw[i + j].lower() != sub[j].lower():
+          match = False
+          break
+      if match:
+        return i
+    return -1