epub-translator 0.0.6__py3-none-any.whl → 0.1.0__py3-none-any.whl

epub_translator/translation/translation.py

@@ -1,231 +0,0 @@
-from math import ceil
-from typing import Callable, Iterator, Generator
-from pathlib import Path
-from concurrent.futures import as_completed, ThreadPoolExecutor
-from xml.etree.ElementTree import Element
-
-from ..llm import LLM
-from ..xml import encode_friendly
-
-from .types import language_chinese_name, Fragment, Language
-from .store import Store
-from .splitter import split_into_chunks
-from .chunk import match_fragments, Chunk
-from .utils import is_empty, clean_spaces
-
-
-ProgressReporter = Callable[[float], None]
-
-def translate(
-      llm: LLM,
-      gen_fragments_iter: Callable[[], Iterator[Fragment]],
-      cache_path: Path | None,
-      target_language: Language,
-      user_prompt: str | None,
-      max_chunk_tokens_count: int,
-      max_threads_count: int,
-      report_progress: ProgressReporter,
-    ) -> Generator[str, None, None]:
-
-  if user_prompt is not None:
-    user_prompt = _normalize_user_input(user_prompt.splitlines())
-
-  store = Store(cache_path) if cache_path else None
-  chunk_ranges = list(split_into_chunks(
-    llm=llm,
-    fragments_iter=gen_fragments_iter(),
-    max_chunk_tokens_count=max_chunk_tokens_count,
-  ))
-  with ThreadPoolExecutor(max_workers=max_threads_count) as executor:
-    futures = [
-      executor.submit(lambda chunk=chunk: (chunk, _translate_chunk(
-        llm=llm,
-        store=store,
-        chunk=chunk,
-        target_language=target_language,
-        user_prompt=user_prompt,
-      )))
-      for chunk in match_fragments(
-        llm=llm,
-        target_language=target_language,
-        chunk_ranges_iter=iter(chunk_ranges),
-        fragments_iter=gen_fragments_iter(),
-      )
-    ]
-    def _generate_chunks_from_futures():
-      try:
-        for future in as_completed(futures):
-          yield future.result()
-      except Exception as err:
-        for future in futures:
-          if not future.done():
-            future.cancel()
-        raise err
-
-    yield from _sort_translated_texts_by_chunk(
-      target=_generate_chunks_from_futures(),
-      total_tokens_count=sum(chunk.tokens_count for chunk in chunk_ranges),
-      report_progress=report_progress,
-    )
-
-def _sort_translated_texts_by_chunk(
-      target: Iterator[tuple[Chunk, list[str]]],
-      total_tokens_count: int,
-      report_progress: ProgressReporter,
-    ) -> Generator[str, None, None]:
-
-  buffer: list[tuple[Chunk, list[str]]] = []
-  wanna_next_index: int = 0
-  translated_tokens_count: int = 0
-
-  for chunk, translated_texts in target:
-    buffer.append((chunk, translated_texts))
-    if wanna_next_index == chunk.index:
-      buffer.sort(key=lambda e: e[0].index)
-      to_clear: list[list[str]] = []
-
-      for chunk, translated_texts in buffer:
-        if chunk.index > wanna_next_index:
-          break
-        to_clear.append(translated_texts)
-        if chunk.index == wanna_next_index:
-          wanna_next_index += 1
-
-      if to_clear:
-        buffer = buffer[len(to_clear):]
-        for translated_texts in to_clear:
-          yield from translated_texts
-
-    translated_tokens_count += chunk.tokens_count
-    report_progress(float(translated_tokens_count) / total_tokens_count)
-
-def _translate_chunk(
-      llm: LLM,
-      store: Store | None,
-      chunk: Chunk,
-      target_language: Language,
-      user_prompt: str | None,
-    ) -> list[str]:
-
-  translated_texts: list[str] | None = None
-  source_texts = chunk.head + chunk.body + chunk.tail
-  if store is not None:
-    translated_texts = store.get(chunk.hash)
-    if translated_texts is not None and \
-        len(source_texts) != len(translated_texts):
-      translated_texts = None
-      print(f"Warning: Mismatched lengths in cached translation for chunk: {chunk.hash.hex()}",)
-
-  if translated_texts is None:
-    translated_texts = [
-      clean_spaces(text)
-      for text in _translate_texts(
-        llm=llm,
-        texts=source_texts,
-        texts_tokens=chunk.tokens_count,
-        target_language=target_language,
-        user_prompt=user_prompt,
-      )
-    ]
-    if store is not None:
-      store.put(chunk.hash, translated_texts)
-
-  head_length = len(chunk.head)
-  translated_texts = translated_texts[head_length:head_length + len(chunk.body)]
-
-  return translated_texts
-
-_PLAIN_TEXT_SCALE = 2.0
-_XML_TEXT_SCALE = 2.5
-
-def _translate_texts(
-      llm: LLM,
-      texts: list[str],
-      texts_tokens: int,
-      target_language: Language,
-      user_prompt: str | None,
-    ) -> list[str]:
-
-  original_text = _normalize_user_input(texts)
-  if original_text is None:
-    return [""] * len(texts)
-
-  user_data = original_text
-  if user_prompt is not None:
-    user_data = f"<rules>{user_prompt}</rules>\n\n{original_text}"
-
-  translated_text = llm.request_text(
-    template_name="translate",
-    text_tag="TXT",
-    user_data=user_data,
-    parser=lambda r: r,
-    max_tokens=ceil(texts_tokens * _PLAIN_TEXT_SCALE),
-    params={
-      "target_language": language_chinese_name(target_language),
-      "user_prompt": user_prompt,
-    },
-  )
-  request_element = Element("request")
-
-  for i, fragment in enumerate(texts):
-    fragment_element = Element("fragment", attrib={
-      "id": str(i + 1),
-    })
-    fragment_element.text = clean_spaces(fragment)
-    request_element.append(fragment_element)
-
-  request_element_text = encode_friendly(request_element)
-  request_text = f"```XML\n{request_element_text}\n```\n\n{translated_text}"
-
-  return llm.request_xml(
-    template_name="format",
-    user_data=request_text,
-    max_tokens=ceil(texts_tokens * _XML_TEXT_SCALE),
-    parser=lambda r: _parse_translated_response(r, len(texts)),
-    params={
-      "target_language": language_chinese_name(target_language),
-    },
-  )
-
-def _parse_translated_response(resp_element: Element, sources_count: int) -> list[str]:
-  fragments: list[str | None] = [None] * sources_count
-  for fragment_element in resp_element:
-    if fragment_element.text is None:
-      continue
-    id = fragment_element.get("id", None)
-    if id is None:
-      continue
-    index = int(id) - 1
-    if index < 0 or index >= len(fragments):
-      raise ValueError(f"invalid fragment id: {id}")
-    fragments[index] = fragment_element.text.strip()
-
-  # 有时 LLM 会将多段融合在一起，这里尽可能让译文靠后，将空白段留在前面。
-  # 这样看起来一大段的译文对应若干小段原文，观感更好。
-  for i in range(len(fragments)):
-    fragment = fragments[i]
-    if fragment is not None and i < len(fragments) - 1:
-      next_fragment = fragments[i + 1]
-      if next_fragment is None:
-        fragments[i] = None
-        fragments[i + 1] = fragment
-
-  return [f or "" for f in fragments]
-
-def _normalize_user_input(user_lines: list[str]) -> str | None:
-  empty_lines_count: int = 0
-  lines: list[str] = []
-  for line in user_lines:
-    if is_empty(line):
-      empty_lines_count += 1
-    else:
-      if lines:
-        if empty_lines_count >= 2:
-          lines.append("")
-          lines.append("")
-        elif empty_lines_count == 1:
-          lines.append("")
-      lines.append(clean_spaces(line))
-  if not lines:
-    return None
-  return "\n".join(lines)

epub_translator/translation/types.py

@@ -1,45 +0,0 @@
-from enum import Enum, IntEnum
-from dataclasses import dataclass
-
-
-class Incision(IntEnum):
-  MUST_BE = 3
-  MOST_LIKELY = 2
-  IMPOSSIBLE = 0
-  UNCERTAIN = 1
-
-@dataclass
-class Fragment:
-  text: str
-  start_incision: Incision
-  end_incision: Incision
-
-class Language(Enum):
-  SIMPLIFIED_CHINESE = "zh-Hans"
-  TRADITIONAL_CHINESE = "zh-Hant"
-  ENGLISH = "en"
-  FRENCH = "fr"
-  GERMAN = "de"
-  SPANISH = "es"
-  RUSSIAN = "ru"
-  ITALIAN = "it"
-  PORTUGUESE = "pt"
-  JAPANESE = "ja"
-  KOREAN = "ko"
-
-_LANGUAGE_NAMES = {
-  Language.SIMPLIFIED_CHINESE: "简体中文",
-  Language.TRADITIONAL_CHINESE: "繁体中文",
-  Language.ENGLISH: "英语",
-  Language.FRENCH: "法语",
-  Language.GERMAN: "德语",
-  Language.SPANISH: "西班牙语",
-  Language.RUSSIAN: "俄语",
-  Language.ITALIAN: "意大利语",
-  Language.PORTUGUESE: "葡萄牙语",
-  Language.JAPANESE: "日语",
-  Language.KOREAN: "韩语",
-}
-
-def language_chinese_name(language: Language) -> str:
-  return _LANGUAGE_NAMES[language]

epub_translator/translation/utils.py

@@ -1,11 +0,0 @@
-import re
-
-
-_EMPTY_LINE = re.compile(r"^\s*$")
-_SPACE = re.compile(r"\s+")
-
-def is_empty(text: str) -> bool:
-  return bool(_EMPTY_LINE.match(text))
-
-def clean_spaces(text: str) -> str:
-  return _SPACE.sub(" ", text.strip())

epub_translator/xml/decoder.py

@@ -1,71 +0,0 @@
-from typing import Generator, Iterable
-from xml.etree.ElementTree import Element
-
-from .tag import Tag, TagKind
-from .parser import parse_tags
-from .transform import tag_to_element
-from .utils import clone
-
-# why implement XML decoding?
-# https://github.com/oomol-lab/pdf-craft/issues/149
-def decode_friendly(chars: Iterable[str], tags: Iterable[str] | str = ()) -> Generator[Element, None, None]:
-  if isinstance(tags, str):
-    tags = set((tags,))
-  else:
-    tags = set(tags)
-
-  for element in _collect_elements(chars):
-    if element.tag in tags or len(tags) == 0:
-      yield clone(element)
-
-def _collect_elements(chars: Iterable[str]) -> Generator[Element, None, None]:
-  opening_stack: list[Element] = []
-  last_closed_element: Element | None = None
-
-  for cell in parse_tags(chars):
-    if isinstance(cell, Tag):
-      tag: Tag = cell
-      element = tag_to_element(tag)
-      if tag.kind == TagKind.CLOSING:
-        popped = _pop_element(tag.name, opening_stack)
-        if popped is not None:
-          yield popped
-          last_closed_element = popped
-        elif last_closed_element is not None:
-          _append_to_tail(last_closed_element, tag.proto)
-      else:
-        if opening_stack:
-          opening_stack[-1].append(element)
-        if tag.kind == TagKind.SELF_CLOSING:
-          yield element
-          last_closed_element = element
-        elif tag.kind == TagKind.OPENING:
-          opening_stack.append(element)
-          last_closed_element = None
-
-    elif last_closed_element is not None:
-      _append_to_tail(last_closed_element, cell)
-
-    elif opening_stack:
-      opening_stack[-1].text = cell
-
-def _append_to_tail(element: Element, text: str) -> None:
-  if element.tail:
-    element.tail += text
-  else:
-    element.tail = text
-
-def _pop_element(tag_name: str, opening_stack: list[Element]) -> Element | None:
-  index = -1
-  for i in range(len(opening_stack) - 1, -1, -1):
-    opening_element = opening_stack[i]
-    if tag_name == opening_element.tag:
-      index = i
-      break
-  if index == -1:
-    return None
-
-  popped: Element | None = None
-  for _ in range(len(opening_stack) - index):
-    popped = opening_stack.pop()
-  return popped

epub_translator/xml/encoder.py

@@ -1,95 +0,0 @@
-from io import StringIO
-from typing import Callable
-from html import escape as escape_html
-from xml.etree.ElementTree import Element
-
-from .tag import Tag, TagKind
-from .parser import parse_tags
-from .transform import element_to_tag
-
-# why implement XML encoding?
-# https://github.com/oomol-lab/pdf-craft/issues/149
-def encode_friendly(element: Element, indent: int = 2) -> str:
-  buffer = StringIO()
-  _encode_element(
-    buffer=buffer,
-    element=element,
-    indent=indent,
-    depth=0,
-    escape=_escape_text,
-  )
-  return buffer.getvalue()
-
-def _escape_text(text: str) -> str:
-  buffer = StringIO()
-  for cell in parse_tags(text):
-    if isinstance(cell, Tag):
-      cell = escape_html(str(cell))
-    buffer.write(cell)
-  return buffer.getvalue()
-
-def encode(element: Element, indent: int = 2) -> str:
-  buffer = StringIO()
-  _encode_element(
-    buffer=buffer,
-    element=element,
-    indent=indent,
-    depth=0,
-    escape=escape_html,
-  )
-  return buffer.getvalue()
-
-_TINY_TEXT_LEN = 35
-
-def _encode_element(
-      buffer: StringIO,
-      element: Element,
-      indent: int,
-      depth: int,
-      escape: Callable[[str], str],
-    ) -> None:
-
-  _write_indent(buffer, indent, depth)
-  if len(element) == 0 and not element.text:
-    tag = element_to_tag(element, TagKind.SELF_CLOSING)
-    buffer.write(str(tag))
-  else:
-    text = (element.text or "").strip()
-    opening_tag = element_to_tag(element, TagKind.OPENING)
-    closing_tag = element_to_tag(element, TagKind.CLOSING)
-    buffer.write(str(opening_tag))
-    is_one_line = (
-      len(text) <= _TINY_TEXT_LEN and
-      len(element) == 0 and
-      "\n" not in text
-    )
-    if text:
-      if not is_one_line:
-        buffer.write("\n")
-        _write_indent(buffer, indent, depth + 1)
-      buffer.write(escape(text))
-
-    for child in element:
-      buffer.write("\n")
-      _encode_element(
-        buffer=buffer,
-        element=child,
-        indent=indent,
-        depth=depth + 1,
-        escape=escape,
-      )
-      child_tail = (child.tail or "").strip()
-      if child_tail:
-        buffer.write("\n")
-        _write_indent(buffer, indent, depth + 1)
-        buffer.write(escape(child_tail))
-
-    if not is_one_line:
-      buffer.write("\n")
-      _write_indent(buffer, indent, depth)
-
-    buffer.write(str(closing_tag))
-
-def _write_indent(buffer: StringIO, indent: int, depth: int) -> None:
-  for _ in range(indent * depth):
-    buffer.write(" ")

epub_translator/xml/parser.py

@@ -1,172 +0,0 @@
-from io import StringIO
-from typing import Generator, Iterable
-from enum import auto, Enum
-from .tag import is_valid_name_char, is_valid_value_char, Tag, TagKind
-
-
-_SPACES = (" ", "\n")
-
-class _Phase(Enum):
-  OUTSIDE = auto()
-  LEFT_BRACKET = auto()
-  LEFT_SLASH = auto()
-  TAG_NAME = auto()
-  TAG_GAP = auto()
-  ATTRIBUTE_NAME = auto()
-  ATTRIBUTE_NAME_EQUAL = auto()
-  ATTRIBUTE_VALUE = auto()
-  MUST_CLOSING_SIGN = auto()
-
-class _ParsedResult(Enum):
-  Continue = auto()
-  Success = auto()
-  Failed = auto()
-
-def parse_tags(chars: Iterable[str]) -> Generator[str | Tag, None, None]:
-  yield from _XMLTagsParser().do(chars)
-
-class _XMLTagsParser:
-  def __init__(self):
-    self._outside_buffer: StringIO = StringIO()
-    self._tag_buffer: StringIO = StringIO()
-    self._tag: Tag | None = None
-    self._phase: _Phase = _Phase.OUTSIDE
-
-  def do(self, chars: Iterable[str]) -> Generator[str | Tag, None, None]:
-    for char in chars:
-      parsed_result = self._parse_char(char)
-      yield from self._generate_by_result(parsed_result)
-
-    self._outside_buffer.write(self._tag_buffer.getvalue())
-    outside_text = self._outside_buffer.getvalue()
-    if outside_text != "":
-      yield outside_text
-
-  def _parse_char(self, char: str) -> _ParsedResult:
-    parsed_result: _ParsedResult = _ParsedResult.Continue
-
-    if self._phase == _Phase.OUTSIDE:
-      if char != "<":
-        self._outside_buffer.write(char)
-      else:
-        self._phase = _Phase.LEFT_BRACKET
-        self._tag_buffer.write(char)
-        self._tag = Tag(
-          kind=TagKind.OPENING,
-          name="",
-          proto="",
-          attributes=[],
-        )
-    else:
-      self._tag_buffer.write(char)
-
-      if self._phase == _Phase.LEFT_BRACKET:
-        if char == "/":
-          self._tag.kind = TagKind.CLOSING
-          self._phase = _Phase.LEFT_SLASH
-        elif is_valid_name_char(char):
-          self._tag.name += char
-          self._phase = _Phase.TAG_NAME
-        else:
-          parsed_result = _ParsedResult.Failed
-
-      elif self._phase == _Phase.LEFT_SLASH:
-        if is_valid_name_char(char):
-          self._tag.name += char
-          self._phase = _Phase.TAG_NAME
-        else:
-          parsed_result = _ParsedResult.Failed
-
-      elif self._phase == _Phase.TAG_NAME:
-        if char in _SPACES:
-          self._phase = _Phase.TAG_GAP
-        elif is_valid_name_char(char):
-          self._tag.name += char
-        elif char == ">":
-          parsed_result = _ParsedResult.Success
-        elif char == "/" and self._tag.kind == TagKind.OPENING:
-          self._tag.kind = TagKind.SELF_CLOSING
-          self._phase = _Phase.MUST_CLOSING_SIGN
-        else:
-          parsed_result = _ParsedResult.Failed
-
-      elif self._phase == _Phase.TAG_GAP:
-        if char in _SPACES:
-          pass
-        elif is_valid_name_char(char):
-          self._tag.attributes.append((char, ""))
-          self._phase = _Phase.ATTRIBUTE_NAME
-        elif char == ">":
-          parsed_result = _ParsedResult.Success
-        elif char == "/" and self._tag.kind == TagKind.OPENING:
-          self._tag.kind = TagKind.SELF_CLOSING
-          self._phase = _Phase.MUST_CLOSING_SIGN
-        else:
-          parsed_result = _ParsedResult.Failed
-
-      elif self._phase == _Phase.ATTRIBUTE_NAME:
-        if is_valid_name_char(char):
-          attr_name, attr_value = self._tag.attributes[-1]
-          attr_name = attr_name + char
-          self._tag.attributes[-1] = (attr_name, attr_value)
-        elif char == "=":
-          self._phase = _Phase.ATTRIBUTE_NAME_EQUAL
-        else:
-          parsed_result = _ParsedResult.Failed
-
-      elif self._phase == _Phase.ATTRIBUTE_NAME_EQUAL:
-        if char == "\"":
-          self._phase = _Phase.ATTRIBUTE_VALUE
-        else:
-          parsed_result = _ParsedResult.Failed
-
-      elif self._phase == _Phase.ATTRIBUTE_VALUE:
-        if is_valid_value_char(char):
-          attr_name, attr_value = self._tag.attributes[-1]
-          attr_value = attr_value + char
-          self._tag.attributes[-1] = (attr_name, attr_value)
-        elif char == "\"":
-          self._phase = _Phase.TAG_GAP
-        else:
-          parsed_result = _ParsedResult.Failed
-
-      elif self._phase == _Phase.MUST_CLOSING_SIGN:
-        if char == ">":
-          parsed_result = _ParsedResult.Success
-        else:
-          parsed_result = _ParsedResult.Failed
-
-    return parsed_result
-
-  def _generate_by_result(self, parsed_result: _ParsedResult) -> Generator[str | Tag, None, None]:
-    if parsed_result == _ParsedResult.Success:
-      assert self._tag is not None
-      if self._is_tag_valid(self._tag):
-        outside_text = self._outside_buffer.getvalue()
-        self._clear_buffer(self._outside_buffer)
-        self._clear_buffer(self._tag_buffer)
-        if outside_text != "":
-          yield outside_text
-        yield self._tag
-      else:
-        self._tag.proto = self._tag_buffer.getvalue()
-        self._outside_buffer.write(self._tag.proto)
-        self._clear_buffer(self._tag_buffer)
-      self._tag = None
-      self._phase = _Phase.OUTSIDE
-
-    elif parsed_result == _ParsedResult.Failed:
-      self._outside_buffer.write(self._tag_buffer.getvalue())
-      self._clear_buffer(self._tag_buffer)
-      self._phase = _Phase.OUTSIDE
-
-  def _is_tag_valid(self, tag: Tag) -> bool:
-    if tag.kind == TagKind.CLOSING and len(tag.attributes) > 0:
-      return False
-    if tag.find_invalid_name() is not None:
-      return False
-    return True
-
-  def _clear_buffer(self, buffer: StringIO):
-    buffer.truncate(0)
-    buffer.seek(0)